From 4318b6cec5ba939e261655c46b0ea886a4799549 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 31 Dec 2025 23:05:57 -0400 Subject: [PATCH] Add SEO files and real documentation content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add permissive robots.txt welcoming all crawlers including AI - Add dynamic sitemap.xml with all pages and tools - Add real documentation content (Getting Started, Installation, First Tool, Publishing, Providers) - Update docs route to use content from docs_content.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/smarttools/web/docs_content.py | 400 +++++++++++++++++++++++++++ src/smarttools/web/routes.py | 127 ++++++++- src/smarttools/web/static/robots.txt | 47 ++++ 3 files changed, 559 insertions(+), 15 deletions(-) create mode 100644 src/smarttools/web/docs_content.py create mode 100644 src/smarttools/web/static/robots.txt diff --git a/src/smarttools/web/docs_content.py b/src/smarttools/web/docs_content.py new file mode 100644 index 0000000..e1c86da --- /dev/null +++ b/src/smarttools/web/docs_content.py @@ -0,0 +1,400 @@ +"""Documentation content for SmartTools web UI. + +This module contains the actual documentation text that gets rendered +on the /docs pages. Content is stored as markdown-ish HTML for simplicity. +""" + +DOCS = { + "getting-started": { + "title": "Getting Started", + "description": "Learn how to install SmartTools and create your first AI-powered CLI tool", + "content": """ +

SmartTools lets you build custom AI-powered CLI commands using simple YAML configuration. +Create tools that work with any AI provider and compose them like Unix pipes.

+ +

What is SmartTools?

+

SmartTools is a lightweight personal tool builder that lets you:

+ + +

Quick Start

+

Get up and running in under a minute:

+ +
# Install SmartTools
+pip install smarttools
+
+# Create your first tool interactively
+smarttools create
+
+# Or install a tool from the registry
+smarttools registry install official/summarize
+
+# Use it!
+cat article.txt | summarize
+ +

How It Works

+

Each tool is a YAML file that defines:

+
    +
  1. Arguments - Custom flags your tool accepts
    2. +
  2. Steps - Prompts to send to AI or Python code to run
    3. +
  3. Output - How to format the final result
    4. +
+ +

Here's a simple example:

+
name: summarize
+version: "1.0.0"
+description: Summarize text using AI
+
+arguments:
+  - flag: --max-length
+    variable: max_length
+    default: "200"
+    description: Maximum summary length in words
+
+steps:
+  - type: prompt
+    provider: claude
+    prompt: |
+      Summarize the following text in {max_length} words or less:
+
+      {input}
+    output_var: summary
+
+output: "{summary}"
+ +

Next Steps

+ +""", + "headings": [ + ("what-is-smarttools", "What is SmartTools?"), + ("quick-start", "Quick Start"), + ("how-it-works", "How It Works"), + ("next-steps", "Next Steps"), + ], + }, + + "installation": { + "title": "Installation", + "description": "How to install SmartTools on your system", + "parent": "getting-started", + "content": """ +

SmartTools requires Python 3.8+ and works on Linux, macOS, and Windows.

+ +

Install with pip

+

The simplest way to install SmartTools:

+
pip install smarttools
+ +

Or with pipx for isolated installation:

+
pipx install smarttools
+ +

Verify Installation

+
smarttools --version
+smarttools --help
+ +

Configure a Provider

+

SmartTools needs at least one AI provider configured. The easiest is Claude CLI:

+ +
# Install Claude CLI (if you have an Anthropic API key)
+pip install claude-cli
+
+# Or use OpenAI
+pip install openai
+
+# Configure your provider
+smarttools config
+ +

Wrapper Scripts Location

+

SmartTools installs wrapper scripts to ~/.local/bin/. Make sure this is in your PATH:

+
# Add to ~/.bashrc or ~/.zshrc
+export PATH="$HOME/.local/bin:$PATH"
+ +

Development Installation

+

To contribute or modify SmartTools:

+
git clone https://gitea.brrd.tech/rob/SmartTools.git
+cd SmartTools
+pip install -e ".[dev]"
+""", + "headings": [ + ("pip-install", "Install with pip"), + ("verify", "Verify Installation"), + ("configure-provider", "Configure a Provider"), + ("wrapper-scripts", "Wrapper Scripts Location"), + ("development-install", "Development Installation"), + ], + }, + + "first-tool": { + "title": "Your First Tool", + "description": "Create your first SmartTools command step by step", + "parent": "getting-started", + "content": """ +

Let's create a simple tool that explains code. You'll learn the basics of tool configuration.

+ +

Create the Tool

+

Run the interactive creator:

+
smarttools create
+ +

Or create the file manually at ~/.smarttools/explain/config.yaml:

+
name: explain
+version: "1.0.0"
+description: Explain code or concepts in simple terms
+category: code
+
+arguments:
+  - flag: --level
+    variable: level
+    default: "beginner"
+    description: "Explanation level: beginner, intermediate, or expert"
+
+steps:
+  - type: prompt
+    provider: claude
+    prompt: |
+      Explain the following in simple terms suitable for a {level}:
+
+      {input}
+
+      Be concise but thorough. Use examples where helpful.
+    output_var: explanation
+
+output: "{explanation}"
+ +

Test Your Tool

+
# Explain some code
+echo "def fib(n): return n if n < 2 else fib(n-1) + fib(n-2)" | explain
+
+# Explain for an expert
+cat complex_algorithm.py | explain --level expert
+ +

Understanding the Config

+ +

Arguments

+

Each argument becomes a CLI flag. The variable name is used in templates:

+
arguments:
+  - flag: --level        # CLI flag: --level beginner
+    variable: level      # Use as {level} in prompts
+    default: "beginner"  # Default if not specified
+ +

Steps

+

Steps run in order. Each step can be a prompt or Python code:

+
steps:
+  - type: prompt
+    provider: claude     # Which AI to use
+    prompt: "..."        # The prompt template
+    output_var: result   # Store response in {result}
+ +

Output

+

The output template formats the final result:

+
output: "{explanation}"  # Print the explanation variable
+ +

Next Steps

+ +""", + "headings": [ + ("create-tool", "Create the Tool"), + ("test-it", "Test Your Tool"), + ("understanding-config", "Understanding the Config"), + ("next", "Next Steps"), + ], + }, + + "publishing": { + "title": "Publishing Tools", + "description": "Share your tools with the SmartTools community", + "content": """ +

Share your tools with the community by publishing to the SmartTools Registry.

+ +

Before Publishing

+

Make sure your tool has:

+ + +

Create an Account

+

Register at the registry to get your publisher namespace.

+ +

Get an API Token

+
    +
  1. Go to your Dashboard → Tokens
    2. +
  2. Click "Create New Token"
    3. +
  3. Copy the token (shown only once!)
    4. +
+ +

Publish Your Tool

+
# Navigate to your tool directory
+cd ~/.smarttools/my-tool/
+
+# First time: enter your token when prompted
+smarttools registry publish
+
+# Dry run to validate without publishing
+smarttools registry publish --dry-run
+ +

Versioning

+

Published versions are immutable. To update a tool:

+
    +
  1. Make your changes
    2. +
  2. Bump the version in config.yaml
    3. +
  3. Run smarttools registry publish
    4. +
+ +

Best Practices

+ +""", + "headings": [ + ("before-publishing", "Before Publishing"), + ("create-account", "Create an Account"), + ("get-token", "Get an API Token"), + ("publish", "Publish Your Tool"), + ("versioning", "Versioning"), + ("best-practices", "Best Practices"), + ], + }, + + "providers": { + "title": "AI Providers", + "description": "Configure different AI providers for your tools", + "content": """ +

SmartTools works with any AI provider that has a CLI interface. Configure providers in +~/.smarttools/providers.yaml.

+ +

Provider Configuration

+

Create or edit ~/.smarttools/providers.yaml:

+ +
providers:
+  - name: claude
+    command: "claude -p"
+
+  - name: openai
+    command: "openai-cli"
+
+  - name: ollama
+    command: "ollama run llama2"
+
+  - name: mock
+    command: "echo '[MOCK RESPONSE]'"
+ +

Using Providers in Tools

+

Specify the provider in your step:

+
steps:
+  - type: prompt
+    provider: claude  # Uses the "claude" provider from config
+    prompt: "..."
+    output_var: response
+ + + +

Claude (Anthropic)

+
# Install
+pip install claude-cli
+
+# Configure with your API key
+export ANTHROPIC_API_KEY="sk-ant-..."
+ +
# providers.yaml
+providers:
+  - name: claude
+    command: "claude -p"
+ +

OpenAI

+
# Install
+pip install openai-cli
+
+# Configure
+export OPENAI_API_KEY="sk-..."
+ +

Ollama (Local)

+
# Install Ollama from ollama.ai
+# Pull a model
+ollama pull llama2
+ +
# providers.yaml
+providers:
+  - name: ollama
+    command: "ollama run llama2"
+ +

Testing with Mock Provider

+

Use the mock provider to test tools without API calls:

+
providers:
+  - name: mock
+    command: "echo 'This is a mock response for testing'"
+ +

Choosing a Provider

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ProviderBest ForCost
ClaudeComplex reasoning, long contextPay per token
OpenAIGeneral purpose, fastPay per token
OllamaPrivacy, offline useFree (local)
+""", + "headings": [ + ("provider-config", "Provider Configuration"), + ("using-providers", "Using Providers in Tools"), + ("popular-providers", "Popular Providers"), + ("testing", "Testing with Mock Provider"), + ("provider-selection", "Choosing a Provider"), + ], + }, +} + + +def get_doc(path: str) -> dict: + """Get documentation content by path.""" + # Normalize path + path = path.strip("/").replace("docs/", "") or "getting-started" + return DOCS.get(path, None) + + +def get_toc(): + """Get table of contents structure.""" + from types import SimpleNamespace + return [ + SimpleNamespace(slug="getting-started", title="Getting Started", children=[ + SimpleNamespace(slug="installation", title="Installation"), + SimpleNamespace(slug="first-tool", title="Your First Tool"), + ]), + SimpleNamespace(slug="publishing", title="Publishing", children=[]), + SimpleNamespace(slug="providers", title="Providers", children=[]), + ] diff --git a/src/smarttools/web/routes.py b/src/smarttools/web/routes.py index 0da3bbf..2997288 100644 --- a/src/smarttools/web/routes.py +++ b/src/smarttools/web/routes.py @@ -455,22 +455,30 @@ def dashboard_settings(): @web_bp.route("/docs", defaults={"path": ""}, endpoint="docs") @web_bp.route("/docs/", endpoint="docs") def docs(path: str): - toc = [ - SimpleNamespace(slug="getting-started", title="Getting Started", children=[ - SimpleNamespace(slug="installation", title="Installation"), - SimpleNamespace(slug="first-tool", title="Your First Tool"), - ]), - SimpleNamespace(slug="publishing", title="Publishing", children=[]), - SimpleNamespace(slug="providers", title="Providers", children=[]), - ] + from .docs_content import get_doc, get_toc + + toc = get_toc() current = path or "getting-started" - page = SimpleNamespace( - title=_title_case(current), - description="SmartTools documentation", - content_html=f"

Documentation for {escape(current)} is coming soon.

", - headings=[], - parent=None, - ) + + # Get content from docs_content module + doc = get_doc(current) + if doc: + page = SimpleNamespace( + title=doc["title"], + description=doc["description"], + content_html=Markup(doc["content"]), + headings=doc.get("headings", []), + parent=doc.get("parent"), + ) + else: + page = SimpleNamespace( + title=_title_case(current), + description="SmartTools documentation", + content_html=Markup(f"

Documentation for {escape(current)} is coming soon.

"), + headings=[], + parent=None, + ) + show_ads = current_app.config.get("SHOW_ADS", False) return render_template( "pages/docs.html", @@ -538,3 +546,92 @@ def forgot_password(): title="Reset Password", body="Password resets are not available yet. Please contact support if needed.", ) + + +@web_bp.route("/robots.txt", endpoint="robots") +def robots(): + """Serve robots.txt - we welcome all crawlers including AI.""" + from flask import send_from_directory + return send_from_directory( + web_bp.static_folder, + "robots.txt", + mimetype="text/plain" + ) + + +@web_bp.route("/sitemap.xml", endpoint="sitemap") +def sitemap(): + """Generate dynamic sitemap.xml with all public pages and tools.""" + from flask import Response + from datetime import datetime + + conn = connect_db() + try: + tools = query_all( + conn, + """ + WITH latest AS ( + SELECT owner, name, MAX(id) AS max_id + FROM tools + WHERE version NOT LIKE '%-%' + GROUP BY owner, name + ) + SELECT t.owner, t.name, t.published_at + FROM tools t + JOIN latest ON t.owner = latest.owner AND t.name = latest.name AND t.id = latest.max_id + ORDER BY t.published_at DESC + """, + [], + ) + tools = [dict(row) for row in tools] + finally: + conn.close() + + base_url = request.url_root.rstrip("/") + today = datetime.utcnow().strftime("%Y-%m-%d") + + # Static pages + static_pages = [ + ("/", "1.0", "daily"), + ("/tools", "0.9", "daily"), + ("/docs", "0.8", "weekly"), + ("/docs/getting-started", "0.8", "weekly"), + ("/docs/installation", "0.7", "weekly"), + ("/docs/first-tool", "0.7", "weekly"), + ("/docs/publishing", "0.7", "weekly"), + ("/docs/providers", "0.7", "weekly"), + ("/tutorials", "0.7", "weekly"), + ("/about", "0.5", "monthly"), + ("/community", "0.5", "weekly"), + ("/donate", "0.4", "monthly"), + ("/privacy", "0.3", "monthly"), + ("/terms", "0.3", "monthly"), + ] + + xml_parts = [''] + xml_parts.append('') + + # Add static pages + for path, priority, freq in static_pages: + xml_parts.append(f""" + {base_url}{path} + {today} + {freq} + {priority} + """) + + # Add tool pages + for tool in tools: + pub_date = tool.get("published_at", today) + if pub_date and "T" in str(pub_date): + pub_date = str(pub_date).split("T")[0] + xml_parts.append(f""" + {base_url}/tools/{tool['owner']}/{tool['name']} + {pub_date or today} + weekly + 0.6 + """) + + xml_parts.append("") + + return Response("\n".join(xml_parts), mimetype="application/xml") diff --git a/src/smarttools/web/static/robots.txt b/src/smarttools/web/static/robots.txt new file mode 100644 index 0000000..f1002c3 --- /dev/null +++ b/src/smarttools/web/static/robots.txt @@ -0,0 +1,47 @@ +# SmartTools Registry - robots.txt +# We welcome all crawlers! Index everything to help users find us +# and enable AI assistants to help solve SmartTools issues. + +User-agent: * +Allow: / + +# Encourage full indexing of documentation and tutorials +Allow: /docs/ +Allow: /tutorials/ +Allow: /tools/ +Allow: /categories/ + +# Allow AI crawlers explicitly (we want AI to learn about SmartTools) +User-agent: GPTBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: Anthropic-AI +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +# Only block private user areas (not useful to index) +Disallow: /dashboard/ +Disallow: /api/v1/me/ +Disallow: /api/v1/tokens + +# Allow API docs to be indexed (helpful for developers) +Allow: /api/ + +# Sitemap location +Sitemap: https://registry.smarttools.dev/sitemap.xml + +# Crawl-delay: none - crawl as fast as you want!