Refactor setup: new/update detection, split Python servers, add MCP server

- laptop_full_setup.sh now handles both fresh install and updates cleanly
  (detects existing install, --force flag to overwrite config files)
- server.py: standalone RAG server with AST-aware code chunking (Python),
  pattern-split for JS/TS/Go, /ingest/repo and /webhook/gitea|github endpoints
- mcp_server.py: new MCP server (port 8002/SSE) with Claude Code-equivalent
  tools: bash, file ops, ripgrep search, git ops, Gitea API, GitHub API,
  RAG repo ingest
- docker-compose: adds mcp-server service, workspace volume, env_file for tokens
- .env preserved on update (tokens never overwritten)
- GPU: OLLAMA_NUM_GPU=999 auto-adapts to any VRAM size (no hard-coded 6GB)
- ZIM downloads remain in kiwix_download.sh (separate, large files)

https://claude.ai/code/session_012gDnantBmFTWZGCiKyjazx

mcp_server.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+MCP Server — Claude Code-equivalent tools for Open WebUI / Claude Code CLI.
+Tools: bash, file read/write/list, code search, git ops, Gitea API, repo ingest.
+Connects via SSE on port 8002 — add to Open WebUI Tools or ~/.claude/mcp.json
+"""
+import os, subprocess, textwrap
+from pathlib import Path
+
+import httpx
+from mcp.server.fastmcp import FastMCP
+
+WORKSPACE   = Path(os.getenv("WORKSPACE_DIR", "/workspace"))
+REPOS_DIR   = Path(os.getenv("REPOS_DIR",     "/repos"))
+GITEA_URL   = os.getenv("GITEA_URL",   "http://gitea:3000")
+GITEA_TOKEN = os.getenv("GITEA_TOKEN", "")
+GITHUB_TOKEN= os.getenv("GITHUB_TOKEN","")
+RAG_URL     = os.getenv("RAG_URL",     "http://rag-server:8001")
+
+mcp = FastMCP("local-dev-tools")
+
+# ── bash ──────────────────────────────────────────────────────────────────────
+@mcp.tool()
+def bash(command: str, cwd: str = "") -> str:
+    """Run a shell command. Default cwd is /workspace."""
+    work = Path(cwd) if cwd else WORKSPACE
+    work.mkdir(parents=True, exist_ok=True)
+    try:
+        r = subprocess.run(command, shell=True, cwd=work, timeout=120,
+                           capture_output=True, text=True)
+        out = r.stdout + (f"\n[stderr]\n{r.stderr}" if r.stderr else "")
+        if r.returncode != 0:
+            out += f"\n[exit {r.returncode}]"
+        return out or "(no output)"
+    except subprocess.TimeoutExpired:
+        return "[timeout after 120s]"
+    except Exception as e:
+        return f"[error] {e}"
+
+# ── file ops ──────────────────────────────────────────────────────────────────
+@mcp.tool()
+def read_file(path: str) -> str:
+    """Read a file. Use absolute path or relative to /workspace."""
+    p = Path(path) if Path(path).is_absolute() else WORKSPACE / path
+    if not p.exists():
+        return f"[not found] {p}"
+    if p.stat().st_size > 500_000:
+        return f"[too large — {p.stat().st_size//1024}KB]"
+    return p.read_text(encoding="utf-8", errors="replace")
+
+@mcp.tool()
+def write_file(path: str, content: str) -> str:
+    """Write content to a file (creates parent dirs). Relative to /workspace."""
+    p = Path(path) if Path(path).is_absolute() else WORKSPACE / path
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(content, encoding="utf-8")
+    return f"Wrote {len(content)} chars to {p}"
+
+@mcp.tool()
+def list_files(path: str = "", pattern: str = "**/*") -> str:
+    """List files matching a glob pattern."""
+    base = Path(path) if path else WORKSPACE
+    if not base.exists():
+        return f"[not found] {base}"
+    files = sorted(str(f.relative_to(base)) for f in base.glob(pattern) if f.is_file())
+    return "\n".join(files[:500]) or "(empty)"
+
+@mcp.tool()
+def search_code(query: str, path: str = "", glob: str = "",
+                case_sensitive: bool = False) -> str:
+    """Search file contents with ripgrep. Returns file:line matches."""
+    base = path or str(WORKSPACE)
+    cmd = ["rg", "--line-number", "--no-heading"]
+    if not case_sensitive:
+        cmd.append("-i")
+    if glob:
+        cmd += ["-g", glob]
+    cmd += [query, base]
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        lines = r.stdout.strip().splitlines()
+        if len(lines) > 200:
+            lines = lines[:200] + [f"… ({len(r.stdout.splitlines())-200} more)"]
+        return "\n".join(lines) or "(no matches)"
+    except FileNotFoundError:
+        # ripgrep not installed, fall back to grep
+        r = subprocess.run(["grep", "-rn", query, base],
+                           capture_output=True, text=True, timeout=30)
+        return r.stdout[:8000] or "(no matches)"
+    except Exception as e:
+        return f"[error] {e}"
+
+# ── git ───────────────────────────────────────────────────────────────────────
+def _git(args: list[str], repo: str = "") -> str:
+    cwd = Path(repo) if repo else WORKSPACE
+    r = subprocess.run(["git"] + args, cwd=cwd,
+                       capture_output=True, text=True, timeout=60)
+    return (r.stdout + r.stderr).strip() or "(no output)"
+
+@mcp.tool()
+def git_status(repo: str = "") -> str:
+    """Show git status of a repo (default: /workspace)."""
+    return _git(["status", "--short"], repo)
+
+@mcp.tool()
+def git_diff(repo: str = "", cached: bool = False) -> str:
+    """Show git diff (staged if cached=True)."""
+    args = ["diff", "--stat", "--cached"] if cached else ["diff", "--stat"]
+    return _git(args, repo) + "\n\n" + _git(
+        ["diff", "--cached"] if cached else ["diff"], repo)
+
+@mcp.tool()
+def git_log(repo: str = "", n: int = 10) -> str:
+    """Show last n git commits."""
+    return _git(["log", f"-{n}", "--oneline", "--decorate"], repo)
+
+@mcp.tool()
+def git_commit(message: str, repo: str = "", add_all: bool = True) -> str:
+    """Stage all changes and create a commit."""
+    if add_all:
+        _git(["add", "-A"], repo)
+    return _git(["commit", "-m", message], repo)
+
+@mcp.tool()
+def git_checkout(branch: str, repo: str = "", create: bool = False) -> str:
+    """Checkout a branch, optionally creating it."""
+    args = ["checkout", "-b", branch] if create else ["checkout", branch]
+    return _git(args, repo)
+
+# ── Gitea API ─────────────────────────────────────────────────────────────────
+def _gitea(method: str, path: str, body: dict = {}) -> dict:
+    if not GITEA_TOKEN:
+        return {"error": "GITEA_TOKEN not set in .env"}
+    url = f"{GITEA_URL}/api/v1{path}"
+    headers = {"Authorization": f"token {GITEA_TOKEN}",
+               "Content-Type": "application/json"}
+    r = httpx.request(method, url, json=body or None, headers=headers, timeout=30)
+    try:
+        return r.json()
+    except Exception:
+        return {"status": r.status_code, "text": r.text}
+
+@mcp.tool()
+def gitea_list_repos() -> str:
+    """List your Gitea repos."""
+    repos = _gitea("GET", "/repos/search?limit=50")
+    if "error" in repos:
+        return repos["error"]
+    return "\n".join(f"{r['full_name']} — {r.get('description','')}"
+                     for r in repos.get("data", []))
+
+@mcp.tool()
+def gitea_create_repo(name: str, private: bool = True, description: str = "") -> str:
+    """Create a new Gitea repository."""
+    r = _gitea("POST", "/user/repos",
+               {"name": name, "private": private, "description": description,
+                "auto_init": True, "default_branch": "main"})
+    return r.get("html_url") or str(r)
+
+@mcp.tool()
+def gitea_create_issue(repo: str, title: str, body: str = "") -> str:
+    """Create an issue on a Gitea repo (format: owner/repo)."""
+    r = _gitea("POST", f"/repos/{repo}/issues", {"title": title, "body": body})
+    return r.get("html_url") or str(r)
+
+# ── GitHub API ────────────────────────────────────────────────────────────────
+@mcp.tool()
+def github_api(method: str, endpoint: str, body: str = "") -> str:
+    """Call the GitHub REST API. endpoint e.g. /repos/owner/repo/issues"""
+    if not GITHUB_TOKEN:
+        return "GITHUB_TOKEN not set in .env"
+    import json as _json
+    headers = {"Authorization": f"Bearer {GITHUB_TOKEN}",
+               "Accept": "application/vnd.github+json"}
+    r = httpx.request(method.upper(), f"https://api.github.com{endpoint}",
+                      json=_json.loads(body) if body else None,
+                      headers=headers, timeout=30)
+    try:
+        return _json.dumps(r.json(), indent=2)
+    except Exception:
+        return r.text
+
+# ── RAG ingest ────────────────────────────────────────────────────────────────
+@mcp.tool()
+def ingest_repo(url: str, name: str = "", branch: str = "main") -> str:
+    """Clone a git repo and index it in the RAG code collection."""
+    r = httpx.post(f"{RAG_URL}/ingest/repo",
+                   json={"url": url, "name": name, "branch": branch}, timeout=300)
+    return r.text
+
+@mcp.tool()
+def rag_health() -> str:
+    """Check RAG server status and indexed document counts."""
+    try:
+        r = httpx.get(f"{RAG_URL}/health", timeout=10)
+        return r.text
+    except Exception as e:
+        return f"RAG server unreachable: {e}"
+
+if __name__ == "__main__":
+    import uvicorn
+    app = mcp.get_asgi_app()
+    uvicorn.run(app, host="0.0.0.0", port=8002)

server.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+"""
+RAG Server — code-aware chunking, multi-collection, repo ingest, webhooks.
+Collections: papers (PDFs/text), code (source files, AST-split for Python)
+"""
+import ast, fnmatch, hashlib, json, logging, os, re, subprocess, threading, time
+from pathlib import Path
+from typing import Any, Optional
+
+import chromadb
+import httpx
+from chromadb.utils.embedding_functions import OllamaEmbeddingFunction
+from fastapi import FastAPI, HTTPException, Request, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("rag")
+
+OLLAMA_URL  = os.getenv("OLLAMA_URL",  "http://ollama:11434")
+CHROMA_URL  = os.getenv("CHROMA_URL",  "http://chromadb:8000")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "nomic-embed-text")
+CHAT_MODEL  = os.getenv("CHAT_MODEL",  "qwen2.5:14b")
+PAPERS_DIR  = Path(os.getenv("PAPERS_DIR", "/papers"))
+REPOS_DIR   = Path(os.getenv("REPOS_DIR",  "/repos"))
+TOP_K       = int(os.getenv("TOP_K", "6"))
+
+CODE_EXTS  = {".py",".js",".ts",".tsx",".jsx",".go",".rs",".java",".c",".cpp",
+              ".h",".hpp",".cs",".rb",".sh",".yaml",".yml",".toml",".sql",".md"}
+SKIP_DIRS  = {"node_modules",".git","__pycache__","dist","build",".venv",
+              "venv","env",".next","vendor","target","bin","obj"}
+SKIP_FILES = {"package-lock.json","yarn.lock","pnpm-lock.yaml","Cargo.lock"}
+MAX_BYTES  = 400_000
+
+app = FastAPI(title="RAG Server")
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
+
+# ── ChromaDB ──────────────────────────────────────────────────────────────────
+def _embed_fn():
+    return OllamaEmbeddingFunction(
+        url=f"{OLLAMA_URL}/api/embeddings", model_name=EMBED_MODEL)
+
+def _chroma():
+    host, port = CHROMA_URL.replace("http://","").split(":")
+    return chromadb.HttpClient(host=host, port=int(port))
+
+def get_col(name: str):
+    return _chroma().get_or_create_collection(name, embedding_function=_embed_fn())
+
+# ── chunkers ─────────────────────────────────────────────────────────────────
+def _doc_id(text: str, key: str) -> str:
+    return hashlib.md5(f"{key}|{text[:200]}".encode()).hexdigest()
+
+def _sliding(text: str, size=1000, overlap=150) -> list[str]:
+    chunks, i = [], 0
+    while i < len(text):
+        chunks.append(text[i:i+size])
+        i += size - overlap
+    return [c for c in chunks if c.strip()]
+
+def _chunk_python(src: str) -> list[tuple[str,str]]:
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return []
+    lines = src.splitlines()
+    out = []
+    for node in ast.iter_child_nodes(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+            chunk = "\n".join(lines[node.lineno-1:node.end_lineno])
+            out.append((node.name, chunk[:4000]))
+    return out
+
+def _chunk_file(path: Path, src: str) -> list[tuple[str,str]]:
+    if path.suffix == ".py":
+        pairs = _chunk_python(src)
+        if pairs:
+            return pairs
+    # function/class boundary split for JS/TS/Go/Rust etc.
+    pat = re.compile(
+        r'(?:^|\n)(?=(?:export\s+)?(?:async\s+)?(?:function|class|const\s+\w+\s*=\s*(?:async\s+)?\()'
+        r'|^func |^type |^impl |^pub fn |^fn )',
+        re.MULTILINE)
+    parts = [p.strip() for p in pat.split(src) if p.strip()]
+    if len(parts) > 1:
+        return [(f"s{i}", p[:4000]) for i, p in enumerate(parts)]
+    return [(f"c{i}", c) for i, c in enumerate(_sliding(src, 1200, 200))]
+
+# ── ingest helpers ────────────────────────────────────────────────────────────
+def ingest_file(col, fpath: Path, repo: str = ""):
+    if fpath.stat().st_size > MAX_BYTES or fpath.name in SKIP_FILES:
+        return
+    if any(fnmatch.fnmatch(fpath.name, p) for p in ("*.min.js","*.min.css","*.map")):
+        return
+    try:
+        src = fpath.read_text(encoding="utf-8", errors="ignore")
+    except Exception:
+        return
+    if not src.strip():
+        return
+    rel = str(fpath)
+    pairs = _chunk_file(fpath, src) if fpath.suffix in CODE_EXTS else \
+            [(f"c{i}", c) for i, c in enumerate(_sliding(src))]
+    ids, docs, metas = [], [], []
+    for label, chunk in pairs:
+        if not chunk.strip():
+            continue
+        ids.append(_doc_id(chunk, rel+label))
+        docs.append(chunk)
+        metas.append({"source": rel, "label": label, "repo": repo,
+                       "lang": fpath.suffix.lstrip(".")})
+    if ids:
+        col.upsert(ids=ids, documents=docs, metadatas=metas)
+
+def ingest_dir(col, directory: Path, repo: str = "") -> int:
+    count = 0
+    for f in directory.rglob("*"):
+        if not f.is_file():
+            continue
+        if any(p in f.parts for p in SKIP_DIRS):
+            continue
+        ingest_file(col, f, repo)
+        count += 1
+    log.info("Indexed %d files from %s", count, directory)
+    return count
+
+def ingest_pdfs(col) -> int:
+    try:
+        import pypdf
+    except ImportError:
+        log.warning("pypdf not installed — skipping PDFs")
+        return 0
+    n = 0
+    for pdf in PAPERS_DIR.glob("*.pdf"):
+        try:
+            text = "\n".join(p.extract_text() or ""
+                             for p in pypdf.PdfReader(str(pdf)).pages)
+            for i, chunk in enumerate(_sliding(text)):
+                col.upsert(ids=[_doc_id(chunk, str(pdf)+str(i))],
+                           documents=[chunk],
+                           metadatas=[{"source": str(pdf), "label": f"p{i}",
+                                       "repo": "", "lang": "pdf"}])
+            n += 1
+        except Exception as e:
+            log.warning("PDF %s: %s", pdf.name, e)
+    return n
+
+# ── startup ───────────────────────────────────────────────────────────────────
+def _startup_index():
+    # wait for embed model
+    for _ in range(40):
+        try:
+            r = httpx.get(f"{OLLAMA_URL}/api/tags", timeout=5)
+            if any(EMBED_MODEL in m["name"] for m in r.json().get("models", [])):
+                break
+        except Exception:
+            pass
+        log.info("Waiting for embed model %s…", EMBED_MODEL)
+        time.sleep(5)
+
+    code_col   = get_col("code")
+    papers_col = get_col("papers")
+    for d in REPOS_DIR.iterdir():
+        if d.is_dir():
+            ingest_dir(code_col, d, d.name)
+    ingest_pdfs(papers_col)
+    for f in PAPERS_DIR.glob("*.txt"):
+        ingest_file(papers_col, f)
+    log.info("Startup index complete")
+
+@app.on_event("startup")
+async def on_startup():
+    threading.Thread(target=_startup_index, daemon=True).start()
+
+# ── endpoints ─────────────────────────────────────────────────────────────────
+@app.get("/health")
+async def health():
+    try:
+        cc = _chroma()
+        return {"status": "ok",
+                "code":   cc.get_collection("code",   embedding_function=_embed_fn()).count(),
+                "papers": cc.get_collection("papers", embedding_function=_embed_fn()).count(),
+                "embed":  EMBED_MODEL, "chat": CHAT_MODEL}
+    except Exception as e:
+        return {"status": "error", "detail": str(e)}
+
+class RepoRequest(BaseModel):
+    url: str
+    name: str = ""
+    branch: str = "main"
+
+@app.post("/ingest/repo")
+async def ingest_repo(req: RepoRequest):
+    name = req.name or req.url.rstrip("/").split("/")[-1].removesuffix(".git")
+    dest = REPOS_DIR / name
+    try:
+        if dest.exists():
+            subprocess.run(["git","pull"], cwd=dest, check=True, timeout=120)
+        else:
+            subprocess.run(["git","clone","--depth=1","-b",req.branch,
+                            req.url, str(dest)], check=True, timeout=300)
+    except subprocess.CalledProcessError as e:
+        raise HTTPException(400, str(e))
+    n = ingest_dir(get_col("code"), dest, name)
+    return {"status": "ok", "repo": name, "files": n}
+
+@app.post("/ingest/papers")
+async def trigger_papers(bg: BackgroundTasks):
+    bg.add_task(ingest_pdfs, get_col("papers"))
+    return {"status": "queued"}
+
+async def _webhook(payload: dict):
+    repo = payload.get("repository") or {}
+    url  = repo.get("clone_url") or repo.get("html_url","")
+    name = repo.get("name","unknown")
+    if not url:
+        return {"status": "ignored"}
+    dest = REPOS_DIR / name
+    if dest.exists():
+        subprocess.run(["git","pull"], cwd=dest, timeout=120)
+    else:
+        subprocess.run(["git","clone","--depth=1",url,str(dest)], timeout=300)
+    n = ingest_dir(get_col("code"), dest, name)
+    return {"status": "ok", "repo": name, "files": n}
+
+@app.post("/webhook/gitea")
+async def webhook_gitea(r: Request): return await _webhook(await r.json())
+
+@app.post("/webhook/github")
+async def webhook_github(r: Request): return await _webhook(await r.json())
+
+# ── RAG chat ──────────────────────────────────────────────────────────────────
+class ChatRequest(BaseModel):
+    model: str = CHAT_MODEL
+    messages: list[dict[str,Any]]
+    stream: bool = False
+    collections: list[str] = ["code","papers"]
+
+def _context(query: str, cols: list[str]) -> str:
+    parts = []
+    for cname in cols:
+        try:
+            col = get_col(cname)
+            if col.count() == 0:
+                continue
+            res = col.query(query_texts=[query], n_results=min(TOP_K, col.count()))
+            for doc, meta in zip(res["documents"][0], res["metadatas"][0]):
+                parts.append(f"### {meta.get('source','')}:{meta.get('label','')}\n"
+                             f"```{meta.get('lang','')}\n{doc}\n```")
+        except Exception as e:
+            log.warning("col %s: %s", cname, e)
+    return "\n\n".join(parts)
+
+@app.post("/v1/chat/completions")
+async def chat(req: ChatRequest):
+    query   = next((m["content"] for m in reversed(req.messages)
+                    if m.get("role")=="user"), "")
+    context = _context(query, req.collections)
+    msgs = [{"role":"system","content":
+             "You are a helpful coding assistant. Use the retrieved context below.\n\n"
+             f"## Context\n{context}"}] + req.messages
+
+    payload = {"model": req.model, "messages": msgs, "stream": req.stream}
+
+    if req.stream:
+        async def gen():
+            async with httpx.AsyncClient(timeout=300) as client:
+                async with client.stream("POST",
+                        f"{OLLAMA_URL}/v1/chat/completions", json=payload) as r:
+                    async for chunk in r.aiter_bytes():
+                        yield chunk
+        return StreamingResponse(gen(), media_type="text/event-stream")
+
+    async with httpx.AsyncClient(timeout=300) as client:
+        r = await client.post(f"{OLLAMA_URL}/v1/chat/completions", json=payload)
+    return r.json()
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("server:app", host="0.0.0.0", port=8001, reload=False)