From 799fc84875e91696596e9210d56cbfd384dbc487 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Mar 2026 18:15:56 +0000 Subject: [PATCH] remove SearXNG entirely; fix Kiwix cold-start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SearXNG: - Dropped from both setup scripts and docker-compose (Google/Startpage block self-hosted instances by IP — not reliable enough to include) - Removed all interactive safe-search and engine-selection prompts - Removed Open WebUI RAG web-search env vars (ENABLE_RAG_WEB_SEARCH, SEARXNG_QUERY_URL, etc.) - Removed port 8888 from UFW rules, start.sh URLs, done output, and the Caddyfile template - Removed searxng/ from .gitignore (directory no longer created) - configure-searxng-safesearch.sh kept in repo for optional manual use Kiwix: - Replace the blocking wait-loop (`until ls *.zim`) with a one-liner that passes whatever ZIM files exist (or none) directly to kiwix-serve, so the container starts immediately and shows an empty library page rather than hanging until ZIMs are downloaded https://claude.ai/code/session_012gDnantBmFTWZGCiKyjazx --- .gitignore | 1 - laptop_full_setup.sh | 300 +++---------------------------------------- local-ai-setup.sh | 247 +---------------------------------- 3 files changed, 21 insertions(+), 527 deletions(-) diff --git a/.gitignore b/.gitignore index eba77b6..0e8856f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,3 @@ gitea/ portainer-data/ invokeai-data/ invokeai-outputs/ -searxng/ diff --git a/laptop_full_setup.sh b/laptop_full_setup.sh index 3cd8f75..52fc894 100755 --- a/laptop_full_setup.sh +++ b/laptop_full_setup.sh @@ -5,7 +5,7 @@ # # Options (checklist at launch): # • Full system setup — Ubuntu apps, security, backups (ubuntu-post-install.sh) -# • AI stack — Ollama · Open WebUI · RAG · MCP · ChromaDB · SearXNG +# • AI stack — Ollama · Open WebUI · RAG · MCP · ChromaDB # Gitea · InvokeAI · Portainer # • Kiwix — Offline Wikipedia, Stack Overflow, Arch Wiki, etc. # @@ -87,7 +87,7 @@ INSTALL_POSTINSTALL=false INSTALL_AI=true # Per-service flags (all default ON — toggled off in Q1a) -SVC_WEBUI=true; SVC_RAG=true; SVC_MCP=true; SVC_SEARXNG=true +SVC_WEBUI=true; SVC_RAG=true; SVC_MCP=true SVC_GITEA=true; SVC_INVOKEAI=true; SVC_PORTAINER=true; SVC_KIWIX=true POSTINSTALL_SCRIPT="$SCRIPT_DIR/ubuntu-post-install.sh" @@ -144,7 +144,6 @@ if $INSTALL_AI; then "open-webui" "Open WebUI — Chat interface (like ChatGPT, uses Ollama)" "$(existing_svc open-webui)" \ "rag" "RAG + ChromaDB— Code & document search (needed by MCP)" "$(existing_svc rag-server)" \ "mcp" "MCP Server — Claude Code tools (bash, files, git, search)" "$(existing_svc mcp-server)" \ - "searxng" "SearXNG — Private local web search engine" "$(existing_svc searxng)" \ "gitea" "Gitea — Self-hosted Git server" "$(existing_svc gitea)" \ "invokeai" "InvokeAI — Image generation (Stable Diffusion)" "$(existing_svc invokeai)" \ "portainer" "Portainer — Docker management web UI" "$(existing_svc portainer)" \ @@ -154,7 +153,6 @@ if $INSTALL_AI; then SVC_WEBUI=false; [[ "$SELECTED_SVCS" == *'"open-webui"'* ]] && SVC_WEBUI=true SVC_RAG=false; [[ "$SELECTED_SVCS" == *'"rag"'* ]] && SVC_RAG=true SVC_MCP=false; [[ "$SELECTED_SVCS" == *'"mcp"'* ]] && SVC_MCP=true - SVC_SEARXNG=false; [[ "$SELECTED_SVCS" == *'"searxng"'* ]] && SVC_SEARXNG=true SVC_GITEA=false; [[ "$SELECTED_SVCS" == *'"gitea"'* ]] && SVC_GITEA=true SVC_INVOKEAI=false; [[ "$SELECTED_SVCS" == *'"invokeai"'* ]] && SVC_INVOKEAI=true SVC_PORTAINER=false;[[ "$SELECTED_SVCS" == *'"portainer"'* ]] && SVC_PORTAINER=true @@ -164,33 +162,31 @@ if $INSTALL_AI; then echo "" echo -e " ${BOLD}[1a] AI Stack — select services to install${NC}" echo " Ollama always included (required core). Type number to toggle, Enter to confirm." - _W=$SVC_WEBUI; _R=$SVC_RAG; _M=$SVC_MCP; _S=$SVC_SEARXNG + _W=$SVC_WEBUI; _R=$SVC_RAG; _M=$SVC_MCP _G=$SVC_GITEA; _I=$SVC_INVOKEAI; _P=$SVC_PORTAINER; _K=$SVC_KIWIX while true; do echo "" printf " [%s] 1. Open WebUI — Chat interface\n" "$($_W && echo '*' || echo ' ')" printf " [%s] 2. RAG+ChromaDB — Code & document search\n" "$($_R && echo '*' || echo ' ')" printf " [%s] 3. MCP Server — Claude Code tools (needs RAG)\n" "$($_M && echo '*' || echo ' ')" - printf " [%s] 4. SearXNG — Private web search\n" "$($_S && echo '*' || echo ' ')" - printf " [%s] 5. Gitea — Self-hosted Git\n" "$($_G && echo '*' || echo ' ')" - printf " [%s] 6. InvokeAI — Image generation\n" "$($_I && echo '*' || echo ' ')" - printf " [%s] 7. Portainer — Docker management UI\n" "$($_P && echo '*' || echo ' ')" - printf " [%s] 8. Kiwix — Offline Wikipedia, Stack Overflow\n" "$($_K && echo '*' || echo ' ')" + printf " [%s] 4. Gitea — Self-hosted Git\n" "$($_G && echo '*' || echo ' ')" + printf " [%s] 5. InvokeAI — Image generation\n" "$($_I && echo '*' || echo ' ')" + printf " [%s] 6. Portainer — Docker management UI\n" "$($_P && echo '*' || echo ' ')" + printf " [%s] 7. Kiwix — Offline Wikipedia, Stack Overflow\n" "$($_K && echo '*' || echo ' ')" echo "" read -rp " Toggle [number] or Enter to confirm: " T case "$T" in 1) $_W && _W=false || _W=true ;; 2) $_R && _R=false || _R=true ;; 3) $_M && _M=false || _M=true ;; - 4) $_S && _S=false || _S=true ;; - 5) $_G && _G=false || _G=true ;; - 6) $_I && _I=false || _I=true ;; - 7) $_P && _P=false || _P=true ;; - 8) $_K && _K=false || _K=true ;; + 4) $_G && _G=false || _G=true ;; + 5) $_I && _I=false || _I=true ;; + 6) $_P && _P=false || _P=true ;; + 7) $_K && _K=false || _K=true ;; "") break ;; esac done - SVC_WEBUI=$_W; SVC_RAG=$_R; SVC_MCP=$_M; SVC_SEARXNG=$_S + SVC_WEBUI=$_W; SVC_RAG=$_R; SVC_MCP=$_M SVC_GITEA=$_G; SVC_INVOKEAI=$_I; SVC_PORTAINER=$_P; SVC_KIWIX=$_K fi @@ -567,228 +563,6 @@ if $INSTALL_AI; then [[ "${DO_PULL,,}" != "n" ]] && PULL_MODELS=true fi -# ── Q6: SearXNG safe-search & engine selection ──────────────────────────────── -SEARXNG_SAFE_LEVEL="none" -SEARXNG_SAFE_INT=0 -SEARXNG_DISABLE_ENGINES="" -SEARXNG_ENABLE_ENGINES="" - -if $INSTALL_AI && $SVC_SEARXNG; then - echo "" - echo -e " ${BOLD}[6/6] SearXNG safe-search${NC}" - echo " 0) None — all results, no filtering (default)" - echo " 1) Moderate — filter explicit content" - echo " 2) Strict — block all explicit content" - echo "" - read -rp " Choice [0]: " _SAFE_PICK - case "${_SAFE_PICK:-0}" in - 1) SEARXNG_SAFE_LEVEL="moderate"; SEARXNG_SAFE_INT=1 ;; - 2) SEARXNG_SAFE_LEVEL="strict"; SEARXNG_SAFE_INT=2 ;; - *) SEARXNG_SAFE_LEVEL="none"; SEARXNG_SAFE_INT=0 ;; - esac - - # Default ON/OFF for an engine given the chosen safe-search level. - # Engines without safe-search support default OFF for moderate/strict. - _sxst() { - [[ "$SEARXNG_SAFE_LEVEL" == "none" ]] && echo ON && return - case "$1" in - mojeek|"mojeek images"|"mojeek news"|\ - yandex|"yandex images"|\ - baidu|naver|\ - invidious|piped|peertube|sepiasearch|\ - vimeo|bitchute|rumble|odysee|\ - imgur|deviantart|artstation) echo OFF ;; - *) echo ON ;; - esac - } - - # Whiptail checklist — prints selected engine names (one per line). - # ESC returns the engines that were pre-checked ON (keeps defaults unchanged). - # Args: "Title" name "Desc" ON|OFF [name "Desc" ON|OFF ...] - _sxmenu() { - local _t="$1"; shift - local -a _n=() _s=() _a=(); local _i=1 - while [[ $# -ge 3 ]]; do - _n+=("$1"); _s+=("$3") - _a+=("$_i" "$(printf '%-26s %s' "$1" "$2")" "$3") - _i=$(( _i + 1 )); shift 3 - done - local _h=$(( ${#_n[@]} + 9 > 24 ? 24 : ${#_n[@]} + 9 )) - local _lh=$(( ${#_n[@]} < 14 ? ${#_n[@]} : 14 )) - local _o _rc _j - # Note: NO 2>/dev/null after the FD swap — that redirect would overwrite - # the pipe (FD2 after swap) with /dev/null, silently discarding selections. - _o=$(whiptail --backtitle "SearXNG — $SEARXNG_SAFE_LEVEL" \ - --title "$_t" --checklist \ - "SPACE = toggle ENTER = confirm ESC = use defaults" \ - "$_h" 76 "$_lh" "${_a[@]}" 3>&1 1>&2 2>&3) && _rc=0 || _rc=$? - if [[ $_rc -ne 0 ]] || [[ -z "${_o//\"}" ]]; then - # ESC / cancel / empty selection → use pre-checked defaults - for _j in "${!_n[@]}"; do - [[ "${_s[$_j]}" == "ON" ]] && printf '%s\n' "${_n[$_j]}" - done - return - fi - for _id in $_o; do _id="${_id//\"/}"; printf '%s\n' "${_n[$(( _id-1 ))]}"; done - } - - # Engines auto-disabled by configure script for moderate/strict. - # If user re-enables one in the menu we pass --enable-engines to override. - _SX_AUTOFF=(mojeek yandex "yandex images" baidu naver invidious piped peertube sepiasearch) - - if command -v whiptail &>/dev/null && [[ -t 0 ]]; then - # ── Web search ───────────────────────────────────────────────────────── - _SX_W=$(_sxmenu "Web Search Engines" \ - "google" "Google" ON \ - "bing" "Microsoft Bing" ON \ - "duckduckgo" "DuckDuckGo" ON \ - "brave" "Brave Search" ON \ - "startpage" "Startpage (Google proxy)" ON \ - "qwant" "Qwant" ON \ - "yahoo" "Yahoo" ON \ - "ecosia" "Ecosia" ON \ - "presearch" "Presearch" ON \ - "yep" "Yep" ON \ - "wiby" "Wiby (classic/indie web)" ON \ - "mojeek" "Mojeek [no safe-search]" "$(_sxst mojeek)" \ - "yandex" "Yandex [unreliable filter]" "$(_sxst yandex)" \ - "baidu" "Baidu [no safe-search]" "$(_sxst baidu)" \ - "naver" "Naver (Korean) [no safe-search]" "$(_sxst naver)") - - # ── Images ───────────────────────────────────────────────────────────── - _SX_I=$(_sxmenu "Image Search Engines" \ - "google images" "Google Images" ON \ - "bing images" "Bing Images" ON \ - "duckduckgo images" "DuckDuckGo Images" ON \ - "brave images" "Brave Images" ON \ - "qwant images" "Qwant Images" ON \ - "startpage images" "Startpage Images" ON \ - "mojeek images" "Mojeek Images [no safe-search]" "$(_sxst 'mojeek images')" \ - "presearch images" "Presearch Images" ON \ - "openverse" "Openverse (open license)" ON \ - "unsplash" "Unsplash (stock photos)" ON \ - "pexels" "Pexels (stock photos)" ON \ - "pixabay images" "Pixabay Images" ON \ - "pinterest" "Pinterest" ON \ - "flickr" "Flickr" ON \ - "wikcommons.images" "Wikimedia Commons Images" ON \ - "artic" "Art Institute of Chicago" ON \ - "yandex images" "Yandex Images [no safe-search]" "$(_sxst 'yandex images')" \ - "imgur" "Imgur [adult content]" "$(_sxst imgur)" \ - "deviantart" "DeviantArt [adult content]" "$(_sxst deviantart)" \ - "artstation" "ArtStation [adult content]" "$(_sxst artstation)" \ - "adobe stock" "Adobe Stock" ON) - - # ── Videos ───────────────────────────────────────────────────────────── - _SX_V=$(_sxmenu "Video Search Engines" \ - "youtube" "YouTube" ON \ - "bing videos" "Bing Videos" ON \ - "brave videos" "Brave Videos" ON \ - "duckduckgo videos" "DuckDuckGo Videos" ON \ - "google videos" "Google Videos" ON \ - "qwant videos" "Qwant Videos" ON \ - "dailymotion" "Dailymotion" ON \ - "media.ccc.de" "media.ccc.de (tech talks)" ON \ - "wikcommons.videos" "Wikimedia Commons Videos" ON \ - "vimeo" "Vimeo [no safe-search]" "$(_sxst vimeo)" \ - "odysee" "Odysee [no safe-search]" "$(_sxst odysee)" \ - "rumble" "Rumble [no safe-search]" "$(_sxst rumble)" \ - "bitchute" "BitChute [no safe-search]" "$(_sxst bitchute)" \ - "invidious" "Invidious (YT) [no safe-search]" "$(_sxst invidious)" \ - "piped" "Piped (YT) [no safe-search]" "$(_sxst piped)" \ - "peertube" "PeerTube [no safe-search]" "$(_sxst peertube)" \ - "sepiasearch" "SepiaSearch [no safe-search]" "$(_sxst sepiasearch)") - - # ── News ─────────────────────────────────────────────────────────────── - _SX_N=$(_sxmenu "News Search Engines" \ - "google news" "Google News" ON \ - "bing news" "Bing News" ON \ - "duckduckgo news" "DuckDuckGo News" ON \ - "brave news" "Brave News" ON \ - "qwant news" "Qwant News" ON \ - "startpage news" "Startpage News" ON \ - "presearch news" "Presearch News" ON \ - "mojeek news" "Mojeek News [no safe-search]" "$(_sxst 'mojeek news')" \ - "reuters" "Reuters" ON \ - "yahoo news" "Yahoo News" ON \ - "wikinews" "WikiNews" ON \ - "yep news" "Yep News" ON) - - # ── Reference & Knowledge ────────────────────────────────────────────── - _SX_R=$(_sxmenu "Reference & Knowledge" \ - "wikipedia" "Wikipedia" ON \ - "wikidata" "Wikidata" ON \ - "wolframalpha" "Wolfram|Alpha" ON \ - "ask" "Ask.com" ON \ - "ddg definitions" "DuckDuckGo Definitions" ON \ - "encyclopsearch" "Encyclopsearch" ON \ - "wikibooks" "Wikibooks" ON \ - "wikiquote" "Wikiquote" ON \ - "wikisource" "Wikisource" ON \ - "wikispecies" "Wikispecies" ON \ - "wikiversity" "Wikiversity" ON \ - "wikivoyage" "Wikivoyage" ON) - - # ── Science ──────────────────────────────────────────────────────────── - _SX_S=$(_sxmenu "Science / Academic" \ - "arxiv" "arXiv (preprints)" ON \ - "semantic scholar" "Semantic Scholar" ON \ - "pubmed" "PubMed (medical)" ON \ - "crossref" "Crossref (DOI/papers)" ON \ - "base" "BASE (open access)" ON) - - _SX_ALL=( - # Web - "google" "bing" "duckduckgo" "brave" "startpage" "qwant" "yahoo" "ecosia" - "presearch" "yep" "wiby" - "mojeek" "yandex" "baidu" "naver" - # Images - "google images" "bing images" "duckduckgo images" "brave images" "qwant images" - "startpage images" "mojeek images" "presearch images" - "openverse" "unsplash" "pexels" "pixabay images" "pinterest" "flickr" - "wikcommons.images" "artic" "yandex images" "imgur" "deviantart" "artstation" "adobe stock" - # Videos - "youtube" "bing videos" "brave videos" "duckduckgo videos" "google videos" "qwant videos" - "dailymotion" "media.ccc.de" "wikcommons.videos" - "vimeo" "odysee" "rumble" "bitchute" - "invidious" "piped" "peertube" "sepiasearch" - # News - "google news" "bing news" "duckduckgo news" "brave news" "qwant news" - "startpage news" "presearch news" "mojeek news" - "reuters" "yahoo news" "wikinews" "yep news" - # Reference - "wikipedia" "wikidata" "wolframalpha" "ask" "ddg definitions" "encyclopsearch" - "wikibooks" "wikiquote" "wikisource" "wikispecies" "wikiversity" "wikivoyage" - # Science - "arxiv" "semantic scholar" "pubmed" "crossref" "base" - ) - _SX_SEL=$(printf '%s\n' "$_SX_W" "$_SX_I" "$_SX_V" "$_SX_N" "$_SX_R" "$_SX_S") - - for _e in "${_SX_ALL[@]}"; do - if printf '%s\n' "$_SX_SEL" | grep -qxF "$_e"; then - # Selected — if auto-disabled by level, pass --enable-engines to override - for _ao in "${_SX_AUTOFF[@]}"; do - if [[ "$_e" == "$_ao" ]]; then - SEARXNG_ENABLE_ENGINES+="${SEARXNG_ENABLE_ENGINES:+,}${_e}" - break - fi - done - else - SEARXNG_DISABLE_ENGINES+="${SEARXNG_DISABLE_ENGINES:+,}${_e}" - fi - done - else - # Fallback: no whiptail / non-interactive terminal - echo "" - echo " (whiptail not available — text entry)" - echo " Web: google bing duckduckgo brave startpage qwant yahoo presearch" - echo " Imgs: google_images bing_images duckduckgo_images flickr imgur" - echo " Vids: youtube dailymotion vimeo rumble odysee bitchute" - echo " News: google_news bing_news reuters yahoo_news wikinews" - read -rp " Engines to disable (space-separated, Enter for defaults): " _ENGS - SEARXNG_DISABLE_ENGINES="${_ENGS// /,}" - fi -fi # ── Summary ─────────────────────────────────────────────────────────────────── echo "" @@ -803,7 +577,6 @@ if $INSTALL_AI; then $SVC_WEBUI && _svcs+=" · Open WebUI" $SVC_RAG && _svcs+=" · RAG+ChromaDB" $SVC_MCP && _svcs+=" · MCP" - $SVC_SEARXNG && _svcs+=" · SearXNG" $SVC_GITEA && _svcs+=" · Gitea" $SVC_INVOKEAI && _svcs+=" · InvokeAI" $SVC_PORTAINER && _svcs+=" · Portainer" @@ -818,14 +591,6 @@ if $INSTALL_AI; then fi fi $SVC_KIWIX && echo " ✓ Kiwix ZIMs → $KIWIX_DIR" -if $INSTALL_AI && $SVC_SEARXNG; then - _sx="safe_search: $SEARXNG_SAFE_LEVEL" - if [[ -n "$SEARXNG_DISABLE_ENGINES" ]]; then - _ndis=$(tr ',' '\n' <<< "$SEARXNG_DISABLE_ENGINES" | grep -c .) - _sx+=" | ${_ndis} engines disabled" - fi - echo " ✓ SearXNG → $_sx" -fi $PULL_MODELS && echo " ✓ Pull models : $EMBED_MODEL + $FAST_MODEL + $CHAT_MODEL + $CODE_MODEL${REASON_MODEL:+ + $REASON_MODEL}" [[ "$ZIM_CHOICE" == "1" ]] && echo " ✓ Download all ZIMs in background (~130 GB)" if [[ "$ZIM_CHOICE" == "2" ]]; then @@ -938,7 +703,7 @@ fi # ============================================================================= section "Directories" # ============================================================================= -for d in papers repos workspace index searxng invokeai-data invokeai-outputs \ +for d in papers repos workspace index invokeai-data invokeai-outputs \ gitea portainer-data logs; do mkdir -p "$BASE/$d" done @@ -981,18 +746,6 @@ ok "mcp_requirements.txt" fi # INSTALL_AI -# ============================================================================= -if $INSTALL_AI && $SVC_SEARXNG; then -section "SearXNG Config" -# ============================================================================= -mkdir -p "$BASE/searxng" -_SXARGS=("$SEARXNG_SAFE_LEVEL") -[[ -n "$SEARXNG_DISABLE_ENGINES" ]] && _SXARGS+=(--disable-engines "$SEARXNG_DISABLE_ENGINES") -[[ -n "$SEARXNG_ENABLE_ENGINES" ]] && _SXARGS+=(--enable-engines "$SEARXNG_ENABLE_ENGINES") -BASE="$BASE" bash "$SCRIPT_DIR/configure-searxng-safesearch.sh" "${_SXARGS[@]}" - -fi # INSTALL_AI && SVC_SEARXNG - # ============================================================================= section ".env File" # ============================================================================= @@ -1048,7 +801,7 @@ cat > "$BASE/docker-compose.yml" << COMPOSE # Show status: docker compose ps # # Services: ollama open-webui chromadb rag-server mcp-server -# searxng kiwix gitea invokeai portainer +# kiwix gitea invokeai portainer # ─────────────────────────────────────────────────────────────────────────────── services: @@ -1094,11 +847,6 @@ ${OLLAMA_VOLUME_LINE} - OPENAI_API_BASE_URL=http://rag-server:8001/v1 - OPENAI_API_KEY=local-rag-key - ENABLE_OPENAI_API=true - - ENABLE_RAG_WEB_SEARCH=true - - RAG_WEB_SEARCH_ENGINE=searxng - - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=${SEARXNG_SAFE_INT} - - RAG_WEB_SEARCH_RESULT_COUNT=5 - - RAG_WEB_SEARCH_CONCURRENT_REQUESTS=10 - ENABLE_TOOL_SERVERS=true - WEBUI_AUTH=true - WEBUI_URL=${WEBUI_URL:-} @@ -1180,18 +928,6 @@ ${OLLAMA_VOLUME_LINE} depends_on: - rag-server - # ── SearXNG — Private web search ─────────────────────────────────────────── - searxng: - image: searxng/searxng:latest - container_name: searxng - restart: unless-stopped - ports: - - "0.0.0.0:8888:8080" - volumes: - - $BASE/searxng:/etc/searxng - cap_drop: [ALL] - cap_add: [CHOWN, SETGID, SETUID] - # ── Kiwix — Offline Wikipedia/docs ───────────────────────────────────────── kiwix: image: ghcr.io/kiwix/kiwix-serve:latest @@ -1202,7 +938,7 @@ ${OLLAMA_VOLUME_LINE} volumes: - $KIWIX_DIR:/data entrypoint: ["sh", "-c"] - command: ["until ls /data/*.zim 2>/dev/null; do echo 'kiwix: waiting for ZIM files…'; sleep 60; done && exec kiwix-serve /data/*.zim"] + command: ["exec kiwix-serve $(ls /data/*.zim 2>/dev/null | tr '\\n' ' ')"] # ── Gitea — Self-hosted Git ───────────────────────────────────────────────── gitea: @@ -1272,7 +1008,7 @@ if command -v ufw &>/dev/null; then if [[ ! -f "$BASE/.ufw-done" ]] || $FORCE; then for port_comment in \ "3000:Open WebUI" "11434:Ollama" "8001:RAG Server" \ - "8002:MCP Server" "8000:ChromaDB" "8888:SearXNG" \ + "8002:MCP Server" "8000:ChromaDB" \ "8181:Kiwix" "3001:Gitea" "2222:Gitea SSH" \ "9090:InvokeAI" "9000:Portainer" "9443:Portainer S"; do port="${port_comment%%:*}" @@ -1298,7 +1034,6 @@ section "Helper Scripts" _START_URLS="" $SVC_WEBUI && _START_URLS+=$'echo " Open WebUI → http://'"$LOCAL_IP"$':3000"\n' $SVC_INVOKEAI && _START_URLS+=$'echo " InvokeAI → http://'"$LOCAL_IP"$':9090"\n' -$SVC_SEARXNG && _START_URLS+=$'echo " SearXNG → http://'"$LOCAL_IP"$':8888"\n' $SVC_GITEA && _START_URLS+=$'echo " Gitea → http://'"$LOCAL_IP"$':3001"\n' $SVC_RAG && _START_URLS+=$'echo " RAG Health → http://'"$LOCAL_IP"$':8001/health"\n' $SVC_MCP && _START_URLS+=$'echo " MCP SSE → http://'"$LOCAL_IP"$':8002/sse"\n' @@ -1413,7 +1148,6 @@ webui.yourdomain.com { } } invokeai.yourdomain.com { reverse_proxy $LOCAL_IP:9090 } -search.yourdomain.com { reverse_proxy $LOCAL_IP:8888 } git.yourdomain.com { reverse_proxy $LOCAL_IP:3001 } kiwix.yourdomain.com { reverse_proxy $LOCAL_IP:8181 } rag.yourdomain.com { reverse_proxy $LOCAL_IP:8001 } @@ -1462,7 +1196,6 @@ if $INSTALL_AI; then $SVC_WEBUI && COMPOSE_SERVICES+=" open-webui" $SVC_RAG && COMPOSE_SERVICES+=" chromadb rag-server" $SVC_MCP && COMPOSE_SERVICES+=" mcp-server" - $SVC_SEARXNG && COMPOSE_SERVICES+=" searxng" $SVC_GITEA && COMPOSE_SERVICES+=" gitea" $SVC_INVOKEAI && COMPOSE_SERVICES+=" invokeai" $SVC_PORTAINER && COMPOSE_SERVICES+=" portainer" @@ -1609,7 +1342,6 @@ echo "" if $INSTALL_AI; then $SVC_WEBUI && echo -e " ${CYAN}Open WebUI${NC} → http://$LOCAL_IP:3000" $SVC_INVOKEAI && echo -e " ${CYAN}InvokeAI${NC} → http://$LOCAL_IP:9090" - $SVC_SEARXNG && echo -e " ${CYAN}SearXNG${NC} → http://$LOCAL_IP:8888" $SVC_GITEA && echo -e " ${CYAN}Gitea${NC} → http://$LOCAL_IP:3001" $SVC_RAG && echo -e " ${CYAN}RAG Health${NC} → http://$LOCAL_IP:8001/health" $SVC_MCP && echo -e " ${CYAN}MCP SSE${NC} → http://$LOCAL_IP:8002/sse" diff --git a/local-ai-setup.sh b/local-ai-setup.sh index 8e21620..e010ba8 100755 --- a/local-ai-setup.sh +++ b/local-ai-setup.sh @@ -44,219 +44,6 @@ info "Base : $BASE" info "IP : $LOCAL_IP" info "GPU : ${VRAM_GB}GB VRAM → $TIER" -# ── SearXNG safe-search & engine selection ──────────────────────────────────── -SEARXNG_DISABLE_ENGINES="" -SEARXNG_ENABLE_ENGINES="" -echo "" -echo " SearXNG safe-search level:" -echo " 0) None — all results, no filtering (default)" -echo " 1) Moderate — filter explicit content" -echo " 2) Strict — block all explicit content" -echo "" -read -rp " Choice [0]: " _SAFE_PICK -case "${_SAFE_PICK:-0}" in - 1) SEARXNG_SAFE_LEVEL="moderate"; SEARXNG_SAFE_INT=1 ;; - 2) SEARXNG_SAFE_LEVEL="strict"; SEARXNG_SAFE_INT=2 ;; - *) SEARXNG_SAFE_LEVEL="none"; SEARXNG_SAFE_INT=0 ;; -esac - -# Default ON/OFF for an engine given the chosen safe-search level. -# Engines without safe-search support default OFF for moderate/strict. -_sxst() { - [[ "$SEARXNG_SAFE_LEVEL" == "none" ]] && echo ON && return - case "$1" in - mojeek|"mojeek images"|"mojeek news"|\ - yandex|"yandex images"|\ - baidu|naver|\ - invidious|piped|peertube|sepiasearch|\ - vimeo|bitchute|rumble|odysee|\ - imgur|deviantart|artstation) echo OFF ;; - *) echo ON ;; - esac -} - -# Whiptail checklist — prints selected engine names (one per line). -# ESC returns the engines that were pre-checked ON (keeps defaults unchanged). -_sxmenu() { - local _t="$1"; shift - local -a _n=() _s=() _a=(); local _i=1 - while [[ $# -ge 3 ]]; do - _n+=("$1"); _s+=("$3") - _a+=("$_i" "$(printf '%-26s %s' "$1" "$2")" "$3") - _i=$(( _i + 1 )); shift 3 - done - local _h=$(( ${#_n[@]} + 9 > 24 ? 24 : ${#_n[@]} + 9 )) - local _lh=$(( ${#_n[@]} < 14 ? ${#_n[@]} : 14 )) - local _o _rc _j - # Note: NO 2>/dev/null after the FD swap — that redirect would overwrite - # the pipe (FD2 after swap) with /dev/null, silently discarding selections. - _o=$(whiptail --backtitle "SearXNG — $SEARXNG_SAFE_LEVEL" \ - --title "$_t" --checklist \ - "SPACE = toggle ENTER = confirm ESC = use defaults" \ - "$_h" 76 "$_lh" "${_a[@]}" 3>&1 1>&2 2>&3) && _rc=0 || _rc=$? - if [[ $_rc -ne 0 ]] || [[ -z "${_o//\"}" ]]; then - # ESC / cancel / empty selection → use pre-checked defaults - for _j in "${!_n[@]}"; do - [[ "${_s[$_j]}" == "ON" ]] && printf '%s\n' "${_n[$_j]}" - done - return - fi - for _id in $_o; do _id="${_id//\"/}"; printf '%s\n' "${_n[$(( _id-1 ))]}"; done -} - -_SX_AUTOFF=(mojeek yandex "yandex images" baidu naver invidious piped peertube sepiasearch) - -if command -v whiptail &>/dev/null && [[ -t 0 ]]; then - # ── Web search ───────────────────────────────────────────────────────────── - _SX_W=$(_sxmenu "Web Search Engines" \ - "google" "Google" ON \ - "bing" "Microsoft Bing" ON \ - "duckduckgo" "DuckDuckGo" ON \ - "brave" "Brave Search" ON \ - "startpage" "Startpage (Google proxy)" ON \ - "qwant" "Qwant" ON \ - "yahoo" "Yahoo" ON \ - "ecosia" "Ecosia" ON \ - "presearch" "Presearch" ON \ - "yep" "Yep" ON \ - "wiby" "Wiby (classic/indie web)" ON \ - "mojeek" "Mojeek [no safe-search]" "$(_sxst mojeek)" \ - "yandex" "Yandex [unreliable filter]" "$(_sxst yandex)" \ - "baidu" "Baidu [no safe-search]" "$(_sxst baidu)" \ - "naver" "Naver (Korean) [no safe-search]" "$(_sxst naver)") - - # ── Images ───────────────────────────────────────────────────────────────── - _SX_I=$(_sxmenu "Image Search Engines" \ - "google images" "Google Images" ON \ - "bing images" "Bing Images" ON \ - "duckduckgo images" "DuckDuckGo Images" ON \ - "brave images" "Brave Images" ON \ - "qwant images" "Qwant Images" ON \ - "startpage images" "Startpage Images" ON \ - "mojeek images" "Mojeek Images [no safe-search]" "$(_sxst 'mojeek images')" \ - "presearch images" "Presearch Images" ON \ - "openverse" "Openverse (open license)" ON \ - "unsplash" "Unsplash (stock photos)" ON \ - "pexels" "Pexels (stock photos)" ON \ - "pixabay images" "Pixabay Images" ON \ - "pinterest" "Pinterest" ON \ - "flickr" "Flickr" ON \ - "wikcommons.images" "Wikimedia Commons Images" ON \ - "artic" "Art Institute of Chicago" ON \ - "yandex images" "Yandex Images [no safe-search]" "$(_sxst 'yandex images')" \ - "imgur" "Imgur [adult content]" "$(_sxst imgur)" \ - "deviantart" "DeviantArt [adult content]" "$(_sxst deviantart)" \ - "artstation" "ArtStation [adult content]" "$(_sxst artstation)" \ - "adobe stock" "Adobe Stock" ON) - - # ── Videos ───────────────────────────────────────────────────────────────── - _SX_V=$(_sxmenu "Video Search Engines" \ - "youtube" "YouTube" ON \ - "bing videos" "Bing Videos" ON \ - "brave videos" "Brave Videos" ON \ - "duckduckgo videos" "DuckDuckGo Videos" ON \ - "google videos" "Google Videos" ON \ - "qwant videos" "Qwant Videos" ON \ - "dailymotion" "Dailymotion" ON \ - "media.ccc.de" "media.ccc.de (tech talks)" ON \ - "wikcommons.videos" "Wikimedia Commons Videos" ON \ - "vimeo" "Vimeo [no safe-search]" "$(_sxst vimeo)" \ - "odysee" "Odysee [no safe-search]" "$(_sxst odysee)" \ - "rumble" "Rumble [no safe-search]" "$(_sxst rumble)" \ - "bitchute" "BitChute [no safe-search]" "$(_sxst bitchute)" \ - "invidious" "Invidious (YT) [no safe-search]" "$(_sxst invidious)" \ - "piped" "Piped (YT) [no safe-search]" "$(_sxst piped)" \ - "peertube" "PeerTube [no safe-search]" "$(_sxst peertube)" \ - "sepiasearch" "SepiaSearch [no safe-search]" "$(_sxst sepiasearch)") - - # ── News ─────────────────────────────────────────────────────────────────── - _SX_N=$(_sxmenu "News Search Engines" \ - "google news" "Google News" ON \ - "bing news" "Bing News" ON \ - "duckduckgo news" "DuckDuckGo News" ON \ - "brave news" "Brave News" ON \ - "qwant news" "Qwant News" ON \ - "startpage news" "Startpage News" ON \ - "presearch news" "Presearch News" ON \ - "mojeek news" "Mojeek News [no safe-search]" "$(_sxst 'mojeek news')" \ - "reuters" "Reuters" ON \ - "yahoo news" "Yahoo News" ON \ - "wikinews" "WikiNews" ON \ - "yep news" "Yep News" ON) - - # ── Reference & Knowledge ────────────────────────────────────────────────── - _SX_R=$(_sxmenu "Reference & Knowledge" \ - "wikipedia" "Wikipedia" ON \ - "wikidata" "Wikidata" ON \ - "wolframalpha" "Wolfram|Alpha" ON \ - "ask" "Ask.com" ON \ - "ddg definitions" "DuckDuckGo Definitions" ON \ - "encyclopsearch" "Encyclopsearch" ON \ - "wikibooks" "Wikibooks" ON \ - "wikiquote" "Wikiquote" ON \ - "wikisource" "Wikisource" ON \ - "wikispecies" "Wikispecies" ON \ - "wikiversity" "Wikiversity" ON \ - "wikivoyage" "Wikivoyage" ON) - - # ── Science ──────────────────────────────────────────────────────────────── - _SX_S=$(_sxmenu "Science / Academic" \ - "arxiv" "arXiv (preprints)" ON \ - "semantic scholar" "Semantic Scholar" ON \ - "pubmed" "PubMed (medical)" ON \ - "crossref" "Crossref (DOI/papers)" ON \ - "base" "BASE (open access)" ON) - - _SX_ALL=( - # Web - "google" "bing" "duckduckgo" "brave" "startpage" "qwant" "yahoo" "ecosia" - "presearch" "yep" "wiby" - "mojeek" "yandex" "baidu" "naver" - # Images - "google images" "bing images" "duckduckgo images" "brave images" "qwant images" - "startpage images" "mojeek images" "presearch images" - "openverse" "unsplash" "pexels" "pixabay images" "pinterest" "flickr" - "wikcommons.images" "artic" "yandex images" "imgur" "deviantart" "artstation" "adobe stock" - # Videos - "youtube" "bing videos" "brave videos" "duckduckgo videos" "google videos" "qwant videos" - "dailymotion" "media.ccc.de" "wikcommons.videos" - "vimeo" "odysee" "rumble" "bitchute" - "invidious" "piped" "peertube" "sepiasearch" - # News - "google news" "bing news" "duckduckgo news" "brave news" "qwant news" - "startpage news" "presearch news" "mojeek news" - "reuters" "yahoo news" "wikinews" "yep news" - # Reference - "wikipedia" "wikidata" "wolframalpha" "ask" "ddg definitions" "encyclopsearch" - "wikibooks" "wikiquote" "wikisource" "wikispecies" "wikiversity" "wikivoyage" - # Science - "arxiv" "semantic scholar" "pubmed" "crossref" "base" - ) - _SX_SEL=$(printf '%s\n' "$_SX_W" "$_SX_I" "$_SX_V" "$_SX_N" "$_SX_R" "$_SX_S") - - for _e in "${_SX_ALL[@]}"; do - if printf '%s\n' "$_SX_SEL" | grep -qxF "$_e"; then - for _ao in "${_SX_AUTOFF[@]}"; do - if [[ "$_e" == "$_ao" ]]; then - SEARXNG_ENABLE_ENGINES+="${SEARXNG_ENABLE_ENGINES:+,}${_e}" - break - fi - done - else - SEARXNG_DISABLE_ENGINES+="${SEARXNG_DISABLE_ENGINES:+,}${_e}" - fi - done -else - echo "" - echo " (whiptail not available — text entry)" - echo " Web: google bing duckduckgo brave startpage qwant yahoo presearch" - echo " Imgs: google_images bing_images duckduckgo_images flickr imgur" - echo " Vids: youtube dailymotion vimeo rumble odysee bitchute" - echo " News: google_news bing_news reuters yahoo_news wikinews" - read -rp " Engines to disable (space-separated, Enter for defaults): " _ENGS - SEARXNG_DISABLE_ENGINES="${_ENGS// /,}" -fi -echo "" write_if_new() { local dest="$1"; local body; body=$(cat) @@ -295,7 +82,7 @@ fi # ── directories ─────────────────────────────────────────────────────────────── section "Directories" -for d in papers repos workspace index searxng invokeai-data invokeai-outputs kiwix gitea portainer-data logs; do +for d in papers repos workspace index invokeai-data invokeai-outputs kiwix gitea portainer-data logs; do mkdir -p "$BASE/$d" done ok "Ready under $BASE" @@ -693,15 +480,6 @@ httpx REQ ok "requirements.txt + mcp_requirements.txt" -# ============================================================================= -section "SearXNG Config" -# ============================================================================= -mkdir -p "$BASE/searxng" -_SXARGS=("$SEARXNG_SAFE_LEVEL") -[[ -n "$SEARXNG_DISABLE_ENGINES" ]] && _SXARGS+=(--disable-engines "$SEARXNG_DISABLE_ENGINES") -[[ -n "$SEARXNG_ENABLE_ENGINES" ]] && _SXARGS+=(--enable-engines "$SEARXNG_ENABLE_ENGINES") -BASE="$BASE" bash "$SCRIPT_DIR/configure-searxng-safesearch.sh" "${_SXARGS[@]}" - # ============================================================================= section ".env (tokens — never overwritten)" # ============================================================================= @@ -736,7 +514,7 @@ cat > "$BASE/docker-compose.yml" << COMPOSE # Show status: docker compose ps # # Services: ollama open-webui chromadb rag-server mcp-server -# searxng kiwix gitea invokeai portainer +# kiwix gitea invokeai portainer # ─────────────────────────────────────────────────────────────────────────────── services: @@ -774,11 +552,6 @@ services: - OPENAI_API_BASE_URL=http://rag-server:8001/v1 - OPENAI_API_KEY=local-rag - ENABLE_OPENAI_API=true - - ENABLE_RAG_WEB_SEARCH=true - - RAG_WEB_SEARCH_ENGINE=searxng - - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=${SEARXNG_SAFE_INT} - - RAG_WEB_SEARCH_RESULT_COUNT=5 - - RAG_WEB_SEARCH_CONCURRENT_REQUESTS=10 - ENABLE_TOOL_SERVERS=true - WEBUI_AUTH=true depends_on: @@ -845,22 +618,14 @@ services: python mcp_server.py" depends_on: [rag-server] - searxng: - image: searxng/searxng:latest - container_name: searxng - restart: unless-stopped - ports: ["0.0.0.0:8888:8080"] - volumes: [$BASE/searxng:/etc/searxng] - cap_drop: [ALL] - cap_add: [CHOWN, SETGID, SETUID] - kiwix: image: ghcr.io/kiwix/kiwix-serve:latest container_name: kiwix restart: unless-stopped ports: ["0.0.0.0:8181:8080"] volumes: [$BASE/kiwix:/data] - command: "*.zim" + entrypoint: ["sh", "-c"] + command: ["exec kiwix-serve $(ls /data/*.zim 2>/dev/null | tr '\\n' ' ')"] gitea: image: gitea/gitea:latest @@ -922,7 +687,7 @@ if command -v ufw &>/dev/null && [[ ! -f "$BASE/.ufw-done" ]] || $FORCE; then read -rp " LAN subnet [192.168.1.0/24]: " LAN; LAN="${LAN:-192.168.1.0/24}" [[ "$LAN" =~ /[0-9]+$ ]] || LAN="${LAN}/24" for pc in "3000:Open WebUI" "11434:Ollama" "8001:RAG" "8002:MCP" \ - "8000:ChromaDB" "8888:SearXNG" "8181:Kiwix" \ + "8000:ChromaDB" "8181:Kiwix" \ "3001:Gitea" "2222:Gitea SSH" "9090:InvokeAI" \ "9000:Portainer" "9443:Portainer S"; do sudo ufw allow from "$LAN" to any port "${pc%%:*}" proto tcp comment "${pc##*:}" >/dev/null @@ -941,7 +706,6 @@ docker compose up -d echo "" echo " Open WebUI → http://$LOCAL_IP:3000" echo " InvokeAI → http://$LOCAL_IP:9090" -echo " SearXNG → http://$LOCAL_IP:8888" echo " Kiwix → http://$LOCAL_IP:8181 (run kiwix_download.sh first)" echo " Gitea → http://$LOCAL_IP:3001" echo " RAG → http://$LOCAL_IP:8001/health" @@ -1041,7 +805,6 @@ echo -e "${GREEN}${BOLD}━━━━━━━━━━━━━━━━━━ echo "" echo -e " ${CYAN}Open WebUI${NC} → http://$LOCAL_IP:3000" echo -e " ${CYAN}InvokeAI${NC} → http://$LOCAL_IP:9090" -echo -e " ${CYAN}SearXNG${NC} → http://$LOCAL_IP:8888" echo -e " ${CYAN}Kiwix${NC} → http://$LOCAL_IP:8181" echo -e " ${CYAN}Gitea${NC} → http://$LOCAL_IP:3001" echo -e " ${CYAN}RAG${NC} → http://$LOCAL_IP:8001/health"