From 9bcef897194932ab5cd4c6ff38549762197cbf55 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Mar 2026 04:49:50 +0000 Subject: [PATCH] add interactive SearXNG safe-search prompts and category/engine controls Setup (both scripts): - Q6 asks safe-search level: none / moderate / strict - Then asks which categories to disable: videos images news science social - Then asks which engines to disable by name (duckduckgo, bing, etc.) - Choices flow into SEARXNG_QUERY_URL &safesearch=N in docker-compose.yml - SearXNG summary line added to laptop_full_setup.sh install plan configure-searxng-safesearch.sh: - Full rewrite: --disable-categories, --disable-engines, --enable-engines - Category maps: videos / images / news / science / social engine lists - --enable-engines overrides auto-disables (e.g. keep yandex on strict) - Preserves existing secret key on update - Creates settings.yml from scratch if missing (safe for setup use) - Help flag (-h/--help) https://claude.ai/code/session_012gDnantBmFTWZGCiKyjazx --- configure-searxng-safesearch.sh | 237 +++++++++++++++++++++----------- laptop_full_setup.sh | 60 ++++++-- local-ai-setup.sh | 45 ++++-- 3 files changed, 233 insertions(+), 109 deletions(-) diff --git a/configure-searxng-safesearch.sh b/configure-searxng-safesearch.sh index 90f77da..c7bcd52 100755 --- a/configure-searxng-safesearch.sh +++ b/configure-searxng-safesearch.sh @@ -1,17 +1,21 @@ #!/usr/bin/env bash # configure-searxng-safesearch.sh -# Set SearXNG safe-search level and disable engines that can't enforce it. +# Set SearXNG safe-search level; optionally disable categories or engines. # # Usage: -# ./configure-searxng-safesearch.sh strict # block explicit content -# ./configure-searxng-safesearch.sh moderate # filter but not strict -# ./configure-searxng-safesearch.sh none # unfiltered (default) +# ./configure-searxng-safesearch.sh [strict|moderate|none] [OPTIONS] # -# What it does: -# 1. Updates ~/docker/ai-stack/searxng/settings.yml -# 2. Disables engines that don't honour the chosen safe-search level -# 3. Updates SEARXNG_QUERY_URL in docker-compose.yml to pass &safesearch=N -# 4. Restarts the SearXNG container to apply changes +# Options: +# --disable-categories cat1,cat2 videos images news science social +# --disable-engines eng1,eng2 e.g. duckduckgo,bing,yandex +# --enable-engines eng1,eng2 re-enable engines auto-disabled by level +# +# Examples: +# ./configure-searxng-safesearch.sh strict +# ./configure-searxng-safesearch.sh moderate --disable-categories videos,images +# ./configure-searxng-safesearch.sh none --disable-engines bing,duckduckgo +# ./configure-searxng-safesearch.sh strict --disable-categories videos \ +# --disable-engines bing --enable-engines yandex set -euo pipefail @@ -19,104 +23,177 @@ set -euo pipefail red() { printf '\e[31m%s\e[0m\n' "$*"; } grn() { printf '\e[32m%s\e[0m\n' "$*"; } blu() { printf '\e[34m%s\e[0m\n' "$*"; } +yel() { printf '\e[33m%s\e[0m\n' "$*"; } die() { red "ERROR: $*"; exit 1; } ok() { grn " ✓ $*"; } info() { blu " → $*"; } +warn() { yel " ! $*"; } -LEVEL="${1:-moderate}" +# ── Defaults ────────────────────────────────────────────────────────────────── +LEVEL="moderate" +DISABLE_CATS="" +DISABLE_ENGINES_EXTRA="" +ENABLE_ENGINES_EXTRA="" BASE="${BASE:-$HOME/docker/ai-stack}" + +# ── Parse arguments ─────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + strict|moderate|none) LEVEL="$1"; shift ;; + --disable-categories) DISABLE_CATS="$2"; shift 2 ;; + --disable-engines) DISABLE_ENGINES_EXTRA="$2"; shift 2 ;; + --enable-engines) ENABLE_ENGINES_EXTRA="$2"; shift 2 ;; + --base) BASE="$2"; shift 2 ;; + -h|--help) + sed -n '2,20p' "$0" | sed 's/^# \?//' + exit 0 + ;; + *) die "Unknown argument: $1 (run with --help)" ;; + esac +done + SETTINGS="$BASE/searxng/settings.yml" COMPOSE="$BASE/docker-compose.yml" -# ── Validate ────────────────────────────────────────────────────────────────── -case "$LEVEL" in - strict|moderate|none) ;; - *) die "Unknown level '$LEVEL'. Use: strict, moderate, or none" ;; -esac +# Ensure settings directory exists +mkdir -p "$(dirname "$SETTINGS")" -[[ -f "$SETTINGS" ]] || die "SearXNG settings not found: $SETTINGS" - -# ── Map level → SearXNG integer ─────────────────────────────────────────────── +# ── Level → integer ─────────────────────────────────────────────────────────── case "$LEVEL" in none) SAFE_INT=0 ;; moderate) SAFE_INT=1 ;; strict) SAFE_INT=2 ;; esac -info "Setting safe_search = $LEVEL (${SAFE_INT})" +info "Safe search: $LEVEL (${SAFE_INT})" -# ── Engines that cannot enforce safe-search ─────────────────────────────────── -# These are disabled for moderate/strict because they either have no -# safe-search API parameter, or don't reliably honour it. -# -# Torrent/P2P — inherently unfiltered -TORRENT_ENGINES=( - "1337x" - "piratebay" - "nyaa" - "torrentz" - "kickass torrents" -) - -# General web / image / video engines without safe-search support +# ── Engines with no safe-search support ─────────────────────────────────────── +# Auto-disabled when level is moderate or strict. NO_SAFESEARCH_ENGINES=( - "mojeek" # no safe-search parameter - "naver" # Korean engine, no safe-search API - "baidu" # Chinese engine, no safe-search for non-CN queries - "yandex" # nominally supports it but does not reliably enforce - "invidious" # YouTube frontend, no filtering - "piped" # YouTube frontend, no filtering - "peertube" # federated video, no filtering - "sepiasearch" # PeerTube index, no filtering + # Torrent / P2P — no filtering possible + "1337x" "piratebay" "nyaa" "torrentz" "kickass torrents" + # Web engines without safe-search API + "mojeek" "naver" "baidu" + # Yandex: parameter exists but not reliably enforced for non-Russian queries + "yandex" + # Video frontends — no safe-search passthrough + "invidious" "piped" "peertube" "sepiasearch" ) -ALL_SKIP=("${TORRENT_ENGINES[@]}" "${NO_SAFESEARCH_ENGINES[@]}") +# ── Category → engine lists ─────────────────────────────────────────────────── +VIDEOS_ENGINES=( + "youtube" "invidious" "piped" "peertube" "sepiasearch" + "dailymotion" "vimeo" + "bing videos" "duckduckgo videos" "google videos" +) +IMAGES_ENGINES=( + "google images" "bing images" "duckduckgo images" + "brave images" "qwant images" + "flickr" "unsplash" "imgur" "deviantart" "openverse" +) +NEWS_ENGINES=( + "google news" "bing news" "duckduckgo news" "brave news" "qwant news" +) +SCIENCE_ENGINES=( + "arxiv" "semantic scholar" "pubmed" "crossref" "base" +) +SOCIAL_ENGINES=( + "reddit" "lemmy" "mastodon" +) -# ── Extract existing secret key (avoid regenerating on every run) ───────────── +# ── Build disable/enable maps ───────────────────────────────────────────────── +declare -A DISABLE_MAP # engine → 1 +declare -A ENABLE_MAP # engine → 1 (overrides everything) + +# Parse --enable-engines +if [[ -n "$ENABLE_ENGINES_EXTRA" ]]; then + IFS=',' read -ra _engs <<< "$ENABLE_ENGINES_EXTRA" + for e in "${_engs[@]}"; do + e="${e#"${e%%[![:space:]]*}"}"; e="${e%"${e##*[![:space:]]}"}" # trim + [[ -n "$e" ]] && ENABLE_MAP["$e"]=1 + done +fi + +# Helper: add to DISABLE_MAP unless explicitly re-enabled +mark_disabled() { + local eng="$1" + [[ -n "${ENABLE_MAP[$eng]+x}" ]] && return # user said keep it + DISABLE_MAP["$eng"]=1 +} + +# Auto-disable no-safesearch engines for moderate/strict +if [[ "$LEVEL" != "none" ]]; then + for eng in "${NO_SAFESEARCH_ENGINES[@]}"; do + mark_disabled "$eng" + done +fi + +# Category disables +if [[ -n "$DISABLE_CATS" ]]; then + IFS=',' read -ra _cats <<< "$DISABLE_CATS" + for cat in "${_cats[@]}"; do + cat="${cat#"${cat%%[![:space:]]*}"}"; cat="${cat%"${cat##*[![:space:]]}"}" + cat="${cat,,}" + case "$cat" in + videos) for e in "${VIDEOS_ENGINES[@]}"; do mark_disabled "$e"; done ;; + images) for e in "${IMAGES_ENGINES[@]}"; do mark_disabled "$e"; done ;; + news) for e in "${NEWS_ENGINES[@]}"; do mark_disabled "$e"; done ;; + science) for e in "${SCIENCE_ENGINES[@]}"; do mark_disabled "$e"; done ;; + social) for e in "${SOCIAL_ENGINES[@]}"; do mark_disabled "$e"; done ;; + "") ;; + *) warn "Unknown category '$cat' — valid: videos images news science social" ;; + esac + done +fi + +# Extra engine disables +if [[ -n "$DISABLE_ENGINES_EXTRA" ]]; then + IFS=',' read -ra _engs <<< "$DISABLE_ENGINES_EXTRA" + for e in "${_engs[@]}"; do + e="${e#"${e%%[![:space:]]*}"}"; e="${e%"${e##*[![:space:]]}"}" + [[ -n "$e" ]] && mark_disabled "$e" + done +fi + +# ── Preserve existing secret key ───────────────────────────────────────────── SECRET_KEY=$(grep -oP '(?<=secret_key: ")[^"]+' "$SETTINGS" 2>/dev/null || true) [[ -z "$SECRET_KEY" ]] && SECRET_KEY=$(openssl rand -hex 32) -# ── Build engine-override block ─────────────────────────────────────────────── -build_overrides() { - local disabled="$1" # true or false - for engine in "${ALL_SKIP[@]}"; do - printf ' - name: %s\n disabled: %s\n' "$engine" "$disabled" - done -} - -if [[ "$LEVEL" == "none" ]]; then - OVERRIDE_BLOCK=$(build_overrides false) -else - OVERRIDE_BLOCK=$(build_overrides true) -fi +# ── Build engine override block ─────────────────────────────────────────────── +ENGINE_BLOCK="" +for eng in "${!DISABLE_MAP[@]}"; do + ENGINE_BLOCK+=" - name: ${eng}\n disabled: true\n" +done +for eng in "${!ENABLE_MAP[@]}"; do + ENGINE_BLOCK+=" - name: ${eng}\n disabled: false\n" +done # ── Write settings.yml ──────────────────────────────────────────────────────── -cat > "$SETTINGS" << YAML -use_default_settings: true -general: - instance_name: "Local Search" -server: - secret_key: "$SECRET_KEY" - limiter: false -search: - safe_search: $SAFE_INT - default_lang: "en" - formats: [html, json] -engines: -$OVERRIDE_BLOCK -YAML +{ + printf 'use_default_settings: true\n' + printf 'general:\n instance_name: "Local Search"\n' + printf 'server:\n secret_key: "%s"\n limiter: false\n' "$SECRET_KEY" + printf 'search:\n safe_search: %d\n default_lang: "en"\n formats: [html, json]\n' "$SAFE_INT" + if [[ -n "$ENGINE_BLOCK" ]]; then + printf 'engines:\n' + printf '%b' "$ENGINE_BLOCK" + fi +} > "$SETTINGS" ok "Updated settings.yml (safe_search: $SAFE_INT)" +if [[ ${#DISABLE_MAP[@]} -gt 0 ]]; then + info "Disabled (${#DISABLE_MAP[@]}): $(printf '%s, ' "${!DISABLE_MAP[@]}" | sed 's/, $//')" +fi +if [[ ${#ENABLE_MAP[@]} -gt 0 ]]; then + info "Re-enabled: $(printf '%s, ' "${!ENABLE_MAP[@]}" | sed 's/, $//')" +fi # ── Update &safesearch= in SEARXNG_QUERY_URL inside docker-compose.yml ──────── if [[ -f "$COMPOSE" ]]; then - # Strip any existing &safesearch=N, then append the current value sed -i -E \ - "s|(SEARXNG_QUERY_URL=http://searxng:[0-9]+/search\?[^&\n]*)(&safesearch=[0-9])?|\1\&safesearch=${SAFE_INT}|g" \ + "s|(SEARXNG_QUERY_URL=http://searxng:[0-9]+/search\?[^&[:space:]]*)(&safesearch=[0-9])?|\1\&safesearch=${SAFE_INT}|g" \ "$COMPOSE" - ok "Updated SEARXNG_QUERY_URL in docker-compose.yml (&safesearch=${SAFE_INT})" -else - info "docker-compose.yml not found — skipping URL update" + ok "Updated SEARXNG_QUERY_URL (&safesearch=${SAFE_INT})" fi # ── Restart SearXNG ─────────────────────────────────────────────────────────── @@ -125,12 +202,10 @@ if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^searxng$'; then docker restart searxng ok "SearXNG restarted" else - info "SearXNG is not running — changes take effect on next start" + info "SearXNG not running — changes take effect on next start" fi echo -grn "Safe-search level: $LEVEL" -if [[ "$LEVEL" != "none" ]]; then - info "Disabled ${#ALL_SKIP[@]} engines that can't enforce '${LEVEL}'" - for e in "${ALL_SKIP[@]}"; do printf ' - %s\n' "$e"; done -fi +grn "Done — safe search: $LEVEL" +[[ "$LEVEL" != "none" ]] && \ + info "Engines skipped (can't enforce '$LEVEL'): ${#DISABLE_MAP[@]} total" diff --git a/laptop_full_setup.sh b/laptop_full_setup.sh index a1a1033..befc92d 100755 --- a/laptop_full_setup.sh +++ b/laptop_full_setup.sh @@ -493,6 +493,37 @@ if $INSTALL_AI; then [[ "${DO_PULL,,}" != "n" ]] && PULL_MODELS=true fi +# ── Q6: SearXNG safe-search ─────────────────────────────────────────────────── +SEARXNG_SAFE_LEVEL="none" +SEARXNG_SAFE_INT=0 +SEARXNG_DISABLE_CATS="" +SEARXNG_DISABLE_ENGINES="" + +if $INSTALL_AI && $SVC_SEARXNG; then + echo "" + echo -e " ${BOLD}[6/6] SearXNG safe-search${NC}" + echo " 0) None — all results, no filtering (default)" + echo " 1) Moderate — filter explicit content" + echo " 2) Strict — block all explicit content" + echo "" + read -rp " Choice [0]: " _SAFE_PICK + case "${_SAFE_PICK:-0}" in + 1) SEARXNG_SAFE_LEVEL="moderate"; SEARXNG_SAFE_INT=1 ;; + 2) SEARXNG_SAFE_LEVEL="strict"; SEARXNG_SAFE_INT=2 ;; + *) SEARXNG_SAFE_LEVEL="none"; SEARXNG_SAFE_INT=0 ;; + esac + + echo "" + echo " Disable search categories? (Enter to keep all)" + echo " Options: videos images news science social" + read -rp " Categories (space-separated, Enter to skip): " SEARXNG_DISABLE_CATS + + echo "" + echo " Disable specific engines? (Enter to keep defaults)" + echo " Web: google bing duckduckgo brave startpage qwant yahoo" + read -rp " Engines (space-separated, Enter to skip): " SEARXNG_DISABLE_ENGINES +fi + # ── Summary ─────────────────────────────────────────────────────────────────── echo "" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" @@ -521,6 +552,12 @@ if $INSTALL_AI; then fi fi $SVC_KIWIX && echo " ✓ Kiwix ZIMs → $KIWIX_DIR" +if $INSTALL_AI && $SVC_SEARXNG; then + _sx="safe_search: $SEARXNG_SAFE_LEVEL" + [[ -n "$SEARXNG_DISABLE_CATS" ]] && _sx+=" | disabled cats: $SEARXNG_DISABLE_CATS" + [[ -n "$SEARXNG_DISABLE_ENGINES" ]] && _sx+=" | disabled engines: $SEARXNG_DISABLE_ENGINES" + echo " ✓ SearXNG → $_sx" +fi $PULL_MODELS && echo " ✓ Pull models : $EMBED_MODEL + $FAST_MODEL + $CHAT_MODEL + $CODE_MODEL${REASON_MODEL:+ + $REASON_MODEL}" [[ "$ZIM_CHOICE" == "1" ]] && echo " ✓ Download all ZIMs in background (~130GB)" [[ "$ZIM_CHOICE" == "2" ]] && echo " ✓ Select ZIMs to download (prompted after stack starts)" @@ -674,23 +711,16 @@ ok "mcp_requirements.txt" fi # INSTALL_AI # ============================================================================= -if $INSTALL_AI; then +if $INSTALL_AI && $SVC_SEARXNG; then section "SearXNG Config" # ============================================================================= -write_if_new "$BASE/searxng/settings.yml" << SEARXNG -use_default_settings: true -general: - instance_name: "Local Search" -server: - secret_key: "$(openssl rand -hex 32)" - limiter: false -search: - safe_search: 0 - default_lang: "en" - formats: [html, json] -SEARXNG +mkdir -p "$BASE/searxng" +_SXARGS=("$SEARXNG_SAFE_LEVEL") +[[ -n "$SEARXNG_DISABLE_CATS" ]] && _SXARGS+=(--disable-categories "${SEARXNG_DISABLE_CATS// /,}") +[[ -n "$SEARXNG_DISABLE_ENGINES" ]] && _SXARGS+=(--disable-engines "${SEARXNG_DISABLE_ENGINES// /,}") +BASE="$BASE" bash "$SCRIPT_DIR/configure-searxng-safesearch.sh" "${_SXARGS[@]}" -fi # INSTALL_AI +fi # INSTALL_AI && SVC_SEARXNG # ============================================================================= section ".env File" @@ -780,7 +810,7 @@ ${OLLAMA_VOLUME_LINE} - ENABLE_OPENAI_API=true - ENABLE_RAG_WEB_SEARCH=true - RAG_WEB_SEARCH_ENGINE=searxng - - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=0 + - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=${SEARXNG_SAFE_INT} - RAG_WEB_SEARCH_RESULT_COUNT=5 - RAG_WEB_SEARCH_CONCURRENT_REQUESTS=10 - ENABLE_TOOL_SERVERS=true diff --git a/local-ai-setup.sh b/local-ai-setup.sh index 1a5363c..91b5b46 100755 --- a/local-ai-setup.sh +++ b/local-ai-setup.sh @@ -14,6 +14,7 @@ FORCE=false; NO_PULL=false for a in "$@"; do [[ "$a" == "--force" ]] && FORCE=true; [[ "$a" == "--no-pull" ]] && NO_PULL=true; done BASE="$HOME/docker/ai-stack" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LOCAL_IP=$(ip route get 1.1.1.1 2>/dev/null | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}') [[ -z "$LOCAL_IP" ]] && read -rp "Enter LAN IP: " LOCAL_IP @@ -43,6 +44,31 @@ info "Base : $BASE" info "IP : $LOCAL_IP" info "GPU : ${VRAM_GB}GB VRAM → $TIER" +# ── SearXNG safe-search ─────────────────────────────────────────────────────── +echo "" +echo " SearXNG safe-search level:" +echo " 0) None — all results, no filtering (default)" +echo " 1) Moderate — filter explicit content" +echo " 2) Strict — block all explicit content" +echo "" +read -rp " Choice [0]: " _SAFE_PICK +case "${_SAFE_PICK:-0}" in + 1) SEARXNG_SAFE_LEVEL="moderate"; SEARXNG_SAFE_INT=1 ;; + 2) SEARXNG_SAFE_LEVEL="strict"; SEARXNG_SAFE_INT=2 ;; + *) SEARXNG_SAFE_LEVEL="none"; SEARXNG_SAFE_INT=0 ;; +esac + +echo "" +echo " Disable search categories? (Enter to keep all)" +echo " Options: videos images news science social" +read -rp " Categories (space-separated, Enter to skip): " SEARXNG_DISABLE_CATS + +echo "" +echo " Disable specific engines? (Enter to keep defaults)" +echo " Web: google bing duckduckgo brave startpage qwant yahoo" +read -rp " Engines (space-separated, Enter to skip): " SEARXNG_DISABLE_ENGINES +echo "" + write_if_new() { local dest="$1"; local body; body=$(cat) if [[ ! -f "$dest" ]] || $FORCE; then @@ -481,18 +507,11 @@ ok "requirements.txt + mcp_requirements.txt" # ============================================================================= section "SearXNG Config" # ============================================================================= -write_if_new "$BASE/searxng/settings.yml" << SEARXNG -use_default_settings: true -general: - instance_name: "Local Search" -server: - secret_key: "$(openssl rand -hex 32)" - limiter: false -search: - safe_search: 0 - default_lang: "en" - formats: [html, json] -SEARXNG +mkdir -p "$BASE/searxng" +_SXARGS=("$SEARXNG_SAFE_LEVEL") +[[ -n "$SEARXNG_DISABLE_CATS" ]] && _SXARGS+=(--disable-categories "${SEARXNG_DISABLE_CATS// /,}") +[[ -n "$SEARXNG_DISABLE_ENGINES" ]] && _SXARGS+=(--disable-engines "${SEARXNG_DISABLE_ENGINES// /,}") +BASE="$BASE" bash "$SCRIPT_DIR/configure-searxng-safesearch.sh" "${_SXARGS[@]}" # ============================================================================= section ".env (tokens — never overwritten)" @@ -553,7 +572,7 @@ services: - ENABLE_OPENAI_API=true - ENABLE_RAG_WEB_SEARCH=true - RAG_WEB_SEARCH_ENGINE=searxng - - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=0 + - SEARXNG_QUERY_URL=http://searxng:8080/search?q=&format=json&safesearch=${SEARXNG_SAFE_INT} - RAG_WEB_SEARCH_RESULT_COUNT=5 - RAG_WEB_SEARCH_CONCURRENT_REQUESTS=10 - ENABLE_TOOL_SERVERS=true