#!/usr/bin/env bash set -Eeuo pipefail # Complete System Integration Test # Tests the entire RAG stack end-to-end # Color codes RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' # Configuration from JSON output CTID="${1:-769276659}" CT_IP="${2:-192.168.45.45}" CT_HOSTNAME="${3:-sb-1769276659}" echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║ ║${NC}" echo -e "${CYAN}║ Customer Installer - Complete System Test ║${NC}" echo -e "${CYAN}║ ║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}" echo "" print_header() { echo "" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BLUE} $1${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" } print_test() { echo -e "${CYAN}[TEST]${NC} $1"; } print_pass() { echo -e "${GREEN}[✓]${NC} $1"; } print_fail() { echo -e "${RED}[✗]${NC} $1"; } print_info() { echo -e "${BLUE}[ℹ]${NC} $1"; } print_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } TOTAL_TESTS=0 PASSED_TESTS=0 FAILED_TESTS=0 run_test() { ((TOTAL_TESTS++)) if eval "$2"; then print_pass "$1" ((PASSED_TESTS++)) return 0 else print_fail "$1" ((FAILED_TESTS++)) return 1 fi } # ============================================================================ # SECTION 1: Container & Infrastructure # ============================================================================ print_header "1. Container & Infrastructure" run_test "Container is running" \ "pct status ${CTID} 2>/dev/null | grep -q 'running'" run_test "Container has correct IP (${CT_IP})" \ "[[ \$(pct exec ${CTID} -- bash -lc \"ip -4 -o addr show scope global | awk '{print \\\$4}' | cut -d/ -f1 | head -n1\" 2>/dev/null) == '${CT_IP}' ]]" run_test "Docker service is active" \ "pct exec ${CTID} -- bash -lc 'systemctl is-active docker' 2>/dev/null | grep -q 'active'" run_test "Stack directory exists" \ "pct exec ${CTID} -- bash -lc 'test -d /opt/customer-stack' 2>/dev/null" # ============================================================================ # SECTION 2: Docker Containers # ============================================================================ print_header "2. Docker Containers Status" run_test "PostgreSQL container is running" \ "pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps postgres --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'" run_test "PostgREST container is running" \ "pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps postgrest --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'" run_test "n8n container is running" \ "pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps n8n --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'" # ============================================================================ # SECTION 3: Database & Extensions # ============================================================================ print_header "3. Database & Extensions" run_test "PostgreSQL accepts connections" \ "pct exec ${CTID} -- bash -lc 'docker exec customer-postgres pg_isready -U customer -d customer' 2>/dev/null | grep -q 'accepting connections'" run_test "pgvector extension is installed" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT extname FROM pg_extension WHERE extname='vector';\\\"\" 2>/dev/null) == 'vector' ]]" run_test "pg_trgm extension is installed" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT extname FROM pg_extension WHERE extname='pg_trgm';\\\"\" 2>/dev/null) == 'pg_trgm' ]]" run_test "Documents table exists" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename='documents';\\\"\" 2>/dev/null) == 'documents' ]]" run_test "match_documents function exists" \ "pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT proname FROM pg_proc WHERE proname='match_documents';\\\"\" 2>/dev/null | grep -q 'match_documents'" run_test "Vector index exists on documents table" \ "pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT indexname FROM pg_indexes WHERE tablename='documents' AND indexname='documents_embedding_idx';\\\"\" 2>/dev/null | grep -q 'documents_embedding_idx'" # ============================================================================ # SECTION 4: PostgREST API # ============================================================================ print_header "4. PostgREST API" run_test "PostgREST root endpoint (internal)" \ "[[ \$(pct exec ${CTID} -- bash -lc \"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/\" 2>/dev/null) == '200' ]]" run_test "PostgREST root endpoint (external)" \ "[[ \$(curl -s -o /dev/null -w '%{http_code}' http://${CT_IP}:3000/ 2>/dev/null) == '200' ]]" run_test "Documents table accessible via API" \ "curl -s http://${CT_IP}:3000/documents 2>/dev/null | grep -q '\['" run_test "PostgREST accessible from n8n container" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n curl -s -o /dev/null -w '%{http_code}' http://postgrest:3000/\" 2>/dev/null) == '200' ]]" # ============================================================================ # SECTION 5: n8n Service # ============================================================================ print_header "5. n8n Service" run_test "n8n web interface (internal)" \ "[[ \$(pct exec ${CTID} -- bash -lc \"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:5678/\" 2>/dev/null) == '200' ]]" run_test "n8n web interface (external)" \ "[[ \$(curl -s -o /dev/null -w '%{http_code}' http://${CT_IP}:5678/ 2>/dev/null) == '200' ]]" run_test "n8n health endpoint" \ "pct exec ${CTID} -- bash -lc \"curl -s http://127.0.0.1:5678/healthz\" 2>/dev/null | grep -q 'ok'" run_test "n8n uses PostgreSQL database" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n printenv DB_TYPE\" 2>/dev/null) == 'postgresdb' ]]" run_test "n8n encryption key is configured" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n printenv N8N_ENCRYPTION_KEY | wc -c\" 2>/dev/null) -gt 10 ]]" run_test "n8n can connect to PostgreSQL" \ "pct exec ${CTID} -- bash -lc \"docker exec n8n nc -zv postgres 5432 2>&1\" 2>/dev/null | grep -q 'succeeded\\|open'" run_test "n8n can connect to PostgREST" \ "pct exec ${CTID} -- bash -lc \"docker exec n8n nc -zv postgrest 3000 2>&1\" 2>/dev/null | grep -q 'succeeded\\|open'" # ============================================================================ # SECTION 6: Workflow Auto-Reload # ============================================================================ print_header "6. Workflow Auto-Reload System" run_test "Workflow reload service is enabled" \ "[[ \$(pct exec ${CTID} -- bash -lc \"systemctl is-enabled n8n-workflow-reload.service\" 2>/dev/null) == 'enabled' ]]" run_test "Workflow template file exists" \ "pct exec ${CTID} -- bash -lc 'test -f /opt/customer-stack/workflow-template.json' 2>/dev/null" run_test "Reload script exists and is executable" \ "pct exec ${CTID} -- bash -lc 'test -x /opt/customer-stack/reload-workflow.sh' 2>/dev/null" # ============================================================================ # SECTION 7: Network & Connectivity # ============================================================================ print_header "7. Network & Connectivity" run_test "Docker network exists" \ "[[ \$(pct exec ${CTID} -- bash -lc \"docker network ls --format '{{.Name}}' | grep -c 'customer-stack_customer-net'\" 2>/dev/null) -gt 0 ]]" run_test "Container can reach internet" \ "pct exec ${CTID} -- bash -lc 'ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1'" run_test "Container can resolve DNS" \ "pct exec ${CTID} -- bash -lc 'ping -c 1 -W 2 google.com >/dev/null 2>&1'" # ============================================================================ # SECTION 8: Permissions & Security # ============================================================================ print_header "8. Permissions & Security" run_test "n8n volume has correct ownership (uid 1000)" \ "[[ \$(pct exec ${CTID} -- bash -lc \"stat -c '%u' /opt/customer-stack/volumes/n8n-data\" 2>/dev/null) == '1000' ]]" run_test "Environment file exists" \ "pct exec ${CTID} -- bash -lc 'test -f /opt/customer-stack/.env' 2>/dev/null" run_test "Environment file has restricted permissions" \ "pct exec ${CTID} -- bash -lc 'test \$(stat -c %a /opt/customer-stack/.env) -le 644' 2>/dev/null" # ============================================================================ # SECTION 9: External Dependencies # ============================================================================ print_header "9. External Dependencies" OLLAMA_STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://192.168.45.3:11434/api/tags 2>/dev/null || echo "000") if [[ "$OLLAMA_STATUS" == "200" ]]; then print_pass "Ollama API is accessible (HTTP ${OLLAMA_STATUS})" ((PASSED_TESTS++)) else print_warn "Ollama API not accessible (HTTP ${OLLAMA_STATUS}) - External service" fi ((TOTAL_TESTS++)) # ============================================================================ # SECTION 10: Log Files # ============================================================================ print_header "10. Log Files & Documentation" run_test "Installation log exists" \ "test -f logs/${CT_HOSTNAME}.log" if [[ -f "logs/${CT_HOSTNAME}.log" ]]; then LOG_SIZE=$(du -h "logs/${CT_HOSTNAME}.log" 2>/dev/null | cut -f1) print_info "Log file size: ${LOG_SIZE}" fi # ============================================================================ # SUMMARY # ============================================================================ echo "" echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║ TEST SUMMARY ║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}" echo "" PASS_RATE=$((PASSED_TESTS * 100 / TOTAL_TESTS)) echo -e " Total Tests: ${TOTAL_TESTS}" echo -e " ${GREEN}Passed: ${PASSED_TESTS}${NC}" echo -e " ${RED}Failed: ${FAILED_TESTS}${NC}" echo -e " Pass Rate: ${PASS_RATE}%" echo "" if [[ $FAILED_TESTS -eq 0 ]]; then echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${GREEN}║ ║${NC}" echo -e "${GREEN}║ ✓ ALL TESTS PASSED SUCCESSFULLY! ║${NC}" echo -e "${GREEN}║ ║${NC}" echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" echo "" echo -e "${BLUE}System Information:${NC}" echo -e " Container ID: ${CTID}" echo -e " Hostname: ${CT_HOSTNAME}" echo -e " IP Address: ${CT_IP}" echo -e " VLAN: 90" echo "" echo -e "${BLUE}Access URLs:${NC}" echo -e " n8n (internal): http://${CT_IP}:5678/" echo -e " n8n (external): https://${CT_HOSTNAME}.userman.de" echo -e " PostgREST API: http://${CT_IP}:3000/" echo "" echo -e "${BLUE}Next Steps:${NC}" echo -e " 1. Configure NGINX reverse proxy on OPNsense" echo -e " 2. Test RAG workflow with document upload" echo -e " 3. Verify Ollama connectivity for AI features" echo "" exit 0 else echo -e "${RED}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${RED}║ ║${NC}" echo -e "${RED}║ ✗ SOME TESTS FAILED ║${NC}" echo -e "${RED}║ ║${NC}" echo -e "${RED}╚════════════════════════════════════════════════════════════╝${NC}" echo "" echo -e "${YELLOW}Please review the failed tests above and check:${NC}" echo -e " - Container logs: pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose logs'" echo -e " - Installation log: cat logs/${CT_HOSTNAME}.log" echo "" exit 1 fi