277 lines
14 KiB
Bash
277 lines
14 KiB
Bash
|
#!/usr/bin/env bash
|
|||
|
|
set -Eeuo pipefail
|
|||
|
|
|
|||
|
|
# Complete System Integration Test
|
|||
|
|
# Tests the entire RAG stack end-to-end
|
|||
|
|
|
|||
|
|
# Color codes
|
|||
|
|
RED='\033[0;31m'
|
|||
|
|
GREEN='\033[0;32m'
|
|||
|
|
YELLOW='\033[1;33m'
|
|||
|
|
BLUE='\033[0;34m'
|
|||
|
|
CYAN='\033[0;36m'
|
|||
|
|
NC='\033[0m'
|
|||
|
|
|
|||
|
|
# Configuration from JSON output
|
|||
|
|
CTID="${1:-769276659}"
|
|||
|
|
CT_IP="${2:-192.168.45.45}"
|
|||
|
|
CT_HOSTNAME="${3:-sb-1769276659}"
|
|||
|
|
|
|||
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}"
|
|||
|
|
echo -e "${CYAN}║ ║${NC}"
|
|||
|
|
echo -e "${CYAN}║ Customer Installer - Complete System Test ║${NC}"
|
|||
|
|
echo -e "${CYAN}║ ║${NC}"
|
|||
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}"
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
print_header() {
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|||
|
|
echo -e "${BLUE} $1${NC}"
|
|||
|
|
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
print_test() { echo -e "${CYAN}[TEST]${NC} $1"; }
|
|||
|
|
print_pass() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|||
|
|
print_fail() { echo -e "${RED}[✗]${NC} $1"; }
|
|||
|
|
print_info() { echo -e "${BLUE}[ℹ]${NC} $1"; }
|
|||
|
|
print_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
|
|||
|
|
|
|||
|
|
TOTAL_TESTS=0
|
|||
|
|
PASSED_TESTS=0
|
|||
|
|
FAILED_TESTS=0
|
|||
|
|
|
|||
|
|
run_test() {
|
|||
|
|
((TOTAL_TESTS++))
|
|||
|
|
if eval "$2"; then
|
|||
|
|
print_pass "$1"
|
|||
|
|
((PASSED_TESTS++))
|
|||
|
|
return 0
|
|||
|
|
else
|
|||
|
|
print_fail "$1"
|
|||
|
|
((FAILED_TESTS++))
|
|||
|
|
return 1
|
|||
|
|
fi
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 1: Container & Infrastructure
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "1. Container & Infrastructure"
|
|||
|
|
|
|||
|
|
run_test "Container is running" \
|
|||
|
|
"pct status ${CTID} 2>/dev/null | grep -q 'running'"
|
|||
|
|
|
|||
|
|
run_test "Container has correct IP (${CT_IP})" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"ip -4 -o addr show scope global | awk '{print \\\$4}' | cut -d/ -f1 | head -n1\" 2>/dev/null) == '${CT_IP}' ]]"
|
|||
|
|
|
|||
|
|
run_test "Docker service is active" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'systemctl is-active docker' 2>/dev/null | grep -q 'active'"
|
|||
|
|
|
|||
|
|
run_test "Stack directory exists" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'test -d /opt/customer-stack' 2>/dev/null"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 2: Docker Containers
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "2. Docker Containers Status"
|
|||
|
|
|
|||
|
|
run_test "PostgreSQL container is running" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps postgres --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'"
|
|||
|
|
|
|||
|
|
run_test "PostgREST container is running" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps postgrest --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'"
|
|||
|
|
|
|||
|
|
run_test "n8n container is running" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose ps n8n --format \"{{.State}}\"' 2>/dev/null | grep -q 'running'"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 3: Database & Extensions
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "3. Database & Extensions"
|
|||
|
|
|
|||
|
|
run_test "PostgreSQL accepts connections" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'docker exec customer-postgres pg_isready -U customer -d customer' 2>/dev/null | grep -q 'accepting connections'"
|
|||
|
|
|
|||
|
|
run_test "pgvector extension is installed" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT extname FROM pg_extension WHERE extname='vector';\\\"\" 2>/dev/null) == 'vector' ]]"
|
|||
|
|
|
|||
|
|
run_test "pg_trgm extension is installed" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT extname FROM pg_extension WHERE extname='pg_trgm';\\\"\" 2>/dev/null) == 'pg_trgm' ]]"
|
|||
|
|
|
|||
|
|
run_test "Documents table exists" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename='documents';\\\"\" 2>/dev/null) == 'documents' ]]"
|
|||
|
|
|
|||
|
|
run_test "match_documents function exists" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT proname FROM pg_proc WHERE proname='match_documents';\\\"\" 2>/dev/null | grep -q 'match_documents'"
|
|||
|
|
|
|||
|
|
run_test "Vector index exists on documents table" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc \"docker exec customer-postgres psql -U customer -d customer -tAc \\\"SELECT indexname FROM pg_indexes WHERE tablename='documents' AND indexname='documents_embedding_idx';\\\"\" 2>/dev/null | grep -q 'documents_embedding_idx'"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 4: PostgREST API
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "4. PostgREST API"
|
|||
|
|
|
|||
|
|
run_test "PostgREST root endpoint (internal)" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/\" 2>/dev/null) == '200' ]]"
|
|||
|
|
|
|||
|
|
run_test "PostgREST root endpoint (external)" \
|
|||
|
|
"[[ \$(curl -s -o /dev/null -w '%{http_code}' http://${CT_IP}:3000/ 2>/dev/null) == '200' ]]"
|
|||
|
|
|
|||
|
|
run_test "Documents table accessible via API" \
|
|||
|
|
"curl -s http://${CT_IP}:3000/documents 2>/dev/null | grep -q '\['"
|
|||
|
|
|
|||
|
|
run_test "PostgREST accessible from n8n container" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n curl -s -o /dev/null -w '%{http_code}' http://postgrest:3000/\" 2>/dev/null) == '200' ]]"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 5: n8n Service
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "5. n8n Service"
|
|||
|
|
|
|||
|
|
run_test "n8n web interface (internal)" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:5678/\" 2>/dev/null) == '200' ]]"
|
|||
|
|
|
|||
|
|
run_test "n8n web interface (external)" \
|
|||
|
|
"[[ \$(curl -s -o /dev/null -w '%{http_code}' http://${CT_IP}:5678/ 2>/dev/null) == '200' ]]"
|
|||
|
|
|
|||
|
|
run_test "n8n health endpoint" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc \"curl -s http://127.0.0.1:5678/healthz\" 2>/dev/null | grep -q 'ok'"
|
|||
|
|
|
|||
|
|
run_test "n8n uses PostgreSQL database" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n printenv DB_TYPE\" 2>/dev/null) == 'postgresdb' ]]"
|
|||
|
|
|
|||
|
|
run_test "n8n encryption key is configured" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker exec n8n printenv N8N_ENCRYPTION_KEY | wc -c\" 2>/dev/null) -gt 10 ]]"
|
|||
|
|
|
|||
|
|
run_test "n8n can connect to PostgreSQL" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc \"docker exec n8n nc -zv postgres 5432 2>&1\" 2>/dev/null | grep -q 'succeeded\\|open'"
|
|||
|
|
|
|||
|
|
run_test "n8n can connect to PostgREST" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc \"docker exec n8n nc -zv postgrest 3000 2>&1\" 2>/dev/null | grep -q 'succeeded\\|open'"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 6: Workflow Auto-Reload
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "6. Workflow Auto-Reload System"
|
|||
|
|
|
|||
|
|
run_test "Workflow reload service is enabled" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"systemctl is-enabled n8n-workflow-reload.service\" 2>/dev/null) == 'enabled' ]]"
|
|||
|
|
|
|||
|
|
run_test "Workflow template file exists" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'test -f /opt/customer-stack/workflow-template.json' 2>/dev/null"
|
|||
|
|
|
|||
|
|
run_test "Reload script exists and is executable" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'test -x /opt/customer-stack/reload-workflow.sh' 2>/dev/null"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 7: Network & Connectivity
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "7. Network & Connectivity"
|
|||
|
|
|
|||
|
|
run_test "Docker network exists" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"docker network ls --format '{{.Name}}' | grep -c 'customer-stack_customer-net'\" 2>/dev/null) -gt 0 ]]"
|
|||
|
|
|
|||
|
|
run_test "Container can reach internet" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1'"
|
|||
|
|
|
|||
|
|
run_test "Container can resolve DNS" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'ping -c 1 -W 2 google.com >/dev/null 2>&1'"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 8: Permissions & Security
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "8. Permissions & Security"
|
|||
|
|
|
|||
|
|
run_test "n8n volume has correct ownership (uid 1000)" \
|
|||
|
|
"[[ \$(pct exec ${CTID} -- bash -lc \"stat -c '%u' /opt/customer-stack/volumes/n8n-data\" 2>/dev/null) == '1000' ]]"
|
|||
|
|
|
|||
|
|
run_test "Environment file exists" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'test -f /opt/customer-stack/.env' 2>/dev/null"
|
|||
|
|
|
|||
|
|
run_test "Environment file has restricted permissions" \
|
|||
|
|
"pct exec ${CTID} -- bash -lc 'test \$(stat -c %a /opt/customer-stack/.env) -le 644' 2>/dev/null"
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 9: External Dependencies
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "9. External Dependencies"
|
|||
|
|
|
|||
|
|
OLLAMA_STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://192.168.45.3:11434/api/tags 2>/dev/null || echo "000")
|
|||
|
|
if [[ "$OLLAMA_STATUS" == "200" ]]; then
|
|||
|
|
print_pass "Ollama API is accessible (HTTP ${OLLAMA_STATUS})"
|
|||
|
|
((PASSED_TESTS++))
|
|||
|
|
else
|
|||
|
|
print_warn "Ollama API not accessible (HTTP ${OLLAMA_STATUS}) - External service"
|
|||
|
|
fi
|
|||
|
|
((TOTAL_TESTS++))
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SECTION 10: Log Files
|
|||
|
|
# ============================================================================
|
|||
|
|
print_header "10. Log Files & Documentation"
|
|||
|
|
|
|||
|
|
run_test "Installation log exists" \
|
|||
|
|
"test -f logs/${CT_HOSTNAME}.log"
|
|||
|
|
|
|||
|
|
if [[ -f "logs/${CT_HOSTNAME}.log" ]]; then
|
|||
|
|
LOG_SIZE=$(du -h "logs/${CT_HOSTNAME}.log" 2>/dev/null | cut -f1)
|
|||
|
|
print_info "Log file size: ${LOG_SIZE}"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# ============================================================================
|
|||
|
|
# SUMMARY
|
|||
|
|
# ============================================================================
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}"
|
|||
|
|
echo -e "${CYAN}║ TEST SUMMARY ║${NC}"
|
|||
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}"
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
PASS_RATE=$((PASSED_TESTS * 100 / TOTAL_TESTS))
|
|||
|
|
|
|||
|
|
echo -e " Total Tests: ${TOTAL_TESTS}"
|
|||
|
|
echo -e " ${GREEN}Passed: ${PASSED_TESTS}${NC}"
|
|||
|
|
echo -e " ${RED}Failed: ${FAILED_TESTS}${NC}"
|
|||
|
|
echo -e " Pass Rate: ${PASS_RATE}%"
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
if [[ $FAILED_TESTS -eq 0 ]]; then
|
|||
|
|
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
|
|||
|
|
echo -e "${GREEN}║ ║${NC}"
|
|||
|
|
echo -e "${GREEN}║ ✓ ALL TESTS PASSED SUCCESSFULLY! ║${NC}"
|
|||
|
|
echo -e "${GREEN}║ ║${NC}"
|
|||
|
|
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${BLUE}System Information:${NC}"
|
|||
|
|
echo -e " Container ID: ${CTID}"
|
|||
|
|
echo -e " Hostname: ${CT_HOSTNAME}"
|
|||
|
|
echo -e " IP Address: ${CT_IP}"
|
|||
|
|
echo -e " VLAN: 90"
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${BLUE}Access URLs:${NC}"
|
|||
|
|
echo -e " n8n (internal): http://${CT_IP}:5678/"
|
|||
|
|
echo -e " n8n (external): https://${CT_HOSTNAME}.userman.de"
|
|||
|
|
echo -e " PostgREST API: http://${CT_IP}:3000/"
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${BLUE}Next Steps:${NC}"
|
|||
|
|
echo -e " 1. Configure NGINX reverse proxy on OPNsense"
|
|||
|
|
echo -e " 2. Test RAG workflow with document upload"
|
|||
|
|
echo -e " 3. Verify Ollama connectivity for AI features"
|
|||
|
|
echo ""
|
|||
|
|
exit 0
|
|||
|
|
else
|
|||
|
|
echo -e "${RED}╔════════════════════════════════════════════════════════════╗${NC}"
|
|||
|
|
echo -e "${RED}║ ║${NC}"
|
|||
|
|
echo -e "${RED}║ ✗ SOME TESTS FAILED ║${NC}"
|
|||
|
|
echo -e "${RED}║ ║${NC}"
|
|||
|
|
echo -e "${RED}╚════════════════════════════════════════════════════════════╝${NC}"
|
|||
|
|
echo ""
|
|||
|
|
echo -e "${YELLOW}Please review the failed tests above and check:${NC}"
|
|||
|
|
echo -e " - Container logs: pct exec ${CTID} -- bash -lc 'cd /opt/customer-stack && docker compose logs'"
|
|||
|
|
echo -e " - Installation log: cat logs/${CT_HOSTNAME}.log"
|
|||
|
|
echo ""
|
|||
|
|
exit 1
|
|||
|
|
fi
|