customer-installer/test_installation.sh

#!/usr/bin/env bash
set -Eeuo pipefail

# Test script for customer-installer deployment
# This script verifies all components of the deployed LXC container

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Test results tracking
TESTS_PASSED=0
TESTS_FAILED=0
TESTS_TOTAL=0

# Parse JSON from installation output or use provided values
CTID="${1:-769276659}"
CT_IP="${2:-192.168.45.45}"
CT_HOSTNAME="${3:-sb-1769276659}"

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Customer Installer - Test Suite${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo -e "Testing Container: ${GREEN}${CTID}${NC}"
echo -e "IP Address: ${GREEN}${CT_IP}${NC}"
echo -e "Hostname: ${GREEN}${CT_HOSTNAME}${NC}"
echo ""

# Helper functions
print_test() {
    echo -e "${BLUE}[TEST]${NC} $1"
}

print_pass() {
    echo -e "${GREEN}[PASS]${NC} $1"
    ((TESTS_PASSED++))
    ((TESTS_TOTAL++))
}

print_fail() {
    echo -e "${RED}[FAIL]${NC} $1"
    ((TESTS_FAILED++))
    ((TESTS_TOTAL++))
}

print_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

print_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

# Test 1: Container exists and is running
print_test "Checking if container ${CTID} exists and is running..."
if pct status "${CTID}" 2>/dev/null | grep -q "running"; then
    print_pass "Container ${CTID} is running"
else
    print_fail "Container ${CTID} is not running"
    exit 1
fi

# Test 2: Container has correct IP
print_test "Verifying container IP address..."
ACTUAL_IP=$(pct exec "${CTID}" -- bash -lc "ip -4 -o addr show scope global | awk '{print \$4}' | cut -d/ -f1 | head -n1" 2>/dev/null || echo "")
if [[ "${ACTUAL_IP}" == "${CT_IP}" ]]; then
    print_pass "Container has correct IP: ${CT_IP}"
else
    print_fail "Container IP mismatch. Expected: ${CT_IP}, Got: ${ACTUAL_IP}"
fi

# Test 3: Docker is installed and running
print_test "Checking Docker installation..."
if pct exec "${CTID}" -- bash -lc "systemctl is-active docker" 2>/dev/null | grep -q "active"; then
    print_pass "Docker is installed and running"
else
    print_fail "Docker is not running"
fi

# Test 4: Docker Compose is available
print_test "Checking Docker Compose plugin..."
if pct exec "${CTID}" -- bash -lc "docker compose version" >/dev/null 2>&1; then
    COMPOSE_VERSION=$(pct exec "${CTID}" -- bash -lc "docker compose version" 2>/dev/null | head -1)
    print_pass "Docker Compose is available: ${COMPOSE_VERSION}"
else
    print_fail "Docker Compose plugin not found"
fi

# Test 5: Stack directory exists
print_test "Checking stack directory structure..."
if pct exec "${CTID}" -- bash -lc "test -d /opt/customer-stack" 2>/dev/null; then
    print_pass "Stack directory exists: /opt/customer-stack"
else
    print_fail "Stack directory not found"
fi

# Test 6: Docker containers are running
print_test "Checking Docker containers status..."
CONTAINERS=$(pct exec "${CTID}" -- bash -lc "cd /opt/customer-stack && docker compose ps --format json" 2>/dev/null || echo "[]")

# Check PostgreSQL
if echo "$CONTAINERS" | grep -q "customer-postgres"; then
    PG_STATUS=$(pct exec "${CTID}" -- bash -lc "cd /opt/customer-stack && docker compose ps postgres --format '{{.State}}'" 2>/dev/null || echo "")
    if [[ "$PG_STATUS" == "running" ]]; then
        print_pass "PostgreSQL container is running"
    else
        print_fail "PostgreSQL container is not running (status: ${PG_STATUS})"
    fi
else
    print_fail "PostgreSQL container not found"
fi

# Check PostgREST
if echo "$CONTAINERS" | grep -q "customer-postgrest"; then
    POSTGREST_STATUS=$(pct exec "${CTID}" -- bash -lc "cd /opt/customer-stack && docker compose ps postgrest --format '{{.State}}'" 2>/dev/null || echo "")
    if [[ "$POSTGREST_STATUS" == "running" ]]; then
        print_pass "PostgREST container is running"
    else
        print_fail "PostgREST container is not running (status: ${POSTGREST_STATUS})"
    fi
else
    print_fail "PostgREST container not found"
fi

# Check n8n
if echo "$CONTAINERS" | grep -q "n8n"; then
    N8N_STATUS=$(pct exec "${CTID}" -- bash -lc "cd /opt/customer-stack && docker compose ps n8n --format '{{.State}}'" 2>/dev/null || echo "")
    if [[ "$N8N_STATUS" == "running" ]]; then
        print_pass "n8n container is running"
    else
        print_fail "n8n container is not running (status: ${N8N_STATUS})"
    fi
else
    print_fail "n8n container not found"
fi

# Test 7: PostgreSQL health check
print_test "Testing PostgreSQL database connectivity..."
PG_HEALTH=$(pct exec "${CTID}" -- bash -lc "docker exec customer-postgres pg_isready -U customer -d customer" 2>/dev/null || echo "failed")
if echo "$PG_HEALTH" | grep -q "accepting connections"; then
    print_pass "PostgreSQL is accepting connections"
else
    print_fail "PostgreSQL health check failed: ${PG_HEALTH}"
fi

# Test 8: pgvector extension
print_test "Checking pgvector extension..."
PGVECTOR_CHECK=$(pct exec "${CTID}" -- bash -lc "docker exec customer-postgres psql -U customer -d customer -tAc \"SELECT extname FROM pg_extension WHERE extname='vector';\"" 2>/dev/null || echo "")
if [[ "$PGVECTOR_CHECK" == "vector" ]]; then
    print_pass "pgvector extension is installed"
else
    print_fail "pgvector extension not found"
fi

# Test 9: Documents table exists
print_test "Checking documents table for vector storage..."
DOCS_TABLE=$(pct exec "${CTID}" -- bash -lc "docker exec customer-postgres psql -U customer -d customer -tAc \"SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename='documents';\"" 2>/dev/null || echo "")
if [[ "$DOCS_TABLE" == "documents" ]]; then
    print_pass "Documents table exists"
else
    print_fail "Documents table not found"
fi

# Test 10: PostgREST API accessibility
print_test "Testing PostgREST API endpoint..."
POSTGREST_RESPONSE=$(pct exec "${CTID}" -- bash -lc "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/" 2>/dev/null || echo "000")
if [[ "$POSTGREST_RESPONSE" == "200" ]]; then
    print_pass "PostgREST API is accessible (HTTP ${POSTGREST_RESPONSE})"
else
    print_fail "PostgREST API not accessible (HTTP ${POSTGREST_RESPONSE})"
fi

# Test 11: PostgREST external accessibility
print_test "Testing PostgREST external accessibility..."
POSTGREST_EXT=$(curl -s -o /dev/null -w '%{http_code}' "http://${CT_IP}:3000/" 2>/dev/null || echo "000")
if [[ "$POSTGREST_EXT" == "200" ]]; then
    print_pass "PostgREST is externally accessible (HTTP ${POSTGREST_EXT})"
else
    print_fail "PostgREST not externally accessible (HTTP ${POSTGREST_EXT})"
fi

# Test 12: n8n web interface
print_test "Testing n8n web interface..."
N8N_RESPONSE=$(pct exec "${CTID}" -- bash -lc "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:5678/" 2>/dev/null || echo "000")
if [[ "$N8N_RESPONSE" == "200" ]]; then
    print_pass "n8n web interface is accessible (HTTP ${N8N_RESPONSE})"
else
    print_fail "n8n web interface not accessible (HTTP ${N8N_RESPONSE})"
fi

# Test 13: n8n external accessibility
print_test "Testing n8n external accessibility..."
N8N_EXT=$(curl -s -o /dev/null -w '%{http_code}' "http://${CT_IP}:5678/" 2>/dev/null || echo "000")
if [[ "$N8N_EXT" == "200" ]]; then
    print_pass "n8n is externally accessible (HTTP ${N8N_EXT})"
else
    print_fail "n8n not externally accessible (HTTP ${N8N_EXT})"
fi

# Test 14: n8n API health
print_test "Testing n8n API health endpoint..."
N8N_HEALTH=$(pct exec "${CTID}" -- bash -lc "curl -s http://127.0.0.1:5678/healthz" 2>/dev/null || echo "")
if echo "$N8N_HEALTH" | grep -q "ok"; then
    print_pass "n8n health check passed"
else
    print_warn "n8n health endpoint returned: ${N8N_HEALTH}"
fi

# Test 15: Check n8n database connection
print_test "Checking n8n database configuration..."
N8N_DB_TYPE=$(pct exec "${CTID}" -- bash -lc "docker exec n8n printenv DB_TYPE" 2>/dev/null || echo "")
if [[ "$N8N_DB_TYPE" == "postgresdb" ]]; then
    print_pass "n8n is configured to use PostgreSQL"
else
    print_fail "n8n database type incorrect: ${N8N_DB_TYPE}"
fi

# Test 16: Workflow auto-reload service
print_test "Checking workflow auto-reload systemd service..."
RELOAD_SERVICE=$(pct exec "${CTID}" -- bash -lc "systemctl is-enabled n8n-workflow-reload.service" 2>/dev/null || echo "disabled")
if [[ "$RELOAD_SERVICE" == "enabled" ]]; then
    print_pass "Workflow auto-reload service is enabled"
else
    print_fail "Workflow auto-reload service not enabled: ${RELOAD_SERVICE}"
fi

# Test 17: Workflow template file exists
print_test "Checking workflow template file..."
if pct exec "${CTID}" -- bash -lc "test -f /opt/customer-stack/workflow-template.json" 2>/dev/null; then
    print_pass "Workflow template file exists"
else
    print_fail "Workflow template file not found"
fi

# Test 18: Reload script exists and is executable
print_test "Checking reload script..."
if pct exec "${CTID}" -- bash -lc "test -x /opt/customer-stack/reload-workflow.sh" 2>/dev/null; then
    print_pass "Reload script exists and is executable"
else
    print_fail "Reload script not found or not executable"
fi

# Test 19: Environment file exists
print_test "Checking environment configuration..."
if pct exec "${CTID}" -- bash -lc "test -f /opt/customer-stack/.env" 2>/dev/null; then
    print_pass "Environment file exists"
else
    print_fail "Environment file not found"
fi

# Test 20: Docker network exists
print_test "Checking Docker network..."
NETWORK_EXISTS=$(pct exec "${CTID}" -- bash -lc "docker network ls --format '{{.Name}}' | grep -c 'customer-stack_customer-net'" 2>/dev/null || echo "0")
if [[ "$NETWORK_EXISTS" -gt 0 ]]; then
    print_pass "Docker network 'customer-stack_customer-net' exists"
else
    print_fail "Docker network not found"
fi

# Test 21: Volume permissions (n8n runs as uid 1000)
print_test "Checking n8n volume permissions..."
N8N_VOLUME_OWNER=$(pct exec "${CTID}" -- bash -lc "stat -c '%u' /opt/customer-stack/volumes/n8n-data" 2>/dev/null || echo "")
if [[ "$N8N_VOLUME_OWNER" == "1000" ]]; then
    print_pass "n8n volume has correct ownership (uid 1000)"
else
    print_fail "n8n volume ownership incorrect: ${N8N_VOLUME_OWNER}"
fi

# Test 22: Check for running workflows
print_test "Checking n8n workflows..."
WORKFLOW_COUNT=$(pct exec "${CTID}" -- bash -lc "curl -s http://127.0.0.1:5678/rest/workflows 2>/dev/null | grep -o '\"id\"' | wc -l" 2>/dev/null || echo "0")
if [[ "$WORKFLOW_COUNT" -gt 0 ]]; then
    print_pass "Found ${WORKFLOW_COUNT} workflow(s) in n8n"
else
    print_warn "No workflows found in n8n (this may be expected if setup is still in progress)"
fi

# Test 23: Check Ollama connectivity (external service)
print_test "Testing Ollama API connectivity..."
OLLAMA_RESPONSE=$(curl -s -o /dev/null -w '%{http_code}' "http://192.168.45.3:11434/api/tags" 2>/dev/null || echo "000")
if [[ "$OLLAMA_RESPONSE" == "200" ]]; then
    print_pass "Ollama API is accessible (HTTP ${OLLAMA_RESPONSE})"
else
    print_warn "Ollama API not accessible (HTTP ${OLLAMA_RESPONSE}) - this is an external dependency"
fi

# Test 24: Container resource usage
print_test "Checking container resource usage..."
MEMORY_USAGE=$(pct exec "${CTID}" -- bash -lc "free -m | awk 'NR==2{printf \"%.0f\", \$3}'" 2>/dev/null || echo "0")
if [[ "$MEMORY_USAGE" -gt 0 ]]; then
    print_pass "Container memory usage: ${MEMORY_USAGE}MB"
else
    print_warn "Could not determine memory usage"
fi

# Test 25: Log file exists
print_test "Checking installation log file..."
if [[ -f "logs/${CT_HOSTNAME}.log" ]]; then
    LOG_SIZE=$(du -h "logs/${CT_HOSTNAME}.log" | cut -f1)
    print_pass "Installation log exists: logs/${CT_HOSTNAME}.log (${LOG_SIZE})"
else
    print_fail "Installation log not found"
fi

# Summary
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Test Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo -e "Total Tests: ${TESTS_TOTAL}"
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
echo ""

if [[ $TESTS_FAILED -eq 0 ]]; then
    echo -e "${GREEN}✓ All tests passed!${NC}"
    echo ""
    echo -e "${BLUE}Access Information:${NC}"
    echo -e "  n8n (internal): http://${CT_IP}:5678/"
    echo -e "  n8n (external): https://${CT_HOSTNAME}.userman.de"
    echo -e "  PostgREST API: http://${CT_IP}:3000/"
    echo ""
    exit 0
else
    echo -e "${RED}✗ Some tests failed. Please review the output above.${NC}"
    echo ""
    exit 1
fi