site11/scripts/status-check.sh

#!/bin/bash
#
# Site11 System Status Check Script
# Comprehensive status check for both Docker and Kubernetes services
#

set -e

# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Site11 System Status Check${NC}"
echo -e "${BLUE}========================================${NC}"

# Function to check service status
check_url() {
    local url=$1
    local name=$2
    local timeout=${3:-5}

    if curl -s --max-time $timeout "$url" > /dev/null 2>&1; then
        echo -e "  ${GREEN}✓${NC} $name: $url"
        return 0
    else
        echo -e "  ${RED}✗${NC} $name: $url"
        return 1
    fi
}

# Function to check Docker service
check_docker_service() {
    local service=$1
    if docker ps --format "table {{.Names}}" | grep -q "$service"; then
        echo -e "  ${GREEN}✓${NC} $service"
        return 0
    else
        echo -e "  ${RED}✗${NC} $service"
        return 1
    fi
}

# Function to check Kubernetes deployment
check_k8s_deployment() {
    local deployment=$1
    local namespace=${2:-site11-pipeline}

    if kubectl -n "$namespace" get deployment "$deployment" >/dev/null 2>&1; then
        local ready=$(kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{.status.readyReplicas}')
        local desired=$(kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{.spec.replicas}')

        if [ "$ready" = "$desired" ] && [ "$ready" != "" ]; then
            echo -e "  ${GREEN}✓${NC} $deployment ($ready/$desired ready)"
            return 0
        else
            echo -e "  ${YELLOW}⚠${NC} $deployment ($ready/$desired ready)"
            return 1
        fi
    else
        echo -e "  ${RED}✗${NC} $deployment (not found)"
        return 1
    fi
}

# 1. Docker Infrastructure Services
echo -e "\n${YELLOW}1. Docker Infrastructure Services${NC}"
docker_services=(
    "site11_mongodb"
    "site11_redis"
    "site11_kafka"
    "site11_zookeeper"
    "site11_pipeline_scheduler"
    "site11_pipeline_monitor"
    "site11_language_sync"
)

docker_healthy=0
for service in "${docker_services[@]}"; do
    if check_docker_service "$service"; then
        ((docker_healthy++))
    fi
done

echo -e "Docker Services: ${GREEN}$docker_healthy${NC}/${#docker_services[@]} healthy"

# 2. Kubernetes Application Services
echo -e "\n${YELLOW}2. Kubernetes Application Services${NC}"
k8s_deployments=(
    "console-backend"
    "console-frontend"
    "pipeline-rss-collector"
    "pipeline-google-search"
    "pipeline-translator"
    "pipeline-ai-article-generator"
    "pipeline-image-generator"
)

k8s_healthy=0
if kubectl cluster-info >/dev/null 2>&1; then
    for deployment in "${k8s_deployments[@]}"; do
        if check_k8s_deployment "$deployment"; then
            ((k8s_healthy++))
        fi
    done
    echo -e "Kubernetes Services: ${GREEN}$k8s_healthy${NC}/${#k8s_deployments[@]} healthy"
else
    echo -e "  ${RED}✗${NC} Kubernetes cluster not accessible"
fi

# 3. Health Check Endpoints
echo -e "\n${YELLOW}3. Health Check Endpoints${NC}"
health_endpoints=(
    "http://localhost:8000/health|Console Backend"
    "http://localhost:8000/api/health|Console API Health"
    "http://localhost:8000/api/users/health|Users Service"
    "http://localhost:8080/|Console Frontend"
    "http://localhost:8100/health|Pipeline Monitor"
    "http://localhost:8099/health|Pipeline Scheduler"
)

health_count=0
for endpoint in "${health_endpoints[@]}"; do
    IFS='|' read -r url name <<< "$endpoint"
    if check_url "$url" "$name"; then
        ((health_count++))
    fi
done

echo -e "Health Endpoints: ${GREEN}$health_count${NC}/${#health_endpoints[@]} accessible"

# 4. Port Forward Status
echo -e "\n${YELLOW}4. Port Forward Status${NC}"
port_forwards=()
while IFS= read -r line; do
    if [[ $line == *"kubectl"* && $line == *"port-forward"* ]]; then
        # Extract port from the command
        if [[ $line =~ ([0-9]+):([0-9]+) ]]; then
            local_port="${BASH_REMATCH[1]}"
            service_port="${BASH_REMATCH[2]}"
            service_name=$(echo "$line" | grep -o 'service/[^ ]*' | cut -d'/' -f2)
            port_forwards+=("$local_port:$service_port|$service_name")
        fi
    fi
done < <(ps aux | grep "kubectl.*port-forward" | grep -v grep)

if [ ${#port_forwards[@]} -eq 0 ]; then
    echo -e "  ${RED}✗${NC} No port forwards running"
    echo -e "  ${YELLOW}ℹ${NC} Run: ./scripts/start-k8s-port-forward.sh"
else
    for pf in "${port_forwards[@]}"; do
        IFS='|' read -r ports service <<< "$pf"
        echo -e "  ${GREEN}✓${NC} $service: localhost:$ports"
    done
fi

# 5. Resource Usage
echo -e "\n${YELLOW}5. Resource Usage${NC}"

# Docker resource usage
if command -v docker &> /dev/null; then
    docker_containers=$(docker ps --filter "name=site11_" --format "table {{.Names}}" | wc -l)
    echo -e "  Docker Containers: ${GREEN}$docker_containers${NC} running"
fi

# Kubernetes resource usage
if kubectl cluster-info >/dev/null 2>&1; then
    k8s_pods=$(kubectl -n site11-pipeline get pods --no-headers 2>/dev/null | wc -l)
    k8s_running=$(kubectl -n site11-pipeline get pods --no-headers 2>/dev/null | grep -c "Running" || echo "0")
    echo -e "  Kubernetes Pods: ${GREEN}$k8s_running${NC}/$k8s_pods running"

    # HPA status
    if kubectl -n site11-pipeline get hpa >/dev/null 2>&1; then
        hpa_count=$(kubectl -n site11-pipeline get hpa --no-headers 2>/dev/null | wc -l)
        echo -e "  HPA Controllers: ${GREEN}$hpa_count${NC} active"
    fi
fi

# 6. Queue Status (Redis)
echo -e "\n${YELLOW}6. Queue Status${NC}"
if check_docker_service "site11_redis"; then
    queues=(
        "queue:rss_collection"
        "queue:google_search"
        "queue:ai_generation"
        "queue:translation"
        "queue:image_generation"
    )

    for queue in "${queues[@]}"; do
        length=$(docker exec site11_redis redis-cli LLEN "$queue" 2>/dev/null || echo "0")
        if [ "$length" -gt 0 ]; then
            echo -e "  ${YELLOW}⚠${NC} $queue: $length items"
        else
            echo -e "  ${GREEN}✓${NC} $queue: empty"
        fi
    done
else
    echo -e "  ${RED}✗${NC} Redis not available"
fi

# 7. Database Status
echo -e "\n${YELLOW}7. Database Status${NC}"
if check_docker_service "site11_mongodb"; then
    # Check MongoDB collections
    collections=$(docker exec site11_mongodb mongosh ai_writer_db --quiet --eval "db.getCollectionNames()" 2>/dev/null | grep -o '"articles_[^"]*"' | wc -l || echo "0")
    echo -e "  ${GREEN}✓${NC} MongoDB: $collections collections"

    # Check article counts
    ko_count=$(docker exec site11_mongodb mongosh ai_writer_db --quiet --eval "db.articles_ko.countDocuments({})" 2>/dev/null || echo "0")
    echo -e "  ${GREEN}✓${NC} Korean articles: $ko_count"
else
    echo -e "  ${RED}✗${NC} MongoDB not available"
fi

# 8. Summary
echo -e "\n${BLUE}========================================${NC}"
echo -e "${BLUE}Summary${NC}"
echo -e "${BLUE}========================================${NC}"

total_services=$((${#docker_services[@]} + ${#k8s_deployments[@]}))
total_healthy=$((docker_healthy + k8s_healthy))

if [ $total_healthy -eq $total_services ] && [ $health_count -eq ${#health_endpoints[@]} ]; then
    echo -e "${GREEN}✓ All systems operational${NC}"
    echo -e "  Services: $total_healthy/$total_services"
    echo -e "  Health checks: $health_count/${#health_endpoints[@]}"
    exit 0
elif [ $total_healthy -gt $((total_services / 2)) ]; then
    echo -e "${YELLOW}⚠ System partially operational${NC}"
    echo -e "  Services: $total_healthy/$total_services"
    echo -e "  Health checks: $health_count/${#health_endpoints[@]}"
    exit 1
else
    echo -e "${RED}✗ System issues detected${NC}"
    echo -e "  Services: $total_healthy/$total_services"
    echo -e "  Health checks: $health_count/${#health_endpoints[@]}"
    echo -e "\n${YELLOW}Troubleshooting:${NC}"
    echo -e "  1. Check Docker: docker-compose -f docker-compose-hybrid.yml ps"
    echo -e "  2. Check Kubernetes: kubectl -n site11-pipeline get pods"
    echo -e "  3. Check port forwards: ./scripts/start-k8s-port-forward.sh"
    echo -e "  4. Check logs: docker-compose -f docker-compose-hybrid.yml logs"
    exit 2
fi