#!/bin/bash # # Site11 System Status Check Script # Comprehensive status check for both Docker and Kubernetes services # set -e # Colors for output GREEN='\033[0;32m' YELLOW='\033[1;33m' RED='\033[0;31m' BLUE='\033[0;34m' NC='\033[0m' # No Color echo -e "${BLUE}========================================${NC}" echo -e "${BLUE}Site11 System Status Check${NC}" echo -e "${BLUE}========================================${NC}" # Function to check service status check_url() { local url=$1 local name=$2 local timeout=${3:-5} if curl -s --max-time $timeout "$url" > /dev/null 2>&1; then echo -e " ${GREEN}✓${NC} $name: $url" return 0 else echo -e " ${RED}✗${NC} $name: $url" return 1 fi } # Function to check Docker service check_docker_service() { local service=$1 if docker ps --format "table {{.Names}}" | grep -q "$service"; then echo -e " ${GREEN}✓${NC} $service" return 0 else echo -e " ${RED}✗${NC} $service" return 1 fi } # Function to check Kubernetes deployment check_k8s_deployment() { local deployment=$1 local namespace=${2:-site11-pipeline} if kubectl -n "$namespace" get deployment "$deployment" >/dev/null 2>&1; then local ready=$(kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{.status.readyReplicas}') local desired=$(kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{.spec.replicas}') if [ "$ready" = "$desired" ] && [ "$ready" != "" ]; then echo -e " ${GREEN}✓${NC} $deployment ($ready/$desired ready)" return 0 else echo -e " ${YELLOW}⚠${NC} $deployment ($ready/$desired ready)" return 1 fi else echo -e " ${RED}✗${NC} $deployment (not found)" return 1 fi } # 1. Docker Infrastructure Services echo -e "\n${YELLOW}1. Docker Infrastructure Services${NC}" docker_services=( "site11_mongodb" "site11_redis" "site11_kafka" "site11_zookeeper" "site11_pipeline_scheduler" "site11_pipeline_monitor" "site11_language_sync" ) docker_healthy=0 for service in "${docker_services[@]}"; do if check_docker_service "$service"; then ((docker_healthy++)) fi done echo -e "Docker Services: ${GREEN}$docker_healthy${NC}/${#docker_services[@]} healthy" # 2. Kubernetes Application Services echo -e "\n${YELLOW}2. Kubernetes Application Services${NC}" k8s_deployments=( "console-backend" "console-frontend" "pipeline-rss-collector" "pipeline-google-search" "pipeline-translator" "pipeline-ai-article-generator" "pipeline-image-generator" ) k8s_healthy=0 if kubectl cluster-info >/dev/null 2>&1; then for deployment in "${k8s_deployments[@]}"; do if check_k8s_deployment "$deployment"; then ((k8s_healthy++)) fi done echo -e "Kubernetes Services: ${GREEN}$k8s_healthy${NC}/${#k8s_deployments[@]} healthy" else echo -e " ${RED}✗${NC} Kubernetes cluster not accessible" fi # 3. Health Check Endpoints echo -e "\n${YELLOW}3. Health Check Endpoints${NC}" health_endpoints=( "http://localhost:8000/health|Console Backend" "http://localhost:8000/api/health|Console API Health" "http://localhost:8000/api/users/health|Users Service" "http://localhost:8080/|Console Frontend" "http://localhost:8100/health|Pipeline Monitor" "http://localhost:8099/health|Pipeline Scheduler" ) health_count=0 for endpoint in "${health_endpoints[@]}"; do IFS='|' read -r url name <<< "$endpoint" if check_url "$url" "$name"; then ((health_count++)) fi done echo -e "Health Endpoints: ${GREEN}$health_count${NC}/${#health_endpoints[@]} accessible" # 4. Port Forward Status echo -e "\n${YELLOW}4. Port Forward Status${NC}" port_forwards=() while IFS= read -r line; do if [[ $line == *"kubectl"* && $line == *"port-forward"* ]]; then # Extract port from the command if [[ $line =~ ([0-9]+):([0-9]+) ]]; then local_port="${BASH_REMATCH[1]}" service_port="${BASH_REMATCH[2]}" service_name=$(echo "$line" | grep -o 'service/[^ ]*' | cut -d'/' -f2) port_forwards+=("$local_port:$service_port|$service_name") fi fi done < <(ps aux | grep "kubectl.*port-forward" | grep -v grep) if [ ${#port_forwards[@]} -eq 0 ]; then echo -e " ${RED}✗${NC} No port forwards running" echo -e " ${YELLOW}ℹ${NC} Run: ./scripts/start-k8s-port-forward.sh" else for pf in "${port_forwards[@]}"; do IFS='|' read -r ports service <<< "$pf" echo -e " ${GREEN}✓${NC} $service: localhost:$ports" done fi # 5. Resource Usage echo -e "\n${YELLOW}5. Resource Usage${NC}" # Docker resource usage if command -v docker &> /dev/null; then docker_containers=$(docker ps --filter "name=site11_" --format "table {{.Names}}" | wc -l) echo -e " Docker Containers: ${GREEN}$docker_containers${NC} running" fi # Kubernetes resource usage if kubectl cluster-info >/dev/null 2>&1; then k8s_pods=$(kubectl -n site11-pipeline get pods --no-headers 2>/dev/null | wc -l) k8s_running=$(kubectl -n site11-pipeline get pods --no-headers 2>/dev/null | grep -c "Running" || echo "0") echo -e " Kubernetes Pods: ${GREEN}$k8s_running${NC}/$k8s_pods running" # HPA status if kubectl -n site11-pipeline get hpa >/dev/null 2>&1; then hpa_count=$(kubectl -n site11-pipeline get hpa --no-headers 2>/dev/null | wc -l) echo -e " HPA Controllers: ${GREEN}$hpa_count${NC} active" fi fi # 6. Queue Status (Redis) echo -e "\n${YELLOW}6. Queue Status${NC}" if check_docker_service "site11_redis"; then queues=( "queue:rss_collection" "queue:google_search" "queue:ai_generation" "queue:translation" "queue:image_generation" ) for queue in "${queues[@]}"; do length=$(docker exec site11_redis redis-cli LLEN "$queue" 2>/dev/null || echo "0") if [ "$length" -gt 0 ]; then echo -e " ${YELLOW}⚠${NC} $queue: $length items" else echo -e " ${GREEN}✓${NC} $queue: empty" fi done else echo -e " ${RED}✗${NC} Redis not available" fi # 7. Database Status echo -e "\n${YELLOW}7. Database Status${NC}" if check_docker_service "site11_mongodb"; then # Check MongoDB collections collections=$(docker exec site11_mongodb mongosh ai_writer_db --quiet --eval "db.getCollectionNames()" 2>/dev/null | grep -o '"articles_[^"]*"' | wc -l || echo "0") echo -e " ${GREEN}✓${NC} MongoDB: $collections collections" # Check article counts ko_count=$(docker exec site11_mongodb mongosh ai_writer_db --quiet --eval "db.articles_ko.countDocuments({})" 2>/dev/null || echo "0") echo -e " ${GREEN}✓${NC} Korean articles: $ko_count" else echo -e " ${RED}✗${NC} MongoDB not available" fi # 8. Summary echo -e "\n${BLUE}========================================${NC}" echo -e "${BLUE}Summary${NC}" echo -e "${BLUE}========================================${NC}" total_services=$((${#docker_services[@]} + ${#k8s_deployments[@]})) total_healthy=$((docker_healthy + k8s_healthy)) if [ $total_healthy -eq $total_services ] && [ $health_count -eq ${#health_endpoints[@]} ]; then echo -e "${GREEN}✓ All systems operational${NC}" echo -e " Services: $total_healthy/$total_services" echo -e " Health checks: $health_count/${#health_endpoints[@]}" exit 0 elif [ $total_healthy -gt $((total_services / 2)) ]; then echo -e "${YELLOW}⚠ System partially operational${NC}" echo -e " Services: $total_healthy/$total_services" echo -e " Health checks: $health_count/${#health_endpoints[@]}" exit 1 else echo -e "${RED}✗ System issues detected${NC}" echo -e " Services: $total_healthy/$total_services" echo -e " Health checks: $health_count/${#health_endpoints[@]}" echo -e "\n${YELLOW}Troubleshooting:${NC}" echo -e " 1. Check Docker: docker-compose -f docker-compose-hybrid.yml ps" echo -e " 2. Check Kubernetes: kubectl -n site11-pipeline get pods" echo -e " 3. Check port forwards: ./scripts/start-k8s-port-forward.sh" echo -e " 4. Check logs: docker-compose -f docker-compose-hybrid.yml logs" exit 2 fi