Securing CloudPanel on Ubuntu 24.04 Part 2

14/11/2024 14/11/2024 security 12 mins read
Table Of Contents

Part 2: Service Security and Advanced Monitoring #

1. Service Security Integration

Let’s implement comprehensive service monitoring:

Terminal window
# Create service monitoring script
sudo nano /usr/local/bin/cloudpanel-service-monitor.sh

# 

#!/bin/bash


set -euo pipefail


# Configuration
ENABLE_EMAIL="false"
ENABLE_DISCORD="true"
DISCORD_WEBHOOK_URL="https://discord.com/api/webhooks/your/yourapi"
GMAIL_USER="[email protected]"
GMAIL_APP_PASSWORD="your-app-password"
EMAIL_TO="[email protected]"


# CloudPanel specific paths
CP_HOME="/home/clp"
CP_DB="${CP_HOME}/htdocs/app/data/db.sq3"
SITES_ROOT="/home"


# Thresholds
CPU_THRESHOLD=80
MEMORY_THRESHOLD=90
PROCESS_THRESHOLD=500
DISK_THRESHOLD=90
ERROR_LOG_THRESHOLD=$((1024*1024*10))  # 10MB
HTTP_ERROR_THRESHOLD=100
CHECK_INTERVAL=300


# Setup
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
OUTPUT_DIR="/var/log/cloudpanel_monitoring"
ALERT_LOG="${OUTPUT_DIR}/alerts.log"
mkdir -p "${OUTPUT_DIR}"


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'


log() {
    echo -e "${2}[$(date '+%Y-%m-%d %H:%M:%S')] ${1}${NC}" | tee -a "${OUTPUT_DIR}/monitoring.log"
}


log_event() {
    local event_name="$1"
    local event_data="$2"
    local severity="${3:-normal}"


    # Log to CloudPanel database
    sqlite3 "$CP_DB" "
        INSERT INTO event (
            created_at,
            user_name,
            user_role,
            event_name,
            event_data,
            source_ip_address,
            user_agent
        ) VALUES (
            datetime('now'),
            'system',
            'ROLE_ADMIN',
            '$event_name',
            '$event_data',
            'localhost',
            'system-monitor'
        );"


    # Create notification if severity is high
    if [ "$severity" = "high" ]; then
        # Generate a unique hash for the notification
        local hash=$(echo "${event_name}${event_data}$(date +%s)" | md5sum | cut -d' ' -f1)


        sqlite3 "$CP_DB" "
            INSERT INTO notification (
                created_at,
                updated_at,
                hash,
                severity,
                subject,
                message,
                is_read
            ) VALUES (
                datetime('now'),
                datetime('now'),
                '$hash',
                'high',
                '$event_name',
                '$event_data',
                0
            );"
    fi
}


validate_config() {
    local has_error=false


    if [ "$ENABLE_EMAIL" = "true" ]; then
        if [ -z "$GMAIL_USER" ] || [ "$GMAIL_USER" = "[email protected]" ]; then
            log "Error: GMAIL_USER not configured" "${RED}"
            has_error=true
        fi
        if [ -z "$GMAIL_APP_PASSWORD" ] || [ "$GMAIL_APP_PASSWORD" = "your-app-password" ]; then
            log "Error: GMAIL_APP_PASSWORD not configured" "${RED}"
            has_error=true
        fi
        if [ -z "$EMAIL_TO" ] || [ "$EMAIL_TO" = "[email protected]" ]; then
            log "Error: EMAIL_TO not configured" "${RED}"
            has_error=true
        fi
    fi


    if [ "$ENABLE_DISCORD" = "true" ]; then
        if [ -z "$DISCORD_WEBHOOK_URL" ] || \
           [ "$DISCORD_WEBHOOK_URL" = "https://discord.com/api/webhooks/your/yourapi" ] || \
           ! echo "$DISCORD_WEBHOOK_URL" | grep -q "^https://discord.com/api/webhooks/[0-9]*/"; then
            log "Error: DISCORD_WEBHOOK_URL not properly configured" "${RED}"
            log "Please set ENABLE_DISCORD=false or configure a valid webhook URL" "${RED}"
            ENABLE_DISCORD=false  # Disable Discord if URL is invalid
        fi
    fi


    if [ "$has_error" = "true" ]; then
        log "Configuration validation failed. Please check settings." "${RED}"
        exit 1
    fi
}


send_discord_alert() {
    local title="$1"
    local message="$2"
    local priority="$3"


    local escaped_title=$(echo "$title" | sed 's/"/\\"/g')
    local escaped_message=$(echo "$message" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
    local escaped_hostname=$(hostname | sed 's/"/\\"/g')


    local color
    if [ "$priority" = "high" ]; then
        color="16711680"
    else
        color="15844367"
    fi


    local json_data
    printf -v json_data '{
        "embeds": [{
            "title": "%s",
            "description": "%s",
            "color": %d,
            "footer": {
                "text": "CloudPanel Monitoring - %s"
            },
            "timestamp": "%s"
        }]
    }' "$escaped_title" "$escaped_message" "$color" "$escaped_hostname" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"


    curl -s -H "Content-Type: application/json" -X POST -d "${json_data}" "${DISCORD_WEBHOOK_URL}"
}


send_gmail() {
    local subject="$1"
    local message="$2"
    local priority="$3"


    local email_content=$(mktemp)


    cat > "${email_content}" << EOF
From: ${GMAIL_USER}
To: ${EMAIL_TO}
Subject: $([ "$priority" = "high" ] && echo "[URGENT] ")${subject}
MIME-Version: 1.0
Content-Type: text/plain; charset="UTF-8"


${message}
EOF


    curl --url "smtps://smtp.gmail.com:465" \
         --ssl-reqd \
         --mail-from "${GMAIL_USER}" \
         --mail-rcpt "${EMAIL_TO}" \
         --upload-file "${email_content}" \
         --user "${GMAIL_USER}:${GMAIL_APP_PASSWORD}" \
         --silent


    rm -f "${email_content}"
}


send_alert() {
    local subject="$1"
    local message="$2"
    local priority="$3"


    local formatted_message="
Server: $(hostname)
Time: $(date)


${message}


This is an automated alert from CloudPanel Monitoring System."


    if [ "$ENABLE_EMAIL" = "true" ]; then
        send_gmail "${subject}" "${formatted_message}" "${priority}"
        log "Email alert sent: ${subject}" "${YELLOW}"
    fi


    if [ "$ENABLE_DISCORD" = "true" ]; then
        send_discord_alert "${subject}" "${formatted_message}" "${priority}"
        log "Discord alert sent: ${subject}" "${YELLOW}"
    fi


    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${subject}" >> "${ALERT_LOG}"
    log_event "system_alert" "${subject}: ${message}" "${priority}"
}


check_system_resources() {
    log "Checking system resources..." "${BLUE}"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')


    # CPU Usage
    local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d. -f1)
    sqlite3 "$CP_DB" "INSERT INTO instance_cpu (created_at, value) VALUES ('$timestamp', $cpu_usage);"


    if [ "${cpu_usage:-0}" -gt "$CPU_THRESHOLD" ]; then
        local top_processes=$(ps aux --sort=-%cpu | head -n 6)
        send_alert "High CPU Usage Alert" "CPU Usage: ${cpu_usage}%\n\nTop Processes:\n${top_processes}" "high"
    fi


    # Memory Usage
    local memory_usage=$(free | grep Mem | awk '{print ($3/$2 * 100)}' | cut -d. -f1)
    sqlite3 "$CP_DB" "INSERT INTO instance_memory (created_at, value) VALUES ('$timestamp', $memory_usage);"


    if [ "${memory_usage:-0}" -gt "$MEMORY_THRESHOLD" ]; then
        local memory_processes=$(ps aux --sort=-%mem | head -n 6)
        send_alert "High Memory Usage Alert" "Memory Usage: ${memory_usage}%\n\nTop Memory Processes:\n${memory_processes}" "high"
    fi


    # Load Average
    # Read and clean load average values
    read -r load1 load5 load15 _ _ < /proc/loadavg


    # Convert to float values and remove any extra content
    load1=$(printf "%.2f" "${load1%%/*}")
    load5=$(printf "%.2f" "${load5%%/*}")
    load15=$(printf "%.2f" "${load15%%/*}")


    # Insert cleaned values into database
    sqlite3 "$CP_DB" "
        INSERT INTO instance_load_average (created_at, period, value)
        VALUES
            ('$timestamp', 1, $load1),
            ('$timestamp', 5, $load5),
            ('$timestamp', 15, $load15);"


    # Process Count
    local process_count=$(ps aux | wc -l)
    if [ "${process_count:-0}" -gt "$PROCESS_THRESHOLD" ]; then
        send_alert "High Process Count Alert" "Process Count: ${process_count}\n\nProcess List:\n$(ps aux | head -n 6)" "high"
    fi


    # Disk Usage
    df -h | grep '^/dev/' | while read -r device size used avail use_percent mount; do
        local usage=${use_percent%\%}
        sqlite3 "$CP_DB" "INSERT INTO instance_disk_usage (created_at, disk, value) VALUES ('$timestamp', '$mount', $usage);"


        if [ "${usage:-0}" -gt "$DISK_THRESHOLD" ]; then
            local disk_info=$(df -h)
            send_alert "High Disk Usage Alert" "Disk Usage: ${usage}%\n\nDisk Information:\n${disk_info}" "high"
        fi
    done
}


check_services() {
    log "Checking critical services..." "${BLUE}"


    # Check basic services
    local basic_services=("nginx" "mysql" "varnish" "redis-server")
    for service in "${basic_services[@]}"; do
        if ! systemctl is-active --quiet "$service"; then
            send_alert "Service Down Alert" "Service ${service} is not running!" "high"
            log_event "service_down" "Service ${service} is not running" "high"
        fi
    done


    # Check PHP-FPM services from database
    local php_versions=$(sqlite3 "$CP_DB" "SELECT DISTINCT php_version FROM php_settings WHERE php_version IS NOT NULL;")
    for version in $php_versions; do
        local service_name="php${version}-fpm"
        if systemctl list-unit-files "${service_name}.service" &>/dev/null; then
            if ! systemctl is-active --quiet "$service_name"; then
                local status_details=$(systemctl status "${service_name}" 2>&1 | head -n 4)
                send_alert "PHP-FPM Service Issue" "PHP ${version} FPM service is down!\n\nStatus Details:\n${status_details}" "high"
                log_event "php_fpm_down" "PHP-FPM ${version} is not running" "high"
            fi
        fi
    done
}


check_website_resources() {
    log "Checking website resources..." "${BLUE}"


    # Query all active sites from database
    sqlite3 "$CP_DB" "SELECT id, domain_name, user FROM site WHERE type = 'vhost';" | while IFS='|' read -r site_id domain user; do
        # Skip if user directory doesn't exist
        if [ ! -d "${SITES_ROOT}/${user}" ]; then
            continue
        fi


        # Check nginx logs
        local nginx_log_dir="${SITES_ROOT}/${user}/logs/nginx"
        local access_log="${nginx_log_dir}/${domain}-access.log"
        local error_log="${nginx_log_dir}/${domain}-error.log"


        if [ -f "$access_log" ]; then
            # Check for high traffic IPs
            local high_traffic_ips=$(tail -n 1000 "$access_log" | awk '{print $1}' | sort | uniq -c | sort -nr | head -n 5)
            local top_ip_count=$(echo "$high_traffic_ips" | head -n 1 | awk '{print $1}')


            if [ "${top_ip_count:-0}" -gt 500 ]; then
                send_alert "High Traffic Alert" "High traffic detected for ${domain}:\n${high_traffic_ips}" "normal"
                log_event "high_traffic" "High traffic on ${domain}" "normal"
            fi


            # Check for high error rates
            local error_count=$(tail -n 1000 "$access_log" | awk '$9 ~ /^[45]/ {print $9}' | wc -l)
            if [ "${error_count:-0}" -gt "$HTTP_ERROR_THRESHOLD" ]; then
                local error_details=$(tail -n 1000 "$access_log" | awk '$9 ~ /^[45]/ {print $1,$9,$7}' | tail -n 10)
                send_alert "High Error Rate Alert" "Domain: ${domain}\nError count: ${error_count}\n\nLast errors:\n${error_details}" "high"
                log_event "high_error_rate" "High error rate on ${domain}: ${error_count} errors" "high"
            fi
        fi


        # Check PHP logs
        local php_log_dir="${SITES_ROOT}/${user}/logs/php"
        local php_error_log="${php_log_dir}/error.log"
        if [ -f "$php_error_log" ]; then
            local log_size=$(stat -c %s "$php_error_log")
            if [ "${log_size:-0}" -gt "$ERROR_LOG_THRESHOLD" ]; then
                send_alert "Large PHP Error Log" "Domain: ${domain}\nPHP error log size: $(numfmt --to=iec-i --suffix=B ${log_size})" "high"
                log_event "large_php_log" "Large PHP error log for ${domain}" "high"
            fi


            # Check for recent fatal errors
            local fatal_errors=$(tail -n 1000 "$php_error_log" | grep -i "fatal error" | tail -n 5)
            if [ -n "$fatal_errors" ]; then
                send_alert "PHP Fatal Errors Detected" "Domain: ${domain}\n\nRecent fatal errors:\n${fatal_errors}" "high"
                log_event "php_fatal_errors" "PHP fatal errors detected on ${domain}" "high"
            fi
        fi


        # Check varnish cache if enabled
        if [ -d "${SITES_ROOT}/${user}/.varnish-cache" ]; then
            local varnish_log="${SITES_ROOT}/${user}/logs/varnish-cache/purge.log"
            if [ -f "$varnish_log" ]; then
                local purge_count=$(tail -n 1000 "$varnish_log" | wc -l)
                if [ "${purge_count:-0}" -gt 1000 ]; then
                    send_alert "High Varnish Purge Rate" "Domain: ${domain}\nHigh number of cache purges: ${purge_count} in recent log" "normal"
                    log_event "high_purge_rate" "High varnish purge rate on ${domain}" "normal"
                fi
            fi
        fi


        # Check application-specific settings
        local app_type=$(sqlite3 "$CP_DB" "SELECT application FROM site WHERE id = $site_id;")
        case "$app_type" in
            "wordpress")
                local wp_config="${SITES_ROOT}/${user}/htdocs/${domain}/wp-config.php"
                if [ -f "$wp_config" ]; then
                    if grep -q "WP_DEBUG.*true" "$wp_config"; then
                        send_alert "WordPress Debug Enabled" "Domain: ${domain}\nWP_DEBUG is enabled in production" "normal"
                        log_event "wp_debug_enabled" "WordPress debug mode enabled on ${domain}" "normal"
                    fi
                fi
                ;;
        esac
    done
}


check_security() {
    log "Checking security..." "${BLUE}"


    # Check failed SSH attempts
    local failed_ssh=$(grep "Failed password" /var/log/auth.log 2>/dev/null | awk '{print $11}' | sort | uniq -c | sort -nr | head -n 5)
    if [ -n "$failed_ssh" ]; then
        send_alert "Security Alert - Failed SSH Attempts" "Recent failed SSH attempts:\n${failed_ssh}" "high"
        log_event "failed_ssh_attempts" "Multiple failed SSH attempts detected" "high"
    fi


    # Check for modified system files
    local modified_files=$(find /etc /usr/bin /usr/sbin -mmin -60 -type f 2>/dev/null)
    if [ -n "$modified_files" ]; then
        send_alert "Security Alert - Modified System Files" "Recently modified system files:\n${modified_files}" "high"
        log_event "modified_system_files" "System files modified in last hour" "high"
    fi


    # Check CloudPanel specific security
    local blocked_ips=$(sqlite3 "$CP_DB" "SELECT COUNT(*) FROM blocked_ip;")
    local blocked_bots=$(sqlite3 "$CP_DB" "SELECT COUNT(*) FROM blocked_bot;")


    if [ "${blocked_ips:-0}" -gt 0 ] || [ "${blocked_bots:-0}" -gt 0 ]; then
        send_alert "Security Info - Blocked Entities" "Current blocks:\nIPs: ${blocked_ips}\nBots: ${blocked_bots}" "normal"
    fi


    # Check for expired SSL certificates
    local near_expiry=$(sqlite3 "$CP_DB" "
        SELECT s.domain_name, c.expires_at
        FROM certificate c
        JOIN site s ON c.site_id = s.id
        WHERE c.expires_at <= datetime('now', '+30 days')
        AND c.expires_at > datetime('now');"
    )


    if [ -n "$near_expiry" ]; then
        send_alert "SSL Certificates Near Expiry" "Certificates expiring soon:\n${near_expiry}" "normal"
        log_event "ssl_near_expiry" "SSL certificates approaching expiration" "normal"
    fi
}


check_scheduled_tasks() {
    log "Checking scheduled tasks..." "${BLUE}"


    # Check cron jobs from database
    local failed_crons=$(sqlite3 "$CP_DB" "
        SELECT s.domain_name, cj.command
        FROM cron_job cj
        JOIN site s ON cj.site_id = s.id;"
    )


    if [ -n "$failed_crons" ]; then
        while IFS='|' read -r domain command; do
            # Basic validation of cron command existence
            local cmd_path=$(echo "$command" | awk '{print $1}')
            if [ ! -f "$cmd_path" ] && ! command -v "$cmd_path" >/dev/null 2>&1; then
                send_alert "Invalid Cron Job" "Domain: ${domain}\nInvalid command path: ${cmd_path}" "normal"
                log_event "invalid_cron" "Invalid cron command for ${domain}" "normal"
            fi
        done <<< "$failed_crons"
    fi
}


main() {
    log "Starting CloudPanel monitoring system..." "${GREEN}"
    validate_config


    while true; do
        check_system_resources
        check_services
        check_website_resources
        check_security
        check_scheduled_tasks


        log "Sleeping for ${CHECK_INTERVAL} seconds..." "${BLUE}"
        sleep "${CHECK_INTERVAL}"
    done
}


main "$@"

This script:

  1. Uses CloudPanel’s actual database schema for storing events and metrics
  2. Monitors all services configured in the database
  3. Records system metrics in CloudPanel’s instance_* tables
  4. Monitors website resources based on CloudPanel’s site structure
  5. Maintains proper logging in CloudPanel’s event system

To implement:

Terminal window
# Make script executable
chmod +x /usr/local/bin/cloudpanel-service-monitor.sh


# Create systemd service
sudo nano /etc/systemd/system/cloudpanel-monitor.service

# 

[Unit]
Description=CloudPanel Service Monitor
After=network.target


[Service]
Type=simple
ExecStart=/usr/local/bin/cloudpanel-service-monitor.sh
Restart=always
User=root


[Install]
WantedBy=multi-user.target

#

Terminal window
# Enable and start the service
sudo systemctl enable cloudpanel-monitor
sudo systemctl start cloudpanel-monitor