Bash: Monitor a systemd service and auto-restart (with logging)

#!/bin/bash

# SystemD Service Monitor & Auto-Restart Script
# Usage: ./sysd-monitor.sh <service_name> [check_interval] [log_file]

set -euo pipefail

# Configuration
SERVICE_NAME="${1:-}"
CHECK_INTERVAL="${2:-30}"  # seconds between checks
LOG_FILE="${3:-/var/log/sysd-monitor-${SERVICE_NAME}.log}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Function to log messages with timestamp
log_message() {
    local level="$1"
    local message="$2"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local log_entry="[$timestamp] [$level] $message"

    # Log to file (create directory if needed)
    mkdir -p "$(dirname "$LOG_FILE")"
    echo "$log_entry" >> "$LOG_FILE"

    # Also output to terminal with colors
    case "$level" in
        "INFO")  echo -e "${BLUE}$log_entry${NC}" ;;
        "WARN")  echo -e "${YELLOW}$log_entry${NC}" ;;
        "ERROR") echo -e "${RED}$log_entry${NC}" ;;
        "SUCCESS") echo -e "${GREEN}$log_entry${NC}" ;;
        *) echo "$log_entry" ;;
    esac
}

# Function to check if service exists
check_service_exists() {
    if ! systemctl list-unit-files | grep -q "^${SERVICE_NAME}.service"; then
        log_message "ERROR" "Service '$SERVICE_NAME' not found in systemd"
        return 1
    fi
    return 0
}

# Function to get service status
get_service_status() {
    systemctl is-active "$SERVICE_NAME" 2>/dev/null || echo "unknown"
}

# Function to restart service
restart_service() {
    log_message "WARN" "Attempting to restart service '$SERVICE_NAME'"

    if sudo systemctl restart "$SERVICE_NAME"; then
        log_message "SUCCESS" "Service '$SERVICE_NAME' restarted successfully"

        # Wait a moment and verify it's running
        sleep 5
        local status=$(get_service_status)
        if [[ "$status" == "active" ]]; then
            log_message "SUCCESS" "Service '$SERVICE_NAME' is now running (status: $status)"
            return 0
        else
            log_message "ERROR" "Service '$SERVICE_NAME' failed to start properly (status: $status)"
            return 1
        fi
    else
        log_message "ERROR" "Failed to restart service '$SERVICE_NAME'"
        return 1
    fi
}

# Function to show usage
show_usage() {
    echo "SystemD Service Monitor & Auto-Restart Script"
    echo ""
    echo "Usage: $0 <service_name> [check_interval] [log_file]"
    echo ""
    echo "Parameters:"
    echo "  service_name    - Name of the systemd service to monitor (required)"
    echo "  check_interval  - Seconds between checks (default: 30)"
    echo "  log_file       - Path to log file (default: /var/log/sysd-monitor-<service>.log)"
    echo ""
    echo "Examples:"
    echo "  $0 nginx                           # Monitor nginx every 30 seconds"
    echo "  $0 apache2 60                     # Monitor apache2 every 60 seconds"
    echo "  $0 my-app 15 /home/user/app.log   # Monitor my-app every 15 seconds"
    echo ""
    echo "Note: This script requires sudo privileges to restart services."
    exit 1
}

# Function to handle graceful shutdown
cleanup() {
    log_message "INFO" "Monitor stopped by user (PID: $$)"
    exit 0
}

# Main monitoring function
monitor_service() {
    local restart_count=0
    local last_restart=0
    local max_restart_attempts=3
    local restart_window=300  # 5 minutes

    log_message "INFO" "Starting monitor for service '$SERVICE_NAME' (PID: $$)"
    log_message "INFO" "Check interval: ${CHECK_INTERVAL}s, Log file: $LOG_FILE"

    while true; do
        local status=$(get_service_status)
        local current_time=$(date +%s)

        case "$status" in
            "active")
                # Service is running - reset restart count if enough time has passed
                if [[ $((current_time - last_restart)) -gt $restart_window ]]; then
                    restart_count=0
                fi
                log_message "INFO" "Service '$SERVICE_NAME' is running (status: $status)"
                ;;

            "inactive"|"failed"|"unknown")
                log_message "ERROR" "Service '$SERVICE_NAME' is not running (status: $status)"

                # Check if we've exceeded restart attempts in the time window
                if [[ $restart_count -ge $max_restart_attempts ]] && [[ $((current_time - last_restart)) -lt $restart_window ]]; then
                    log_message "ERROR" "Maximum restart attempts ($max_restart_attempts) reached within $restart_window seconds. Waiting before trying again..."
                    sleep $restart_window
                    restart_count=0
                fi

                # Attempt restart
                if restart_service; then
                    restart_count=$((restart_count + 1))
                    last_restart=$current_time
                else
                    log_message "ERROR" "Restart failed, will try again in ${CHECK_INTERVAL} seconds"
                fi
                ;;

            *)
                log_message "WARN" "Service '$SERVICE_NAME' has unexpected status: $status"
                ;;
        esac

        sleep "$CHECK_INTERVAL"
    done
}

# Main script execution
main() {
    # Check if help is requested
    if [[ "${1:-}" =~ ^(-h|--help|help)$ ]]; then
        show_usage
    fi

    # Validate service name
    if [[ -z "$SERVICE_NAME" ]]; then
        echo -e "${RED}Error: Service name is required${NC}"
        show_usage
    fi

    # Validate check interval
    if ! [[ "$CHECK_INTERVAL" =~ ^[0-9]+$ ]] || [[ "$CHECK_INTERVAL" -lt 1 ]]; then
        echo -e "${RED}Error: Check interval must be a positive integer${NC}"
        exit 1
    fi

    # Check if running as root for logging to /var/log
    if [[ "$LOG_FILE" == /var/log/* ]] && [[ $EUID -ne 0 ]]; then
        echo -e "${YELLOW}Warning: Not running as root. Using local log file instead.${NC}"
        LOG_FILE="./sysd-monitor-${SERVICE_NAME}.log"
    fi

    # Set up signal handlers for graceful shutdown
    trap cleanup SIGTERM SIGINT

    # Verify service exists
    if ! check_service_exists; then
        exit 1
    fi

    # Check if we have sudo access for service restart
    if ! sudo -n systemctl status "$SERVICE_NAME" >/dev/null 2>&1; then
        echo -e "${YELLOW}Note: This script requires sudo privileges to restart services.${NC}"
        echo "You may be prompted for your password when a restart is needed."
    fi

    # Start monitoring
    monitor_service
}

# Run main function with all arguments
main "$@"