12.2. Maintenance & Backup

12.2.1. Common Pitfalls

1. Backup files getting too large

# Bad: No size limits, fills disk
tar -czf full_backup.tar.gz /
# Disk fills up, backup fails

# Better: Check space before backing up
available=$(df /backups | awk 'NR==2 {print $4}')
needed=$(($(du -s /home | awk '{print $1}') * 2))
[[ $available -gt $needed ]] || {
  echo "Insufficient space for backup" >&2
  exit 1
}

2. Not testing backup restoration

# Bad: Never verify backups work
tar -czf backup.tar.gz /data

# Better: Periodically test restore
tar -tzf backup.tar.gz > /dev/null  # At minimum
# Even better: Actually restore to test system

3. Backup strategy mismatch

# Bad: Backing up too frequently (disk waste) or too infrequently (data loss)
# Weekly backup with growing data = slow, expensive

# Better: Tiered approach
# - Daily incremental (fast, cheap)
# - Weekly full (can restore completely)
# - Monthly archival (compliance/long-term)

4. Storing backups in same location as data

# Bad: Both in /home (if disk fails, backup is gone)
tar -czf /home/backup.tar.gz /home

# Better: Different disk/location
tar -czf /backups/backup.tar.gz /home

# Better: Off-site backup
rsync -az /backups/ remote:/backups/

5. Not logging backup results

# Bad: No record of backup success
0 2 * * * /backup.sh

# Better: Log results
0 2 * * * /backup.sh >> /var/log/backup.log 2>&1
# Even better: Email on failure
0 2 * * * /backup.sh >> /var/log/backup.log 2>&1 || \
  mail -s "Backup Failed" admin@example.com

12.2.2. Real-World Example: Complete Maintenance Suite

#!/bin/bash
set -euo pipefail

# COMPREHENSIVE MAINTENANCE SCRIPT FOR SERVERS
# Run daily via cron: 0 2 * * * /usr/local/bin/maintenance.sh

readonly LOG_FILE="/var/log/maintenance.log"
readonly BACKUP_DIR="/backups"
readonly RETENTION_DAYS=30
readonly REPORT_EMAIL="admin@example.com"

log() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# 1. Backup system
backup_system() {
  log "Starting system backup..."
  local backup_file="$BACKUP_DIR/system_$(date +%Y%m%d).tar.gz"
  
  mkdir -p "$BACKUP_DIR"
  tar -czf "$backup_file" /etc /home /opt 2>/dev/null || {
    log "ERROR: Backup failed"
    return 1
  }
  
  # Verify
  tar -tzf "$backup_file" > /dev/null || {
    log "ERROR: Backup corrupted"
    rm -f "$backup_file"
    return 1
  }
  
  log "Backup completed: $(du -h $backup_file | cut -f1)"
}

# 2. Cleanup old backups
cleanup_backups() {
  log "Cleaning up backups older than $RETENTION_DAYS days..."
  find "$BACKUP_DIR" -name "system_*.tar.gz" -mtime +$RETENTION_DAYS -delete
}

# 3. Rotate logs
rotate_logs() {
  log "Rotating system logs..."
  find /var/log -name "*.log" -mtime +7 ! -name "*.gz" | \
    while read -r f; do gzip "$f"; done
}

# 4. Database maintenance
maintain_db() {
  if command -v mysqld &>/dev/null; then
    log "Optimizing MySQL databases..."
    mysql -u root -e "SELECT @@version" &>/dev/null && {
      mysqlcheck -u root --all-databases --auto-repair
      log "Database maintenance completed"
    }
  fi
}

# 5. Disk usage report
report_usage() {
  log "Disk usage summary:"
  df -h | grep -E "^/dev" | awk '{print $6 " " $5}'
}

# Run all tasks
{
  log "=== Daily Maintenance Started ==="
  backup_system
  cleanup_backups
  rotate_logs
  maintain_db
  report_usage
  log "=== Daily Maintenance Completed ==="
} | tee -a "$LOG_FILE"

# Email report on errors
grep ERROR "$LOG_FILE" | tail -1 | \
  mail -s "Maintenance Report" "$REPORT_EMAIL" 2>/dev/null || true

12.2.3. System Maintenance Tasks

12.2.3.1. Log Rotation and Cleanup

#!/bin/bash

cleanup_logs() {
  local log_dir=$1
  local retention_days=${2:-30}
  
  echo "Cleaning logs in $log_dir older than $retention_days days..."
  
  # Delete old logs
  find "$log_dir" -name "*.log" -mtime +$retention_days -delete
  
  # Compress recent logs
  find "$log_dir" -name "*.log" -mtime +7 ! -name "*.gz" | \
    while read -r logfile; do
      gzip "$logfile"
    done
  
  # Report disk space saved
  du -sh "$log_dir"
}

# Cron entry
# @daily /usr/local/bin/cleanup_logs.sh /var/log 30

12.2.3.2. Database Maintenance

#!/bin/bash

maintain_database() {
  local db_name=$1
  local db_user="root"
  
  echo "Optimizing database: $db_name"
  
  # Check and optimize all tables
  mysql -u "$db_user" -e "
    USE $db_name;
    SELECT CONCAT('CHECK TABLE ', table_name, ';') 
    FROM information_schema.tables 
    WHERE table_schema = '$db_name' 
    INTO OUTFILE '/tmp/check.sql';
  "
  
  mysql -u "$db_user" < /tmp/check.sql
  
  # Repair any corrupted tables
  mysqlcheck -u "$db_user" --auto-repair "$db_name"
  
  # Optimize tables
  mysql -u "$db_user" -e "
    SELECT CONCAT('OPTIMIZE TABLE ', table_name, ';')
    FROM information_schema.tables
    WHERE table_schema = '$db_name'
    INTO OUTFILE '/tmp/optimize.sql';
  "
  
  mysql -u "$db_user" < /tmp/optimize.sql
  
  rm -f /tmp/check.sql /tmp/optimize.sql
}

12.2.4. Retention and Cleanup

12.2.4.1. Implementing Retention Policies

#!/bin/bash

cleanup_old_backups() {
  local backup_dir=$1
  local retention_days=${2:-30}
  local max_backups=${3:-10}
  
  echo "Cleaning up backups older than $retention_days days..."
  
  # Delete by age
  find "$backup_dir" -name "backup_*.tar.gz" -mtime +$retention_days -delete
  
  # Delete excess backups (keep only most recent N)
  local count=$(find "$backup_dir" -name "backup_*.tar.gz" | wc -l)
  if [[ $count -gt $max_backups ]]; then
    # Delete oldest backups
    find "$backup_dir" -name "backup_*.tar.gz" -type f | \
      sort | head -n $((count - max_backups)) | xargs rm -f
  fi
  
  # Report remaining backups
  ls -lh "$backup_dir"/backup_*.tar.gz 2>/dev/null | wc -l | xargs echo "Backups remaining:"
}

cleanup_old_backups /backups 30 10

12.2.4.2. Backup with Rotation

# Keep only last 7 daily backups, 4 weekly, 12 monthly
backup_with_rotation() {
  local backup_dir=$1
  
  mkdir -p "$backup_dir"/{daily,weekly,monthly}
  
  local today=$(date +%Y%m%d)
  local daily_backup="$backup_dir/daily/backup_$today.tar.gz"
  
  # Create daily backup
  tar -czf "$daily_backup" /home /etc || return 1
  
  # Keep only last 7 daily backups
  find "$backup_dir/daily" -name "backup_*.tar.gz" -mtime +7 -delete
  
  # Weekly backup on Sunday
  if [[ $(date +%u) -eq 7 ]]; then
    local weekly_backup="$backup_dir/weekly/backup_week_$(date +%Y_W%V).tar.gz"
    cp "$daily_backup" "$weekly_backup"
    find "$backup_dir/weekly" -mtime +30 -delete  # Keep 4+ weeks
  fi
  
  # Monthly backup on 1st of month
  if [[ $(date +%d) -eq 1 ]]; then
    local monthly_backup="$backup_dir/monthly/backup_$(date +%Y_%m).tar.gz"
    cp "$daily_backup" "$monthly_backup"
    find "$backup_dir/monthly" -mtime +365 -delete  # Keep 12+ months
  fi
}

12.2.5. Building Reliable Backup Scripts

12.2.5.1. Backup Strategy and Types

# Full backup: Copy everything
backup_full() {
  local timestamp=$(date +%Y%m%d_%H%M%S)
  tar -czf "full_backup_$timestamp.tar.gz" /home /etc /opt
}

# Incremental backup: Only files changed since last backup
backup_incremental() {
  local timestamp=$(date +%Y%m%d_%H%M%S)
  local level_file="/var/log/backup_level"
  
  # Create marker for next incremental
  tar -czf "incremental_$timestamp.tar.gz" \
    --newer="$level_file" /home /etc
  
  touch "$level_file"
}

# Differential backup: Changed since last full backup
backup_differential() {
  local timestamp=$(date +%Y%m%d_%H%M%S)
  local last_full="/var/log/last_full_backup"
  
  tar -czf "differential_$timestamp.tar.gz" \
    --newer="$last_full" /home /etc
}

12.2.5.2. Backup with Verification

#!/bin/bash
set -euo pipefail

backup_with_verify() {
  local source=$1
  local dest=$2
  local backup_file="$dest/backup_$(date +%Y%m%d_%H%M%S).tar.gz"
  
  # Create backup
  echo "Creating backup..."
  tar -czf "$backup_file" "$source" || {
    echo "Backup failed" >&2
    return 1
  }
  
  # Verify integrity
  echo "Verifying backup..."
  tar -tzf "$backup_file" > /dev/null || {
    echo "Backup verification failed" >&2
    rm -f "$backup_file"
    return 1
  }
  
  # Check size (ensure not empty)
  local size=$(stat -c%s "$backup_file" 2>/dev/null || echo 0)
  [[ $size -gt 1000 ]] || {
    echo "Backup file suspiciously small: $size bytes" >&2
    return 1
  }
  
  echo "Backup verified: $backup_file ($size bytes)"
  return 0
}