Foreword

Traditional workflow requires manual execution of the rclone copy command on the local server each time new resources are downloaded to cloud storage, resulting in a cumbersome and repetitive process. To address this inefficiency, this script was engineered to provide automated monitoring capabilities for specified directories and seamless retrieval of newly updated files without manual intervention.

Script

#!/bin/bash

# 配置文件路径
CONFIG_DIR="/root/.av_sync"
TASKS_FILE="$CONFIG_DIR/tasks.conf"
LOG_FILE="$CONFIG_DIR/sync.log"
PID_FILE="$CONFIG_DIR/monitor.pid"

# 创建配置目录
mkdir -p "$CONFIG_DIR"

# 日志函数
log_message() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# 显示主菜单
show_menu() {
    clear
    echo "======================================"
    echo "    文件夹监测同步管理系统"
    echo "======================================"
    
    # 显示监测状态
    if is_monitoring_running; then
        echo "状态: 监测服务运行中 (PID: $(cat $PID_FILE))"
    else
        echo "状态: 监测服务未运行"
    fi
    
    echo "======================================"
    echo "1. 添加监测任务"
    echo "2. 查看所有任务"
    echo "3. 删除监测任务"
    echo "4. 停止监测服务"
    echo "5. 手动启动监测服务"
    echo "6. 查看运行日志"
    echo "7. 退出"
    echo "======================================"
    echo -n "请选择操作 [1-7]: "
}

# 验证路径是否存在
validate_path() {
    local path="$1"
    if [ ! -d "$path" ]; then
        echo "错误: 路径 $path 不存在"
        return 1
    fi
    return 0
}

# 检查监测服务是否运行
is_monitoring_running() {
    if [ -f "$PID_FILE" ]; then
        local pid=$(cat "$PID_FILE")
        if ps -p "$pid" > /dev/null 2>&1; then
            return 0
        else
            rm -f "$PID_FILE"
            return 1
        fi
    fi
    return 1
}

# 停止监测服务
stop_monitoring() {
    if [ -f "$PID_FILE" ]; then
        local pid=$(cat "$PID_FILE")
        if ps -p "$pid" > /dev/null 2>&1; then
            kill "$pid" 2>/dev/null
            rm -f "$PID_FILE"
            log_message "监测服务已停止 (PID: $pid)"
            return 0
        else
            rm -f "$PID_FILE"
        fi
    fi
    return 1
}

# 检查并复制单个任务的新文件
check_task() {
    local task_id="$1"
    local source_dir="$2"
    local target_dir="$3"
    local scan_file="$4"
    
    local temp_scan="/tmp/scan_${task_id}_$$.txt"
    
    # 当前扫描
    find "$source_dir" -type f > "$temp_scan" 2>/dev/null
    
    # 找出新文件
    local new_files=$(comm -13 <(sort "$scan_file") <(sort "$temp_scan"))
    
    if [ -n "$new_files" ]; then
        local new_count=$(echo "$new_files" | wc -l)
        log_message "任务 [$task_id] 发现 $new_count 个新文件"
        
        # 逐个复制新文件
        while IFS= read -r file; do
            if [ -f "$file" ]; then
                local filename=$(basename "$file")
                log_message "任务 [$task_id] 正在复制: $filename"
                
                rclone copy "$file" "$target_dir" --progress >> "$LOG_FILE" 2>&1
                
                if [ $? -eq 0 ]; then
                    log_message "任务 [$task_id] 复制成功: $filename"
                    
                    # 立即将该文件添加到扫描记录中
                    echo "$file" >> "$scan_file"
                    
                    # 对扫描文件排序去重，保持文件整洁
                    sort -u "$scan_file" -o "$scan_file"
                    
                    log_message "任务 [$task_id] 已更新扫描记录"
                else
                    log_message "任务 [$task_id] 复制失败: $filename"
                fi
            fi
        done <<< "$new_files"
    fi
    
    # 清理临时文件
    rm -f "$temp_scan"
}

# 监测服务主循环
monitoring_loop() {
    log_message "===== 监测服务启动 ====="
    
    # 检查rclone
    if ! command -v rclone &> /dev/null; then
        log_message "错误: rclone 未安装"
        exit 1
    fi
    
    while true; do
        if [ -f "$TASKS_FILE" ] && [ -s "$TASKS_FILE" ]; then
            while IFS='|' read -r task_id source_dir target_dir scan_file; do
                if [ -d "$source_dir" ] && [ -d "$target_dir" ]; then
                    check_task "$task_id" "$source_dir" "$target_dir" "$scan_file"
                else
                    log_message "任务 [$task_id] 警告: 源或目标目录不存在"
                fi
            done < "$TASKS_FILE"
        fi
        
        sleep 600
    done
}

# 启动监测服务
start_monitoring() {
    # 检查是否已经运行
    if is_monitoring_running; then
        return 0
    fi
    
    # 检查是否有任务
    if [ ! -f "$TASKS_FILE" ] || [ ! -s "$TASKS_FILE" ]; then
        return 1
    fi
    
    # 后台启动监测服务
    monitoring_loop >> "$LOG_FILE" 2>&1 &
    local pid=$!
    echo $pid > "$PID_FILE"
    
    log_message "监测服务已启动 (PID: $pid)"
    return 0
}

# 添加监测任务
add_task() {
    echo ""
    echo "=== 添加新的监测任务 ==="
    echo ""
    
    # 输入源文件夹
    while true; do
        read -p "请输入要监测的源文件夹绝对路径: " source_dir
        if validate_path "$source_dir"; then
            break
        fi
    done
    
    # 输入目标文件夹
    while true; do
        read -p "请输入复制到的目标文件夹绝对路径: " target_dir
        if validate_path "$target_dir"; then
            break
        fi
    done
    
    # 生成任务ID
    local task_id=$(date +%s)
    local scan_file="$CONFIG_DIR/scan_${task_id}.txt"
    
    # 保存任务配置
    echo "$task_id|$source_dir|$target_dir|$scan_file" >> "$TASKS_FILE"
    
    # 执行初始扫描
    echo ""
    echo "正在执行初始扫描..."
    find "$source_dir" -type f | sort > "$scan_file"
    local file_count=$(wc -l < "$scan_file")
    
    echo "任务添加成功！"
    echo "任务ID: $task_id"
    echo "源文件夹: $source_dir"
    echo "目标文件夹: $target_dir"
    echo "初始文件数: $file_count"
    
    log_message "添加任务 [$task_id]: $source_dir -> $target_dir (初始文件: $file_count)"
    
    # 自动启动或重启监测服务
    echo ""
    echo "正在启动监测服务..."
    
    if is_monitoring_running; then
        echo "检测到监测服务正在运行，正在重启以加载新任务..."
        stop_monitoring
        sleep 1
    fi
    
    if start_monitoring; then
        echo "监测服务已启动，将每10分钟自动检查一次"
    else
        echo "警告: 监测服务启动失败"
    fi
    
    echo ""
    read -p "按Enter键继续..."
}

# 查看所有任务
view_tasks() {
    echo ""
    echo "=== 当前所有监测任务 ==="
    echo ""
    
    if [ ! -f "$TASKS_FILE" ] || [ ! -s "$TASKS_FILE" ]; then
        echo "暂无监测任务"
    else
        echo "任务ID       源文件夹                              目标文件夹"
        echo "------------------------------------------------------------------------"
        while IFS='|' read -r task_id source_dir target_dir scan_file; do
            printf "%-12s %-38s %s\n" "$task_id" "$source_dir" "$target_dir"
        done < "$TASKS_FILE"
    fi
    
    echo ""
    read -p "按Enter键继续..."
}

# 删除监测任务
delete_task() {
    echo ""
    echo "=== 删除监测任务 ==="
    echo ""
    
    if [ ! -f "$TASKS_FILE" ] || [ ! -s "$TASKS_FILE" ]; then
        echo "暂无监测任务"
        read -p "按Enter键继续..."
        return
    fi
    
    # 显示任务列表
    echo "任务ID       源文件夹"
    echo "----------------------------------------"
    while IFS='|' read -r task_id source_dir target_dir scan_file; do
        printf "%-12s %s\n" "$task_id" "$source_dir"
    done < "$TASKS_FILE"
    
    echo ""
    read -p "请输入要删除的任务ID: " task_id
    
    if grep -q "^$task_id|" "$TASKS_FILE"; then
        # 删除扫描文件
        local scan_file=$(grep "^$task_id|" "$TASKS_FILE" | cut -d'|' -f4)
        rm -f "$scan_file"
        
        # 从配置文件中删除
        sed -i "/^$task_id|/d" "$TASKS_FILE"
        
        echo "任务 $task_id 已删除"
        log_message "删除任务 [$task_id]"
        
        # 如果没有任务了，停止监测服务
        if [ ! -s "$TASKS_FILE" ]; then
            echo "所有任务已删除，正在停止监测服务..."
            stop_monitoring
        fi
    else
        echo "任务ID不存在"
    fi
    
    echo ""
    read -p "按Enter键继续..."
}

# 手动停止监测
manual_stop() {
    echo ""
    if stop_monitoring; then
        echo "监测服务已停止"
    else
        echo "监测服务未运行"
    fi
    echo ""
    read -p "按Enter键继续..."
}

# 手动启动监测
manual_start() {
    echo ""
    if is_monitoring_running; then
        echo "监测服务已在运行中"
    else
        if start_monitoring; then
            echo "监测服务已启动"
        else
            echo "启动失败: 请先添加监测任务"
        fi
    fi
    echo ""
    read -p "按Enter键继续..."
}

# 查看运行日志
view_logs() {
    echo ""
    echo "=== 最近50条运行日志 ==="
    echo ""
    
    if [ -f "$LOG_FILE" ]; then
        tail -n 50 "$LOG_FILE"
    else
        echo "暂无日志"
    fi
    
    echo ""
    read -p "按Enter键继续..."
}

# 主程序
main() {
    # 启动时检查是否需要恢复监测服务
    if [ -f "$TASKS_FILE" ] && [ -s "$TASKS_FILE" ]; then
        if ! is_monitoring_running; then
            start_monitoring
        fi
    fi
    
    while true; do
        show_menu
        read choice
        
        case $choice in
            1) add_task ;;
            2) view_tasks ;;
            3) delete_task ;;
            4) manual_stop ;;
            5) manual_start ;;
            6) view_logs ;;
            7) 
                echo "退出程序"
                exit 0
                ;;
            *) 
                echo "无效选择，请重新输入"
                sleep 1
                ;;
        esac
    done
}

# 捕获退出信号
trap "exit 0" SIGINT SIGTERM

# 运行主程序
main

Detailed Explanation

📋 Overall Script Functionality

This script implements a background file monitoring service capable of:

Monitoring specified source folders for new files
Automatically copying new files to target folders using rclone
Providing an interactive menu for managing multiple monitoring tasks

🔧 Core Component Analysis

1. Configuration and Initialization Section

CONFIG_DIR="/root/.av_sync"
TASKS_FILE="$CONFIG_DIR/tasks.conf"    # Stores all monitoring tasks
LOG_FILE="$CONFIG_DIR/sync.log"        # Runtime logs
PID_FILE="$CONFIG_DIR/monitor.pid"     # Monitoring service process ID

2. Key Function Explanations

`log_message()` - Logging

echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"

Outputs to both terminal and log file simultaneously
Formatted with timestamp

`is_monitoring_running()` - Service Status Check

if [ -f "$PID_FILE" ]; then
    local pid=$(cat "$PID_FILE")
    if ps -p "$pid" > /dev/null 2>&1; then  # Check if process exists
        return 0  # Running
    else
        rm -f "$PID_FILE"  # Clean up stale PID file
        return 1
    fi
fi

`check_task()` - Core Monitoring Logic ⭐

check_task() {
    local task_id="$1"
    local source_dir="$2"
    local target_dir="$3"
    local scan_file="$4"  # Previous scan file list
    
    # 1. Current scan of all files in source folder
    find "$source_dir" -type f > "$temp_scan"
    
    # 2. Compare to identify new files (comm -13 displays lines unique to second file)
    local new_files=$(comm -13 <(sort "$scan_file") <(sort "$temp_scan"))
    
    # 3. Copy each new file individually
    while IFS= read -r file; do
        rclone copy "$file" "$target_dir" --progress
        
        if [ $? -eq 0 ]; then
            # 4. Update scan record immediately after successful copy
            echo "$file" >> "$scan_file"
            sort -u "$scan_file" -o "$scan_file"  # Deduplicate and sort
        fi
    done <<< "$new_files"
}

Working Principle:

Uses find to retrieve all current files in source folder
Compares with previous scan results (scan_file)
Uses comm -13 command to identify newly added files
Copies each file individually using rclone
Updates scan record after successful completion

`monitoring_loop()` - Main Monitoring Loop

monitoring_loop() {
    while true; do
        # Iterate through all tasks
        while IFS='|' read -r task_id source_dir target_dir scan_file; do
            check_task "$task_id" "$source_dir" "$target_dir" "$scan_file"
        done < "$TASKS_FILE"
        
        sleep 600  # Check every 10 minutes
    done
}

`start_monitoring()` - Launch Background Service

start_monitoring() {
    # Run monitoring loop in background
    monitoring_loop >> "$LOG_FILE" 2>&1 &
    local pid=$!  # Capture background process PID
    echo $pid > "$PID_FILE"  # Save PID
}

Option 1: Add Monitoring Task

add_task() {
    # 1. Validate source folder path
    read -p "Please enter absolute path of source folder to monitor: " source_dir
    
    # 2. Validate target folder path
    read -p "Please enter absolute path of target folder: " target_dir
    
    # 3. Generate unique task ID (using timestamp)
    local task_id=$(date +%s)
    local scan_file="$CONFIG_DIR/scan_${task_id}.txt"
    
    # 4. Save task configuration (format: task_id|source|target|scan_file)
    echo "$task_id|$source_dir|$target_dir|$scan_file" >> "$TASKS_FILE"
    
    # 5. Perform initial scan
    find "$source_dir" -type f | sort > "$scan_file"
    
    # 6. Restart monitoring service to load new task
    stop_monitoring
    start_monitoring
}

Option 3: Delete Monitoring Task

delete_task() {
    # Delete scan file
    rm -f "$scan_file"
    
    # Remove from configuration file (using sed)
    sed -i "/^$task_id|/d" "$TASKS_FILE"
    
    # Stop service if no tasks remain
    if [ ! -s "$TASKS_FILE" ]; then
        stop_monitoring
    fi
}

🔍 Key Technical Points

1. Process Management

# Launch in background
command &
pid=$!

# Check if process exists
ps -p "$pid" > /dev/null 2>&1

# Terminate process
kill "$pid"

2. File Comparison Technique

# comm command compares two sorted files
comm -13 file1 file2  # Display lines unique to file2
# -1: suppress lines unique to file1
# -3: suppress common lines

3. Configuration File Format

task_id|source_dir|target_dir|scan_file
1698765432|/home/data|/backup|/root/.av_sync/scan_1698765432.txt

Uses | delimiter, parsed via IFS='|' read

4. Signal Trapping

trap "exit 0" SIGINT SIGTERM
# Captures Ctrl+C (SIGINT) and termination signals (SIGTERM)

⚙️ Workflow Diagram

Script Launch
    ↓
Check for task configuration
    ↓
Auto-start monitoring service (background)
    ↓
Enter interactive menu
    ↓
User adds task → Initial scan → Save configuration
    ↓
Monitoring loop (every 10 minutes):
    ├─ Read all tasks
    ├─ Scan source folders
    ├─ Compare to identify new files
    ├─ Copy new files via rclone
    └─ Update scan records

💡 Use Cases

This script is suitable for:

Automated Backup: Monitor working directories and automatically backup new files to remote storage
Log Collection: Monitor application log directories and automatically archive new logs
File Distribution: Monitor upload directories and automatically distribute to multiple target locations

⚠️ Important Considerations

Requires rclone: Must pre-install and configure rclone
Permission Requirements: Configuration directory is in /root/, requires root privileges
Performance Considerations: find command may be slow in large directories
Network Dependency: If rclone target is remote, requires stable network connection

This is a production-grade Bash script with complete functionality and clear structure! 🚀

Sources

Automatically Sync Cloud Storage

This script facilitates the automatic monitoring and synchronization of content from PikPak and other cloud storage platforms to local server infrastructure via rclone.

Foreword

Script

Detailed Explanation

📋 Overall Script Functionality

🔧 Core Component Analysis

1. Configuration and Initialization Section

2. Key Function Explanations

`log_message()` - Logging

`is_monitoring_running()` - Service Status Check

`check_task()` - Core Monitoring Logic ⭐

`monitoring_loop()` - Main Monitoring Loop

`start_monitoring()` - Launch Background Service

🎯 Interactive Menu Functions

Option 1: Add Monitoring Task

Option 3: Delete Monitoring Task

🔍 Key Technical Points

1. Process Management

2. File Comparison Technique

3. Configuration File Format

4. Signal Trapping

⚙️ Workflow Diagram

💡 Use Cases

⚠️ Important Considerations

Automatically Sync Cloud Storage

This script facilitates the automatic monitoring and synchronization of content from PikPak and other cloud storage platforms to local server infrastructure via rclone.

Foreword

Script

Detailed Explanation

📋 Overall Script Functionality

🔧 Core Component Analysis

1. Configuration and Initialization Section

2. Key Function Explanations

log_message() - Logging

is_monitoring_running() - Service Status Check

check_task() - Core Monitoring Logic ⭐

monitoring_loop() - Main Monitoring Loop

start_monitoring() - Launch Background Service

🎯 Interactive Menu Functions

Option 1: Add Monitoring Task

Option 3: Delete Monitoring Task

🔍 Key Technical Points

1. Process Management

2. File Comparison Technique

3. Configuration File Format

4. Signal Trapping

⚙️ Workflow Diagram

💡 Use Cases

⚠️ Important Considerations

`log_message()` - Logging

`is_monitoring_running()` - Service Status Check

`check_task()` - Core Monitoring Logic ⭐

`monitoring_loop()` - Main Monitoring Loop

`start_monitoring()` - Launch Background Service