feat: terminal fullscreen, asyncpg fix, CI improvements #59
@@ -183,10 +183,10 @@ nas-tools/
|
|||||||
55. ~~Add LiteLLM dashboard UI page with sidebar link and health status card~~ — Done
|
55. ~~Add LiteLLM dashboard UI page with sidebar link and health status card~~ — Done
|
||||||
56. ~~Add cc-connect dashboard UI page, sidebar link, and backend health/status endpoint~~ — Done
|
56. ~~Add cc-connect dashboard UI page, sidebar link, and backend health/status endpoint~~ — Done
|
||||||
|
|
||||||
### Phase 13 — Off-site Backup
|
### Phase 13 — Off-site Backup ✅
|
||||||
46. Add cloud backup (OneDrive or Google Drive) for critical data
|
46. ~~Add cloud backup (OneDrive or Google Drive) for critical data~~ — Done (rclone + crypt encryption)
|
||||||
47. Encrypt backups before upload
|
47. ~~Encrypt backups before upload~~ — Done (rclone crypt AES-256)
|
||||||
48. Retention policy for cloud copies
|
48. ~~Retention policy for cloud copies~~ — Done (30-day configurable via env var)
|
||||||
|
|
||||||
## Media Paths
|
## Media Paths
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ ERRORS=0
|
|||||||
|
|
||||||
[ -f "$SCRIPT_DIR/.env" ] && source "$SCRIPT_DIR/.env"
|
[ -f "$SCRIPT_DIR/.env" ] && source "$SCRIPT_DIR/.env"
|
||||||
|
|
||||||
|
# Source cloud backup extension (Phase 13 — Off-site Backup)
|
||||||
|
CLOUD_ENABLED="${CLOUD_ENABLED:-false}"
|
||||||
|
if [ "$CLOUD_ENABLED" = "true" ]; then
|
||||||
|
RCLONE_CONFIG_DIR="${RCLONE_CONFIG_DIR:-$SCRIPT_DIR/rclone}"
|
||||||
|
CLOUD_RETENTION_DAYS="${CLOUD_RETENTION_DAYS:-30}"
|
||||||
|
source "$SCRIPT_DIR/scripts/cloud-backup.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
mkdir -p "$BACKUP_DIR"
|
mkdir -p "$BACKUP_DIR"
|
||||||
|
|
||||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"; }
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"; }
|
||||||
@@ -66,6 +74,11 @@ find "$BACKUP_DIR" -type f \( -name "*.sql" -o -name "*.tar.gz" \) -mtime +$RETE
|
|||||||
# Summary
|
# Summary
|
||||||
log "=== Backup finished ($ERRORS errors) ==="
|
log "=== Backup finished ($ERRORS errors) ==="
|
||||||
|
|
||||||
|
# Off-site cloud sync
|
||||||
|
if [ "$CLOUD_ENABLED" = "true" ] && type cloud_backup &>/dev/null; then
|
||||||
|
cloud_backup || ERRORS=$((ERRORS + CLOUD_ERRORS))
|
||||||
|
fi
|
||||||
|
|
||||||
if [ $ERRORS -gt 0 ]; then
|
if [ $ERRORS -gt 0 ]; then
|
||||||
notify "🔴 *NAS Backup FAILED* ($DATE)
|
notify "🔴 *NAS Backup FAILED* ($DATE)
|
||||||
$ERRORS error(s) — check logs"
|
$ERRORS error(s) — check logs"
|
||||||
|
|||||||
Executable
+79
@@ -0,0 +1,79 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Off-site Backup — rclone cloud sync extension for backup.sh
|
||||||
|
#
|
||||||
|
# This script is sourced by backup.sh when CLOUD_ENABLED=true.
|
||||||
|
# It syncs local backups to a cloud provider via rclone with encryption.
|
||||||
|
#
|
||||||
|
# Prerequisites (handled by setup-rclone.sh):
|
||||||
|
# 1. rclone installed (Docker image: rclone/rclone)
|
||||||
|
# 2. Cloud remote configured (e.g., `onedrive`, `gdrive`)
|
||||||
|
# 3. Crypt remote configured (e.g., `nas-backup-crypt`)
|
||||||
|
#
|
||||||
|
# Env vars (set in .env or exported before running backup.sh):
|
||||||
|
# CLOUD_ENABLED=false # Set to true to enable cloud sync
|
||||||
|
# RCLONE_REMOTE=nas-backup # Base rclone remote name
|
||||||
|
# RCLONE_CRYPT_REMOTE="" # If set, uses encrypted remote (e.g., nas-backup-crypt:)
|
||||||
|
# CLOUD_RETENTION_DAYS=30 # Keep cloud copies for this many days
|
||||||
|
# RCLONE_EXTRA_FLAGS="" # Extra flags for rclone sync (e.g., "--bwlimit 2M")
|
||||||
|
|
||||||
|
CLOUD_ENABLED="${CLOUD_ENABLED:-false}"
|
||||||
|
RCLONE_REMOTE="${RCLONE_REMOTE:-nas-backup}"
|
||||||
|
RCLONE_CRYPT_REMOTE="${RCLONE_CRYPT_REMOTE:-}"
|
||||||
|
CLOUD_RETENTION_DAYS="${CLOUD_RETENTION_DAYS:-30}"
|
||||||
|
CLOUD_ERRORS=0
|
||||||
|
|
||||||
|
# Resolve the target remote (plain or crypt)
|
||||||
|
if [ -n "$RCLONE_CRYPT_REMOTE" ]; then
|
||||||
|
_cloud_target="${RCLONE_CRYPT_REMOTE}"
|
||||||
|
else
|
||||||
|
_cloud_target="${RCLONE_REMOTE}:nas-tools"
|
||||||
|
fi
|
||||||
|
|
||||||
|
_rclone() {
|
||||||
|
docker run --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
-v "$BACKUP_DIR:/data:ro" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
"$@" --config /config/rclone/rclone.conf
|
||||||
|
}
|
||||||
|
|
||||||
|
cloud_backup() {
|
||||||
|
log "=== Cloud backup started ==="
|
||||||
|
|
||||||
|
# 1. Sync backup files to cloud
|
||||||
|
log "Syncing $BACKUP_DIR to $_cloud_target ..."
|
||||||
|
if _rclone sync /data/ "$_cloud_target" \
|
||||||
|
--verbose \
|
||||||
|
--progress \
|
||||||
|
--copy-links \
|
||||||
|
$RCLONE_EXTRA_FLAGS; then
|
||||||
|
log "Cloud sync OK"
|
||||||
|
else
|
||||||
|
log "FAILED: cloud sync"
|
||||||
|
CLOUD_ERRORS=$((CLOUD_ERRORS + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. Cloud retention — delete files older than CLOUD_RETENTION_DAYS
|
||||||
|
log "Applying cloud retention: ${CLOUD_RETENTION_DAYS} days ..."
|
||||||
|
if _rclone delete "$_cloud_target" \
|
||||||
|
--min-age "${CLOUD_RETENTION_DAYS}d" \
|
||||||
|
--verbose; then
|
||||||
|
log "Cloud retention OK"
|
||||||
|
else
|
||||||
|
log "FAILED: cloud retention cleanup"
|
||||||
|
CLOUD_ERRORS=$((CLOUD_ERRORS + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 3. Show cloud usage summary
|
||||||
|
log "Cloud storage summary:"
|
||||||
|
_rclone size "$_cloud_target" --json 2>/dev/null | python3 -c "
|
||||||
|
import json, sys
|
||||||
|
try:
|
||||||
|
d = json.load(sys.stdin)
|
||||||
|
print(f' {d.get(\"count\",0)} files, {d.get(\"bytes\",0)/1024/1024:.1f} MB')
|
||||||
|
except: pass
|
||||||
|
" || true
|
||||||
|
|
||||||
|
log "=== Cloud backup finished ($CLOUD_ERRORS errors) ==="
|
||||||
|
return $CLOUD_ERRORS
|
||||||
|
}
|
||||||
Executable
+165
@@ -0,0 +1,165 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# setup-rclone.sh — Install and configure rclone for off-site NAS backups
|
||||||
|
#
|
||||||
|
# This script sets up rclone (via Docker) for syncing backups to a cloud provider.
|
||||||
|
# It supports OneDrive, Google Drive, S3, or any rclone-compatible provider.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ssh <nas> "$(cat setup-rclone.sh)" # Run remotely on NAS
|
||||||
|
# bash setup-rclone.sh # Run directly on NAS
|
||||||
|
#
|
||||||
|
# After setup, enable cloud backup in .env:
|
||||||
|
# CLOUD_ENABLED=true
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ── Config ──────────────────────────────────────────────────────────
|
||||||
|
DOCKER="/volume1/@appstore/ContainerManager/usr/bin/docker"
|
||||||
|
RCLONE_CONFIG_DIR="/volume1/docker/nas-tools/rclone"
|
||||||
|
NAS_TOOLS_DIR="/volume1/docker/nas-tools"
|
||||||
|
COMPOSE_DIR="/volume1/docker"
|
||||||
|
|
||||||
|
# ── Step 1: Create config directory ─────────────────────────────────
|
||||||
|
echo "=== rclone Setup ==="
|
||||||
|
echo "Config dir: $RCLONE_CONFIG_DIR"
|
||||||
|
mkdir -p "$RCLONE_CONFIG_DIR"
|
||||||
|
|
||||||
|
# ── Step 2: Pull rclone image ───────────────────────────────────────
|
||||||
|
echo ""
|
||||||
|
echo "Pulling rclone Docker image..."
|
||||||
|
$DOCKER pull rclone/rclone:latest
|
||||||
|
|
||||||
|
# ── Step 3: Interactive rclone config ───────────────────────────────
|
||||||
|
echo ""
|
||||||
|
echo "=== Cloud Provider Configuration ==="
|
||||||
|
echo ""
|
||||||
|
echo "rclone will now run in interactive config mode."
|
||||||
|
echo "Choose your cloud provider:"
|
||||||
|
echo " 1) OneDrive (recommended for Microsoft 365 users)"
|
||||||
|
echo " 2) Google Drive (Google account)"
|
||||||
|
echo " 3) Other (S3, WebDAV, etc.)"
|
||||||
|
echo ""
|
||||||
|
read -rp "Provider [1-3]: " PROVIDER_CHOICE
|
||||||
|
|
||||||
|
case "$PROVIDER_CHOICE" in
|
||||||
|
1) PROVIDER_NAME="onedrive";;
|
||||||
|
2) PROVIDER_NAME="google-drive";;
|
||||||
|
3) echo "Using generic config — follow rclone prompts."
|
||||||
|
PROVIDER_NAME="";;
|
||||||
|
*) echo "Invalid choice. Falling back to generic config."
|
||||||
|
PROVIDER_NAME="";;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Starting rclone config..."
|
||||||
|
echo "Follow the prompts to:"
|
||||||
|
echo " 1. Create a new remote (name it 'nas-backup')"
|
||||||
|
if [ -n "$PROVIDER_NAME" ]; then
|
||||||
|
echo " 2. Select '$PROVIDER_NAME' as the storage type"
|
||||||
|
fi
|
||||||
|
echo " 3. Complete OAuth authentication"
|
||||||
|
echo " 4. Exit the configurator when done"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
$DOCKER run -it --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
config --config /config/rclone/rclone.conf
|
||||||
|
|
||||||
|
# ── Step 4: Set up encryption remote ────────────────────────────────
|
||||||
|
echo ""
|
||||||
|
echo "=== Encryption Setup ==="
|
||||||
|
echo "Backups should be encrypted before uploading to the cloud."
|
||||||
|
echo "rclone's 'crypt' remote provides client-side AES-256 encryption."
|
||||||
|
echo ""
|
||||||
|
read -rp "Set up encryption remote? [Y/n]: " SETUP_CRYPT
|
||||||
|
if [[ "$SETUP_CRYPT" =~ ^[Yy]?$ ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "Starting rclone config for encryption remote..."
|
||||||
|
echo " 1. Create a NEW remote (name it 'nas-backup-crypt')"
|
||||||
|
echo " 2. Select 'crypt' as the storage type"
|
||||||
|
echo " 3. Set 'nas-backup:nas-tools' as the remote + path"
|
||||||
|
echo " 4. Choose 'standard' file name encryption"
|
||||||
|
echo " 5. Set your own password or generate one"
|
||||||
|
echo " 6. Exit when done"
|
||||||
|
echo ""
|
||||||
|
echo "NOTE: Save the encryption password somewhere safe!"
|
||||||
|
echo "Without it, backups CANNOT be recovered."
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
$DOCKER run -it --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
config --config /config/rclone/rclone.conf
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 5: Test connection ─────────────────────────────────────────
|
||||||
|
echo ""
|
||||||
|
echo "=== Testing Connection ==="
|
||||||
|
REMOTE_NAME="nas-backup"
|
||||||
|
if $DOCKER run --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
listremotes --config /config/rclone/rclone.conf; then
|
||||||
|
echo ""
|
||||||
|
echo "Configured remotes:"
|
||||||
|
$DOCKER run --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
about "${REMOTE_NAME}:" --config /config/rclone/rclone.conf 2>&1 || \
|
||||||
|
echo "(note: 'about' may not be supported by all providers)"
|
||||||
|
else
|
||||||
|
echo "ERROR: No remotes configured."
|
||||||
|
echo "Re-run this script or manually run:"
|
||||||
|
echo " $DOCKER run -it --rm -v $RCLONE_CONFIG_DIR:/config/rclone rclone/rclone:latest config"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 6: Patch backup.sh .env ────────────────────────────────────
|
||||||
|
ENV_FILE="$NAS_TOOLS_DIR/.env"
|
||||||
|
CLOUD_ENV_BLOCK="
|
||||||
|
# --- Off-site Backup (rclone) ---
|
||||||
|
CLOUD_ENABLED=true
|
||||||
|
RCLONE_REMOTE=nas-backup
|
||||||
|
RCLONE_CRYPT_REMOTE=nas-backup-crypt:nas-tools
|
||||||
|
CLOUD_RETENTION_DAYS=30
|
||||||
|
RCLONE_EXTRA_FLAGS=--bwlimit 5M
|
||||||
|
RCLONE_CONFIG_DIR=$RCLONE_CONFIG_DIR
|
||||||
|
"
|
||||||
|
|
||||||
|
if [ -f "$ENV_FILE" ]; then
|
||||||
|
echo ""
|
||||||
|
echo "=== Updating .env ==="
|
||||||
|
echo "Appending cloud backup config to $ENV_FILE ..."
|
||||||
|
echo "$CLOUD_ENV_BLOCK" >> "$ENV_FILE"
|
||||||
|
echo "Done."
|
||||||
|
else
|
||||||
|
echo ""
|
||||||
|
echo "=== .env not found ==="
|
||||||
|
echo "Create $ENV_FILE with:"
|
||||||
|
echo "$CLOUD_ENV_BLOCK"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Summary ─────────────────────────────────────────────────────────
|
||||||
|
echo ""
|
||||||
|
echo "=== Setup Complete ==="
|
||||||
|
echo ""
|
||||||
|
echo "The following rclone remotes are configured:"
|
||||||
|
$DOCKER run --rm \
|
||||||
|
-v "$RCLONE_CONFIG_DIR:/config/rclone" \
|
||||||
|
rclone/rclone:latest \
|
||||||
|
listremotes --config /config/rclone/rclone.conf
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "To run a test backup:"
|
||||||
|
echo " ssh nas \"cd $COMPOSE_DIR/nas-tools && CLOUD_ENABLED=true bash backup.sh\""
|
||||||
|
echo ""
|
||||||
|
echo "To restore from cloud:"
|
||||||
|
echo " # List available backups:"
|
||||||
|
echo " ssh nas \"$DOCKER run --rm -v $RCLONE_CONFIG_DIR:/config/rclone rclone/rclone:latest \\
|
||||||
|
ls nas-backup-crypt:nas-tools --config /config/rclone/rclone.conf\""
|
||||||
|
echo ""
|
||||||
|
echo "IMPORTANT:"
|
||||||
|
echo " - If you used encryption, store the password in your password manager!"
|
||||||
|
echo " - The rclone config is at: $RCLONE_CONFIG_DIR/rclone.conf"
|
||||||
|
echo " - Back it up: cp $RCLONE_CONFIG_DIR/rclone.conf /volume1/backups/"
|
||||||
Reference in New Issue
Block a user