Added option to run local (rsync) or cloud (rclone) backups, in case you prefer a "full" copy with hardlinks via rsync, or latest/incremental snapshots via rclone

2026-09-03 13:21:54 -04:00
parent fe6101a532
commit 0144cc2b4e
11 changed files with 1428 additions and 0 deletions
@@ -0,0 +1,134 @@
+package backup
+
+import (
+  "context"
+  "errors"
+  "fmt"
+  "path/filepath"
+  "sort"
+  "time"
+
+  "git.metaunix.net/bitgoblin/goblin-store/lib/config"
+
+  "github.com/rclone/rclone/fs"
+  "github.com/rclone/rclone/fs/operations"
+  "github.com/rclone/rclone/fs/sync"
+
+  // Register desired backends
+  _ "github.com/rclone/rclone/backend/local"
+  _ "github.com/rclone/rclone/backend/s3"
+)
+
+func RunIncrementalBackup(ctx context.Context, cfg *config.Config) error {
+  // Configure global Rclone concurrency parameters
+  ci := fs.GetConfig(ctx)
+  ci.Transfers = cfg.Workers
+  ci.Checkers = cfg.Workers * 2
+
+  // Setup directories:
+  // - destination/latest: holds current full mirror
+  // - destination/increments/<timestamp>: holds files altered or deleted during sync
+  timestamp := time.Now().Format("2006-01-02_15-04-05")
+  latestDir := filepath.Join(cfg.Destination, "latest")
+  incrementDir := filepath.Join(cfg.Destination, "increments", timestamp)
+  incrementsBase := filepath.Join(cfg.Destination, "increments")
+
+  // 1. Resolve source and target filesystems
+  fSrc, err := fs.NewFs(ctx, cfg.Source)
+  if err != nil {
+    return fmt.Errorf("failed to open source %q: %w", cfg.Source, err)
+  }
+
+  fDstLatest, err := fs.NewFs(ctx, latestDir)
+  if err != nil {
+    return fmt.Errorf("failed to open destination %q: %w", latestDir, err)
+  }
+
+  fDstBackup, err := fs.NewFs(ctx, incrementDir)
+  if err != nil {
+    return fmt.Errorf("failed to initialize increment directory %q: %w", incrementDir, err)
+  }
+
+  // 2. Set BackupDir on the config context
+  ci.BackupDir = fDstBackup.Root()
+
+  if cfg.Verbose {
+    fmt.Printf("Starting sync from %s to %s with %d workers...\n",
+      cfg.Source, latestDir, cfg.Workers)
+  }
+
+  if err := sync.Sync(ctx, fDstLatest, fSrc, false); err != nil {
+    return fmt.Errorf("sync operation failed: %w", err)
+  }
+
+  // 3. Clean up the timestamp directory if nothing changed during sync
+  entries, err := fDstBackup.List(ctx, "")
+  if err == nil && len(entries) == 0 {
+    _ = operations.TryRmdir(ctx, fDstBackup, "")
+    if cfg.Verbose {
+      fmt.Println("No modifications detected; no increment folder created.")
+    }
+  } else if err == nil {
+    fmt.Printf("Created incremental snapshot: %s\n", incrementDir)
+  }
+
+  // 4. Prune excess incremental folders
+  if cfg.KeepIncrementals > 0 {
+    if err := pruneIncrementals(ctx, incrementsBase, cfg.KeepIncrementals, cfg.Verbose); err != nil {
+      return fmt.Errorf("retention pruning failed: %w", err)
+    }
+  }
+
+  return nil
+}
+
+func pruneIncrementals(ctx context.Context, incrementsPath string, keep int, verbose bool) error {
+  fIncrements, err := fs.NewFs(ctx, incrementsPath)
+  if err != nil {
+    return nil
+  }
+
+  entries, err := fIncrements.List(ctx, "")
+  if err != nil {
+    // If the folder hasn't been created yet, there is nothing to prune
+    if errors.Is(err, fs.ErrorDirNotFound) {
+      return nil
+    }
+    return err
+  }
+
+  var dirs []string
+  for _, entry := range entries {
+    if _, isDir := entry.(fs.Directory); isDir {
+      dirs = append(dirs, entry.Remote())
+    }
+  }
+
+  sort.Strings(dirs)
+
+  if len(dirs) <= keep {
+    return nil
+  }
+
+  excess := len(dirs) - keep
+  toDelete := dirs[:excess]
+
+  if verbose {
+    fmt.Printf("Pruning %d expired incremental snapshots (retention: %d)...\n", excess, keep)
+  }
+
+  for _, dirName := range toDelete {
+    subDirFs, err := fs.NewFs(ctx, filepath.Join(incrementsPath, dirName))
+    if err != nil {
+      continue
+    }
+    if err := operations.Purge(ctx, subDirFs, ""); err != nil {
+      return fmt.Errorf("failed to purge %s: %w", dirName, err)
+    }
+    if verbose {
+      fmt.Printf("Deleted snapshot: %s\n", dirName)
+    }
+  }
+
+  return nil
+}
@@ -0,0 +1,156 @@
+package backup
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"git.metaunix.net/bitgoblin/goblin-store/lib/config"
+)
+
+// RunRsyncIncremental executes a local hardlink-based incremental backup.
+func RunRsyncIncremental(ctx context.Context, cfg *config.Config) error {
+	// Verify that the rsync binary exists in PATH
+	if _, err := exec.LookPath("rsync"); err != nil {
+		return fmt.Errorf("rsync binary not found in PATH: %w", err)
+	}
+
+	// 1. Ensure the base destination directory exists
+	if err := os.MkdirAll(cfg.Destination, 0755); err != nil {
+		return fmt.Errorf("failed to create destination directory %q: %w", cfg.Destination, err)
+	}
+
+	// 2. Identify the most recent snapshot for link-dest deduplication
+	latestSnapshot, err := findLatestSnapshot(cfg.Destination)
+	if err != nil {
+		return fmt.Errorf("failed scanning for previous snapshots: %w", err)
+	}
+
+	timestamp := time.Now().Format("2006-01-02_15-04-05")
+	currentBackupDir := filepath.Join(cfg.Destination, timestamp)
+
+	// Ensure the source ends with a trailing slash so rsync copies directory
+	// contents directly rather than nesting the parent folder
+	src := filepath.Clean(cfg.Source) + string(filepath.Separator)
+
+	// 3. Assemble rsync arguments
+	args := []string{
+		"-a",       // Archive mode: preserves permissions, symlinks, timestamps, etc.
+		"--delete", // Delete files in dest that no longer exist in source
+	}
+
+	if cfg.Verbose {
+		args = append(args, "-v")
+	}
+
+	if latestSnapshot != "" {
+		// Use absolute path for --link-dest to prevent path resolution issues
+		absLinkDest, err := filepath.Abs(latestSnapshot)
+		if err != nil {
+			return fmt.Errorf("failed resolving link-dest path: %w", err)
+		}
+		args = append(args, fmt.Sprintf("--link-dest=%s", absLinkDest))
+
+		if cfg.Verbose {
+			fmt.Printf("Linking unchanged files against baseline: %s\n", absLinkDest)
+		}
+	} else if cfg.Verbose {
+		fmt.Println("No previous snapshots found. Generating initial full baseline...")
+	}
+
+	args = append(args, src, currentBackupDir)
+
+	if cfg.Verbose {
+		fmt.Printf("Executing: rsync %s\n", args)
+	}
+
+	// 4. Run the rsync command with context cancellation
+	cmd := exec.CommandContext(ctx, "rsync", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("rsync execution failed: %w", err)
+	}
+
+	fmt.Printf("Successfully created snapshot at: %s\n", currentBackupDir)
+
+	// 5. Enforce retention policy
+	if cfg.KeepIncrementals > 0 {
+		if err := pruneSnapshots(cfg.Destination, cfg.KeepIncrementals, cfg.Verbose); err != nil {
+			return fmt.Errorf("retention pruning failed: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// findLatestSnapshot scans the destination directory and returns the path
+// to the newest directory matching the timestamp pattern.
+func findLatestSnapshot(destRoot string) (string, error) {
+	entries, err := os.ReadDir(destRoot)
+	if err != nil {
+		return "", err
+	}
+
+	var snapshots []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			snapshots = append(snapshots, entry.Name())
+		}
+	}
+
+	if len(snapshots) == 0 {
+		return "", nil
+	}
+
+	// Lexicographical sort works chronologically because of "YYYY-MM-DD_HH-MM-SS" format
+	sort.Strings(snapshots)
+	return filepath.Join(destRoot, snapshots[len(snapshots)-1]), nil
+}
+
+// pruneSnapshots deletes the oldest snapshots when the total count exceeds the retention limit.
+func pruneSnapshots(destRoot string, keep int, verbose bool) error {
+	entries, err := os.ReadDir(destRoot)
+	if err != nil {
+		return err
+	}
+
+	var snapshots []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			snapshots = append(snapshots, entry.Name())
+		}
+	}
+
+	sort.Strings(snapshots)
+
+	if len(snapshots) <= keep {
+		return nil
+	}
+
+	excessCount := len(snapshots) - keep
+	toDelete := snapshots[:excessCount]
+
+	if verbose {
+		fmt.Printf("Pruning %d snapshot(s) exceeding retention limit (%d)...\n", excessCount, keep)
+	}
+
+	for _, dirName := range toDelete {
+		targetPath := filepath.Join(destRoot, dirName)
+		if verbose {
+			fmt.Printf("Deleting expired snapshot: %s\n", targetPath)
+		}
+
+		// os.RemoveAll removes directory entries and unlinks hardlinked inodes safely
+		if err := os.RemoveAll(targetPath); err != nil {
+			return fmt.Errorf("failed to remove snapshot %s: %w", targetPath, err)
+		}
+	}
+
+	return nil
+}
@@ -0,0 +1,36 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/spf13/viper"
+)
+
+type Config struct {
+	Source           string `mapstructure:"source"`
+	Destination      string `mapstructure:"destination"`
+	Workers          int    `mapstructure:"workers"`
+	KeepIncrementals int    `mapstructure:"keep_incrementals"`
+	Verbose          bool   `mapstructure:"verbose"`
+}
+
+// Load unmarshals the current Viper state into a Config struct and validates it.
+func Load() (*Config, error) {
+	var cfg Config
+	if err := viper.Unmarshal(&cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal configuration: %w", err)
+	}
+
+	if cfg.Source == "" {
+		return nil, errors.New("source path must be specified")
+	}
+	if cfg.Destination == "" {
+		return nil, errors.New("destination path must be specified")
+	}
+	if cfg.KeepIncrementals < 0 {
+		cfg.KeepIncrementals = 0
+	}
+
+	return &cfg, nil
+}