Added option to run local (rsync) or cloud (rclone) backups, in case you prefer a "full" copy with hardlinks via rsync, or latest/incremental snapshots via rclone

lib/backup/rclone.go

@@ -0,0 +1,134 @@
+package backup
+
+import (
+  "context"
+  "errors"
+  "fmt"
+  "path/filepath"
+  "sort"
+  "time"
+
+  "git.metaunix.net/bitgoblin/goblin-store/lib/config"
+
+  "github.com/rclone/rclone/fs"
+  "github.com/rclone/rclone/fs/operations"
+  "github.com/rclone/rclone/fs/sync"
+
+  // Register desired backends
+  _ "github.com/rclone/rclone/backend/local"
+  _ "github.com/rclone/rclone/backend/s3"
+)
+
+func RunIncrementalBackup(ctx context.Context, cfg *config.Config) error {
+  // Configure global Rclone concurrency parameters
+  ci := fs.GetConfig(ctx)
+  ci.Transfers = cfg.Workers
+  ci.Checkers = cfg.Workers * 2
+
+  // Setup directories:
+  // - destination/latest: holds current full mirror
+  // - destination/increments/<timestamp>: holds files altered or deleted during sync
+  timestamp := time.Now().Format("2006-01-02_15-04-05")
+  latestDir := filepath.Join(cfg.Destination, "latest")
+  incrementDir := filepath.Join(cfg.Destination, "increments", timestamp)
+  incrementsBase := filepath.Join(cfg.Destination, "increments")
+
+  // 1. Resolve source and target filesystems
+  fSrc, err := fs.NewFs(ctx, cfg.Source)
+  if err != nil {
+    return fmt.Errorf("failed to open source %q: %w", cfg.Source, err)
+  }
+
+  fDstLatest, err := fs.NewFs(ctx, latestDir)
+  if err != nil {
+    return fmt.Errorf("failed to open destination %q: %w", latestDir, err)
+  }
+
+  fDstBackup, err := fs.NewFs(ctx, incrementDir)
+  if err != nil {
+    return fmt.Errorf("failed to initialize increment directory %q: %w", incrementDir, err)
+  }
+
+  // 2. Set BackupDir on the config context
+  ci.BackupDir = fDstBackup.Root()
+
+  if cfg.Verbose {
+    fmt.Printf("Starting sync from %s to %s with %d workers...\n",
+      cfg.Source, latestDir, cfg.Workers)
+  }
+
+  if err := sync.Sync(ctx, fDstLatest, fSrc, false); err != nil {
+    return fmt.Errorf("sync operation failed: %w", err)
+  }
+
+  // 3. Clean up the timestamp directory if nothing changed during sync
+  entries, err := fDstBackup.List(ctx, "")
+  if err == nil && len(entries) == 0 {
+    _ = operations.TryRmdir(ctx, fDstBackup, "")
+    if cfg.Verbose {
+      fmt.Println("No modifications detected; no increment folder created.")
+    }
+  } else if err == nil {
+    fmt.Printf("Created incremental snapshot: %s\n", incrementDir)
+  }
+
+  // 4. Prune excess incremental folders
+  if cfg.KeepIncrementals > 0 {
+    if err := pruneIncrementals(ctx, incrementsBase, cfg.KeepIncrementals, cfg.Verbose); err != nil {
+      return fmt.Errorf("retention pruning failed: %w", err)
+    }
+  }
+
+  return nil
+}
+
+func pruneIncrementals(ctx context.Context, incrementsPath string, keep int, verbose bool) error {
+  fIncrements, err := fs.NewFs(ctx, incrementsPath)
+  if err != nil {
+    return nil
+  }
+
+  entries, err := fIncrements.List(ctx, "")
+  if err != nil {
+    // If the folder hasn't been created yet, there is nothing to prune
+    if errors.Is(err, fs.ErrorDirNotFound) {
+      return nil
+    }
+    return err
+  }
+
+  var dirs []string
+  for _, entry := range entries {
+    if _, isDir := entry.(fs.Directory); isDir {
+      dirs = append(dirs, entry.Remote())
+    }
+  }
+
+  sort.Strings(dirs)
+
+  if len(dirs) <= keep {
+    return nil
+  }
+
+  excess := len(dirs) - keep
+  toDelete := dirs[:excess]
+
+  if verbose {
+    fmt.Printf("Pruning %d expired incremental snapshots (retention: %d)...\n", excess, keep)
+  }
+
+  for _, dirName := range toDelete {
+    subDirFs, err := fs.NewFs(ctx, filepath.Join(incrementsPath, dirName))
+    if err != nil {
+      continue
+    }
+    if err := operations.Purge(ctx, subDirFs, ""); err != nil {
+      return fmt.Errorf("failed to purge %s: %w", dirName, err)
+    }
+    if verbose {
+      fmt.Printf("Deleted snapshot: %s\n", dirName)
+    }
+  }
+
+  return nil
+}