Added option to run local (rsync) or cloud (rclone) backups, in case you prefer a "full" copy with hardlinks via rsync, or latest/incremental snapshots via rclone
This commit is contained in:
@@ -0,0 +1,134 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.metaunix.net/bitgoblin/goblin-store/lib/config"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
|
||||
// Register desired backends
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
)
|
||||
|
||||
func RunIncrementalBackup(ctx context.Context, cfg *config.Config) error {
|
||||
// Configure global Rclone concurrency parameters
|
||||
ci := fs.GetConfig(ctx)
|
||||
ci.Transfers = cfg.Workers
|
||||
ci.Checkers = cfg.Workers * 2
|
||||
|
||||
// Setup directories:
|
||||
// - destination/latest: holds current full mirror
|
||||
// - destination/increments/<timestamp>: holds files altered or deleted during sync
|
||||
timestamp := time.Now().Format("2006-01-02_15-04-05")
|
||||
latestDir := filepath.Join(cfg.Destination, "latest")
|
||||
incrementDir := filepath.Join(cfg.Destination, "increments", timestamp)
|
||||
incrementsBase := filepath.Join(cfg.Destination, "increments")
|
||||
|
||||
// 1. Resolve source and target filesystems
|
||||
fSrc, err := fs.NewFs(ctx, cfg.Source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source %q: %w", cfg.Source, err)
|
||||
}
|
||||
|
||||
fDstLatest, err := fs.NewFs(ctx, latestDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open destination %q: %w", latestDir, err)
|
||||
}
|
||||
|
||||
fDstBackup, err := fs.NewFs(ctx, incrementDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize increment directory %q: %w", incrementDir, err)
|
||||
}
|
||||
|
||||
// 2. Set BackupDir on the config context
|
||||
ci.BackupDir = fDstBackup.Root()
|
||||
|
||||
if cfg.Verbose {
|
||||
fmt.Printf("Starting sync from %s to %s with %d workers...\n",
|
||||
cfg.Source, latestDir, cfg.Workers)
|
||||
}
|
||||
|
||||
if err := sync.Sync(ctx, fDstLatest, fSrc, false); err != nil {
|
||||
return fmt.Errorf("sync operation failed: %w", err)
|
||||
}
|
||||
|
||||
// 3. Clean up the timestamp directory if nothing changed during sync
|
||||
entries, err := fDstBackup.List(ctx, "")
|
||||
if err == nil && len(entries) == 0 {
|
||||
_ = operations.TryRmdir(ctx, fDstBackup, "")
|
||||
if cfg.Verbose {
|
||||
fmt.Println("No modifications detected; no increment folder created.")
|
||||
}
|
||||
} else if err == nil {
|
||||
fmt.Printf("Created incremental snapshot: %s\n", incrementDir)
|
||||
}
|
||||
|
||||
// 4. Prune excess incremental folders
|
||||
if cfg.KeepIncrementals > 0 {
|
||||
if err := pruneIncrementals(ctx, incrementsBase, cfg.KeepIncrementals, cfg.Verbose); err != nil {
|
||||
return fmt.Errorf("retention pruning failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func pruneIncrementals(ctx context.Context, incrementsPath string, keep int, verbose bool) error {
|
||||
fIncrements, err := fs.NewFs(ctx, incrementsPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
entries, err := fIncrements.List(ctx, "")
|
||||
if err != nil {
|
||||
// If the folder hasn't been created yet, there is nothing to prune
|
||||
if errors.Is(err, fs.ErrorDirNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var dirs []string
|
||||
for _, entry := range entries {
|
||||
if _, isDir := entry.(fs.Directory); isDir {
|
||||
dirs = append(dirs, entry.Remote())
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(dirs)
|
||||
|
||||
if len(dirs) <= keep {
|
||||
return nil
|
||||
}
|
||||
|
||||
excess := len(dirs) - keep
|
||||
toDelete := dirs[:excess]
|
||||
|
||||
if verbose {
|
||||
fmt.Printf("Pruning %d expired incremental snapshots (retention: %d)...\n", excess, keep)
|
||||
}
|
||||
|
||||
for _, dirName := range toDelete {
|
||||
subDirFs, err := fs.NewFs(ctx, filepath.Join(incrementsPath, dirName))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := operations.Purge(ctx, subDirFs, ""); err != nil {
|
||||
return fmt.Errorf("failed to purge %s: %w", dirName, err)
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("Deleted snapshot: %s\n", dirName)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.metaunix.net/bitgoblin/goblin-store/lib/config"
|
||||
)
|
||||
|
||||
// RunRsyncIncremental executes a local hardlink-based incremental backup.
|
||||
func RunRsyncIncremental(ctx context.Context, cfg *config.Config) error {
|
||||
// Verify that the rsync binary exists in PATH
|
||||
if _, err := exec.LookPath("rsync"); err != nil {
|
||||
return fmt.Errorf("rsync binary not found in PATH: %w", err)
|
||||
}
|
||||
|
||||
// 1. Ensure the base destination directory exists
|
||||
if err := os.MkdirAll(cfg.Destination, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create destination directory %q: %w", cfg.Destination, err)
|
||||
}
|
||||
|
||||
// 2. Identify the most recent snapshot for link-dest deduplication
|
||||
latestSnapshot, err := findLatestSnapshot(cfg.Destination)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed scanning for previous snapshots: %w", err)
|
||||
}
|
||||
|
||||
timestamp := time.Now().Format("2006-01-02_15-04-05")
|
||||
currentBackupDir := filepath.Join(cfg.Destination, timestamp)
|
||||
|
||||
// Ensure the source ends with a trailing slash so rsync copies directory
|
||||
// contents directly rather than nesting the parent folder
|
||||
src := filepath.Clean(cfg.Source) + string(filepath.Separator)
|
||||
|
||||
// 3. Assemble rsync arguments
|
||||
args := []string{
|
||||
"-a", // Archive mode: preserves permissions, symlinks, timestamps, etc.
|
||||
"--delete", // Delete files in dest that no longer exist in source
|
||||
}
|
||||
|
||||
if cfg.Verbose {
|
||||
args = append(args, "-v")
|
||||
}
|
||||
|
||||
if latestSnapshot != "" {
|
||||
// Use absolute path for --link-dest to prevent path resolution issues
|
||||
absLinkDest, err := filepath.Abs(latestSnapshot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed resolving link-dest path: %w", err)
|
||||
}
|
||||
args = append(args, fmt.Sprintf("--link-dest=%s", absLinkDest))
|
||||
|
||||
if cfg.Verbose {
|
||||
fmt.Printf("Linking unchanged files against baseline: %s\n", absLinkDest)
|
||||
}
|
||||
} else if cfg.Verbose {
|
||||
fmt.Println("No previous snapshots found. Generating initial full baseline...")
|
||||
}
|
||||
|
||||
args = append(args, src, currentBackupDir)
|
||||
|
||||
if cfg.Verbose {
|
||||
fmt.Printf("Executing: rsync %s\n", args)
|
||||
}
|
||||
|
||||
// 4. Run the rsync command with context cancellation
|
||||
cmd := exec.CommandContext(ctx, "rsync", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("rsync execution failed: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Successfully created snapshot at: %s\n", currentBackupDir)
|
||||
|
||||
// 5. Enforce retention policy
|
||||
if cfg.KeepIncrementals > 0 {
|
||||
if err := pruneSnapshots(cfg.Destination, cfg.KeepIncrementals, cfg.Verbose); err != nil {
|
||||
return fmt.Errorf("retention pruning failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findLatestSnapshot scans the destination directory and returns the path
|
||||
// to the newest directory matching the timestamp pattern.
|
||||
func findLatestSnapshot(destRoot string) (string, error) {
|
||||
entries, err := os.ReadDir(destRoot)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var snapshots []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
snapshots = append(snapshots, entry.Name())
|
||||
}
|
||||
}
|
||||
|
||||
if len(snapshots) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Lexicographical sort works chronologically because of "YYYY-MM-DD_HH-MM-SS" format
|
||||
sort.Strings(snapshots)
|
||||
return filepath.Join(destRoot, snapshots[len(snapshots)-1]), nil
|
||||
}
|
||||
|
||||
// pruneSnapshots deletes the oldest snapshots when the total count exceeds the retention limit.
|
||||
func pruneSnapshots(destRoot string, keep int, verbose bool) error {
|
||||
entries, err := os.ReadDir(destRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var snapshots []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
snapshots = append(snapshots, entry.Name())
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(snapshots)
|
||||
|
||||
if len(snapshots) <= keep {
|
||||
return nil
|
||||
}
|
||||
|
||||
excessCount := len(snapshots) - keep
|
||||
toDelete := snapshots[:excessCount]
|
||||
|
||||
if verbose {
|
||||
fmt.Printf("Pruning %d snapshot(s) exceeding retention limit (%d)...\n", excessCount, keep)
|
||||
}
|
||||
|
||||
for _, dirName := range toDelete {
|
||||
targetPath := filepath.Join(destRoot, dirName)
|
||||
if verbose {
|
||||
fmt.Printf("Deleting expired snapshot: %s\n", targetPath)
|
||||
}
|
||||
|
||||
// os.RemoveAll removes directory entries and unlinks hardlinked inodes safely
|
||||
if err := os.RemoveAll(targetPath); err != nil {
|
||||
return fmt.Errorf("failed to remove snapshot %s: %w", targetPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user