Commit 59e40b7e authored by ale's avatar ale

Use a temp file for the path list to deal with very large lists properly

parent c335bf2c
Pipeline #3570 passed with stages
in 1 minute and 57 seconds
......@@ -2,6 +2,8 @@ package tabacco
import (
"context"
"fmt"
"io/ioutil"
"path/filepath"
"git.autistici.org/ai3/tools/tabacco/jobs"
......@@ -30,17 +32,33 @@ func atomPath(a Atom, root string) string {
return filepath.Join(root, a.Name)
}
func (h *fileHandler) BackupJob(rctx RuntimeContext, backup *Backup, ds *Dataset) jobs.Job {
// Build the list of filesystem paths to pass to the
// Repository.Backup method.
var paths []string
for _, a := range ds.Atoms {
paths = append(paths, atomPath(a, h.path))
func (h *fileHandler) saveAtomsToFile(ctx context.Context, atoms []Atom) (string, error) {
tmpf, err := ioutil.TempFile(getWorkDir(ctx), "file-list-")
if err != nil {
return "", err
}
for _, a := range atoms {
path := atomPath(a, h.path)
if _, err := fmt.Fprintf(tmpf, "%s\n", path); err != nil {
return "", err
}
}
cmd := rctx.Repo().BackupCmd(backup, ds, paths)
return tmpf.Name(), tmpf.Close()
}
// Now pass those paths to the Backup method.
func (h *fileHandler) BackupJob(rctx RuntimeContext, backup *Backup, ds *Dataset) jobs.Job {
return jobs.JobFunc(func(ctx context.Context) error {
// Build the list of filesystem paths to pass to the
// Repository.Backup method. Use a temporary file, in case
// the path list is too long for the command line.
tmpf, err := h.saveAtomsToFile(ctx, ds.Atoms)
if err != nil {
return err
}
cmd := rctx.Repo().BackupCmd(backup, ds, tmpf)
// Now pass those paths to the Backup method.
return rctx.Shell().Run(ctx, cmd)
})
}
......
......@@ -152,12 +152,12 @@ func resticBackupTags(backup *Backup, ds *Dataset) string {
return fmt.Sprintf("--tag dataset_id=%s --tag backup_id=%s", ds.ID, backup.ID)
}
func (r *resticRepository) BackupCmd(backup *Backup, ds *Dataset, sourcePaths []string) string {
func (r *resticRepository) BackupCmd(backup *Backup, ds *Dataset, inputFile string) string {
return fmt.Sprintf(
"%s backup --no-cache --exclude-caches --one-file-system %s %s",
"%s backup --no-cache --exclude-caches --one-file-system %s --files-from %s",
r.resticCmd(),
resticBackupTags(backup, ds),
strings.Join(sourcePaths, " "),
inputFile,
)
}
......
......@@ -125,7 +125,7 @@ type Handler interface {
// Repository is the interface to a remote repository.
type Repository interface {
Init(context.Context, RuntimeContext) error
BackupCmd(*Backup, *Dataset, []string) string
BackupCmd(*Backup, *Dataset, string) string
RestoreCmd(context.Context, RuntimeContext, *Backup, *Dataset, []string, string) (string, error)
BackupStreamCmd(*Backup, *Dataset) string
RestoreStreamCmd(context.Context, RuntimeContext, *Backup, *Dataset, string) (string, error)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment