Give each backup job its temporary working directory

manager.go

@@ -16,6 +16,7 @@ type tabaccoManager struct {
 	*jobs.ExclusiveLockManager
 	*jobs.QueueManager
 	*jobs.StateManager
+	*workdirManager
 
 	configMgr *ConfigManager
 	ms        MetadataStore
@@ -23,11 +24,19 @@ type tabaccoManager struct {
 
 // NewManager creates a new Manager.
 func NewManager(ctx context.Context, configMgr *ConfigManager, ms MetadataStore) (Manager, error) {
+	// If we can't create a workdirManager, it probably means we
+	// don't have permissions to the WorkDir, which is bad.
+	wm, err := newWorkdirManager(configMgr.getWorkDir())
+	if err != nil {
+		return nil, err
+	}
+
 	// Note: the queue configuration won't be reloaded.
 	return &tabaccoManager{
 		ExclusiveLockManager: jobs.NewExclusiveLockManager(),
 		QueueManager:         jobs.NewQueueManager(configMgr.getQueueSpec()),
 		StateManager:         jobs.NewStateManager(),
+		workdirManager:       wm,
 
 		configMgr: configMgr,
 		ms:        ms,
@@ -127,9 +136,11 @@ func (m *tabaccoManager) backupDatasetJob(h Handler, backup Backup, ds Dataset)
 	return m.WithQueue(
 		m.WithStatus(
 			m.WithExclusiveLock(
-				withInstrumentation(
-					jobs.SyncGroup(out),
-					ds.Name,
+				m.withWorkDir(
+					withInstrumentation(
+						jobs.SyncGroup(out),
+						ds.Name,
+					),
 				),
 				id,
 				false),

shell.go

@@ -4,10 +4,14 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
+
+	"git.autistici.org/ale/tabacco/jobs"
 )
 
 // Shell runs commands, with some options (a global dry-run flag
@@ -39,6 +43,9 @@ func (s *Shell) SetIOClass(n int) {
 	s.ioniceClass = n
 }
 
+// command builds an exec.Cmd and gets some parameters from the
+// context - notably it sets log output and working directory to be in
+// the working dir if the job has been wrapped in WithWorkDir().
 func (s *Shell) command(ctx context.Context, arg string) *exec.Cmd {
 	log.Printf("sh: %s", arg)
 
@@ -51,19 +58,20 @@ func (s *Shell) command(ctx context.Context, arg string) *exec.Cmd {
 
 	if s.niceLevel != 0 {
 		args = append(
-			[]string{"/usr/bin/nice", "-n", strconv.Itoa(s.niceLevel)},
+			[]string{"nice", "-n", strconv.Itoa(s.niceLevel)},
 			args...,
 		)
 	}
 	if s.ioniceClass != 0 {
 		args = append(
-			[]string{"/usr/bin/ionice", "-c", strconv.Itoa(s.ioniceClass)},
+			[]string{"ionice", "-c", strconv.Itoa(s.ioniceClass)},
 			args...,
 		)
 	}
 
 	c := exec.CommandContext(ctx, args[0], args[1:]...) // #nosec
 	c.Stderr = os.Stderr
+	c.Dir = getWorkDir(ctx)
 	return c
 }
 
@@ -127,3 +135,56 @@ func (s *Shell) RunWithEnv(ctx context.Context, arg string, envMap map[string]st
 func (s *Shell) Output(ctx context.Context, arg string) ([]byte, error) {
 	return s.command(ctx, arg).Output()
 }
+
+// Manage a work directory (temporary scratch with lots of space).
+type workdirManager struct {
+	dir string
+}
+
+func newWorkdirManager(tmpdir string) (*workdirManager, error) {
+	dir, err := ioutil.TempDir(tmpdir, "tabacco-")
+	if err != nil {
+		return nil, err
+	}
+	return &workdirManager{dir}, nil
+}
+
+func (w *workdirManager) Close() {
+	os.RemoveAll(w.dir) // nolint
+}
+
+func (w *workdirManager) withWorkDir(j jobs.Job) jobs.Job {
+	jobDir := filepath.Join(w.dir, "job-"+j.ID())
+	return &workdirJob{
+		Job: j,
+		dir: jobDir,
+	}
+}
+
+type workdirJob struct {
+	jobs.Job
+	dir string
+}
+
+type workdirKeyType int
+
+var workdirKey workdirKeyType = 1
+
+func (j *workdirJob) RunContext(ctx context.Context) error {
+	if err := os.Mkdir(j.dir, 0700); err != nil {
+		return err
+	}
+
+	ctx = context.WithValue(ctx, workdirKey, j.dir)
+	return j.Job.RunContext(ctx)
+}
+
+// getWorkDir returns the current working directory, provided the
+// WithWorkDir wrapper has been called.
+func getWorkDir(ctx context.Context) string {
+	dir, ok := ctx.Value(workdirKey).(string)
+	if ok {
+		return dir
+	}
+	return "/tmp"
+}