Merge branch 'excludes' into 'master'

Support per-source excludes (for file handlers)

Closes #4

See merge request !3

.gitlab-ci.yml

 
 stages:
-  - test
   - build_pkgsrc
   - build_pkg
   - upload_pkg
 
-# Do not run tests on master, they are executed by the
-# Debian package builder.
-# TODO: currently broken
-test:
-  stage: test
-  image: "debian:stretch"
-  script:
-    - "./install_restic_for_tests.sh"
-    - "apt-get install -y liblz4-tool golang"
-    - "go test -v ./..."
-  except:
-    - master
-
 build_pkgsrc:stretch:
   stage: build_pkgsrc
   image: "registry.git.autistici.org/ai3/build-deb:stretch-amd64"

handler_file.go

@@ -10,11 +10,15 @@ import (
 )
 
 type fileHandler struct {
-	path string
+	path    string
+	exclude []string
 }
 
 func newFileHandler(name string, params Params) (Handler, error) {
-	return &fileHandler{path: params.Get("path")}, nil
+	return &fileHandler{
+		path:    params.Get("path"),
+		exclude: params.GetList("exclude"),
+	}, nil
 }
 
 // Convert the atom to a path.
@@ -58,7 +62,7 @@ func (h *fileHandler) BackupJob(rctx RuntimeContext, backup *Backup, ds *Dataset
 
 		// Invoke the backup command (path-based).
 		repo := rctx.Repo()
-		cmd := repo.BackupCmd(backup, ds, tmpf)
+		cmd := repo.BackupCmd(backup, ds, tmpf, h.exclude)
 		return repo.RunBackup(ctx, rctx.Shell(), backup, ds, cmd)
 	})
 }

repository.go

@@ -7,9 +7,9 @@ import (
 
 // RepositorySpec defines the configuration of a repository.
 type RepositorySpec struct {
-	Name   string                 `yaml:"name"`
-	Type   string                 `yaml:"type"`
-	Params map[string]interface{} `yaml:"params"`
+	Name   string `yaml:"name"`
+	Type   string `yaml:"type"`
+	Params Params `yaml:"params"`
 }
 
 // Parse a RepositorySpec and return a Repository instance.

repository_restic.go

@@ -66,6 +66,7 @@ func checkResticVersion(bin string) error {
 	if err != nil {
 		return err
 	}
+	log.Printf("detected restic version %s", v)
 	minV, _ := version.NewVersion(resticMinGoodVersion) // nolint
 	if v.LessThan(minV) {
 		return fmt.Errorf("restic should be at least version %s (is %s)", minV, v)
@@ -74,34 +75,23 @@ func checkResticVersion(bin string) error {
 }
 
 // newResticRepository returns a restic repository.
-func newResticRepository(params map[string]interface{}) (Repository, error) {
-	uri, ok := params["uri"].(string)
-	if !ok || uri == "" {
+func newResticRepository(params Params) (Repository, error) {
+	uri := params.Get("uri")
+	if uri == "" {
 		return nil, errors.New("missing uri")
 	}
-	password, ok := params["password"].(string)
-	if !ok || password == "" {
+	password := params.Get("password")
+	if password == "" {
 		return nil, errors.New("missing password")
 	}
 
-	ex, _ := params["exclude"].([]string)
-	exf, _ := params["exclude_files"].([]string)
 	bin := "restic"
-	if s, ok := params["restic_binary"].(string); ok {
+	if s := params.Get("restic_binary"); s != "" {
 		bin = s
 	}
 	if err := checkResticVersion(bin); err != nil {
 		return nil, err
 	}
-	autoPrune := true
-	if b, ok := params["autoprune"].(bool); ok {
-		autoPrune = b
-	}
-
-	var cacheDir string
-	if s, ok := params["cache_dir"].(string); ok {
-		cacheDir = s
-	}
 
 	tmpf, err := ioutil.TempFile("", "restic-pw-")
 	if err != nil {
@@ -116,14 +106,18 @@ func newResticRepository(params map[string]interface{}) (Repository, error) {
 		return nil, err
 	}
 
+	autoPrune := true
+	if b, ok := params.GetBool("autoprune"); ok {
+		autoPrune = b
+	}
 	return &resticRepository{
 		bin:          bin,
 		uri:          uri,
 		passwordFile: tmpf.Name(),
-		excludes:     ex,
-		excludeFiles: exf,
+		excludes:     params.GetList("exclude"),
+		excludeFiles: params.GetList("exclude_files"),
 		autoPrune:    autoPrune,
-		cacheDirPath: cacheDir,
+		cacheDirPath: params.Get("cache_dir"),
 	}, nil
 }
 
@@ -166,19 +160,26 @@ func (r *resticRepository) cacheArgs(ds *Dataset) string {
 	}
 	dir := filepath.Join(r.cacheDirPath, strings.Replace(ds.Source, "/", "_", -1))
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		os.MkdirAll(dir, 0700)
+		if err := os.MkdirAll(dir, 0700); err != nil {
+			log.Printf("warning: could not create cache directory (%s), running without a local cache", err)
+			return "--no-cache"
+		}
 	}
 	return fmt.Sprintf("--cache-dir %s", dir)
 }
 
-func (r *resticRepository) BackupCmd(backup *Backup, ds *Dataset, inputFile string) string {
-	return fmt.Sprintf(
+func (r *resticRepository) BackupCmd(backup *Backup, ds *Dataset, inputFile string, exclude []string) string {
+	cmd := fmt.Sprintf(
 		"%s backup %s --json --exclude-caches --one-file-system %s --files-from %s",
 		r.resticCmd(),
 		r.cacheArgs(ds),
 		resticBackupTags(backup, ds),
 		inputFile,
 	)
+	for _, ex := range exclude {
+		cmd += fmt.Sprintf(" --exclude='%s'", ex)
+	}
+	return cmd
 }
 
 func (r *resticRepository) getSnapshotID(ctx context.Context, rctx RuntimeContext, backup *Backup, ds *Dataset) (string, error) {

types.go

@@ -22,6 +22,14 @@ func (p Params) Get(key string) string {
 	return ""
 }
 
+// GetList returns a string list value for a parameter.
+func (p Params) GetList(key string) []string {
+	if l, ok := p[key].([]string); ok {
+		return l
+	}
+	return nil
+}
+
 // GetBool returns a boolean value for a parameter (may be a string).
 // Returns value and presence.
 func (p Params) GetBool(key string) (bool, bool) {
@@ -139,7 +147,7 @@ type Handler interface {
 // Repository is the interface to a remote repository.
 type Repository interface {
 	Init(context.Context, RuntimeContext) error
-	BackupCmd(*Backup, *Dataset, string) string
+	BackupCmd(*Backup, *Dataset, string, []string) string
 	RestoreCmd(context.Context, RuntimeContext, *Backup, *Dataset, []string, string) (string, error)
 	BackupStreamCmd(*Backup, *Dataset) string
 	RestoreStreamCmd(context.Context, RuntimeContext, *Backup, *Dataset, string) (string, error)