repository_restic.go 6.52 KB
Newer Older
ale's avatar
ale committed
1 2 3 4 5 6 7 8 9
package tabacco

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
ale's avatar
ale committed
10
	"log"
ale's avatar
ale committed
11
	"os"
ale's avatar
ale committed
12
	"os/exec"
ale's avatar
ale committed
13
	"path/filepath"
ale's avatar
ale committed
14
	"regexp"
ale's avatar
ale committed
15
	"strings"
ale's avatar
ale committed
16
	"sync"
ale's avatar
ale committed
17
	"time"
ale's avatar
ale committed
18 19

	"github.com/hashicorp/go-version"
ale's avatar
ale committed
20 21 22
)

type resticRepository struct {
ale's avatar
ale committed
23
	bin          string
ale's avatar
ale committed
24 25 26 27
	uri          string
	passwordFile string
	excludes     []string
	excludeFiles []string
28
	autoPrune    bool
ale's avatar
ale committed
29

ale's avatar
ale committed
30
	initialized sync.Once
ale's avatar
ale committed
31 32
}

ale's avatar
ale committed
33 34
func (r *resticRepository) resticCmd() string {
	args := []string{r.bin, "-r", r.uri}
ale's avatar
ale committed
35
	if r.passwordFile != "" {
ale's avatar
ale committed
36
		args = append(args, "--password-file", r.passwordFile)
ale's avatar
ale committed
37 38
	}
	for _, x := range r.excludes {
ale's avatar
ale committed
39
		args = append(args, "--exclude", x)
ale's avatar
ale committed
40 41
	}
	for _, x := range r.excludeFiles {
ale's avatar
ale committed
42
		args = append(args, "--exclude-file", x)
ale's avatar
ale committed
43
	}
ale's avatar
ale committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
	return strings.Join(args, " ")
}

// We need to check that we're running at least restic 0.9, or the
// restore functionality won't work as expected.
var (
	resticVersionRx      = regexp.MustCompile(`^restic ([0-9.]+)`)
	resticMinGoodVersion = "0.9"
)

func checkResticVersion(bin string) error {
	output, err := exec.Command(bin, "version").Output() // #nosec
	if err != nil {
		return err
	}
	m := resticVersionRx.FindStringSubmatch(string(output))
	if len(m) < 2 {
		return errors.New("could not parse restic version")
	}
	v, err := version.NewVersion(m[1])
	if err != nil {
		return err
	}
	minV, _ := version.NewVersion(resticMinGoodVersion) // nolint
	if v.LessThan(minV) {
		return fmt.Errorf("restic should be at least version %s (is %s)", minV, v)
	}
	return nil
ale's avatar
ale committed
72 73 74
}

// newResticRepository returns a restic repository.
ale's avatar
ale committed
75
func newResticRepository(params map[string]interface{}) (Repository, error) {
ale's avatar
ale committed
76 77 78 79 80 81 82 83 84 85 86
	uri, ok := params["uri"].(string)
	if !ok || uri == "" {
		return nil, errors.New("missing uri")
	}
	password, ok := params["password"].(string)
	if !ok || password == "" {
		return nil, errors.New("missing password")
	}

	ex, _ := params["exclude"].([]string)
	exf, _ := params["exclude_files"].([]string)
87 88 89
	bin := "restic"
	if s, ok := params["restic_binary"].(string); ok {
		bin = s
ale's avatar
ale committed
90 91 92 93
	}
	if err := checkResticVersion(bin); err != nil {
		return nil, err
	}
94 95 96 97
	autoPrune := true
	if b, ok := params["autoprune"].(bool); ok {
		autoPrune = b
	}
ale's avatar
ale committed
98 99 100 101 102 103

	tmpf, err := ioutil.TempFile("", "restic-pw-")
	if err != nil {
		return nil, err
	}
	if _, err := io.WriteString(tmpf, password); err != nil {
104
		os.Remove(tmpf.Name()) // nolint
ale's avatar
ale committed
105 106 107
		return nil, err
	}
	if err := tmpf.Close(); err != nil {
108
		os.Remove(tmpf.Name()) // nolint
ale's avatar
ale committed
109 110 111 112
		return nil, err
	}

	return &resticRepository{
ale's avatar
ale committed
113
		bin:          bin,
ale's avatar
ale committed
114 115 116 117
		uri:          uri,
		passwordFile: tmpf.Name(),
		excludes:     ex,
		excludeFiles: exf,
118
		autoPrune:    autoPrune,
ale's avatar
ale committed
119 120 121 122 123 124 125
	}, nil
}

func (r *resticRepository) Close() error {
	return os.Remove(r.passwordFile)
}

ale's avatar
ale committed
126
func (r *resticRepository) Init(ctx context.Context, rctx RuntimeContext) error {
ale's avatar
ale committed
127 128 129 130 131 132 133 134 135 136 137
	r.initialized.Do(func() {
		// Restic init will fail if the repository is already
		// initialized, ignore errors (but log them).
		if err := rctx.Shell().Run(ctx, fmt.Sprintf(
			"%s init --quiet || true",
			r.resticCmd(),
		)); err != nil {
			log.Printf("restic repository init failed (likely harmless): %v", err)
		}
	})
	return nil
ale's avatar
ale committed
138 139
}

ale's avatar
ale committed
140
func (r *resticRepository) Prepare(ctx context.Context, rctx RuntimeContext, backup *Backup) error {
141 142 143
	if !r.autoPrune {
		return nil
	}
ale's avatar
ale committed
144
	return rctx.Shell().Run(ctx, fmt.Sprintf(
ale's avatar
ale committed
145 146
		"%s forget --host %s --keep-last 10 --prune",
		r.resticCmd(),
ale's avatar
ale committed
147 148 149 150
		backup.Host,
	))
}

151 152 153 154
func resticBackupTags(backup *Backup, ds *Dataset) string {
	return fmt.Sprintf("--tag dataset_id=%s --tag backup_id=%s", ds.ID, backup.ID)
}

ale's avatar
ale committed
155 156
func (r *resticRepository) BackupCmd(backup *Backup, ds *Dataset, sourcePaths []string) string {
	return fmt.Sprintf(
157
		"%s backup --cleanup-cache --exclude-caches --one-file-system %s %s",
ale's avatar
ale committed
158
		r.resticCmd(),
159
		resticBackupTags(backup, ds),
ale's avatar
ale committed
160
		strings.Join(sourcePaths, " "),
ale's avatar
ale committed
161 162 163
	)
}

ale's avatar
ale committed
164 165
func (r *resticRepository) getSnapshotID(ctx context.Context, rctx RuntimeContext, backup *Backup, ds *Dataset) (string, error) {
	data, err := rctx.Shell().Output(ctx, fmt.Sprintf(
166
		"%s snapshots --json %s",
ale's avatar
ale committed
167
		r.resticCmd(),
168
		resticBackupTags(backup, ds),
ale's avatar
ale committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182
	))
	if err != nil {
		return "", err
	}
	snaps, err := parseResticSnapshots(data)
	if err != nil {
		return "", err
	}
	if len(snaps) < 1 {
		return "", fmt.Errorf("could not find snapshot for backup id %s", backup.ID)
	}
	return snaps[0].ShortID, nil
}

ale's avatar
ale committed
183 184
func (r *resticRepository) RestoreCmd(ctx context.Context, rctx RuntimeContext, backup *Backup, ds *Dataset, paths []string, target string) (string, error) {
	snap, err := r.getSnapshotID(ctx, rctx, backup, ds)
ale's avatar
ale committed
185
	if err != nil {
ale's avatar
ale committed
186 187 188 189 190
		return "", err
	}

	cmd := []string{
		fmt.Sprintf("%s restore", r.resticCmd()),
ale's avatar
ale committed
191
	}
ale's avatar
ale committed
192 193 194 195 196 197 198 199 200 201

	for _, path := range paths {
		cmd = append(cmd, fmt.Sprintf("--include %s", path))
	}

	cmd = append(cmd, fmt.Sprintf("--target %s", target))
	cmd = append(cmd, snap)
	return strings.Join(cmd, " "), nil
}

202 203 204
// A special path for stdin datasets that is likely to be unused by the
// rest of the filesystem (the path namespace in Restic is global).
func datasetStdinPath(ds *Dataset) string {
205
	dsPath := filepath.Join(ds.Source, ds.ID)
206 207 208
	return fmt.Sprintf("/STDIN_%s", strings.Replace(dsPath, "/", "_", -1))
}

ale's avatar
ale committed
209
func (r *resticRepository) BackupStreamCmd(backup *Backup, ds *Dataset) string {
210
	fakePath := datasetStdinPath(ds)
ale's avatar
ale committed
211
	return fmt.Sprintf(
212
		"%s backup --cleanup-cache --exclude-caches %s --stdin --stdin-filename %s",
ale's avatar
ale committed
213
		r.resticCmd(),
214
		resticBackupTags(backup, ds),
ale's avatar
ale committed
215
		fakePath,
ale's avatar
ale committed
216 217 218
	)
}

ale's avatar
ale committed
219 220 221 222
func (r *resticRepository) RestoreStreamCmd(ctx context.Context, rctx RuntimeContext, backup *Backup, ds *Dataset, target string) (string, error) {
	snap, err := r.getSnapshotID(ctx, rctx, backup, ds)
	if err != nil {
		return "", err
ale's avatar
ale committed
223 224
	}

225
	fakePath := datasetStdinPath(ds)
ale's avatar
ale committed
226 227 228 229 230 231 232 233 234 235
	targetPath := filepath.Base(fakePath)

	// Restore the file to a temporary directory, then pipe it.
	return fmt.Sprintf(
		"(%s restore --target %s %s 1>&2 && cat %s)",
		r.resticCmd(),
		target,
		snap,
		filepath.Join(target, targetPath),
	), nil
ale's avatar
ale committed
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
}

// Data about a snapshot, obtained from 'restic snapshots --json'.
type resticSnapshot struct {
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
	Time     time.Time `json:"time"`
	Parent   string    `json:"parent"`
	Tree     string    `json:"tree"`
	Hostname string    `json:"hostname"`
	Username string    `json:"username"`
	UID      int       `json:"uid"`
	GID      int       `json:"gid"`
	Paths    []string  `json:"paths"`
	Tags     []string  `json:"tags"`
}

func parseResticSnapshots(output []byte) (snapshots []resticSnapshot, err error) {
254 255 256 257
	// An empty input is not necessarily an error.
	if len(output) > 0 {
		err = json.Unmarshal(output, &snapshots)
	}
ale's avatar
ale committed
258 259
	return
}