repository_restic_test.go 4.7 KB
Newer Older
ale's avatar
ale committed
1 2 3 4 5 6 7 8 9 10
package tabacco

import (
	"bytes"
	"context"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
ale's avatar
ale committed
11

ale's avatar
ale committed
12
	"git.autistici.org/ai3/tools/tabacco/jobs"
ale's avatar
ale committed
13 14 15 16 17
)

// Create a temporary directory with two subdirs: 'data', for
// the test backup data, and 'repo' to store the (remote)
// repository. Populate 'data' with two tiny files.
18 19
//
// nolint
ale's avatar
ale committed
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
func createTempDirWithData(t *testing.T) string {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	for _, d := range []string{"data", "repo", "restore"} {
		os.Mkdir(filepath.Join(tmpdir, d), 0700)
	}
	ioutil.WriteFile(
		filepath.Join(tmpdir, "data", "file1"),
		[]byte("this is file number one."),
		0600,
	)
	ioutil.WriteFile(
		filepath.Join(tmpdir, "data", "file2"),
		[]byte("this is file number two."),
		0600,
	)
	return tmpdir
}

// nolint: gocyclo
ale's avatar
ale committed
42
func runResticTest(t *testing.T, tmpdir string, source *SourceSpec, restorePattern string, checkFn func(testing.TB, string)) {
ale's avatar
ale committed
43 44 45 46 47
	// Check that we can actually run restic.
	if err := checkResticVersion("restic"); err != nil {
		t.Skip("can't run restic: ", err)
	}

ale's avatar
ale committed
48 49 50 51 52 53 54 55 56 57
	store := &dummyMetadataStore{}

	repoSpec := RepositorySpec{
		Name: "main",
		Type: "restic",
		Params: map[string]interface{}{
			"uri":      tmpdir + "/repo",
			"password": "testpass",
		},
	}
ale's avatar
ale committed
58
	handlerSpecs := []*HandlerSpec{
ale's avatar
ale committed
59
		// 'file' is predefined.
ale's avatar
ale committed
60
		&HandlerSpec{
ale's avatar
ale committed
61
			Name: "data",
ale's avatar
ale committed
62
			Type: "pipe",
ale's avatar
ale committed
63
			Params: map[string]interface{}{
ale's avatar
ale committed
64 65 66
				"backup_command": "echo data",
				// The restore command also verifies the data.
				"restore_command": "read row ; test \"x$$row\" = xdata",
ale's avatar
ale committed
67 68 69
			},
		},
	}
ale's avatar
ale committed
70
	queueSpec := &jobs.QueueSpec{
ale's avatar
ale committed
71
		Concurrency: 2,
ale's avatar
ale committed
72
	}
ale's avatar
ale committed
73
	sourceSpecs := []*SourceSpec{source}
ale's avatar
ale committed
74 75

	// Run the backup.
ale's avatar
ale committed
76
	configMgr, err := NewConfigManager(&Config{
ale's avatar
ale committed
77 78 79 80 81 82 83 84 85 86 87
		Queue:        queueSpec,
		Repository:   repoSpec,
		HandlerSpecs: handlerSpecs,
		SourceSpecs:  sourceSpecs,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer configMgr.Close()

	m, err := NewManager(context.TODO(), configMgr, store)
ale's avatar
ale committed
88 89 90 91 92
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

ale's avatar
ale committed
93
	backup, err := m.Backup(context.TODO(), configMgr.current().SourceSpecs()[0])
ale's avatar
ale committed
94 95 96 97 98 99 100 101
	if err != nil {
		t.Fatal(err)
	}
	if backup.ID == "" || backup.Host == "" {
		t.Fatalf("empty fields in backup: %+v", backup)
	}

	// Check the 'restic snapshots' output.
ale's avatar
ale committed
102
	output, err := exec.Command("env", "RESTIC_REPOSITORY=", "RESTIC_PASSWORD_FILE=", "RESTIC_PASSWORD=testpass", "restic", "-r", tmpdir+"/repo", "snapshots", "--json").Output()
103 104 105
	if err != nil {
		t.Fatalf("'restic snapshots' failed: %v", err)
	}
ale's avatar
ale committed
106 107
	snaps, err := parseResticSnapshots(output)
	if err != nil {
108
		t.Fatalf("parsing restic snaphots output: %v, output:\n%s", err, string(output))
ale's avatar
ale committed
109 110 111 112 113 114 115 116 117 118 119 120
	}
	if len(snaps) != 1 {
		t.Fatalf("wrong number of snapshots: %+v", snaps)
	}
	snap := snaps[0]
	if len(snap.Tags) != 2 {
		t.Fatalf("woops, bad number of tags: %+v", snap)
	}

	// Now try to restore.
	err = m.Restore(
		context.TODO(),
ale's avatar
ale committed
121
		&FindRequest{Pattern: restorePattern},
ale's avatar
ale committed
122 123 124 125 126
		tmpdir+"/restore",
	)
	if err != nil {
		t.Fatal("Restore", err)
	}
ale's avatar
ale committed
127 128 129 130 131 132 133

	if checkFn != nil {
		checkFn(t, tmpdir)
	}
}

func checkRestoredData(t testing.TB, tmpdir string) {
ale's avatar
ale committed
134 135 136 137 138 139 140 141
	data, err := ioutil.ReadFile(filepath.Join(tmpdir, "restore", tmpdir, "data", "file1"))
	if err != nil {
		t.Fatalf("data/file1 has not been restored: %v", err)
	}
	if !bytes.Equal(data, []byte("this is file number one.")) {
		t.Fatalf("data/file1 has bad restored contents: %s", string(data))
	}
}
ale's avatar
ale committed
142

ale's avatar
ale committed
143
func TestRestic(t *testing.T) {
ale's avatar
ale committed
144 145 146
	tmpdir := createTempDirWithData(t)
	defer os.RemoveAll(tmpdir)

ale's avatar
ale committed
147 148 149 150 151 152
	runResticTest(
		t, tmpdir,
		&SourceSpec{
			Name:     "source1",
			Handler:  "file",
			Schedule: "@random_every 1h",
ale's avatar
ale committed
153
			Params: map[string]interface{}{
ale's avatar
ale committed
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
				"path": filepath.Join(tmpdir, "data"),
			},
			Datasets: []*DatasetSpec{
				&DatasetSpec{
					Atoms: []Atom{
						{
							Name: "f1",
							Path: "file1",
						},
						{
							Name: "f2",
							Path: "file2",
						},
					},
				},
ale's avatar
ale committed
169 170
			},
		},
ale's avatar
ale committed
171 172 173 174 175 176 177 178 179 180 181
		"source1/*",
		checkRestoredData,
	)
}

func TestRestic_Stream(t *testing.T) {
	tmpdir := createTempDirWithData(t)
	defer os.RemoveAll(tmpdir)

	runResticTest(
		t, tmpdir,
ale's avatar
ale committed
182 183 184 185 186 187 188 189 190 191 192 193 194 195
		&SourceSpec{
			Name:     "source1",
			Handler:  "data",
			Schedule: "@random_every 1h",
			Datasets: []*DatasetSpec{
				&DatasetSpec{
					Atoms: []Atom{
						{
							Name: "f1",
						},
					},
				},
			},
		},
ale's avatar
ale committed
196 197 198 199
		"source1/*",
		nil,
	)
}
ale's avatar
ale committed
200

ale's avatar
ale committed
201 202 203
func TestRestic_Stream_Compress(t *testing.T) {
	tmpdir := createTempDirWithData(t)
	defer os.RemoveAll(tmpdir)
ale's avatar
ale committed
204

ale's avatar
ale committed
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
	runResticTest(
		t, tmpdir,
		&SourceSpec{
			Name:     "source1",
			Handler:  "data",
			Schedule: "@random_every 1h",
			Datasets: []*DatasetSpec{
				&DatasetSpec{
					Atoms: []Atom{
						{
							Name: "f1",
						},
					},
				},
			},
			Params: map[string]interface{}{
				"compress": true,
			},
		},
		"source1/*",
		nil,
ale's avatar
ale committed
226 227
	)
}