manager_test.go 5.66 KB
Newer Older
ale's avatar
ale committed
1 2 3 4
package tabacco

import (
	"context"
ale's avatar
ale committed
5
	"log"
6
	"path/filepath"
ale's avatar
ale committed
7 8
	"testing"
	"time"
ale's avatar
ale committed
9

ale's avatar
ale committed
10
	"git.autistici.org/ai3/tools/tabacco/jobs"
ale's avatar
ale committed
11 12 13 14 15 16 17
)

type dummyMetadataEntry struct {
	backupID string
	backupTS time.Time
	dsName   string
	host     string
ale's avatar
ale committed
18
	source   string
19
	path     string
ale's avatar
ale committed
20 21 22
	atom     Atom
}

ale's avatar
ale committed
23
func (e dummyMetadataEntry) match(req *FindRequest) bool {
24 25 26 27
	if req.Pattern != "" {
		if !req.matchPattern(e.path) {
			return false
		}
ale's avatar
ale committed
28 29 30 31 32 33 34
	}
	if req.Host != "" && req.Host != e.host {
		return false
	}
	return true
}

ale's avatar
ale committed
35 36 37 38
func (e dummyMetadataEntry) toDataset() *Dataset {
	return &Dataset{
		Name:   e.dsName,
		Source: e.source,
ale's avatar
ale committed
39 40 41
	}
}

ale's avatar
ale committed
42 43
func (e dummyMetadataEntry) toBackup() *Backup {
	return &Backup{
ale's avatar
ale committed
44 45 46 47 48 49 50 51 52 53
		ID:        e.backupID,
		Timestamp: e.backupTS,
		Host:      e.host,
	}
}

type dummyMetadataStore struct {
	log []dummyMetadataEntry
}

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
// Argh! This is copy&pasted from server/service.go, but with minor
// modifications due to the different types... terrible.
func keepNumVersions(dbAtoms []dummyMetadataEntry, numVersions int) []dummyMetadataEntry {
	// numVersions == 0 is remapped to 1.
	if numVersions < 1 {
		numVersions = 1
	}

	count := 0
	tmp := make(map[string][]dummyMetadataEntry)
	for _, a := range dbAtoms {
		l := tmp[a.path]
		if len(l) < numVersions {
			l = append(l, a)
			count++
ale's avatar
ale committed
69
		}
70 71 72 73 74 75 76 77
		tmp[a.path] = l
	}
	out := make([]dummyMetadataEntry, 0, count)
	for _, l := range tmp {
		out = append(out, l...)
	}
	return out
}
ale's avatar
ale committed
78

79 80 81 82 83 84 85 86
func groupByBackup(dbAtoms []dummyMetadataEntry) []*Backup {
	// As we scan through dbAtoms, aggregate into Backups and Datasets.
	backups := make(map[string]*Backup)
	dsm := make(map[string]map[string]*Dataset)

	for _, atom := range dbAtoms {
		// Create the Backup object if it does not exist.
		b, ok := backups[atom.backupID]
ale's avatar
ale committed
87
		if !ok {
88 89
			b = atom.toBackup()
			backups[atom.backupID] = b
ale's avatar
ale committed
90
		}
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110

		// Create the Dataset object for this Backup in the
		// two-level map (creating the intermediate map if
		// necessary).
		tmp, ok := dsm[atom.backupID]
		if !ok {
			tmp = make(map[string]*Dataset)
			dsm[atom.backupID] = tmp
		}
		// Match datasets by their full path.
		dsPath := filepath.Join(atom.source, atom.dsName)
		ds, ok := tmp[dsPath]
		if !ok {
			ds = atom.toDataset()
			tmp[dsPath] = ds
			b.Datasets = append(b.Datasets, ds)
		}

		// Finally, add the atom to the dataset.
		ds.Atoms = append(ds.Atoms, atom.atom)
ale's avatar
ale committed
111 112
	}

113 114 115
	out := make([]*Backup, 0, len(backups))
	for _, b := range backups {
		out = append(out, b)
ale's avatar
ale committed
116
	}
117 118
	return out
}
ale's avatar
ale committed
119

120 121 122 123 124
func (d *dummyMetadataStore) FindAtoms(_ context.Context, req *FindRequest) ([]*Backup, error) {
	var tmp []dummyMetadataEntry
	for _, l := range d.log {
		if !l.match(req) {
			continue
ale's avatar
ale committed
125
		}
126
		tmp = append(tmp, l)
ale's avatar
ale committed
127
	}
128 129

	return groupByBackup(keepNumVersions(tmp, req.NumVersions)), nil
ale's avatar
ale committed
130 131
}

ale's avatar
ale committed
132 133
func (d *dummyMetadataStore) AddDataset(_ context.Context, backup *Backup, ds *Dataset) error {
	log.Printf("AddDataset: %+v", *ds)
ale's avatar
ale committed
134
	for _, atom := range ds.Atoms {
135
		path := filepath.Join(ds.Source, ds.Name, atom.Name)
ale's avatar
ale committed
136 137 138 139
		d.log = append(d.log, dummyMetadataEntry{
			backupID: backup.ID,
			backupTS: backup.Timestamp,
			host:     backup.Host,
140
			path:     path,
ale's avatar
ale committed
141
			dsName:   ds.Name,
ale's avatar
ale committed
142
			source:   ds.Source,
ale's avatar
ale committed
143 144 145 146 147 148
			atom:     atom,
		})
	}
	return nil
}

ale's avatar
ale committed
149
func TestManager_Backup(t *testing.T) {
ale's avatar
ale committed
150 151 152 153 154 155 156 157 158
	store := &dummyMetadataStore{}
	repoSpec := RepositorySpec{
		Name: "main",
		Type: "restic",
		Params: map[string]interface{}{
			"uri":      "/tmp/restic/repo",
			"password": "testpass",
		},
	}
ale's avatar
ale committed
159 160
	handlerSpecs := []*HandlerSpec{
		&HandlerSpec{
ale's avatar
ale committed
161 162 163 164 165
			Name: "file1",
			Type: "file",
			Params: map[string]interface{}{
				"path": "/source/of/file1",
			},
ale's avatar
ale committed
166
			//PreBackupCommand: "echo hello",
ale's avatar
ale committed
167
		},
ale's avatar
ale committed
168
		&HandlerSpec{
ale's avatar
ale committed
169 170 171
			Name: "dbpipe",
			Type: "pipe",
			Params: map[string]interface{}{
ale's avatar
ale committed
172
				"backup_command":  "echo ${backup.id} ${ds.name} ${atom.names}",
ale's avatar
ale committed
173
				"restore_command": "cat",
ale's avatar
ale committed
174 175 176
			},
		},
	}
ale's avatar
ale committed
177 178 179 180 181 182 183 184 185
	sourceSpecs := []*SourceSpec{
		&SourceSpec{
			Name:     "source1",
			Handler:  "file1",
			Schedule: "@random_every 1h",
			Datasets: []*DatasetSpec{
				&DatasetSpec{
					Name:  "user1",
					Atoms: []Atom{{Name: "user1"}},
ale's avatar
ale committed
186
				},
ale's avatar
ale committed
187 188 189
				&DatasetSpec{
					Name:  "user2",
					Atoms: []Atom{{Name: "user2"}},
ale's avatar
ale committed
190 191 192
				},
			},
		},
ale's avatar
ale committed
193 194 195 196 197
		&SourceSpec{
			Name:            "source2",
			Handler:         "dbpipe",
			Schedule:        "@random_every 1h",
			DatasetsCommand: "echo '[{name: users, atoms: [{name: user1}, {name: user2}]}]'",
ale's avatar
ale committed
198 199
		},
	}
ale's avatar
ale committed
200
	queueSpec := jobs.QueueSpec{
ale's avatar
ale committed
201 202
		Workers: map[string]int{"backup": 2},
	}
ale's avatar
ale committed
203 204

	// Run the backup.
ale's avatar
ale committed
205
	configMgr, err := NewConfigManager(&Config{
ale's avatar
ale committed
206 207 208 209 210 211 212 213 214 215 216 217
		Queue:        queueSpec,
		Repository:   repoSpec,
		HandlerSpecs: handlerSpecs,
		SourceSpecs:  sourceSpecs,
		DryRun:       true,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer configMgr.Close()

	m, err := NewManager(context.TODO(), configMgr, store)
ale's avatar
ale committed
218 219 220 221 222
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

ale's avatar
ale committed
223 224 225 226 227 228 229 230
	for _, src := range configMgr.getSourceSpecs() {
		backup, err := m.Backup(context.TODO(), src)
		if err != nil {
			t.Fatal(err)
		}
		if backup.ID == "" || backup.Host == "" {
			t.Fatalf("empty fields in backup: %+v", backup)
		}
ale's avatar
ale committed
231 232 233 234
	}

	// Try to find atoms in the metadata store.
	// Let's try with a pattern first.
ale's avatar
ale committed
235
	resp, err := store.FindAtoms(context.TODO(), &FindRequest{Pattern: "source1/*", NumVersions: 1})
ale's avatar
ale committed
236 237 238
	if err != nil {
		t.Fatal("FindAtoms", err)
	}
ale's avatar
ale committed
239 240 241 242 243
	if len(resp) != 1 {
		t.Fatalf("bad FindAtoms(source1/*) response: %+v", resp)
	}
	if l := len(resp[0].Datasets); l != 2 {
		t.Fatalf("bad number of datasets returned by FindAtoms(source1/*): got %d, expected 2", l)
ale's avatar
ale committed
244 245 246
	}

	// A pattern matching a single atom.
ale's avatar
ale committed
247
	resp, err = store.FindAtoms(context.TODO(), &FindRequest{Pattern: "source1/user2/user2"})
ale's avatar
ale committed
248 249 250 251
	if err != nil {
		t.Fatal("FindAtoms", err)
	}
	if len(resp) != 1 {
ale's avatar
ale committed
252
		t.Fatalf("bad FindAtoms(source1/user2/user2) response: %+v", resp)
ale's avatar
ale committed
253 254
	}
}