source.go 4.68 KB
Newer Older
ale's avatar
ale committed
1 2 3 4 5
package tabacco

import (
	"context"
	"errors"
ale's avatar
ale committed
6
	"fmt"
ale's avatar
ale committed
7
	"os"
ale's avatar
ale committed
8
	"os/exec"
ale's avatar
ale committed
9 10
	"time"

11
	"git.autistici.org/ai3/tools/tabacco/util"
ale's avatar
ale committed
12
	"gopkg.in/yaml.v2"
ale's avatar
ale committed
13 14
)

ale's avatar
ale committed
15 16 17 18 19 20 21 22 23 24
// DatasetSpec describes a dataset in the configuration.
type DatasetSpec struct {
	Name string `yaml:"name"`

	Atoms        []Atom `yaml:"atoms"`
	AtomsCommand string `yaml:"atoms_command"`
}

// Parse a DatasetSpec and return a Dataset.
func (spec *DatasetSpec) Parse(ctx context.Context, src *SourceSpec) (*Dataset, error) {
25
	// Build the atoms list, invoking the atoms_command if necessary.
ale's avatar
ale committed
26
	var atoms []Atom
27
	atoms = append(atoms, spec.Atoms...)
ale's avatar
ale committed
28 29 30 31 32
	if spec.AtomsCommand != "" {
		var cmdAtoms []Atom
		if err := runYAMLCommand(ctx, spec.AtomsCommand, &cmdAtoms); err != nil {
			return nil, fmt.Errorf("source %s: dataset %s: error in atoms command: %v", src.Name, spec.Name, err)
		}
33
		atoms = append(atoms, cmdAtoms...)
ale's avatar
ale committed
34 35 36
	}

	return &Dataset{
37
		Name:   spec.Name,
ale's avatar
ale committed
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
		Source: src.Name,
		Atoms:  atoms,
	}, nil
}

// Check syntactical validity of the DatasetSpec.
func (spec *DatasetSpec) Check() error {
	if spec.Name == "" {
		return errors.New("dataset name is not set")
	}
	if len(spec.Atoms) > 0 && spec.AtomsCommand != "" {
		return errors.New("can't specify both 'atoms' and 'atoms_command'")
	}
	if len(spec.Atoms) == 0 && spec.AtomsCommand == "" {
		return errors.New("must specify one of 'atoms' or 'atoms_command'")
	}
	return nil
}

// SourceSpec defines the configuration for a data source. Data
// sources can dynamically or statically generate one or more
// Datasets, each containing one or more Atoms.
//
// Handlers are launched once per Dataset, and they know how to deal
// with backing up / restoring individual Atoms.
ale's avatar
ale committed
63
type SourceSpec struct {
ale's avatar
ale committed
64 65
	Name    string `yaml:"name"`
	Handler string `yaml:"handler"`
ale's avatar
ale committed
66 67

	// Schedule to run the backup on.
ale's avatar
ale committed
68
	Schedule string `yaml:"schedule"`
ale's avatar
ale committed
69

ale's avatar
ale committed
70
	// Define Datasets statically, or use a script to generate them
ale's avatar
ale committed
71
	// dynamically on every new backup.
ale's avatar
ale committed
72 73
	Datasets        []*DatasetSpec `yaml:"datasets"`
	DatasetsCommand string         `yaml:"datasets_command"`
ale's avatar
ale committed
74

ale's avatar
ale committed
75 76 77 78 79 80 81 82 83 84
	// Commands to run before and after operations on the source.
	PreBackupCommand   string `yaml:"pre_backup_command"`
	PostBackupCommand  string `yaml:"post_backup_command"`
	PreRestoreCommand  string `yaml:"pre_restore_command"`
	PostRestoreCommand string `yaml:"post_restore_command"`

	Params Params `yaml:"params"`

	// Timeout for execution of the entire backup operation.
	Timeout time.Duration `yaml:"timeout"`
ale's avatar
ale committed
85 86
}

ale's avatar
ale committed
87 88 89 90 91 92 93 94 95
// Parse a SourceSpec and return one or more Datasets.
func (spec *SourceSpec) Parse(ctx context.Context) ([]*Dataset, error) {
	// Build the atoms list, invoking the atoms_command if
	// necessary, and creating actual atoms with absolute names.
	dspecs := append([]*DatasetSpec{}, spec.Datasets...)
	if spec.DatasetsCommand != "" {
		var cmdSpecs []*DatasetSpec
		if err := runYAMLCommand(ctx, spec.DatasetsCommand, &cmdSpecs); err != nil {
			return nil, fmt.Errorf("error in datasets command: %v", err)
ale's avatar
ale committed
96
		}
ale's avatar
ale committed
97
		dspecs = append(dspecs, cmdSpecs...)
ale's avatar
ale committed
98 99
	}

ale's avatar
ale committed
100 101 102 103 104 105 106 107 108 109
	// Call Parse on all datasets.
	datasets := make([]*Dataset, 0, len(dspecs))
	for _, dspec := range dspecs {
		ds, err := dspec.Parse(ctx, spec)
		if err != nil {
			return nil, fmt.Errorf("error parsing dataset %s: %v", dspec.Name, err)
		}
		datasets = append(datasets, ds)
	}
	return datasets, nil
ale's avatar
ale committed
110 111
}

ale's avatar
ale committed
112
// Check syntactical validity of the SourceSpec. Not an alternative to
ale's avatar
ale committed
113 114 115
// validation at usage time, but it provides an early warning to the
// user. Checks the handler name against a string set of handler
// names.
ale's avatar
ale committed
116 117 118 119 120
func (spec *SourceSpec) Check(handlers map[string]*HandlerSpec) error {
	if spec.Timeout == 0 {
		spec.Timeout = 24 * time.Hour
	}

ale's avatar
ale committed
121
	if spec.Name == "" {
ale's avatar
ale committed
122 123 124 125 126 127 128
		return errors.New("source name is not set")
	}
	if spec.Schedule == "" {
		return errors.New("schedule is not set")
	}
	if spec.Handler == "" {
		return errors.New("handler is not set")
ale's avatar
ale committed
129 130 131 132
	}
	if _, ok := handlers[spec.Handler]; !ok {
		return fmt.Errorf("unknown handler '%s'", spec.Handler)
	}
ale's avatar
ale committed
133 134 135 136 137
	if len(spec.Datasets) > 0 && spec.DatasetsCommand != "" {
		return errors.New("can't specify both 'datasets' and 'datasets_command'")
	}
	if len(spec.Datasets) == 0 && spec.DatasetsCommand == "" {
		return errors.New("must specify one of 'datasets' or 'datasets_command'")
ale's avatar
ale committed
138
	}
139 140 141 142 143 144 145 146 147 148

	// Check the datasets, at least those that are provided
	// statically.
	merr := new(util.MultiError)
	for _, ds := range spec.Datasets {
		if err := ds.Check(); err != nil {
			merr.Add(fmt.Errorf("dataset %s: %v", ds.Name, err))
		}
	}
	return merr.OrNil()
ale's avatar
ale committed
149 150
}

ale's avatar
ale committed
151
func runYAMLCommand(ctx context.Context, cmd string, obj interface{}) error {
ale's avatar
ale committed
152
	c := exec.Command("/bin/sh", "-c", cmd) // #nosec
ale's avatar
ale committed
153 154
	c.Stderr = os.Stderr
	output, err := c.Output()
ale's avatar
ale committed
155
	if err != nil {
ale's avatar
ale committed
156
		return err
ale's avatar
ale committed
157 158
	}

ale's avatar
ale committed
159
	return yaml.Unmarshal(output, obj)
ale's avatar
ale committed
160
}