Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
ai3
tools
tabacco
Commits
04055321
Commit
04055321
authored
Jun 19, 2019
by
ale
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'v2' into 'master'
V2 See merge request
!1
parents
a277f63f
25f279d0
Pipeline
#3491
failed with stages
in 1 minute and 23 seconds
Changes
27
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
27 changed files
with
1181 additions
and
852 deletions
+1181
-852
.gitlab-ci.yml
.gitlab-ci.yml
+1
-0
agent.go
agent.go
+5
-4
agent_test.go
agent_test.go
+26
-22
cmd/tabacco/query.go
cmd/tabacco/query.go
+89
-0
cmd/tabacco/restore.go
cmd/tabacco/restore.go
+2
-2
config.go
config.go
+111
-90
config_test.go
config_test.go
+67
-53
handler_file.go
handler_file.go
+37
-30
handler_pipe.go
handler_pipe.go
+83
-89
handlers.go
handlers.go
+15
-10
jobs/job.go
jobs/job.go
+1
-1
manager.go
manager.go
+135
-115
manager_test.go
manager_test.go
+136
-83
metadb/client/client.go
metadb/client/client.go
+12
-6
metadb/migrations/1_initialize_schema.down.sql
metadb/migrations/1_initialize_schema.down.sql
+1
-1
metadb/migrations/1_initialize_schema.up.sql
metadb/migrations/1_initialize_schema.up.sql
+6
-7
metadb/migrations/bindata.go
metadb/migrations/bindata.go
+9
-10
metadb/server/http.go
metadb/server/http.go
+1
-1
metadb/server/service.go
metadb/server/service.go
+72
-64
metadb/server/service_test.go
metadb/server/service_test.go
+9
-10
repository.go
repository.go
+2
-2
repository_restic.go
repository_restic.go
+77
-56
repository_restic_test.go
repository_restic_test.go
+105
-27
shell.go
shell.go
+4
-67
source.go
source.go
+114
-66
testdata/sources/source.yml
testdata/sources/source.yml
+4
-0
types.go
types.go
+57
-36
No files found.
.gitlab-ci.yml
View file @
04055321
...
...
@@ -12,6 +12,7 @@ test:
image
:
"
ai/test:go"
script
:
-
"
./install_restic_for_tests.sh"
-
"
apt-get
install
-y
liblz4-tool"
-
"
go-test-runner
."
except
:
-
master
...
...
agent.go
View file @
04055321
...
...
@@ -35,7 +35,8 @@ func NewAgent(ctx context.Context, configMgr *ConfigManager, ms MetadataStore) (
case
<-
stopCh
:
return
case
<-
notifyCh
:
schedule
,
err
:=
makeSchedule
(
ctx
,
mgr
,
configMgr
.
getSourceSpecs
(),
configMgr
.
getSeed
())
config
:=
configMgr
.
current
()
schedule
,
err
:=
makeSchedule
(
ctx
,
mgr
,
config
.
SourceSpecs
(),
config
.
Seed
())
if
err
!=
nil
{
log
.
Printf
(
"error updating scheduler: %v"
,
err
)
}
...
...
@@ -63,7 +64,7 @@ func (a *Agent) Close() {
// Create a new jobs.Schedule that will trigger a separate backup for
// each configured data source that includes a 'schedule' attribute.
func
makeSchedule
(
ctx
context
.
Context
,
m
Manager
,
sourceSpecs
[]
SourceSpec
,
hostSeed
int64
)
(
*
jobs
.
Schedule
,
error
)
{
func
makeSchedule
(
ctx
context
.
Context
,
m
Manager
,
sourceSpecs
[]
*
SourceSpec
,
hostSeed
int64
)
(
*
jobs
.
Schedule
,
error
)
{
sched
:=
jobs
.
NewSchedule
(
ctx
,
hostSeed
)
merr
:=
new
(
util
.
MultiError
)
var
good
int
...
...
@@ -72,9 +73,9 @@ func makeSchedule(ctx context.Context, m Manager, sourceSpecs []SourceSpec, host
continue
}
// Bind spec to a new closure.
err
:=
func
(
spec
SourceSpec
)
error
{
err
:=
func
(
spec
*
SourceSpec
)
error
{
return
sched
.
Add
(
spec
.
Name
,
spec
.
Schedule
,
func
()
jobs
.
Job
{
_
,
j
,
err
:=
m
.
BackupJob
(
ctx
,
[]
SourceSpec
{
spec
}
)
_
,
j
,
err
:=
m
.
BackupJob
(
ctx
,
spec
)
if
err
!=
nil
{
log
.
Printf
(
"%s: can't create backup job: %v"
,
spec
.
Name
,
err
)
}
...
...
agent_test.go
View file @
04055321
...
...
@@ -9,19 +9,19 @@ import (
type
fakeManager
struct
{}
func
(
m
*
fakeManager
)
BackupJob
(
context
.
Context
,
[]
SourceSpec
)
(
Backup
,
jobs
.
Job
,
error
)
{
return
Backup
{},
nil
,
nil
func
(
m
*
fakeManager
)
BackupJob
(
context
.
Context
,
*
SourceSpec
)
(
*
Backup
,
jobs
.
Job
,
error
)
{
return
&
Backup
{},
nil
,
nil
}
func
(
m
*
fakeManager
)
Backup
(
context
.
Context
,
[]
SourceSpec
)
(
Backup
,
error
)
{
return
Backup
{},
nil
func
(
m
*
fakeManager
)
Backup
(
context
.
Context
,
*
SourceSpec
)
(
*
Backup
,
error
)
{
return
&
Backup
{},
nil
}
func
(
m
*
fakeManager
)
RestoreJob
(
context
.
Context
,
FindRequest
,
string
)
(
jobs
.
Job
,
error
)
{
func
(
m
*
fakeManager
)
RestoreJob
(
context
.
Context
,
*
FindRequest
,
string
)
(
jobs
.
Job
,
error
)
{
return
nil
,
nil
}
func
(
m
*
fakeManager
)
Restore
(
context
.
Context
,
FindRequest
,
string
)
error
{
func
(
m
*
fakeManager
)
Restore
(
context
.
Context
,
*
FindRequest
,
string
)
error
{
return
nil
}
...
...
@@ -34,27 +34,31 @@ func (m *fakeManager) GetStatus() ([]jobs.Status, []jobs.Status, []jobs.Status)
}
func
TestMakeSchedule
(
t
*
testing
.
T
)
{
sourceSpecs
:=
[]
SourceSpec
{
{
Name
:
"source1"
,
sourceSpecs
:=
[]
*
SourceSpec
{
&
SourceSpec
{
Name
:
"source1
/users
"
,
Handler
:
"file1"
,
Schedule
:
"@random_every 1d"
,
Atoms
:
[]
Atom
{
{
Name
:
"user1"
,
RelativePath
:
"user1"
,
},
{
Name
:
"user2"
,
RelativePath
:
"user2"
,
Datasets
:
[]
*
DatasetSpec
{
&
DatasetSpec
{
Atoms
:
[]
Atom
{
{
Name
:
"user1"
,
Path
:
"user1"
,
},
{
Name
:
"user2"
,
Path
:
"user2"
,
},
},
},
},
},
{
Name
:
"source2"
,
Handler
:
"dbpipe"
,
Schedule
:
"35 3 * * *"
,
Atom
sCommand
:
"echo user1 user1 ; echo user2 user2"
,
&
SourceSpec
{
Name
:
"source2"
,
Handler
:
"dbpipe"
,
Schedule
:
"35 3 * * *"
,
Dataset
sCommand
:
"echo user1 user1 ; echo user2 user2"
,
},
}
...
...
cmd/tabacco/query.go
0 → 100644
View file @
04055321
package
main
import
(
"context"
"encoding/json"
"errors"
"flag"
"log"
"os"
"time"
"github.com/google/subcommands"
"git.autistici.org/ai3/tools/tabacco"
mdbc
"git.autistici.org/ai3/tools/tabacco/metadb/client"
)
var
rpcTimeout
=
120
*
time
.
Second
type
queryCommand
struct
{
configPath
string
host
string
numVersions
int
}
func
(
c
*
queryCommand
)
Name
()
string
{
return
"query"
}
func
(
c
*
queryCommand
)
Synopsis
()
string
{
return
"query the backup metadata database"
}
func
(
c
*
queryCommand
)
Usage
()
string
{
return
`query [<flags>] <atom_pattern>
Query the backup metadata database.
`
}
func
(
c
*
queryCommand
)
SetFlags
(
f
*
flag
.
FlagSet
)
{
f
.
StringVar
(
&
c
.
configPath
,
"config"
,
"/etc/tabacco/agent.yml"
,
"configuration `file`"
)
f
.
StringVar
(
&
c
.
host
,
"host"
,
""
,
"filter by host"
)
f
.
IntVar
(
&
c
.
numVersions
,
"num-versions"
,
1
,
"return the most recent `N` versions"
)
}
func
(
c
*
queryCommand
)
buildRequest
(
f
*
flag
.
FlagSet
)
(
*
tabacco
.
FindRequest
,
error
)
{
if
f
.
NArg
()
!=
1
{
return
nil
,
errors
.
New
(
"error: wrong number of arguments"
)
}
return
&
tabacco
.
FindRequest
{
Pattern
:
f
.
Arg
(
0
),
Host
:
c
.
host
,
NumVersions
:
c
.
numVersions
,
},
nil
}
func
(
c
*
queryCommand
)
Execute
(
ctx
context
.
Context
,
f
*
flag
.
FlagSet
,
args
...
interface
{})
subcommands
.
ExitStatus
{
req
,
err
:=
c
.
buildRequest
(
f
)
if
err
!=
nil
{
log
.
Printf
(
"error in request: %v"
,
err
)
return
subcommands
.
ExitUsageError
}
// Parse configuration and connect to the metadata store.
config
,
err
:=
tabacco
.
ReadConfig
(
c
.
configPath
)
if
err
!=
nil
{
log
.
Printf
(
"error reading config: %v"
,
err
)
return
subcommands
.
ExitFailure
}
store
,
err
:=
mdbc
.
New
(
config
.
MetadataStoreBackend
)
if
err
!=
nil
{
log
.
Printf
(
"error in metadata client config: %v"
,
err
)
return
subcommands
.
ExitFailure
}
// Make the RPC.
rctx
,
cancel
:=
context
.
WithTimeout
(
ctx
,
rpcTimeout
)
defer
cancel
()
result
,
err
:=
store
.
FindAtoms
(
rctx
,
req
)
if
err
!=
nil
{
log
.
Printf
(
"FindAtoms() error: %v"
,
err
)
return
subcommands
.
ExitFailure
}
data
,
_
:=
json
.
MarshalIndent
(
result
,
""
,
" "
)
os
.
Stdout
.
Write
(
data
)
return
subcommands
.
ExitSuccess
}
func
init
()
{
subcommands
.
Register
(
&
queryCommand
{},
""
)
}
cmd/tabacco/restore.go
View file @
04055321
...
...
@@ -77,8 +77,8 @@ func (c *restoreCommand) buildRestoreJob(ctx context.Context, mgr tabacco.Manage
return
jobs
.
AsyncGroup
(
restoreJobs
),
nil
}
func
(
c
*
restoreCommand
)
newFindRequest
(
s
string
)
tabacco
.
FindRequest
{
return
tabacco
.
FindRequest
{
func
(
c
*
restoreCommand
)
newFindRequest
(
s
string
)
*
tabacco
.
FindRequest
{
return
&
tabacco
.
FindRequest
{
Pattern
:
s
,
}
}
...
...
config.go
View file @
04055321
...
...
@@ -23,7 +23,7 @@ var defaultSeedFile = "/var/tmp/.tabacco_scheduler_seed"
// holds it all together.
type
Config
struct
{
Hostname
string
`yaml:"hostname"`
Queue
jobs
.
QueueSpec
`yaml:"queue_config"`
Queue
*
jobs
.
QueueSpec
`yaml:"queue_config"`
Repository
RepositorySpec
`yaml:"repository"`
DryRun
bool
`yaml:"dry_run"`
DefaultNiceLevel
int
`yaml:"default_nice_level"`
...
...
@@ -32,43 +32,83 @@ type Config struct {
RandomSeedFile
string
`yaml:"random_seed_file"`
MetadataStoreBackend
*
clientutil
.
BackendConfig
`yaml:"metadb"`
HandlerSpecs
[]
HandlerSpec
SourceSpecs
[]
SourceSpec
HandlerSpecs
[]
*
HandlerSpec
SourceSpecs
[]
*
SourceSpec
}
type
runtimeAssets
struct
{
handlerMap
map
[
string
]
Handler
repo
Repository
seed
int64
shell
*
Shell
// RuntimeContext provides access to runtime objects whose lifetime is
// ultimately tied to the configuration. Configuration can change
// during the lifetime of the process, but we want backup jobs to have
// a consistent view of the configuration while they execute, so
// access to the current version of the configuration is controlled to
// the ConfigManager.
type
RuntimeContext
interface
{
Shell
()
*
Shell
Repo
()
Repository
QueueSpec
()
*
jobs
.
QueueSpec
Seed
()
int64
WorkDir
()
string
SourceSpecs
()
[]
*
SourceSpec
FindSource
(
string
)
*
SourceSpec
HandlerSpec
(
string
)
*
HandlerSpec
Close
()
}
func
(
a
*
runtimeAssets
)
Close
()
{
// The set of objects that are created from a Config and that the main
// code cares about.
type
parsedConfig
struct
{
handlerMap
map
[
string
]
*
HandlerSpec
sourceSpecs
[]
*
SourceSpec
sourceSpecsByName
map
[
string
]
*
SourceSpec
queue
*
jobs
.
QueueSpec
repo
Repository
seed
int64
shell
*
Shell
workDir
string
}
func
(
a
*
parsedConfig
)
Close
()
{
a
.
repo
.
Close
()
// nolint
}
func
buildHandlerMap
(
specs
[]
HandlerSpec
,
shell
*
Shell
)
(
map
[
string
]
Handler
,
error
)
{
m
:=
make
(
map
[
string
]
Handler
)
merr
:=
new
(
util
.
MultiError
)
for
_
,
spec
:=
range
specs
{
h
,
err
:=
spec
.
Parse
(
shell
)
if
err
!=
nil
{
merr
.
Add
(
err
)
continue
}
m
[
spec
.
Name
]
=
h
func
(
a
*
parsedConfig
)
Shell
()
*
Shell
{
return
a
.
shell
}
func
(
a
*
parsedConfig
)
Repo
()
Repository
{
return
a
.
repo
}
func
(
a
*
parsedConfig
)
QueueSpec
()
*
jobs
.
QueueSpec
{
return
a
.
queue
}
func
(
a
*
parsedConfig
)
Seed
()
int64
{
return
a
.
seed
}
func
(
a
*
parsedConfig
)
WorkDir
()
string
{
return
a
.
workDir
}
func
(
a
*
parsedConfig
)
SourceSpecs
()
[]
*
SourceSpec
{
return
a
.
sourceSpecs
}
func
(
a
*
parsedConfig
)
HandlerSpec
(
name
string
)
*
HandlerSpec
{
return
a
.
handlerMap
[
name
]
}
func
(
a
*
parsedConfig
)
FindSource
(
name
string
)
*
SourceSpec
{
return
a
.
sourceSpecsByName
[
name
]
}
func
buildHandlerMap
(
specs
[]
*
HandlerSpec
)
map
[
string
]
*
HandlerSpec
{
// Create a handler map with a default 'file' spec.
m
:=
map
[
string
]
*
HandlerSpec
{
"file"
:
&
HandlerSpec
{
Name
:
"file"
,
Type
:
"file"
,
},
}
for
_
,
h
:=
range
specs
{
m
[
h
.
Name
]
=
h
}
return
m
,
merr
.
OrNil
()
return
m
}
func
(
c
*
Config
)
parse
()
(
*
runtimeAssets
,
error
)
{
func
(
c
*
Config
)
parse
()
(
*
parsedConfig
,
error
)
{
shell
:=
NewShell
(
c
.
DryRun
)
shell
.
SetNiceLevel
(
c
.
DefaultNiceLevel
)
shell
.
SetIOClass
(
c
.
DefaultIOClass
)
// Parse the repository config. An error here is fatal, as we
// don't have a way to operate without a repository.
repo
,
err
:=
c
.
Repository
.
Parse
(
shell
)
repo
,
err
:=
c
.
Repository
.
Parse
()
if
err
!=
nil
{
return
nil
,
err
}
...
...
@@ -76,21 +116,26 @@ func (c *Config) parse() (*runtimeAssets, error) {
merr
:=
new
(
util
.
MultiError
)
// Build the handlers.
handlerMap
,
err
:=
buildHandlerMap
(
c
.
HandlerSpecs
,
shell
)
if
err
!=
nil
{
merr
.
Add
(
err
)
}
handlerMap
:=
buildHandlerMap
(
c
.
HandlerSpecs
)
// Validate the sources (Parse is called later at runtime).
var
srcs
[]
SourceSpec
// Sources that fail the check are removed from the
// SourceSpecs array. We also check that sources have unique
// names.
srcMap
:=
make
(
map
[
string
]
*
SourceSpec
)
var
srcs
[]
*
SourceSpec
for
_
,
spec
:=
range
c
.
SourceSpecs
{
if
err
:=
spec
.
Check
(
handlerMap
);
err
!=
nil
{
merr
.
Add
(
err
)
merr
.
Add
(
fmt
.
Errorf
(
"source %s: %v"
,
spec
.
Name
,
err
)
)
continue
}
if
_
,
ok
:=
srcMap
[
spec
.
Name
];
ok
{
merr
.
Add
(
fmt
.
Errorf
(
"duplicated source %s"
,
spec
.
Name
))
continue
}
srcMap
[
spec
.
Name
]
=
spec
srcs
=
append
(
srcs
,
spec
)
}
c
.
SourceSpecs
=
srcs
// Read (or create) the seed file.
seedFile
:=
defaultSeedFile
...
...
@@ -99,28 +144,32 @@ func (c *Config) parse() (*runtimeAssets, error) {
}
seed
:=
mustGetSeed
(
seedFile
)
return
&
runtimeAssets
{
shell
:
shell
,
repo
:
repo
,
handlerMap
:
handlerMap
,
seed
:
seed
,
return
&
parsedConfig
{
handlerMap
:
handlerMap
,
sourceSpecs
:
srcs
,
sourceSpecsByName
:
srcMap
,
queue
:
c
.
Queue
,
shell
:
shell
,
repo
:
repo
,
seed
:
seed
,
workDir
:
c
.
WorkDir
,
},
merr
.
OrNil
()
}
// The following functions read YAML files from .d-style directories. To be nice
// to the user, each file can contain either a single object or a list of
// multiple objects.
func
readHandlersFromDir
(
dir
string
)
([]
HandlerSpec
,
error
)
{
var
out
[]
HandlerSpec
func
readHandlersFromDir
(
dir
string
)
([]
*
HandlerSpec
,
error
)
{
var
out
[]
*
HandlerSpec
err
:=
foreachYAMLFile
(
dir
,
func
(
path
string
)
error
{
var
specs
[]
HandlerSpec
var
specs
[]
*
HandlerSpec
log
.
Printf
(
"reading handler: %s"
,
path
)
if
err
:=
readYAMLFile
(
path
,
&
specs
);
err
!=
nil
{
var
spec
HandlerSpec
if
err
:=
readYAMLFile
(
path
,
&
spec
);
err
!=
nil
{
return
err
}
specs
=
[]
HandlerSpec
{
spec
}
specs
=
append
(
specs
,
&
spec
)
}
out
=
append
(
out
,
specs
...
)
return
nil
...
...
@@ -128,17 +177,17 @@ func readHandlersFromDir(dir string) ([]HandlerSpec, error) {
return
out
,
err
}
func
readSourcesFromDir
(
dir
string
)
([]
SourceSpec
,
error
)
{
var
out
[]
SourceSpec
func
readSourcesFromDir
(
dir
string
)
([]
*
SourceSpec
,
error
)
{
var
out
[]
*
SourceSpec
err
:=
foreachYAMLFile
(
dir
,
func
(
path
string
)
error
{
var
specs
[]
SourceSpec
var
specs
[]
*
SourceSpec
log
.
Printf
(
"reading source: %s"
,
path
)
if
err
:=
readYAMLFile
(
path
,
&
specs
);
err
!=
nil
{
var
spec
SourceSpec
if
err
:=
readYAMLFile
(
path
,
&
spec
);
err
!=
nil
{
return
err
}
specs
=
[]
SourceSpec
{
spec
}
specs
=
append
(
specs
,
&
spec
)
}
out
=
append
(
out
,
specs
...
)
return
nil
...
...
@@ -215,6 +264,7 @@ func foreachYAMLFile(dir string, f func(string) error) error {
merr
:=
new
(
util
.
MultiError
)
for
_
,
path
:=
range
files
{
if
err
:=
f
(
path
);
err
!=
nil
{
log
.
Printf
(
"error loading yaml file %s: %v"
,
path
,
err
)
merr
.
Add
(
err
)
}
}
...
...
@@ -228,8 +278,7 @@ func foreachYAMLFile(dir string, f func(string) error) error {
// unregister).
type
ConfigManager
struct
{
mx
sync
.
Mutex
config
*
Config
assets
*
runtimeAssets
parsed
*
parsedConfig
// Listeners are notified on every reload.
notifyCh
chan
struct
{}
...
...
@@ -246,12 +295,14 @@ func NewConfigManager(config *Config) (*ConfigManager, error) {
}
go
func
()
{
for
range
m
.
notifyCh
{
m
.
mx
.
Lock
()
for
_
,
lch
:=
range
m
.
listeners
{
select
{
case
lch
<-
struct
{}{}
:
default
:
}
}
m
.
mx
.
Unlock
()
}
}()
return
m
,
nil
...
...
@@ -260,22 +311,23 @@ func NewConfigManager(config *Config) (*ConfigManager, error) {
// Reload the configuration (at least, the parts of it that can be
// dynamically reloaded).
func
(
m
*
ConfigManager
)
Reload
(
config
*
Config
)
error
{
assets
,
err
:=
config
.
parse
()
if
assets
==
nil
{
parsed
,
err
:=
config
.
parse
()
if
parsed
==
nil
{
return
err
}
else
if
err
!=
nil
{
log
.
Printf
(
"warning: errors in configuration: %v"
,
err
)
}
// Update config and notify listeners (in a separate
// goroutine, that does not hold the lock).
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
if
m
.
assets
!=
nil
{
m
.
assets
.
Close
()
// nolint
if
m
.
parsed
!=
nil
{
m
.
parsed
.
Close
()
// nolint
}
log
.
Printf
(
"loaded new config: %d handlers, %d sources"
,
len
(
assets
.
handlerMap
),
len
(
config
.
SourceSpecs
))
m
.
assets
=
assets
m
.
config
=
config
log
.
Printf
(
"loaded new config: %d handlers, %d sources"
,
len
(
parsed
.
handlerMap
),
len
(
parsed
.
sourceSpecs
))
m
.
parsed
=
parsed
m
.
notifyCh
<-
struct
{}{}
return
nil
}
...
...
@@ -284,8 +336,8 @@ func (m *ConfigManager) Reload(config *Config) error {
func
(
m
*
ConfigManager
)
Close
()
{
m
.
mx
.
Lock
()
close
(
m
.
notifyCh
)
if
m
.
assets
!=
nil
{
m
.
assets
.
Close
()
if
m
.
parsed
!=
nil
{
m
.
parsed
.
Close
()
}
m
.
mx
.
Unlock
()
}
...
...
@@ -303,47 +355,16 @@ func (m *ConfigManager) Notify() <-chan struct{} {
return
ch
}
func
(
m
*
ConfigManager
)
getHandler
(
name
string
)
(
Handler
,
bool
)
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
h
,
ok
:=
m
.
assets
.
handlerMap
[
name
]
return
h
,
ok
}
func
(
m
*
ConfigManager
)
getRepository
()
Repository
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
assets
.
repo
}
func
(
m
*
ConfigManager
)
getQueueSpec
()
jobs
.
QueueSpec
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
config
.
Queue
}
func
(
m
*
ConfigManager
)
getSourceSpecs
()
[]
SourceSpec
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
config
.
SourceSpecs
}
func
(
m
*
ConfigManager
)
getSeed
()
int64
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
assets
.
seed
}
func
(
m
*
ConfigManager
)
getShell
()
*
Shell
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
assets
.
shell
// NewRuntimeContext returns a new RuntimeContext, capturing current
// configuration and runtime assets.
func
(
m
*
ConfigManager
)
NewRuntimeContext
()
RuntimeContext
{
return
m
.
current
()
}
func
(
m
*
ConfigManager
)
getWorkDir
()
strin
g
{
func
(
m
*
ConfigManager
)
current
()
*
parsedConfi
g
{
m
.
mx
.
Lock
()
defer
m
.
mx
.
Unlock
()
return
m
.
config
.
WorkDir
return
m
.
parsed
}
func
mustGetSeed
(
path
string
)
int64
{
...
...
config_test.go
View file @
04055321
...
...
@@ -4,9 +4,10 @@ import (
"context"
"errors"
"fmt"
"log"
"os"
"strings"
"testing"
"time"
)
func
TestReadConfig
(
t
*
testing
.
T
)
{
...
...
@@ -27,6 +28,7 @@ func TestConfigManager(t *testing.T) {
if
err
!=
nil
{
t
.
Fatal
(
"ReadConfig()"
,
err
)
}
log
.
Printf
(
"loaded %d sources"
,
len
(
conf
.
SourceSpecs
))
mgr
,
err
:=
NewConfigManager
(
conf
)
if
err
!=
nil
{
t
.
Fatal
(
"NewConfigManager()"
,
err
)
...
...
@@ -34,13 +36,18 @@ func TestConfigManager(t *testing.T) {
defer
mgr
.
Close
()
// Test one of the accessor methods.
if
s
:=
mgr
.
get
SourceSpecs
();
len
(
s
)
!=
1
{
t
.
Fatalf
(
"
get
SourceSpecs() bad result: %+v"
,
s
)
if
s
:=
mgr
.
current
()
.
SourceSpecs
();
len
(
s
)
!=
1
{
t
.
Fatalf
(
"
current().
SourceSpecs() bad result: %+v"
,
s
)
}
// Test the Notify() mechanism by checking that it triggers
// right away when setting up a new listener.
<-
mgr
.
Notify
()
tmr
:=
time
.
NewTimer
(
1
*
time
.
Second
)
select
{
case
<-
mgr
.
Notify
()
:
case
<-
tmr
.
C
:
t
.
Fatal
(
"Notify() channel did not trigger"
)
}
}
func
TestRandomSeed
(
t
*
testing
.
T
)
{
...
...
@@ -61,7 +68,7 @@ func TestConfig_Parse(t *testing.T) {
type
testdata
struct
{
config
*
Config
expectedOK
bool
checkFn
func
(
*
runtimeAssets
,
[]
Dataset
)
error
checkFn
func
([]
*
Dataset
)
error
}
tdd
:=
[]
testdata
{
// The following tests cover a few ways to generate
...
...
@@ -69,24 +76,27 @@ func TestConfig_Parse(t *testing.T) {
// the README.
{
&
Config
{
SourceSpecs
:
[]
SourceSpec
{
{
Name
:
"users/account1"
,
Handler
:
"file"
,
Atoms
:
[]
Atom
{
{
RelativePath
:
"/data/account1"
},
},
},
{
Name
:
"users/account2"
,
Handler
:
"file"
,
Atoms
:
[]
Atom
{
{
RelativePath
:
"/data/account2"
},
SourceSpecs
:
[]
*
SourceSpec
{