Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
ai3
tools
tabacco
Commits
f3117535
Commit
f3117535
authored
Aug 03, 2018
by
ale
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Give each backup job its temporary working directory
parent
4b038578
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
77 additions
and
5 deletions
+77
-5
manager.go
manager.go
+14
-3
shell.go
shell.go
+63
-2
No files found.
manager.go
View file @
f3117535
...
...
@@ -16,6 +16,7 @@ type tabaccoManager struct {
*
jobs
.
ExclusiveLockManager
*
jobs
.
QueueManager
*
jobs
.
StateManager
*
workdirManager
configMgr
*
ConfigManager
ms
MetadataStore
...
...
@@ -23,11 +24,19 @@ type tabaccoManager struct {
// NewManager creates a new Manager.
func
NewManager
(
ctx
context
.
Context
,
configMgr
*
ConfigManager
,
ms
MetadataStore
)
(
Manager
,
error
)
{
// If we can't create a workdirManager, it probably means we
// don't have permissions to the WorkDir, which is bad.
wm
,
err
:=
newWorkdirManager
(
configMgr
.
getWorkDir
())
if
err
!=
nil
{
return
nil
,
err
}
// Note: the queue configuration won't be reloaded.
return
&
tabaccoManager
{
ExclusiveLockManager
:
jobs
.
NewExclusiveLockManager
(),
QueueManager
:
jobs
.
NewQueueManager
(
configMgr
.
getQueueSpec
()),
StateManager
:
jobs
.
NewStateManager
(),
workdirManager
:
wm
,
configMgr
:
configMgr
,
ms
:
ms
,
...
...
@@ -127,9 +136,11 @@ func (m *tabaccoManager) backupDatasetJob(h Handler, backup Backup, ds Dataset)
return
m
.
WithQueue
(
m
.
WithStatus
(
m
.
WithExclusiveLock
(
withInstrumentation
(
jobs
.
SyncGroup
(
out
),
ds
.
Name
,
m
.
withWorkDir
(
withInstrumentation
(
jobs
.
SyncGroup
(
out
),
ds
.
Name
,
),
),
id
,
false
),
...
...
shell.go
View file @
f3117535
...
...
@@ -4,10 +4,14 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"git.autistici.org/ale/tabacco/jobs"
)
// Shell runs commands, with some options (a global dry-run flag
...
...
@@ -39,6 +43,9 @@ func (s *Shell) SetIOClass(n int) {
s
.
ioniceClass
=
n
}
// command builds an exec.Cmd and gets some parameters from the
// context - notably it sets log output and working directory to be in
// the working dir if the job has been wrapped in WithWorkDir().
func
(
s
*
Shell
)
command
(
ctx
context
.
Context
,
arg
string
)
*
exec
.
Cmd
{
log
.
Printf
(
"sh: %s"
,
arg
)
...
...
@@ -51,19 +58,20 @@ func (s *Shell) command(ctx context.Context, arg string) *exec.Cmd {
if
s
.
niceLevel
!=
0
{
args
=
append
(
[]
string
{
"
/usr/bin/
nice"
,
"-n"
,
strconv
.
Itoa
(
s
.
niceLevel
)},
[]
string
{
"nice"
,
"-n"
,
strconv
.
Itoa
(
s
.
niceLevel
)},
args
...
,
)
}
if
s
.
ioniceClass
!=
0
{
args
=
append
(
[]
string
{
"
/usr/bin/
ionice"
,
"-c"
,
strconv
.
Itoa
(
s
.
ioniceClass
)},
[]
string
{
"ionice"
,
"-c"
,
strconv
.
Itoa
(
s
.
ioniceClass
)},
args
...
,
)
}
c
:=
exec
.
CommandContext
(
ctx
,
args
[
0
],
args
[
1
:
]
...
)
// #nosec
c
.
Stderr
=
os
.
Stderr
c
.
Dir
=
getWorkDir
(
ctx
)
return
c
}
...
...
@@ -127,3 +135,56 @@ func (s *Shell) RunWithEnv(ctx context.Context, arg string, envMap map[string]st
func
(
s
*
Shell
)
Output
(
ctx
context
.
Context
,
arg
string
)
([]
byte
,
error
)
{
return
s
.
command
(
ctx
,
arg
)
.
Output
()
}
// Manage a work directory (temporary scratch with lots of space).
type
workdirManager
struct
{
dir
string
}
func
newWorkdirManager
(
tmpdir
string
)
(
*
workdirManager
,
error
)
{
dir
,
err
:=
ioutil
.
TempDir
(
tmpdir
,
"tabacco-"
)
if
err
!=
nil
{
return
nil
,
err
}
return
&
workdirManager
{
dir
},
nil
}
func
(
w
*
workdirManager
)
Close
()
{
os
.
RemoveAll
(
w
.
dir
)
// nolint
}
func
(
w
*
workdirManager
)
withWorkDir
(
j
jobs
.
Job
)
jobs
.
Job
{
jobDir
:=
filepath
.
Join
(
w
.
dir
,
"job-"
+
j
.
ID
())
return
&
workdirJob
{
Job
:
j
,
dir
:
jobDir
,
}
}
type
workdirJob
struct
{
jobs
.
Job
dir
string
}
type
workdirKeyType
int
var
workdirKey
workdirKeyType
=
1
func
(
j
*
workdirJob
)
RunContext
(
ctx
context
.
Context
)
error
{
if
err
:=
os
.
Mkdir
(
j
.
dir
,
0700
);
err
!=
nil
{
return
err
}
ctx
=
context
.
WithValue
(
ctx
,
workdirKey
,
j
.
dir
)
return
j
.
Job
.
RunContext
(
ctx
)
}
// getWorkDir returns the current working directory, provided the
// WithWorkDir wrapper has been called.
func
getWorkDir
(
ctx
context
.
Context
)
string
{
dir
,
ok
:=
ctx
.
Value
(
workdirKey
)
.
(
string
)
if
ok
{
return
dir
}
return
"/tmp"
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment