Commit 7205304a authored by ale's avatar ale

Add live progress debug info for restic processes

parent cae8ed84
Pipeline #8319 passed with stages
in 1 minute and 38 seconds
......@@ -32,6 +32,7 @@ body { background: white; font-family: "Helvetica", sans-serif; }
<h1>tabacco</h1>
<p><a href="/debug/restic">active backups</a></p>
<p><a href="/debug/jobs">job status</a></p>
<p><a href="/debug/sched">schedule</a></p>
<p><a href="/metrics">metrics</a></p>
......@@ -109,6 +110,48 @@ body { background: white; font-family: "Helvetica", sans-serif; }
<h1>Schedule</h1>
{{template "schedule_status" .Schedule}}
{{template "footer"}}
`
activeDebugTpl = `{{template "header"}}
<h1>Active backups</h1>
<table>
<thead>
<tr>
<th>ID</th>
<th>Dataset</th>
<th>% Done</th>
<th>Elapsed</th>
<th>Files</th>
<th>Bytes</th>
</tr>
</thead>
<tbody>
{{range $id, $a := .Active}}
<tr>
<td>{{$a.Backup.ID}}</td>
<td>{{$a.Dataset.Source}}</td>
<td>{{$a.PercentDone}}</td>
<td>{{$a.SecondsElapsed}}s</td>
<td>{{$a.FilesDone}}/{{$a.TotalFiles}}</td>
<td>{{$a.BytesDone}}/{{$a.TotalBytes}}</td>
{{if $a.HasCurrentFiles}}
</tr>
<tr>
<td colspan="2"></td>
<td colspan="4">
{{range $a.CurrentFiles}}
{{.}}<br>
{{end}}
</td>
{{end}}
</tr>
{{end}}
</tbody>
</table>
{{template "footer"}}
`
......@@ -132,6 +175,7 @@ func init() {
template.Must(debugTpl.New("index").Parse(indexDebugTpl))
template.Must(debugTpl.New("state_manager_debug_page").Parse(stateManagerDebugTpl))
template.Must(debugTpl.New("scheduler_debug_page").Parse(schedulerDebugTpl))
template.Must(debugTpl.New("active_debug_page").Parse(activeDebugTpl))
}
// Job status debug handler.
......@@ -161,6 +205,18 @@ func (a *Agent) handleSchedulerDebug(w http.ResponseWriter, r *http.Request) {
}
}
// Active restic processes debug handler.
func (a *Agent) handleActiveDebug(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
active.mx.Lock()
if err := debugTpl.Lookup("active_debug_page").Execute(w, map[string]interface{}{
"Active": active.active,
}); err != nil {
log.Printf("active_debug_page: %v", err)
}
active.mx.Unlock()
}
// Agent debug handler page, with links to the other two.
func (a *Agent) handleDebugPage(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
......@@ -178,6 +234,7 @@ func (a *Agent) StartHTTPServer(addr string) {
h := http.NewServeMux()
h.HandleFunc("/debug/jobs", a.handleStateManagerDebug)
h.HandleFunc("/debug/sched", a.handleSchedulerDebug)
h.HandleFunc("/debug/restic", a.handleActiveDebug)
// Add all the pprof handlers, they're useful for debugging.
h.HandleFunc("/debug/pprof/", pprof.Index)
......
......@@ -16,6 +16,7 @@ import (
"sync"
"time"
"git.autistici.org/ai3/tools/tabacco/jobs"
"github.com/hashicorp/go-version"
)
......@@ -272,11 +273,62 @@ func (r *resticRepository) RestoreStreamCmd(ctx context.Context, rctx RuntimeCon
), nil
}
// Generic JSON message from 'restic backup'.
type resticMessage struct {
MessageType string `json:"message_type"`
// Global map that keeps track of the currently running restic
// processes and their respective progress, for debugging purposes.
type activeBackupStatus struct {
sync.Mutex
Backup *Backup
Dataset *Dataset
Status *resticStatusMessage
}
type activeBackups struct {
mx sync.Mutex
active map[string]*activeBackupStatus
}
var active *activeBackups
func init() {
active = &activeBackups{
active: make(map[string]*activeBackupStatus),
}
}
func (a *activeBackups) WithResticStatus(jobID string, backup *Backup, ds *Dataset, f func(chan *resticStatusMessage) error) error {
ch := make(chan *resticStatusMessage, 1)
status := &activeBackupStatus{
Backup: backup,
Dataset: ds,
}
a.mx.Lock()
a.active[jobID] = status
a.mx.Unlock()
go func() {
for resticStatus := range ch {
a.mx.Lock()
log.Printf("STATUS: %+v", *resticStatus)
status.Status = resticStatus
a.mx.Unlock()
}
}()
// Status.
err := f(ch)
close(ch)
a.mx.Lock()
delete(a.active, jobID)
a.mx.Unlock()
return err
}
// JSON status message.
type resticStatusMessage struct {
SecondsElapsed int64 `json:"seconds_elapsed"`
PercentDone float64 `json:"percent_done"`
TotalFiles int64 `json:"total_files"`
......@@ -284,8 +336,14 @@ type resticMessage struct {
TotalBytes int64 `json:"total_bytes"`
BytesDone int64 `json:"bytes_done"`
CurrentFiles []string `json:"current_files"`
}
func (m *resticStatusMessage) HasCurrentFiles() bool {
return len(m.CurrentFiles) > 0
}
// Summary.
// JSON summary message.
type resticSummaryMessage struct {
FilesNew int64 `json:"files_new"`
FilesChanged int64 `json:"files_changed"`
FilesUnmodified int64 `json:"files_unmodified"`
......@@ -301,22 +359,34 @@ type resticMessage struct {
SnapshotID string `json:"snapshot_id"`
}
// Generic JSON message from 'restic backup'.
type resticMessage struct {
MessageType string `json:"message_type"`
resticStatusMessage
resticSummaryMessage
}
// Scan the output of 'restic backup' for the snapshot ID. Modifies backup/dataset objects.
func (r *resticRepository) RunBackup(ctx context.Context, shell *Shell, backup *Backup, ds *Dataset, cmd string) error {
return shell.RunWithStdoutCallback(ctx, cmd, func(line []byte) {
var msg resticMessage
if err := json.Unmarshal(line, &msg); err != nil {
log.Printf("error parsing restic JSON message: %v (%s)", err, line)
return
}
if msg.MessageType == "summary" {
log.Printf("dataset %s: total_bytes=%d, new=%d", ds.ID, msg.TotalBytesProcessed, msg.DataAdded)
ds.SnapshotID = msg.SnapshotID
ds.TotalFiles = msg.TotalFilesProcessed
ds.TotalBytes = msg.TotalBytesProcessed
ds.BytesAdded = msg.DataAdded
ds.Duration = int(msg.TotalDuration)
}
return active.WithResticStatus(jobs.GetID(ctx), backup, ds, func(progressCh chan *resticStatusMessage) error {
return shell.RunWithStdoutCallback(ctx, cmd, func(line []byte) {
var msg resticMessage
if err := json.Unmarshal(line, &msg); err != nil {
log.Printf("error parsing restic JSON message: %v (%s)", err, line)
return
}
switch msg.MessageType {
case "status":
progressCh <- &msg.resticStatusMessage
case "summary":
log.Printf("dataset %s: total_bytes=%d, new=%d", ds.ID, msg.TotalBytesProcessed, msg.DataAdded)
ds.SnapshotID = msg.SnapshotID
ds.TotalFiles = msg.TotalFilesProcessed
ds.TotalBytes = msg.TotalBytesProcessed
ds.BytesAdded = msg.DataAdded
ds.Duration = int(msg.TotalDuration)
}
})
})
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment