Commit f2ac045b authored by ale's avatar ale

Fix debug pages and pprof handlers

parent fb23a285
......@@ -4,12 +4,9 @@ import (
"context"
"fmt"
"log"
"net/http"
"net/http/pprof"
"git.autistici.org/ale/tabacco/jobs"
"git.autistici.org/ale/tabacco/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// Agent holds a Manager and a Scheduler together, and runs periodic
......@@ -58,18 +55,6 @@ func (a *Agent) Close() {
a.sched.Stop()
}
// StartHTTPServer starts a HTTP server that exports Prometheus
// metrics and debug information.
func (a *Agent) StartHTTPServer(addr string) error {
http.HandleFunc("/debug/jobs", a.handleStateManagerDebug)
http.HandleFunc("/debug/sched", a.handleSchedulerDebug)
http.Handle("/debug/pprof/", pprof.Handler(""))
http.Handle("/metrics", promhttp.Handler())
http.HandleFunc("/", a.handleDebugPage)
go http.ListenAndServe(addr, nil)
return nil
}
// Create a new jobs.Schedule that will trigger a separate backup for
// each configured data source that includes a 'schedule' attribute.
func makeSchedule(ctx context.Context, m Manager, sourceSpecs []SourceSpec, hostSeed int64) (*jobs.Schedule, error) {
......
......@@ -71,16 +71,22 @@ func (c *agentCommand) Execute(ctx context.Context, f *flag.FlagSet, args ...int
return subcommands.ExitFailure
}
d, err := tabacco.NewAgent(ctx, configMgr, store)
agent, err := tabacco.NewAgent(ctx, configMgr, store)
if err != nil {
log.Printf("error: %v", err)
return subcommands.ExitFailure
}
defer d.Close() // nolint
defer agent.Close() // nolint
// Wait for the outmost Context to terminate (presumably due to SIGTERM).
log.Printf("backup agent started")
if c.httpAddr != "" {
agent.StartHTTPServer(c.httpAddr)
}
// Wait for the outmost Context to terminate (presumably due to SIGTERM).
<-ctx.Done()
log.Printf("backup agent stopped")
return subcommands.ExitSuccess
......
......@@ -42,6 +42,7 @@ func readHandlersFromDir(dir string) ([]HandlerSpec, error) {
var out []HandlerSpec
err := foreachYAMLFile(dir, func(path string) error {
var spec HandlerSpec
log.Printf("reading handler: %s", path)
if err := readYAMLFile(path, &spec); err != nil {
return err
}
......@@ -65,6 +66,7 @@ func readSourcesFromDir(dir string, handlerSpecs []HandlerSpec) ([]SourceSpec, e
var out []SourceSpec
err := foreachYAMLFile(dir, func(path string) error {
var spec SourceSpec
log.Printf("reading source: %s", path)
if err := readYAMLFile(path, &spec); err != nil {
return err
}
......@@ -85,6 +87,7 @@ func readSourcesFromDir(dir string, handlerSpecs []HandlerSpec) ([]SourceSpec, e
func ReadConfig(path string) (*Config, error) {
// Read and validate the main configuration from 'path'.
var config Config
log.Printf("reading config: %s", path)
if err := readYAMLFile(path, &config); err != nil {
return nil, err
}
......@@ -219,6 +222,7 @@ func (m *ConfigManager) Reload(config *Config) error {
if m.repo != nil {
m.repo.Close() // nolint
}
log.Printf("loading new config: %d handlers, %d sources", len(handlerMap), len(config.SourceSpecs))
m.repo = repo
m.handlerMap = handlerMap
m.config = config
......@@ -288,6 +292,7 @@ func mustGetSeed(path string) int64 {
return int64(seed)
}
}
log.Printf("generating new random seed for this host")
seed, data := randomSeed()
if err := ioutil.WriteFile(path, data, 0600); err != nil {
log.Printf("warning: can't write random seed file: %v", err)
......
......@@ -2,7 +2,12 @@ package tabacco
import (
"html/template"
"log"
"net/http"
"net/http/pprof"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
......@@ -13,9 +18,11 @@ var (
<style type="text/css">
body { background: white; font-family: "Helvetica", sans-serif; }
.table th { text-align: left; font-weight: bold; }
.table td { text-align: left; }
.table td { text-align: left; padding-right: 10px; }
.table thead tr { border-bottom: 2px solid #333; }
.error { color: #a00; }
.error { color: #900; }
.ok { color: #090; }
.id { color: #999; }
</style>
</head>
<body>
......@@ -47,12 +54,12 @@ body { background: white; font-family: "Helvetica", sans-serif; }
<tbody>
{{range .}}
<tr>
<td>{{.ID}}</td>
<td>{{.Job.Key}}</td>
<td class="id">{{.ID}}</td>
<td>{{.Name}}</td>
<td>{{.Status}}</td>
<td>{{if not .StartedAt.IsZero}}{{.StartedAt}}{{end}}</td>
<td>{{if not .CompletedAt.IsZero}}{{.CompletedAt}}{{end}}</td>
<td><span class="error">{{.Err}}</span></td>
<td>{{if not .StartedAt.IsZero}}{{timefmt .StartedAt}}{{end}}</td>
<td>{{if not .CompletedAt.IsZero}}{{timefmt .CompletedAt}}{{end}}</td>
<td>{{if .Err}}<span class="error">{{.Err}}</span>{{else if not .CompletedAt.IsZero}}<span class="ok">ok</span>{{end}}</td>
</tr>
{{end}}
</tbody>
......@@ -79,7 +86,6 @@ body { background: white; font-family: "Helvetica", sans-serif; }
<thead>
<tr>
<th>Name</th>
<th>Handler</th>
<th>Schedule</th>
<th>Last Run</th>
<th>Next Run</th>
......@@ -88,11 +94,11 @@ body { background: white; font-family: "Helvetica", sans-serif; }
<tbody>
{{range .}}
<tr>
<td>{{.Spec.Name}}</td>
<td>{{.Spec.Handler}}</td>
<td>{{.Spec.Schedule}}</td>
<td>{{if not .Prev.IsZero}}{{.Prev}}{{end}}</td>
<td>{{if not .Next.IsZero}}{{.Next}}{{end}}</td>
<td>{{.Name}}</td>
<td>{{.Schedule}}</td>
<td>{{if not .Prev.IsZero}}{{timefmt .Prev}}{{end}}</td>
<td>{{if not .Next.IsZero}}{{timefmt .Next}}{{end}}</td>
<td>{{if .LastError}}<span class="error">{{.LastError}}</span>{{else if not .Prev.IsZero}}<span class="ok">ok</span>{{end}}</td>
</tr>
{{end}}
</tbody>
......@@ -112,8 +118,14 @@ body { background: white; font-family: "Helvetica", sans-serif; }
debugTpl *template.Template
)
func timefmt(t time.Time) string {
return t.Format(time.Stamp)
}
func init() {
debugTpl = template.New("")
debugTpl = template.New("").Funcs(template.FuncMap{
"timefmt": timefmt,
})
template.Must(debugTpl.New("header").Parse(headerTpl))
template.Must(debugTpl.New("footer").Parse(footerTpl))
template.Must(debugTpl.New("index").Parse(indexDebugTpl))
......@@ -126,22 +138,26 @@ func (a *Agent) handleStateManagerDebug(w http.ResponseWriter, r *http.Request)
pending, running, done := a.mgr.GetStatus()
w.Header().Set("Content-Type", "text/html")
_ = debugTpl.Lookup("state_manager_debug_page").Execute(w, map[string]interface{}{
if err := debugTpl.Lookup("state_manager_debug_page").Execute(w, map[string]interface{}{
"Pending": pending,
"NumPending": len(pending),
"Running": running,
"NumRunning": len(running),
"Done": done,
"NumDone": len(done),
})
}); err != nil {
log.Printf("state_manager_debug_page: %v", err)
}
}
// Scheduler debug handler.
func (a *Agent) handleSchedulerDebug(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_ = debugTpl.Lookup("scheduler_debug_page").Execute(w, map[string]interface{}{
if err := debugTpl.Lookup("scheduler_debug_page").Execute(w, map[string]interface{}{
"Schedule": a.sched.GetStatus(),
})
}); err != nil {
log.Printf("scheduler_debug_page: %v", err)
}
}
// Agent debug handler page, with links to the other two.
......@@ -152,5 +168,24 @@ func (a *Agent) handleDebugPage(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set("Content-Type", "text/html")
_ = debugTpl.Lookup("index").Execute(w, nil) // nolint
debugTpl.Lookup("index").Execute(w, nil) // nolint
}
// StartHTTPServer starts a HTTP server that exports Prometheus
// metrics and debug information.
func (a *Agent) StartHTTPServer(addr string) {
h := http.NewServeMux()
h.HandleFunc("/debug/jobs", a.handleStateManagerDebug)
h.HandleFunc("/debug/sched", a.handleSchedulerDebug)
// Add all the pprof handlers, they're useful for debugging.
h.HandleFunc("/debug/pprof/", pprof.Index)
h.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
h.HandleFunc("/debug/pprof/profile", pprof.Profile)
h.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
h.HandleFunc("/debug/pprof/trace", pprof.Trace)
h.Handle("/metrics", promhttp.Handler())
h.HandleFunc("/", a.handleDebugPage)
go http.ListenAndServe(addr, h)
}
......@@ -55,10 +55,11 @@ func (s *Schedule) Add(name, schedStr string, jobFn JobGeneratorFunc) error {
return err
}
s.c.Schedule(sched, &cronJob{
name: name,
fn: jobFn,
ctx: s.rootCtx,
schedule: s,
name: name,
fn: jobFn,
ctx: s.rootCtx,
schedule: s,
scheduleStr: schedStr,
})
return nil
}
......@@ -89,15 +90,20 @@ func (s *Schedule) stop() {
// it in a new goroutine, so, from the point of view of the cron
// package, cron jobs are instantaneous.
type cronJob struct {
name string
fn JobGeneratorFunc
ctx context.Context
name string
scheduleStr string
fn JobGeneratorFunc
ctx context.Context
// Link back to the Schedule so we can notify the Scheduler
// about exit status.
schedule *Schedule
}
func (j *cronJob) Name() string { return j.name }
func (j *cronJob) Schedule() string { return j.scheduleStr }
func (j *cronJob) Run() {
job := j.fn()
if job == nil {
......@@ -190,6 +196,7 @@ func (s *Scheduler) Stop() {
// running, or terminated in the past.
type CronJobStatus struct {
Name string
Schedule string
Prev time.Time
Next time.Time
LastError string
......@@ -213,6 +220,11 @@ func cronJobStatusListOrderByName(list []CronJobStatus) *cronJobStatusList {
}
}
type hasNameAndSchedule interface {
Name() string
Schedule() string
}
// GetStatus returns the current status of the scheduled jobs.
func (s *Scheduler) GetStatus() []CronJobStatus {
s.mx.Lock()
......@@ -224,12 +236,14 @@ func (s *Scheduler) GetStatus() []CronJobStatus {
var jobs []CronJobStatus
for _, entry := range s.cur.c.Entries() {
// Get the cronJob behind the cron.Job interface.
if job, ok := entry.Job.(*cronJob); ok {
if jj, ok := entry.Job.(hasNameAndSchedule); ok {
name := jj.Name()
jobs = append(jobs, CronJobStatus{
Name: job.name,
Name: name,
Schedule: jj.Schedule(),
Prev: entry.Prev,
Next: entry.Next,
LastError: s.getLastErrorString(job.name),
LastError: s.getLastErrorString(name),
})
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment