Commit bf4a40db authored by ale's avatar ale

Make log compaction and pruning configurable

Adds the new configuration parameters 'compaction_keep_count' and
'max_age_days'.
parent 5aff704b
......@@ -6,18 +6,6 @@ import (
"time"
)
var (
// How often to run the user log compaction.
compactionInterval = 600 * time.Second
// How many log entries to keep when compacting.
userKeepLogCount = 100
// How often to run the log pruner.
pruneInterval = 24 * time.Hour
// Delete entries older than this many days.
pruneCutoffDays = 365
)
// Call a function at semi-regular intervals.
type cronJob struct {
f func()
......@@ -108,7 +96,7 @@ func compactLogs(db *sql.DB, stmts statementMap, username string, count int) err
}
// Remove old entries (both logs and devices) from the database.
func pruneLogs(db *sql.DB, stmts statementMap) error {
func pruneLogs(db *sql.DB, stmts statementMap, pruneCutoffDays int) error {
// Run each batch deletion in its own transaction, just to be nice.
cutoff := time.Now().AddDate(0, 0, -pruneCutoffDays)
for _, stmt := range []string{"prune_userlog", "prune_device_info"} {
......
......@@ -31,7 +31,7 @@ func TestPruneLogs(t *testing.T) {
t.Fatal("no logs loaded?")
}
if err := pruneLogs(db, stmts); err != nil {
if err := pruneLogs(db, stmts, 365); err != nil {
t.Fatal("pruneLogs():", err)
}
n2 := countLogs(db)
......
......@@ -12,7 +12,9 @@ import (
// Config for the UserMetaServer.
type Config struct {
DBURI string `yaml:"db_uri"`
DBURI string `yaml:"db_uri"`
CompactionKeepCount int `yaml:"compaction_keep_count"`
MaxAgeDays int `yaml:"max_age_days"`
}
// UserMetaServer exposes the analysis service and the user metadata
......@@ -38,7 +40,7 @@ func New(config *Config) (*UserMetaServer, error) {
return nil, err
}
userlog, err := newUserlogDB(db)
userlog, err := newUserlogDB(db, config.CompactionKeepCount, config.MaxAgeDays)
if err != nil {
db.Close() // nolint
return nil, err
......
......@@ -9,6 +9,14 @@ import (
"git.autistici.org/id/usermetadb"
)
var (
// How often to run the user log compaction.
compactionInterval = 600 * time.Second
// How often to run the log pruner.
pruneInterval = 24 * time.Hour
)
var userlogDBStatements = map[string]string{
// Modify the 'devices' table.
"check_device_info": `SELECT 1 FROM devices WHERE username = ? AND id = ?`,
......@@ -113,33 +121,38 @@ type userlogDB struct {
pendingCompaction *usernameSet
}
func newUserlogDB(db *sql.DB) (*userlogDB, error) {
func newUserlogDB(db *sql.DB, compactionKeepCount, maxAgeDays int) (*userlogDB, error) {
stmts, err := newStatementMap(db, userlogDBStatements)
if err != nil {
return nil, err
}
pendingCompaction := newUsernameSet()
cronjobs := []*cronJob{
newCron(func() {
pendingCompaction.foreach(func(username string) {
if err := compactLogs(db, stmts, username, userKeepLogCount); err != nil {
udb := &userlogDB{
db: db,
stmts: stmts,
}
// Selectively add cronjobs if the related parameters are non-zero.
if compactionKeepCount > 0 {
pc := newUsernameSet()
udb.pendingCompaction = pc
udb.cronjobs = append(udb.cronjobs, newCron(func() {
pc.foreach(func(username string) {
if err := compactLogs(db, stmts, username, compactionKeepCount); err != nil {
log.Printf("error cleaning up logs for user %s: %v", username, err)
}
})
}, compactionInterval),
newCron(func() {
if err := pruneLogs(db, stmts); err != nil {
}, compactionInterval))
}
if maxAgeDays > 0 {
udb.cronjobs = append(udb.cronjobs, newCron(func() {
if err := pruneLogs(db, stmts, maxAgeDays); err != nil {
log.Printf("error in log pruning: %v", err)
}
}, pruneInterval),
}, pruneInterval))
}
return &userlogDB{
db: db,
stmts: stmts,
cronjobs: cronjobs,
pendingCompaction: pendingCompaction,
}, nil
return udb, nil
}
func (u *userlogDB) Close() {
......@@ -230,7 +243,7 @@ func (u *userlogDB) AddLog(entry *usermetadb.LogEntry) error {
return err
}
if entry.Username != "" {
if entry.Username != "" && u.pendingCompaction != nil {
u.pendingCompaction.add(entry.Username)
}
......
......@@ -150,7 +150,7 @@ func TestUserlog_AddLog(t *testing.T) {
}
defer db.Close()
ulog, err := newUserlogDB(db)
ulog, err := newUserlogDB(db, 0, 0)
if err != nil {
t.Fatal(err)
}
......@@ -175,7 +175,7 @@ func TestUserlog_GetUserDevices(t *testing.T) {
bulkLoadTestLogs(t, db)
ulog, err := newUserlogDB(db)
ulog, err := newUserlogDB(db, 0, 0)
if err != nil {
t.Fatal(err)
}
......@@ -210,7 +210,7 @@ func TestUserlog_GetUserLogs(t *testing.T) {
bulkLoadTestLogs(t, db)
ulog, err := newUserlogDB(db)
ulog, err := newUserlogDB(db, 0, 0)
if err != nil {
t.Fatal(err)
}
......@@ -242,3 +242,24 @@ func TestUserlog_GetUserLogs(t *testing.T) {
t.Fatal("GetUserLogs(nonexisting) returned non-nil")
}
}
func TestUserLog_HasCronjobs(t *testing.T) {
defer os.Remove("test.db")
db, err := openDB("test.db")
if err != nil {
t.Fatal(err)
}
defer db.Close()
bulkLoadTestLogs(t, db)
ulog, err := newUserlogDB(db, 100, 365)
if err != nil {
t.Fatal(err)
}
defer ulog.Close()
if nc := len(ulog.cronjobs); nc != 2 {
t.Fatalf("unexpected number of cron jobs: got %d, exp %d", nc, 2)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment