Skip to content
Snippets Groups Projects
Commit f69bb3c1 authored by ale's avatar ale
Browse files

Merge branch 'simpler-sql' into 'master' 

Use ai3/go-common/sqlutil as database API

See merge request !25
parents 4956648d feb46a4d
No related branches found
No related tags found
1 merge request!25Use ai3/go-common/sqlutil as database API
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 305 additions and 1707 deletions
go.mod
+ 3
− 3
View file @ f69bb3c1
...@@ -3,9 +3,9 @@ module git.autistici.org/id/usermetadb ...@@ -3,9 +3,9 @@ module git.autistici.org/id/usermetadb
go 1.14 go 1.14
require ( require (
git.autistici.org/ai3/go-common v0.0.0-20220814084314-c7650f356855 git.autistici.org/ai3/go-common v0.0.0-20220814124137-4f7bac42fbdb
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/google/go-cmp v0.5.8 github.com/google/go-cmp v0.5.8
github.com/mattn/go-sqlite3 v1.14.14 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf // indirect
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
go.sum
+ 3
− 992
View file @ f69bb3c1
This diff is collapsed.
migrations/1_initialize_schema.down.sql deleted 100644 → 0
+ 0
− 6
View file @ 4956648d
DROP INDEX idx_devices_id_username;
DROP TABLE devices;
DROP INDEX idx_userlog_username;
DROP INDEX idx_userlog_device_id;
DROP TABLE userlog;
migrations/1_initialize_schema.up.sql deleted 100644 → 0
+ 0
− 39
View file @ 4956648d
-- We store the raw denormalized user logs, because device information
-- might change over time (think a version update, or restoring a
-- backup to a different OS) and it may be useful to see the
-- historical variation. Unique device information is also aggregated
-- incrementally to a separate table, to provide a quick way to obtain
-- the list of devices for a user without a large table scan.
CREATE TABLE devices (
id VARCHAR(64) NOT NULL,
username TEXT NOT NULL,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
first_seen DATETIME,
last_seen DATETIME,
last_device_remote_zone TEXT,
last_device_user_agent TEXT
);
CREATE UNIQUE INDEX idx_devices_id_username ON devices (id, username);
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64) NOT NULL,
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);
CREATE INDEX idx_userlog_username ON userlog (username);
CREATE INDEX idx_userlog_device_id ON userlog (device_id);
migrations/2_lastlogin.down.sql deleted 100644 → 0
+ 0
− 3
View file @ 4956648d
DROP INDEX idx_lastlogin_username_service;
DROP INDEX idx_lastlogin_username;
DROP TABLE lastlogin;
migrations/2_lastlogin.up.sql deleted 100644 → 0
+ 0
− 8
View file @ 4956648d
CREATE TABLE lastlogin (
username TEXT NOT NULL,
service TEXT NOT NULL,
timestamp DATETIME NOT NULL
);
CREATE UNIQUE INDEX idx_lastlogin_username_service ON lastlogin (username, service);
CREATE INDEX idx_lastlogin_username ON lastlogin (username);
migrations/3_userlog_device_nullable.up.sql deleted 100644 → 0
+ 0
− 31
View file @ 4956648d
-- Make the userlog.device_id field nullable. Unfortunately SQLite
-- does not support modifying constraints on a column, so we have to
-- dump and reload the table contents using a temporary table.
PRAGMA foreign_keys=off;
ALTER TABLE userlog RENAME TO _userlog_old;
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64),
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);
INSERT INTO userlog (username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp)
SELECT username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp
FROM _userlog_old;
DROP TABLE _userlog_old;
PRAGMA foreign_keys=on;
migrations/bindata.go deleted 100644 → 0
+ 0
− 375
View file @ 4956648d
// Code generated by go-bindata.
// sources:
// migrations/1_initialize_schema.down.sql
// migrations/1_initialize_schema.up.sql
// migrations/2_lastlogin.down.sql
// migrations/2_lastlogin.up.sql
// migrations/3_userlog_device_nullable.up.sql
// DO NOT EDIT!
package migrations
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
)
type asset struct {
bytes []byte
info os.FileInfo
}
type bindataFileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
}
func (fi bindataFileInfo) Name() string {
return fi.name
}
func (fi bindataFileInfo) Size() int64 {
return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
return fi.modTime
}
func (fi bindataFileInfo) IsDir() bool {
return false
}
func (fi bindataFileInfo) Sys() interface{} {
return nil
}
var __1_initialize_schemaDownSql = []byte(`DROP INDEX idx_devices_id_username;
DROP TABLE devices;
DROP INDEX idx_userlog_username;
DROP INDEX idx_userlog_device_id;
DROP TABLE userlog;
`)
func _1_initialize_schemaDownSqlBytes() ([]byte, error) {
return __1_initialize_schemaDownSql, nil
}
func _1_initialize_schemaDownSql() (*asset, error) {
bytes, err := _1_initialize_schemaDownSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1_initialize_schema.down.sql", size: 144, mode: os.FileMode(420), modTime: time.Unix(1535013291, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __1_initialize_schemaUpSql = []byte(`
-- We store the raw denormalized user logs, because device information
-- might change over time (think a version update, or restoring a
-- backup to a different OS) and it may be useful to see the
-- historical variation. Unique device information is also aggregated
-- incrementally to a separate table, to provide a quick way to obtain
-- the list of devices for a user without a large table scan.
CREATE TABLE devices (
id VARCHAR(64) NOT NULL,
username TEXT NOT NULL,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
first_seen DATETIME,
last_seen DATETIME,
last_device_remote_zone TEXT,
last_device_user_agent TEXT
);
CREATE UNIQUE INDEX idx_devices_id_username ON devices (id, username);
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64) NOT NULL,
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);
CREATE INDEX idx_userlog_username ON userlog (username);
CREATE INDEX idx_userlog_device_id ON userlog (device_id);
`)
func _1_initialize_schemaUpSqlBytes() ([]byte, error) {
return __1_initialize_schemaUpSql, nil
}
func _1_initialize_schemaUpSql() (*asset, error) {
bytes, err := _1_initialize_schemaUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1_initialize_schema.up.sql", size: 1258, mode: os.FileMode(420), modTime: time.Unix(1535013291, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __2_lastloginDownSql = []byte(`DROP INDEX idx_lastlogin_username_service;
DROP INDEX idx_lastlogin_username;
DROP TABLE lastlogin;
`)
func _2_lastloginDownSqlBytes() ([]byte, error) {
return __2_lastloginDownSql, nil
}
func _2_lastloginDownSql() (*asset, error) {
bytes, err := _2_lastloginDownSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "2_lastlogin.down.sql", size: 100, mode: os.FileMode(420), modTime: time.Unix(1551599549, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __2_lastloginUpSql = []byte(`CREATE TABLE lastlogin (
username TEXT NOT NULL,
service TEXT NOT NULL,
timestamp DATETIME NOT NULL
);
CREATE UNIQUE INDEX idx_lastlogin_username_service ON lastlogin (username, service);
CREATE INDEX idx_lastlogin_username ON lastlogin (username);
`)
func _2_lastloginUpSqlBytes() ([]byte, error) {
return __2_lastloginUpSql, nil
}
func _2_lastloginUpSql() (*asset, error) {
bytes, err := _2_lastloginUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "2_lastlogin.up.sql", size: 253, mode: os.FileMode(420), modTime: time.Unix(1551599549, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var __3_userlog_device_nullableUpSql = []byte(`
-- Make the userlog.device_id field nullable. Unfortunately SQLite
-- does not support modifying constraints on a column, so we have to
-- dump and reload the table contents using a temporary table.
PRAGMA foreign_keys=off;
ALTER TABLE userlog RENAME TO _userlog_old;
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64),
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);
INSERT INTO userlog (username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp)
SELECT username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp
FROM _userlog_old;
DROP TABLE _userlog_old;
PRAGMA foreign_keys=on;
`)
func _3_userlog_device_nullableUpSqlBytes() ([]byte, error) {
return __3_userlog_device_nullableUpSql, nil
}
func _3_userlog_device_nullableUpSql() (*asset, error) {
bytes, err := _3_userlog_device_nullableUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "3_userlog_device_nullable.up.sql", size: 1046, mode: os.FileMode(420), modTime: time.Unix(1560152262, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
}
return a.bytes, nil
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
a, err := Asset(name)
if err != nil {
panic("asset: Asset(" + name + "): " + err.Error())
}
return a
}
// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
}
return a.info, nil
}
return nil, fmt.Errorf("AssetInfo %s not found", name)
}
// AssetNames returns the names of the assets.
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"1_initialize_schema.down.sql": _1_initialize_schemaDownSql,
"1_initialize_schema.up.sql": _1_initialize_schemaUpSql,
"2_lastlogin.down.sql": _2_lastloginDownSql,
"2_lastlogin.up.sql": _2_lastloginUpSql,
"3_userlog_device_nullable.up.sql": _3_userlog_device_nullableUpSql,
}
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
cannonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(cannonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
}
}
if node.Func != nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
rv := make([]string, 0, len(node.Children))
for childName := range node.Children {
rv = append(rv, childName)
}
return rv, nil
}
type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"1_initialize_schema.down.sql": &bintree{_1_initialize_schemaDownSql, map[string]*bintree{}},
"1_initialize_schema.up.sql": &bintree{_1_initialize_schemaUpSql, map[string]*bintree{}},
"2_lastlogin.down.sql": &bintree{_2_lastloginDownSql, map[string]*bintree{}},
"2_lastlogin.up.sql": &bintree{_2_lastloginUpSql, map[string]*bintree{}},
"3_userlog_device_nullable.up.sql": &bintree{_3_userlog_device_nullableUpSql, map[string]*bintree{}},
}}
// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
data, err := Asset(name)
if err != nil {
return err
}
info, err := AssetInfo(name)
if err != nil {
return err
}
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}
err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
if err != nil {
return err
}
return nil
}
// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
children, err := AssetDir(name)
// File
if err != nil {
return RestoreAsset(dir, name)
}
// Dir
for _, child := range children {
err = RestoreAssets(dir, filepath.Join(name, child))
if err != nil {
return err
}
}
return nil
}
func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}
server/analysis.go
+ 2
− 1
View file @ f69bb3c1
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"git.autistici.org/ai3/go-common/sqlutil"
"git.autistici.org/id/usermetadb" "git.autistici.org/id/usermetadb"
) )
...@@ -26,7 +27,7 @@ func (d *analysisService) Close() { ...@@ -26,7 +27,7 @@ func (d *analysisService) Close() {
func (d *analysisService) CheckDevice(ctx context.Context, username string, deviceInfo *usermetadb.DeviceInfo) (bool, error) { func (d *analysisService) CheckDevice(ctx context.Context, username string, deviceInfo *usermetadb.DeviceInfo) (bool, error) {
var seen bool var seen bool
err := withReadonlyTX(d.db, func(tx *sql.Tx) error { err := sqlutil.WithReadonlyTx(ctx, d.db, func(tx *sql.Tx) error {
err := tx.QueryRow(analysisStatements["check_device_info"], username, deviceInfo.ID).Scan(&seen) err := tx.QueryRow(analysisStatements["check_device_info"], username, deviceInfo.ID).Scan(&seen)
if err != nil && err != sql.ErrNoRows { if err != nil && err != sql.ErrNoRows {
return err return err
......
server/lastlogin.go
+ 4
− 3
View file @ f69bb3c1
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"strings" "strings"
"time" "time"
"git.autistici.org/ai3/go-common/sqlutil"
"git.autistici.org/id/usermetadb" "git.autistici.org/id/usermetadb"
) )
...@@ -63,7 +64,7 @@ func (l *lastloginDB) AddLastLogin(ctx context.Context, entry *usermetadb.LastLo ...@@ -63,7 +64,7 @@ func (l *lastloginDB) AddLastLogin(ctx context.Context, entry *usermetadb.LastLo
roundTimestamp(entry.Timestamp), roundTimestamp(entry.Timestamp),
} }
return withTX(l.db, func(tx *sql.Tx) error { return sqlutil.WithTx(ctx, l.db, func(tx *sql.Tx) error {
_, err := tx.Exec(lastloginDBStatements["insert_or_replace_last_login"], args...) _, err := tx.Exec(lastloginDBStatements["insert_or_replace_last_login"], args...)
return err return err
}) })
...@@ -86,7 +87,7 @@ func (l *lastloginDB) GetLastLogin(ctx context.Context, username string, service ...@@ -86,7 +87,7 @@ func (l *lastloginDB) GetLastLogin(ctx context.Context, username string, service
} }
var entries []*usermetadb.LastLoginEntry var entries []*usermetadb.LastLoginEntry
err := withReadonlyTX(l.db, func(tx *sql.Tx) error { err := sqlutil.WithReadonlyTx(ctx, l.db, func(tx *sql.Tx) error {
rows, err := tx.Query(lastloginDBStatements[stmt], args...) rows, err := tx.Query(lastloginDBStatements[stmt], args...)
if err != nil { if err != nil {
return err return err
...@@ -131,7 +132,7 @@ func (l *lastloginDB) GetUnusedAccounts(ctx context.Context, usernames []string, ...@@ -131,7 +132,7 @@ func (l *lastloginDB) GetUnusedAccounts(ctx context.Context, usernames []string,
} }
args = append(args, cutoff) args = append(args, cutoff)
err := withReadonlyTX(l.db, func(tx *sql.Tx) error { err := sqlutil.WithReadonlyTx(ctx, l.db, func(tx *sql.Tx) error {
rows, err := tx.Query(q, args...) rows, err := tx.Query(q, args...)
if err != nil { if err != nil {
return err return err
......
server/sql.go
+ 83
− 93
View file @ f69bb3c1
...@@ -2,102 +2,92 @@ package server ...@@ -2,102 +2,92 @@ package server
import ( import (
"database/sql" "database/sql"
"log"
"strings"
migrate "github.com/golang-migrate/migrate/v4" "git.autistici.org/ai3/go-common/sqlutil"
msqlite3 "github.com/golang-migrate/migrate/v4/database/sqlite3"
bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
_ "github.com/mattn/go-sqlite3"
"git.autistici.org/id/usermetadb/migrations"
) )
const dbDriver = "sqlite3" var migrations = []func(*sql.Tx) error{
sqlutil.Statement(`
func openDB(dburi string) (*sql.DB, error) { -- We store the raw denormalized user logs, because device information
if !strings.Contains(dburi, "?") { -- might change over time (think a version update, or restoring a
// Tune the SQLite engine: -- backup to a different OS) and it may be useful to see the
// - activate the WAL journal -- historical variation. Unique device information is also aggregated
// - enable auto-vacuuming since we delete lots of data -- incrementally to a separate table, to provide a quick way to obtain
// - disable syncing, to reduce disk write load at the expense -- the list of devices for a user without a large table scan.
// of durability. CREATE TABLE devices (
dburi += "?_journal=WAL&_sync=OFF&_auto_vacuum=incremental" id VARCHAR(64) NOT NULL,
} username TEXT NOT NULL,
device_browser TEXT,
db, err := sql.Open(dbDriver, dburi) device_os TEXT,
if err != nil { device_mobile BOOL,
return nil, err first_seen DATETIME,
} last_seen DATETIME,
last_device_remote_zone TEXT,
if err = runDatabaseMigrations(db); err != nil { last_device_user_agent TEXT
db.Close() // nolint );`, `
return nil, err CREATE UNIQUE INDEX idx_devices_id_username ON devices (id, username);
`, `
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64) NOT NULL,
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);`, `
CREATE INDEX idx_userlog_username ON userlog (username);
`, `
CREATE INDEX idx_userlog_device_id ON userlog (device_id);
`),
sqlutil.Statement(`
CREATE TABLE lastlogin (
username TEXT NOT NULL,
service TEXT NOT NULL,
timestamp DATETIME NOT NULL
);
`, `
CREATE UNIQUE INDEX idx_lastlogin_username_service ON lastlogin (username, service);
`, `
CREATE INDEX idx_lastlogin_username ON lastlogin (username);
`),
sqlutil.Statement(`
PRAGMA foreign_keys=off;
`, `
ALTER TABLE userlog RENAME TO _userlog_old;
`, `
CREATE TABLE userlog (
username TEXT NOT NULL,
service TEXT NOT NULL,
log_type TEXT NOT NULL,
login_method TEXT,
message TEXT,
device_id VARCHAR(64),
device_remote_zone TEXT,
device_user_agent TEXT,
device_browser TEXT,
device_os TEXT,
device_mobile BOOL,
timestamp DATETIME
);
`, `
INSERT INTO userlog (username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp)
SELECT username, service, log_type, login_method, message, device_id, device_remote_zone, device_user_agent, device_browser, device_os, device_mobile, timestamp
FROM _userlog_old;
`, `
DROP TABLE _userlog_old;
`, `
PRAGMA foreign_keys=on;
`),
} }
// Limit the pool to a single connection. func openDB(dburi string) (*sql.DB, error) {
// https://github.com/mattn/go-sqlite3/issues/209 return sqlutil.OpenDB(dburi, sqlutil.WithMigrations(migrations))
db.SetMaxOpenConns(1)
return db, nil
}
type migrateLogger struct{}
func (l migrateLogger) Printf(format string, v ...interface{}) {
log.Printf("db: "+format, v...)
}
func (l migrateLogger) Verbose() bool { return true }
func runDatabaseMigrations(db *sql.DB) error {
si, err := bindata.WithInstance(bindata.Resource(
migrations.AssetNames(),
func(name string) ([]byte, error) {
return migrations.Asset(name)
}))
if err != nil {
return err
}
di, err := msqlite3.WithInstance(db, &msqlite3.Config{
MigrationsTable: msqlite3.DefaultMigrationsTable,
DatabaseName: "usermetadb",
})
if err != nil {
return err
}
m, err := migrate.NewWithInstance("bindata", si, dbDriver, di)
if err != nil {
return err
}
m.Log = &migrateLogger{}
log.Printf("running database migrations")
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return err
}
return nil
}
func withTX(db *sql.DB, f func(*sql.Tx) error) error {
tx, err := db.Begin()
if err != nil {
return err
}
if err := f(tx); err != nil {
tx.Rollback() // nolint
return err
}
return tx.Commit()
}
func withReadonlyTX(db *sql.DB, f func(*sql.Tx) error) error {
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback() // nolint
return f(tx)
} }
server/userlog.go
+ 4
− 3
View file @ f69bb3c1
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"regexp" "regexp"
"time" "time"
"git.autistici.org/ai3/go-common/sqlutil"
"git.autistici.org/id/usermetadb" "git.autistici.org/id/usermetadb"
) )
...@@ -241,7 +242,7 @@ func (u *userlogDB) AddLog(ctx context.Context, entry *usermetadb.LogEntry) erro ...@@ -241,7 +242,7 @@ func (u *userlogDB) AddLog(ctx context.Context, entry *usermetadb.LogEntry) erro
args = append(args, entry.DeviceInfo.Mobile) args = append(args, entry.DeviceInfo.Mobile)
} }
if err := withTX(u.db, func(tx *sql.Tx) error { if err := sqlutil.WithTx(ctx, u.db, func(tx *sql.Tx) error {
if entry.DeviceInfo != nil { if entry.DeviceInfo != nil {
if err := u.updateDeviceInfo(tx, entry.Username, entry.DeviceInfo); err != nil { if err := u.updateDeviceInfo(tx, entry.Username, entry.DeviceInfo); err != nil {
return err return err
...@@ -318,7 +319,7 @@ func (u *userlogDB) GetUserLogs(ctx context.Context, username string, maxDays, l ...@@ -318,7 +319,7 @@ func (u *userlogDB) GetUserLogs(ctx context.Context, username string, maxDays, l
} }
var out []*usermetadb.LogEntry var out []*usermetadb.LogEntry
err := withReadonlyTX(u.db, func(tx *sql.Tx) error { err := sqlutil.WithReadonlyTx(ctx, u.db, func(tx *sql.Tx) error {
rows, err := tx.Query(userlogDBStatements["get_user_logs"], username, cutoff, limit) rows, err := tx.Query(userlogDBStatements["get_user_logs"], username, cutoff, limit)
if err != nil { if err != nil {
return err return err
...@@ -339,7 +340,7 @@ func (u *userlogDB) GetUserLogs(ctx context.Context, username string, maxDays, l ...@@ -339,7 +340,7 @@ func (u *userlogDB) GetUserLogs(ctx context.Context, username string, maxDays, l
func (u *userlogDB) GetUserDevices(ctx context.Context, username string) ([]*usermetadb.MetaDeviceInfo, error) { func (u *userlogDB) GetUserDevices(ctx context.Context, username string) ([]*usermetadb.MetaDeviceInfo, error) {
var out []*usermetadb.MetaDeviceInfo var out []*usermetadb.MetaDeviceInfo
err := withReadonlyTX(u.db, func(tx *sql.Tx) error { err := sqlutil.WithReadonlyTx(ctx, u.db, func(tx *sql.Tx) error {
rows, err := tx.Query(userlogDBStatements["devices_with_counts"], username) rows, err := tx.Query(userlogDBStatements["devices_with_counts"], username)
if err != nil { if err != nil {
return err return err
......
vendor/git.autistici.org/ai3/go-common/go.mod
+ 1
− 0
View file @ f69bb3c1
...@@ -16,6 +16,7 @@ require ( ...@@ -16,6 +16,7 @@ require (
github.com/gofrs/flock v0.8.0 // indirect github.com/gofrs/flock v0.8.0 // indirect
github.com/google/go-cmp v0.5.8 github.com/google/go-cmp v0.5.8
github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40
github.com/mattn/go-sqlite3 v1.14.14
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_golang v1.12.2
github.com/russross/blackfriday/v2 v2.1.0 github.com/russross/blackfriday/v2 v2.1.0
......
vendor/git.autistici.org/ai3/go-common/go.sum
+ 3
− 0
View file @ f69bb3c1
...@@ -552,7 +552,10 @@ github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxm ...@@ -552,7 +552,10 @@ github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxm
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw=
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
......
vendor/git.autistici.org/ai3/go-common/sqlutil/db.go 0 → 100644
+ 155
− 0
View file @ f69bb3c1
package sqlutil
import (
"context"
"database/sql"
"fmt"
"log"
"strings"
_ "github.com/mattn/go-sqlite3"
)
// DebugMigrations can be set to true to dump statements to stderr.
var DebugMigrations bool
const defaultOptions = "?cache=shared&_busy_timeout=10000&_journal=WAL&_sync=OFF"
type sqlOptions struct {
migrations []func(*sql.Tx) error
sqlopts string
}
type Option func(*sqlOptions)
func WithMigrations(migrations []func(*sql.Tx) error) Option {
return func(opts *sqlOptions) {
opts.migrations = migrations
}
}
func WithSqliteOptions(sqlopts string) Option {
return func(opts *sqlOptions) {
opts.sqlopts = sqlopts
}
}
// OpenDB opens a SQLite database and runs the database migrations.
func OpenDB(dburi string, options ...Option) (*sql.DB, error) {
var opts sqlOptions
opts.sqlopts = defaultOptions
for _, o := range options {
o(&opts)
}
// Add sqlite3-specific parameters if none are already
// specified in the connection URI.
if !strings.Contains(dburi, "?") {
dburi += opts.sqlopts
}
db, err := sql.Open("sqlite3", dburi)
if err != nil {
return nil, err
}
// Limit the pool to a single connection.
// https://github.com/mattn/go-sqlite3/issues/209
db.SetMaxOpenConns(1)
if err = migrate(db, opts.migrations); err != nil {
db.Close() // nolint
return nil, err
}
return db, nil
}
// Fetch legacy (golang-migrate/migrate/v4) schema version.
func getLegacyMigrationVersion(tx *sql.Tx) (int, error) {
var version int
if err := tx.QueryRow(`SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&version); err != nil {
return 0, err
}
return version, nil
}
func migrate(db *sql.DB, migrations []func(*sql.Tx) error) error {
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("DB migration begin transaction: %w", err)
}
defer tx.Rollback() // nolint: errcheck
var idx int
if err = tx.QueryRow("PRAGMA user_version").Scan(&idx); err != nil {
return fmt.Errorf("getting latest applied migration: %w", err)
}
if idx == 0 {
if legacyIdx, err := getLegacyMigrationVersion(tx); err == nil {
idx = legacyIdx
}
}
if idx == len(migrations) {
// Already fully migrated, nothing needed.
return nil
} else if idx > len(migrations) {
return fmt.Errorf("database is at version %d, which is more recent than this binary understands", idx)
}
for i, f := range migrations[idx:] {
if err := f(tx); err != nil {
return fmt.Errorf("migration to version %d failed: %w", i+1, err)
}
}
if n := len(migrations); n > 0 {
// For some reason, ? substitution doesn't work in PRAGMA
// statements, sqlite reports a parse error.
if _, err := tx.Exec(fmt.Sprintf("PRAGMA user_version=%d", n)); err != nil {
return fmt.Errorf("recording new DB version: %w", err)
}
log.Printf("db migration: upgraded schema version %d -> %d", idx, n)
}
return tx.Commit()
}
// Statement for migrations, executes one or more SQL statements.
func Statement(idl ...string) func(*sql.Tx) error {
return func(tx *sql.Tx) error {
for _, stmt := range idl {
if DebugMigrations {
log.Printf("db migration: executing: %s", stmt)
}
if _, err := tx.Exec(stmt); err != nil {
return err
}
}
return nil
}
}
func WithTx(ctx context.Context, db *sql.DB, f func(*sql.Tx) error) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
if err := f(tx); err != nil {
tx.Rollback() // nolint
return err
}
return tx.Commit()
}
func WithReadonlyTx(ctx context.Context, db *sql.DB, f func(*sql.Tx) error) error {
tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return err
}
defer tx.Rollback() // nolint
return f(tx)
}
vendor/git.autistici.org/ai3/go-common/sqlutil/query_builder.go 0 → 100644
+ 47
− 0
View file @ f69bb3c1
package sqlutil
import (
"database/sql"
"strings"
)
// QueryBuilder is a very simple programmatic query builder, to
// simplify the operation of adding WHERE and ORDER BY clauses
// programatically.
type QueryBuilder struct {
base string
tail string
where []string
args []interface{}
}
// NewQuery returns a query builder starting with the given base query.
func NewQuery(s string) *QueryBuilder {
return &QueryBuilder{base: s}
}
// OrderBy adds an ORDER BY clause.
func (q *QueryBuilder) OrderBy(s string) *QueryBuilder {
q.tail += " ORDER BY "
q.tail += s
return q
}
// Where adds a WHERE clause with associated argument(s).
func (q *QueryBuilder) Where(clause string, args ...interface{}) *QueryBuilder {
q.where = append(q.where, clause)
q.args = append(q.args, args...)
return q
}
// Query executes the resulting query in the given transaction.
func (q *QueryBuilder) Query(tx *sql.Tx) (*sql.Rows, error) {
s := q.base
if len(q.where) > 0 {
s += " WHERE "
s += strings.Join(q.where, " AND ")
}
s += q.tail
return tx.Query(s, q.args...)
}
vendor/github.com/golang-migrate/migrate/v4/.dockerignore deleted 100644 → 0
+ 0
− 12
View file @ 4956648d
# Project
FAQ.md
README.md
LICENSE
.gitignore
.travis.yml
CONTRIBUTING.md
MIGRATIONS.md
docker-deploy.sh
# Golang
testing
vendor/github.com/golang-migrate/migrate/v4/.gitignore deleted 100644 → 0
+ 0
− 10
View file @ 4956648d
.DS_Store
cli/build
cli/cli
cli/migrate
.coverage
.godoc.pid
vendor/
.vscode/
.idea
dist/
vendor/github.com/golang-migrate/migrate/v4/.golangci.yml deleted 100644 → 0
+ 0
− 26
View file @ 4956648d
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
timeout: 5m
linters:
enable:
#- golint
- interfacer
- unconvert
#- dupl
- goconst
- gofmt
- misspell
- unparam
- nakedret
- prealloc
#- gosec
linters-settings:
misspell:
locale: US
issues:
max-same-issues: 0
max-issues-per-linter: 0
exclude-use-default: false
exclude:
# gosec: Duplicated errcheck checks
- G104
vendor/github.com/golang-migrate/migrate/v4/.goreleaser.yml deleted 100644 → 0
+ 0
− 102
View file @ 4956648d
project_name: migrate
before:
hooks:
- go mod tidy
builds:
- env:
- CGO_ENABLED=0
goos:
- linux
- windows
- darwin
goarch:
- amd64
- arm
- arm64
- 386
goarm:
- 7
main: ./cmd/migrate
ldflags:
- '-w -s -X main.Version={{ .Version }} -extldflags "static"'
flags:
- "-tags={{ .Env.DATABASE }} {{ .Env.SOURCE }}"
- "-trimpath"
nfpms:
- homepage: "https://github.com/golang-migrate/migrate"
maintainer: "dhui@users.noreply.github.com"
license: MIT
description: "Database migrations"
formats:
- deb
file_name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
dockers:
- goos: linux
goarch: amd64
dockerfile: Dockerfile.github-actions
use: buildx
ids:
- migrate
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
build_flag_templates:
- '--label=org.opencontainers.image.created={{ .Date }}'
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
- '--label=org.opencontainers.image.version={{ .Version }}'
- "--label=org.opencontainers.image.source={{ .GitURL }}"
- "--platform=linux/amd64"
- goos: linux
goarch: arm64
dockerfile: Dockerfile.github-actions
use: buildx
ids:
- migrate
image_templates:
- 'migrate/migrate:{{ .Tag }}-arm64'
build_flag_templates:
- '--label=org.opencontainers.image.created={{ .Date }}'
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
- '--label=org.opencontainers.image.version={{ .Version }}'
- "--label=org.opencontainers.image.source={{ .GitURL }}"
- "--platform=linux/arm64"
docker_manifests:
- name_template: 'migrate/migrate:{{ .Tag }}'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
- name_template: 'migrate/migrate:{{ .Major }}'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
- name_template: 'migrate/migrate:latest'
image_templates:
- 'migrate/migrate:{{ .Tag }}-amd64'
- 'migrate/migrate:{{ .Tag }}-arm64'
archives:
- name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: zip
checksum:
name_template: 'sha256sum.txt'
release:
draft: true
prerelease: auto
source:
enabled: true
format: zip
changelog:
skip: false
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
- Merge pull request
- Merge branch
- go mod tidy
snapshot:
name_template: "{{ .Tag }}-next"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment