diff --git a/cmd/auditd-dbtool/auditd-dbtool.go b/cmd/auditd-dbtool/auditd-dbtool.go
new file mode 100644
index 0000000000000000000000000000000000000000..6803d840f4b38c727cd066ae1268bc37d2620c37
--- /dev/null
+++ b/cmd/auditd-dbtool/auditd-dbtool.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+ "flag"
+ "log"
+ "os"
+
+ "git.autistici.org/ai/audit/server"
+)
+
+var (
+ doDump = flag.Bool("dump", false, "dump all messages in the db")
+ doRestore = flag.Bool("restore", false, "restore messages from a dump")
+ dbDir = flag.String("data-dir", "/var/lib/auditd", "Path to the database directory")
+)
+
+func main() {
+ log.SetFlags(0)
+ flag.Parse()
+
+ if (!*doDump && !*doRestore) || (*doDump && *doRestore) {
+ log.Fatal("Must specify either --dump or --restore")
+ }
+
+ db := server.NewDB(*dbDir, nil)
+ defer db.Close()
+
+ var err error
+ if *doDump {
+ err = db.Dump(os.Stdout)
+ } else {
+ err = db.Restore(os.Stdin)
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/debian/install-golang b/debian/install-golang
index 57e181f082d567942b045fd275abb8647431de9f..f4aa095cfa630abfa05b2325b3b8036e59cac0a0 100755
--- a/debian/install-golang
+++ b/debian/install-golang
@@ -13,7 +13,7 @@ mangle_arch() {
ROOT="$1"
ARCH=`mangle_arch $2`
VERSION="$3"
-DL_URL=https://go.googlecode.com/files/go${VERSION}.linux-${ARCH}.tar.gz
+DL_URL=https://storage.googleapis.com/golang/go${VERSION}.linux-${ARCH}.tar.gz
mkdir -p ${ROOT}
wget -O - ${DL_URL} \
diff --git a/debian/rules b/debian/rules
index 78992644b22dee419a76d5022de70ff58f2484aa..de8fd35631347a035f57536d9427e538af1c72dc 100755
--- a/debian/rules
+++ b/debian/rules
@@ -8,7 +8,7 @@
export DH_OPTIONS
GOPKG = git.autistici.org/ai/audit
-GO_VERSION = 1.2.1
+GO_VERSION = 1.5.3
DEBDIR = $(CURDIR)/debian
BUILDDIR = $(CURDIR)/debian/build
@@ -33,6 +33,7 @@ override_dh_install:
$(GOROOT)/bin/go get -d -v ./$(GOPKG)/... && \
CGO_CFLAGS=-Dleveldb_free=free $(GOROOT)/bin/go install -v ./$(GOPKG)/... && \
install -m 755 -o root -g root $(BUILDDIR)/bin/auditd $(DEBDIR)/ai-auditd/usr/sbin/auditd && \
+ install -m 755 -o root -g root $(BUILDDIR)/bin/auditd-dbtool $(DEBDIR)/ai-auditd/usr/sbin/auditd-dbtool && \
install -m 755 -o root -g root $(BUILDDIR)/bin/localauditd $(DEBDIR)/localauditd/usr/sbin/localauditd && \
install -m 755 -o root -g root $(BUILDDIR)/bin/auditc $(DEBDIR)/auditc/usr/bin/auditc)
diff --git a/server/db.go b/server/db.go
index 355ccd37018a5c079a91aebc52e1c507a6cec8f2..4601799d4c97a2eca4befaa4aa97384eb82f69a3 100644
--- a/server/db.go
+++ b/server/db.go
@@ -1,8 +1,10 @@
package server
import (
+ "bufio"
"bytes"
"fmt"
+ "io"
"log"
"strings"
@@ -116,11 +118,7 @@ func makePrefixKeyRange(prefix []byte) ([]byte, []byte) {
return startKey, endKey
}
-// Write a Message to the database.
-func (db *DB) Write(msg audit.Message) error {
- wb := levigo.NewWriteBatch()
- defer wb.Close()
-
+func (db *DB) writeToBatch(msg audit.Message, wb *levigo.WriteBatch) {
// Generate a unique ID for the message.
msgid := audit.NewUniqueId(msg.Stamp())
wb.Put(makeKey("messages", msgid), msg.ToJSON())
@@ -138,6 +136,60 @@ func (db *DB) Write(msg audit.Message) error {
wb.Put(makeKey("index", []byte(key), []byte(value), msgid), msgid)
}
+}
+
+// Write a Message to the database.
+func (db *DB) Write(msg audit.Message) error {
+ wb := levigo.NewWriteBatch()
+ defer wb.Close()
+
+ db.writeToBatch(msg, wb)
+
+ wo := levigo.NewWriteOptions()
+ defer wo.Close()
+ return db.leveldb.Write(wo, wb)
+}
+
+// Dump contents to a writer.
+func (db *DB) Dump(w io.Writer) error {
+ ro := levigo.NewReadOptions()
+ ro.SetFillCache(false)
+ defer ro.Close()
+
+ i := db.leveldb.NewIterator(ro)
+ defer i.Close()
+
+ startKey, endKey := makePrefixKeyRange(makeKey("messages"))
+ for i.Seek(startKey); i.Valid() && bytes.Compare(i.Key(), endKey) < 0; i.Next() {
+ fmt.Fprintf(w, "%s\n", i.Value())
+ }
+ if err := i.GetError(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// Restore contents from a dump.
+func (db *DB) Restore(r io.Reader) error {
+ wb := levigo.NewWriteBatch()
+ defer wb.Close()
+
+ errs := 0
+ scanner := bufio.NewScanner(r)
+ for scanner.Scan() {
+ msg, err := audit.MessageFromJSON(scanner.Bytes())
+ if err != nil {
+ errs++
+ continue
+ }
+ db.writeToBatch(msg, wb)
+ }
+ if err := scanner.Err(); err != nil {
+ return err
+ }
+ if errs > 0 {
+ return fmt.Errorf("failed to restore %d records", errs)
+ }
wo := levigo.NewWriteOptions()
defer wo.Close()
diff --git a/server/db_test.go b/server/db_test.go
index a945d35c327c551c2916a1ea94d534b3f6b78845..7c197c9f1d64c9927a45af49329549cdef74548c 100644
--- a/server/db_test.go
+++ b/server/db_test.go
@@ -1,6 +1,7 @@
package server
import (
+ "bytes"
"fmt"
"io/ioutil"
"os"
@@ -33,6 +34,44 @@ func testInsertRecords(db *DB, n int) error {
return nil
}
+func TestDB_QueryRealData(t *testing.T) {
+ dbdir, _ := ioutil.TempDir("", "auditdb-")
+ defer os.RemoveAll(dbdir)
+
+ db := NewDB(dbdir, nil)
+ defer db.Close()
+
+ testData := []string{
+ `{"host":"opposizione","message":"created email solemnis@privacyrequired.com","request_type":"email","source":"accountserver","stamp":1.437922949e+09,"user":"solemnis@privacyrequired.com","who":"ai-services"}`,
+ `{"alias":"info1@grrlz.net","host":"opposizione","message":"deleted alias info1@grrlz.net","source":"accountserver","stamp":1.434558486e+09,"user":"elitee@hacari.com","who":"elitee@hacari.com"}`,
+ `{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.439408672e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+ `{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.432290377e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+ `{"alias":"skankindaddy@autistici.org","host":"opposizione","message":"created alias skankindaddy@autistici.org","source":"accountserver","stamp":1.444286071e+09,"user":"ilnonno@autistici.org","who":"ilnonno@autistici.org"}`,
+ `{"alias":"sknkindaddy@autistici.org","host":"opposizione","message":"deleted alias sknkindaddy@autistici.org","source":"accountserver","stamp":1.44428608e+09,"user":"ilnonno@autistici.org","who":"ilnonno@autistici.org"}`,
+ `{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.426104104e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+ `{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.423795607e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+ `{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.445412207e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+ }
+ for _, data := range testData {
+ msg, _ := audit.MessageFromJSON([]byte(data))
+ if err := db.Write(msg); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // Test simple query.
+ result, err := db.Query(map[string]string{"user": "blackwolf@autistici.org"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(result) != 5 {
+ t.Fatalf("bad # of results: %+v", result)
+ }
+ if result[0]["user"].(string) != "blackwolf@autistici.org" {
+ t.Fatalf("bad result: %+v", result[0])
+ }
+}
+
func TestDB_Query(t *testing.T) {
dbdir, _ := ioutil.TempDir("", "auditdb-")
defer os.RemoveAll(dbdir)
@@ -86,6 +125,39 @@ func TestDB_Query(t *testing.T) {
}
}
+func TestDB_DumpAndRestore(t *testing.T) {
+ dbdir, _ := ioutil.TempDir("", "auditdb-")
+ defer os.RemoveAll(dbdir)
+
+ db := NewDB(dbdir, nil)
+ defer db.Close()
+
+ testData := []string{
+ `{"user": "a", "host": "one", "message": "wow", "stamp": 123456789}`,
+ `{"user": "b", "host": "one", "message": "wow", "stamp": 123456790}`,
+ `{"user": "c", "host": "two", "message": "wow", "stamp": 123456790}`,
+ }
+ for _, data := range testData {
+ msg, _ := audit.MessageFromJSON([]byte(data))
+ if err := db.Write(msg); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ var buf bytes.Buffer
+ if err := db.Dump(&buf); err != nil {
+ t.Fatalf("Dump: %v", err)
+ }
+
+ dbdir2, _ := ioutil.TempDir("", "auditdb-")
+ defer os.RemoveAll(dbdir2)
+ db2 := NewDB(dbdir2, nil)
+ defer db2.Close()
+ if err := db2.Restore(&buf); err != nil {
+ t.Fatalf("Restore: %v", err)
+ }
+}
+
// Benchmark straight insertion performance.
func BenchmarkDB_Insert(b *testing.B) {
dbdir, _ := ioutil.TempDir("", "auditdb-")
diff --git a/unique_id.go b/unique_id.go
index e5a8f7994afc86595f3aea47a09b7213bdd21585..686e17c0d3f0e49007f3e324ebeb9b360ff39b77 100644
--- a/unique_id.go
+++ b/unique_id.go
@@ -7,6 +7,12 @@ import (
"time"
)
+func init() {
+ // Just need to differentiate between runs, not aiming for
+ // cryptographic safety here.
+ rand.Seed(time.Now().Unix())
+}
+
// Generate a new time-based unique ID for use as primary key. The
// resulting IDs can be sorted lexicographically preserving the time
// ordering.
@@ -15,7 +21,7 @@ func NewUniqueId(t time.Time) []byte {
// The key consists of serialized time + sufficiently large
// random number. This should allow a high insertion rate.
- binary.Write(&b, binary.BigEndian, t)
+ binary.Write(&b, binary.BigEndian, t.UnixNano())
binary.Write(&b, binary.BigEndian, rand.Int63())
return b.Bytes()