add a tool to dump and restore the db

cmd/auditd-dbtool/auditd-dbtool.go

+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"git.autistici.org/ai/audit/server"
+)
+
+var (
+	doDump = flag.Bool("dump", false, "dump all messages in the db")
+	doRestore = flag.Bool("restore", false, "restore messages from a dump")
+	dbDir = flag.String("data-dir", "/var/lib/auditd", "Path to the database directory")
+)
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	if (!*doDump && !*doRestore) || (*doDump && *doRestore) {
+		log.Fatal("Must specify either --dump or --restore")
+	}
+
+	db := server.NewDB(*dbDir, nil)
+	defer db.Close()
+
+	var err error
+	if *doDump {
+		err = db.Dump(os.Stdout)
+	} else {
+		err = db.Restore(os.Stdin)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+}

debian/install-golang

@@ -13,7 +13,7 @@ mangle_arch() {
 ROOT="$1"
 ARCH=`mangle_arch $2`
 VERSION="$3"
-DL_URL=https://go.googlecode.com/files/go${VERSION}.linux-${ARCH}.tar.gz
+DL_URL=https://storage.googleapis.com/golang/go${VERSION}.linux-${ARCH}.tar.gz
 
 mkdir -p ${ROOT}
 wget -O - ${DL_URL} \

debian/rules

@@ -8,7 +8,7 @@
 export DH_OPTIONS
 
 GOPKG = git.autistici.org/ai/audit
-GO_VERSION = 1.2.1
+GO_VERSION = 1.5.3
 
 DEBDIR = $(CURDIR)/debian
 BUILDDIR = $(CURDIR)/debian/build
@@ -33,6 +33,7 @@ override_dh_install:
 	 $(GOROOT)/bin/go get -d -v ./$(GOPKG)/... && \
 	 CGO_CFLAGS=-Dleveldb_free=free $(GOROOT)/bin/go install -v ./$(GOPKG)/... && \
 	 install -m 755 -o root -g root $(BUILDDIR)/bin/auditd $(DEBDIR)/ai-auditd/usr/sbin/auditd && \
+	 install -m 755 -o root -g root $(BUILDDIR)/bin/auditd-dbtool $(DEBDIR)/ai-auditd/usr/sbin/auditd-dbtool && \
 	 install -m 755 -o root -g root $(BUILDDIR)/bin/localauditd $(DEBDIR)/localauditd/usr/sbin/localauditd && \
 	 install -m 755 -o root -g root $(BUILDDIR)/bin/auditc $(DEBDIR)/auditc/usr/bin/auditc)
 

server/db.go

 package server
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
+	"io"
 	"log"
 	"strings"
 
@@ -116,11 +118,7 @@ func makePrefixKeyRange(prefix []byte) ([]byte, []byte) {
 	return startKey, endKey
 }
 
-// Write a Message to the database.
-func (db *DB) Write(msg audit.Message) error {
-	wb := levigo.NewWriteBatch()
-	defer wb.Close()
-
+func (db *DB) writeToBatch(msg audit.Message, wb *levigo.WriteBatch) {
 	// Generate a unique ID for the message.
 	msgid := audit.NewUniqueId(msg.Stamp())
 	wb.Put(makeKey("messages", msgid), msg.ToJSON())
@@ -138,6 +136,60 @@ func (db *DB) Write(msg audit.Message) error {
 
 		wb.Put(makeKey("index", []byte(key), []byte(value), msgid), msgid)
 	}
+}
+
+// Write a Message to the database.
+func (db *DB) Write(msg audit.Message) error {
+	wb := levigo.NewWriteBatch()
+	defer wb.Close()
+
+	db.writeToBatch(msg, wb)
+
+	wo := levigo.NewWriteOptions()
+	defer wo.Close()
+	return db.leveldb.Write(wo, wb)
+}
+
+// Dump contents to a writer.
+func (db *DB) Dump(w io.Writer) error {
+	ro := levigo.NewReadOptions()
+	ro.SetFillCache(false)
+	defer ro.Close()
+
+	i := db.leveldb.NewIterator(ro)
+	defer i.Close()
+
+	startKey, endKey := makePrefixKeyRange(makeKey("messages"))
+	for i.Seek(startKey); i.Valid() && bytes.Compare(i.Key(), endKey) < 0; i.Next() {
+		fmt.Fprintf(w, "%s\n", i.Value())
+	}
+	if err := i.GetError(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Restore contents from a dump.
+func (db *DB) Restore(r io.Reader) error {
+	wb := levigo.NewWriteBatch()
+	defer wb.Close()
+
+	errs := 0
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		msg, err := audit.MessageFromJSON(scanner.Bytes())
+		if err != nil {
+			errs++
+			continue
+		}
+		db.writeToBatch(msg, wb)
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	if errs > 0 {
+		return fmt.Errorf("failed to restore %d records", errs)
+	}
 
 	wo := levigo.NewWriteOptions()
 	defer wo.Close()

server/db_test.go

 package server
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -33,6 +34,44 @@ func testInsertRecords(db *DB, n int) error {
 	return nil
 }
 
+func TestDB_QueryRealData(t *testing.T) {
+	dbdir, _ := ioutil.TempDir("", "auditdb-")
+	defer os.RemoveAll(dbdir)
+
+	db := NewDB(dbdir, nil)
+	defer db.Close()
+
+	testData := []string{
+		`{"host":"opposizione","message":"created email solemnis@privacyrequired.com","request_type":"email","source":"accountserver","stamp":1.437922949e+09,"user":"solemnis@privacyrequired.com","who":"ai-services"}`,
+		`{"alias":"info1@grrlz.net","host":"opposizione","message":"deleted alias info1@grrlz.net","source":"accountserver","stamp":1.434558486e+09,"user":"elitee@hacari.com","who":"elitee@hacari.com"}`,
+		`{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.439408672e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+		`{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.432290377e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+		`{"alias":"skankindaddy@autistici.org","host":"opposizione","message":"created alias skankindaddy@autistici.org","source":"accountserver","stamp":1.444286071e+09,"user":"ilnonno@autistici.org","who":"ilnonno@autistici.org"}`,
+		`{"alias":"sknkindaddy@autistici.org","host":"opposizione","message":"deleted alias sknkindaddy@autistici.org","source":"accountserver","stamp":1.44428608e+09,"user":"ilnonno@autistici.org","who":"ilnonno@autistici.org"}`,
+		`{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.426104104e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+		`{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.423795607e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+		`{"host":"opposizione","message":"password reset","source":"accountserver","stamp":1.445412207e+09,"user":"blackwolf@autistici.org","who":"accountserver"}`,
+	}
+	for _, data := range testData {
+		msg, _ := audit.MessageFromJSON([]byte(data))
+		if err := db.Write(msg); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Test simple query.
+	result, err := db.Query(map[string]string{"user": "blackwolf@autistici.org"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result) != 5 {
+		t.Fatalf("bad # of results: %+v", result)
+	}
+	if result[0]["user"].(string) != "blackwolf@autistici.org" {
+		t.Fatalf("bad result: %+v", result[0])
+	}
+}
+
 func TestDB_Query(t *testing.T) {
 	dbdir, _ := ioutil.TempDir("", "auditdb-")
 	defer os.RemoveAll(dbdir)
@@ -86,6 +125,39 @@ func TestDB_Query(t *testing.T) {
 	}
 }
 
+func TestDB_DumpAndRestore(t *testing.T) {
+	dbdir, _ := ioutil.TempDir("", "auditdb-")
+	defer os.RemoveAll(dbdir)
+
+	db := NewDB(dbdir, nil)
+	defer db.Close()
+
+	testData := []string{
+		`{"user": "a", "host": "one", "message": "wow", "stamp": 123456789}`,
+		`{"user": "b", "host": "one", "message": "wow", "stamp": 123456790}`,
+		`{"user": "c", "host": "two", "message": "wow", "stamp": 123456790}`,
+	}
+	for _, data := range testData {
+		msg, _ := audit.MessageFromJSON([]byte(data))
+		if err := db.Write(msg); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	var buf bytes.Buffer
+	if err := db.Dump(&buf); err != nil {
+		t.Fatalf("Dump: %v", err)
+	}
+
+	dbdir2, _ := ioutil.TempDir("", "auditdb-")
+	defer os.RemoveAll(dbdir2)
+	db2 := NewDB(dbdir2, nil)
+	defer db2.Close()
+	if err := db2.Restore(&buf); err != nil {
+		t.Fatalf("Restore: %v", err)
+	}
+}
+
 // Benchmark straight insertion performance.
 func BenchmarkDB_Insert(b *testing.B) {
 	dbdir, _ := ioutil.TempDir("", "auditdb-")

unique_id.go

@@ -7,6 +7,12 @@ import (
 	"time"
 )
 
+func init() {
+	// Just need to differentiate between runs, not aiming for
+	// cryptographic safety here.
+	rand.Seed(time.Now().Unix())
+}
+
 // Generate a new time-based unique ID for use as primary key. The
 // resulting IDs can be sorted lexicographically preserving the time
 // ordering.
@@ -15,7 +21,7 @@ func NewUniqueId(t time.Time) []byte {
 
 	// The key consists of serialized time + sufficiently large
 	// random number. This should allow a high insertion rate.
-	binary.Write(&b, binary.BigEndian, t)
+	binary.Write(&b, binary.BigEndian, t.UnixNano())
 	binary.Write(&b, binary.BigEndian, rand.Int63())
 
 	return b.Bytes()