Documentation updates

cmd/iprep/server.go

@@ -36,7 +36,8 @@ func (c *serverCommand) Name() string     { return "server" }
 func (c *serverCommand) Synopsis() string { return "run the rpc server" }
 func (c *serverCommand) Usage() string {
 	return `server [<flags>]:
-        Run the GRPC server.
+
+        Run the main database and analysis service, with a GRPC API.
 
 `
 }
@@ -112,6 +113,7 @@ func (c *serverCommand) run(ctx context.Context) error {
 		return grpcServer.Serve(l)
 	})
 
+	// Reload the scoring script on SIGHUP.
 	reloadSigCh := make(chan os.Signal, 1)
 	go func() {
 		for range reloadSigCh {
@@ -120,9 +122,13 @@ func (c *serverCommand) run(ctx context.Context) error {
 	}()
 	signal.Notify(reloadSigCh, syscall.SIGHUP)
 
+	// Terminate the program on SIGINT/SIGTERM.
 	stopSigCh := make(chan os.Signal, 1)
 	signal.Notify(stopSigCh, syscall.SIGINT, syscall.SIGTERM)
 
+	// At this point we got nothing else to do but wait for the
+	// serving goroutines to return. Notify systemd that we're
+	// ready.
 	daemon.SdNotify(false, daemon.SdNotifyReady) // nolint
 
 	select {
@@ -131,6 +137,8 @@ func (c *serverCommand) run(ctx context.Context) error {
 	case <-ctx.Done():
 	}
 
+	// We got here because of an error (or a termination request),
+	// so orderly shut down all the serving components.
 	cancel()
 
 	if httpServer != nil {

cmd/iprep/tail.go

@@ -53,10 +53,24 @@ func (c *tailCommand) Name() string     { return "tail" }
 func (c *tailCommand) Synopsis() string { return "run the rpc tail" }
 func (c *tailCommand) Usage() string {
 	return `tail [<flags>]:
-        Monitor stdin in realtime and update counters based on
+
+        Monitor stdin in realtime and generate events based on
         user-specified detection rules (regexp-based). Meant to
         process third-party logs.
 
+        The patterns file should contain one rule per line, consisting
+        of a regexp bounded by / characters (Perl- style), followed by
+        whitespace, followed by the event type that should be set on
+        the generated event. Regular expressions must contain one
+        capture group for the IP address.
+
+        As an example, the following rule:
+
+            /auth failure from ip ([.0-9]+)/ auth_failure
+
+        will trigger an 'auth_failure' event with the IP extracted
+        from the message.
+
 `
 }
 

script/script.go

@@ -10,10 +10,28 @@ import (
 	"github.com/d5/tengo/stdlib"
 )
 
+// A Script is a compiled Tengo script that will be executed on every
+// request in order to provide a final reputation score for an IP.
+//
+// The script will have a few global variables available:
+//
+// 'ip' is the IP being looked up
+//
+// 'counts' is a map[string]int64 of counters returned from the
+// database in the chosen time interval
+//
+// 'interval' is the duration in seconds of the time interval chosen
+// for this evaluation, so one can compute rates
+//
+// To return the final reputation score (in the range 0 - 1), the
+// script should set the 'score' variable.
+//
 type Script struct {
 	compiled *script.Compiled
 }
 
+// LoadScript creates a new Script by loading its source from the
+// specified file.
 func LoadScript(path string) (*Script, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
@@ -26,43 +44,64 @@ var globalVars = []string{
 	"ip", "score", "counts", "interval", "ext",
 }
 
+// NewScript creates a new Script with the given source.
 func NewScript(src []byte) (*Script, error) {
 	s := script.New(src)
+
 	// Define all the global variables (input and output), so that
-	// they can be set later on the compiled object.
+	// they can be set later on the compiled object. The object
+	// type we use here does not matter.
 	for _, v := range globalVars {
 		if err := s.Add(v, 0); err != nil {
 			return nil, err
 		}
 	}
 
-	// Add available imports.
-	s.SetImports(stdlib.GetModuleMap("math", "times"))
+	// Load the Tengo standard library.
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 
+	// Compile the script. We're going to Clone the compiled
+	// version on every request.
 	c, err := s.Compile()
 	if err != nil {
 		return nil, err
 	}
+
 	return &Script{compiled: c}, nil
 }
 
+// RunIP evaluates the script once with the provided context and
+// returns the resulting score.
 func (script *Script) RunIP(ctx context.Context, ip string, counts map[string]int64, intervalSecs float64, ext map[string]interface{}) (float64, error) {
 	c := script.compiled.Clone()
-	c.Set("ip", ip)
-	c.Set("score", 0.0)
-	c.Set("interval", intervalSecs)
-	c.Set("ext", ext)
+
+	// Set the global variables that constitute the script
+	// execution context. We expect no errors here even if the
+	// script does not reference some of these variables, as we
+	// have pre-defined them in NewScript.
+	if err := c.Set("ip", ip); err != nil {
+		return 0, err
+	}
+	if err := c.Set("score", 0.0); err != nil {
+		return 0, err
+	}
+	if err := c.Set("interval", intervalSecs); err != nil {
+		return 0, err
+	}
+	if err := c.Set("ext", ext); err != nil {
+		return 0, err
+	}
 	if err := c.Set("counts", intMap(counts)); err != nil {
 		return 0, err
 	}
+
+	// Run the script and extract the result from the global
+	// variable 'score'. Acceptable values for score are anything
+	// that can be converted to float by Tengo.
 	if err := c.RunContext(ctx); err != nil {
 		return 0, err
 	}
-	score := c.Get("score")
-	if i := score.Int(); i != 0 {
-		return float64(i), nil
-	}
-	return score.Float(), nil
+	return c.Get("score").Float(), nil
 }
 
 type Manager struct {

script/script_test.go

@@ -5,12 +5,7 @@ import (
 	"testing"
 )
 
-func TestScript(t *testing.T) {
-	src := `
-score = counts["test"] / 2 + counts["test2"]
-//score = 7
-`
-
+func runTestScript(t *testing.T, src string, expected float64) {
 	s, err := NewScript([]byte(src))
 	if err != nil {
 		t.Fatalf("NewScript: %v", err)
@@ -24,7 +19,32 @@ score = counts["test"] / 2 + counts["test2"]
 	if err != nil {
 		t.Fatalf("runScript: %v", err)
 	}
-	if score != 7 {
-		t.Fatalf("bad score, expected 7 got %f", score)
+	if score != expected {
+		t.Fatalf("bad score, expected %f got %f", expected, score)
 	}
 }
+
+func TestScript_WithLookup(t *testing.T) {
+	runTestScript(t, `
+score = counts["test"] / 2 + counts["test2"]
+//score = 7
+`, 7)
+}
+
+func TestScript_FloatScore(t *testing.T) {
+	runTestScript(t, `
+score = 7.0
+`, 7)
+}
+
+func TestScript_IntScore(t *testing.T) {
+	runTestScript(t, `
+score = 7
+`, 7)
+}
+
+func TestScript_StringScore(t *testing.T) {
+	runTestScript(t, `
+score = "7"
+`, 7)
+}

submission/queue.go

@@ -44,6 +44,10 @@ func (o *Options) withDefaults() *Options {
 	return o
 }
 
+// A Submitter sends events to a remote iprep collector, trying to
+// keep the overall request rate under control: above a certain rate
+// of requests (defined via Options), the Submitter will start
+// aggregating and batching events.
 type Submitter interface {
 	AddEvent(*ippb.Event)
 	AddAggregate(*ippb.Aggregate)
@@ -72,6 +76,7 @@ func newSubmitter(client client.Client, opts *Options) Submitter {
 	return s
 }
 
+// New creates a new Submitter pointing at the specified collector addr.
 func New(addr string, opts *Options) (Submitter, error) {
 	c, err := client.New(addr, opts.ClientOptions)
 	if err != nil {