From c3a78a7ea5638fdae712580980553ff080161a81 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Fri, 1 Nov 2019 10:19:26 +0000
Subject: [PATCH] Documentation updates
---
cmd/iprep/server.go | 10 ++++++-
cmd/iprep/tail.go | 16 ++++++++++-
script/script.go | 63 ++++++++++++++++++++++++++++++++++---------
script/script_test.go | 36 +++++++++++++++++++------
submission/queue.go | 5 ++++
5 files changed, 108 insertions(+), 22 deletions(-)
diff --git a/cmd/iprep/server.go b/cmd/iprep/server.go
index cf0c5b6..c865b29 100644
--- a/cmd/iprep/server.go
+++ b/cmd/iprep/server.go
@@ -36,7 +36,8 @@ func (c *serverCommand) Name() string { return "server" }
func (c *serverCommand) Synopsis() string { return "run the rpc server" }
func (c *serverCommand) Usage() string {
return `server [<flags>]:
- Run the GRPC server.
+
+ Run the main database and analysis service, with a GRPC API.
`
}
@@ -112,6 +113,7 @@ func (c *serverCommand) run(ctx context.Context) error {
return grpcServer.Serve(l)
})
+ // Reload the scoring script on SIGHUP.
reloadSigCh := make(chan os.Signal, 1)
go func() {
for range reloadSigCh {
@@ -120,9 +122,13 @@ func (c *serverCommand) run(ctx context.Context) error {
}()
signal.Notify(reloadSigCh, syscall.SIGHUP)
+ // Terminate the program on SIGINT/SIGTERM.
stopSigCh := make(chan os.Signal, 1)
signal.Notify(stopSigCh, syscall.SIGINT, syscall.SIGTERM)
+ // At this point we got nothing else to do but wait for the
+ // serving goroutines to return. Notify systemd that we're
+ // ready.
daemon.SdNotify(false, daemon.SdNotifyReady) // nolint
select {
@@ -131,6 +137,8 @@ func (c *serverCommand) run(ctx context.Context) error {
case <-ctx.Done():
}
+ // We got here because of an error (or a termination request),
+ // so orderly shut down all the serving components.
cancel()
if httpServer != nil {
diff --git a/cmd/iprep/tail.go b/cmd/iprep/tail.go
index daf5448..27d615a 100644
--- a/cmd/iprep/tail.go
+++ b/cmd/iprep/tail.go
@@ -53,10 +53,24 @@ func (c *tailCommand) Name() string { return "tail" }
func (c *tailCommand) Synopsis() string { return "run the rpc tail" }
func (c *tailCommand) Usage() string {
return `tail [<flags>]:
- Monitor stdin in realtime and update counters based on
+
+ Monitor stdin in realtime and generate events based on
user-specified detection rules (regexp-based). Meant to
process third-party logs.
+ The patterns file should contain one rule per line, consisting
+ of a regexp bounded by / characters (Perl- style), followed by
+ whitespace, followed by the event type that should be set on
+ the generated event. Regular expressions must contain one
+ capture group for the IP address.
+
+ As an example, the following rule:
+
+ /auth failure from ip ([.0-9]+)/ auth_failure
+
+ will trigger an 'auth_failure' event with the IP extracted
+ from the message.
+
`
}
diff --git a/script/script.go b/script/script.go
index 8bfe9d3..55f0be3 100644
--- a/script/script.go
+++ b/script/script.go
@@ -10,10 +10,28 @@ import (
"github.com/d5/tengo/stdlib"
)
+// A Script is a compiled Tengo script that will be executed on every
+// request in order to provide a final reputation score for an IP.
+//
+// The script will have a few global variables available:
+//
+// 'ip' is the IP being looked up
+//
+// 'counts' is a map[string]int64 of counters returned from the
+// database in the chosen time interval
+//
+// 'interval' is the duration in seconds of the time interval chosen
+// for this evaluation, so one can compute rates
+//
+// To return the final reputation score (in the range 0 - 1), the
+// script should set the 'score' variable.
+//
type Script struct {
compiled *script.Compiled
}
+// LoadScript creates a new Script by loading its source from the
+// specified file.
func LoadScript(path string) (*Script, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
@@ -26,43 +44,64 @@ var globalVars = []string{
"ip", "score", "counts", "interval", "ext",
}
+// NewScript creates a new Script with the given source.
func NewScript(src []byte) (*Script, error) {
s := script.New(src)
+
// Define all the global variables (input and output), so that
- // they can be set later on the compiled object.
+ // they can be set later on the compiled object. The object
+ // type we use here does not matter.
for _, v := range globalVars {
if err := s.Add(v, 0); err != nil {
return nil, err
}
}
- // Add available imports.
- s.SetImports(stdlib.GetModuleMap("math", "times"))
+ // Load the Tengo standard library.
+ s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+ // Compile the script. We're going to Clone the compiled
+ // version on every request.
c, err := s.Compile()
if err != nil {
return nil, err
}
+
return &Script{compiled: c}, nil
}
+// RunIP evaluates the script once with the provided context and
+// returns the resulting score.
func (script *Script) RunIP(ctx context.Context, ip string, counts map[string]int64, intervalSecs float64, ext map[string]interface{}) (float64, error) {
c := script.compiled.Clone()
- c.Set("ip", ip)
- c.Set("score", 0.0)
- c.Set("interval", intervalSecs)
- c.Set("ext", ext)
+
+ // Set the global variables that constitute the script
+ // execution context. We expect no errors here even if the
+ // script does not reference some of these variables, as we
+ // have pre-defined them in NewScript.
+ if err := c.Set("ip", ip); err != nil {
+ return 0, err
+ }
+ if err := c.Set("score", 0.0); err != nil {
+ return 0, err
+ }
+ if err := c.Set("interval", intervalSecs); err != nil {
+ return 0, err
+ }
+ if err := c.Set("ext", ext); err != nil {
+ return 0, err
+ }
if err := c.Set("counts", intMap(counts)); err != nil {
return 0, err
}
+
+ // Run the script and extract the result from the global
+ // variable 'score'. Acceptable values for score are anything
+ // that can be converted to float by Tengo.
if err := c.RunContext(ctx); err != nil {
return 0, err
}
- score := c.Get("score")
- if i := score.Int(); i != 0 {
- return float64(i), nil
- }
- return score.Float(), nil
+ return c.Get("score").Float(), nil
}
type Manager struct {
diff --git a/script/script_test.go b/script/script_test.go
index 49946f6..2a0b71a 100644
--- a/script/script_test.go
+++ b/script/script_test.go
@@ -5,12 +5,7 @@ import (
"testing"
)
-func TestScript(t *testing.T) {
- src := `
-score = counts["test"] / 2 + counts["test2"]
-//score = 7
-`
-
+func runTestScript(t *testing.T, src string, expected float64) {
s, err := NewScript([]byte(src))
if err != nil {
t.Fatalf("NewScript: %v", err)
@@ -24,7 +19,32 @@ score = counts["test"] / 2 + counts["test2"]
if err != nil {
t.Fatalf("runScript: %v", err)
}
- if score != 7 {
- t.Fatalf("bad score, expected 7 got %f", score)
+ if score != expected {
+ t.Fatalf("bad score, expected %f got %f", expected, score)
}
}
+
+func TestScript_WithLookup(t *testing.T) {
+ runTestScript(t, `
+score = counts["test"] / 2 + counts["test2"]
+//score = 7
+`, 7)
+}
+
+func TestScript_FloatScore(t *testing.T) {
+ runTestScript(t, `
+score = 7.0
+`, 7)
+}
+
+func TestScript_IntScore(t *testing.T) {
+ runTestScript(t, `
+score = 7
+`, 7)
+}
+
+func TestScript_StringScore(t *testing.T) {
+ runTestScript(t, `
+score = "7"
+`, 7)
+}
diff --git a/submission/queue.go b/submission/queue.go
index 7ffa88f..3e2d30a 100644
--- a/submission/queue.go
+++ b/submission/queue.go
@@ -44,6 +44,10 @@ func (o *Options) withDefaults() *Options {
return o
}
+// A Submitter sends events to a remote iprep collector, trying to
+// keep the overall request rate under control: above a certain rate
+// of requests (defined via Options), the Submitter will start
+// aggregating and batching events.
type Submitter interface {
AddEvent(*ippb.Event)
AddAggregate(*ippb.Aggregate)
@@ -72,6 +76,7 @@ func newSubmitter(client client.Client, opts *Options) Submitter {
return s
}
+// New creates a new Submitter pointing at the specified collector addr.
func New(addr string, opts *Options) (Submitter, error) {
c, err := client.New(addr, opts.ClientOptions)
if err != nil {
--
GitLab