Add a --test option to the tail command

cmd/iprep/tail.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"regexp"
+	"strings"
 
 	ippb "git.autistici.org/ai3/tools/iprep/proto"
 	"git.autistici.org/ai3/tools/iprep/submission"
@@ -16,6 +17,7 @@ import (
 )
 
 type tailCommand struct {
+	testOnly    bool
 	serverAddr  string
 	patternFile string
 }
@@ -33,11 +35,21 @@ func parsePatterns(r io.Reader, filename string) ([]pattern, error) {
 	var lineno int
 	for scanner.Scan() {
 		lineno++
-		m := patternRx.FindSubmatch(scanner.Bytes())
-		if len(m) < 1 {
+		line := scanner.Text()
+		if n := strings.Index(line, "#"); n >= 0 {
+			line = line[:n]
+		}
+		line = strings.TrimSpace(line)
+		if line == "" {
 			continue
 		}
-		rx, err := regexp.Compile(string(m[1]))
+
+		m := patternRx.FindStringSubmatch(line)
+		if len(m) < 1 {
+			return nil, fmt.Errorf("syntax error, %s line %d: no pattern found", filename, lineno)
+		}
+		p := strings.Replace(string(m[1]), "\\/", "/", -1)
+		rx, err := regexp.Compile(p)
 		if err != nil {
 			return nil, fmt.Errorf("syntax error, %s line %d: %v", filename, lineno, err)
 		}
@@ -77,6 +89,7 @@ func (c *tailCommand) Usage() string {
 func (c *tailCommand) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&c.serverAddr, "server", "", "`address` (https://host:port) of the iprep server")
 	f.StringVar(&c.patternFile, "patterns", "", "`file` with patterns to load")
+	f.BoolVar(&c.testOnly, "test", false, "only print matches, do not submit them")
 }
 
 func (c *tailCommand) Execute(ctx context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
@@ -112,24 +125,32 @@ func (c *tailCommand) run(ctx context.Context) error {
 		return err
 	}
 
-	sub, err := submission.New(c.serverAddr, nil)
-	if err != nil {
-		return err
+	var sub submission.Submitter
+	if !c.testOnly {
+		sub, err = submission.New(c.serverAddr, nil)
+		if err != nil {
+			return err
+		}
+		defer sub.Close()
 	}
-	defer sub.Close()
 
 	scanner := bufio.NewScanner(os.Stdin)
 	for scanner.Scan() {
+		b := scanner.Bytes()
 		for _, p := range patterns {
-			m := p.rx.FindSubmatch(scanner.Bytes())
-			if len(m) < 1 {
+			m := p.rx.FindSubmatch(b)
+			if len(m) < 2 {
 				continue
 			}
-			sub.AddEvent(&ippb.Event{
-				Ip:    string(m[1]),
-				Type:  p.eventType,
-				Count: 1,
-			})
+			if c.testOnly {
+				fmt.Printf("ip=%s type=%s\n", string(m[1]), p.eventType)
+			} else {
+				sub.AddEvent(&ippb.Event{
+					Ip:    string(m[1]),
+					Type:  p.eventType,
+					Count: 1,
+				})
+			}
 		}
 	}
 

example.patterns

+# Example patterns.  To be used with "journalctl --output=cat" (which
+# only prints the log message, with no additional metadata).
+
+### SSH authentication failures
+
+# Silly brute-forcers that do not support our kex:
+/^Unable to negotiate with ([.0-9]+) port \d+: no matching host key type found./ ssh
+
+### Email-related rules
+
+# Postscreen failures - protocol errors are (in high volume) characteristic of spammers
+/^NOQUEUE: reject: RCPT from \[([.0-9]+)\]:\d+: 550 5.5.1 Protocol error;/ spammer
+
+# Spammers trying to send email via disabled accounts
+/^NOQUEUE: reject: RCPT from [^[]+\[(.[0-9]+)\]: 553 5.7.1 <[^>]+>: Sender address rejected: not owned by user/ spammer
+
+# Spammers triggering SPF failures
+/^550 5.7.23 Message rejected due to: SPF fail - not authorized. Please see http:\/\/www.openspf.net\/Why?s=mfrom;id=[^;]*;ip=([.0-9]+);/ spammer
+
+### Authentication
+
+# General auth-server errors
+/^auth-server\[\d+\]: auth: user=.* service=smtp status=error ip=([.0-9]+) error=/ auth
+
+### Wordpress-specific rules
+
+/^.*nginx_access: .+ .+ (?:::ffff:)?([.:0-9a-f]+) .*"POST \/wp-login\.php HTTP/ wordpress
+/^.*nginx_access: .+ .+ (?:::ffff:)?([.:0-9a-f]+) .*"POST \/wp-comments-post\.php HTTP/ wordpress
+