Initial commit

README.md

+reports-collector
+===
+
+Proof of concept of an ingestion service for anomaly reports.
+
+Supports receiving reports over HTTPS for the following mechanisms:
+
+* [Browser Reporting API (NEL, CSP, etc)](https://www.w3.org/TR/reporting/)
+* [TLS-RPT](https://tools.ietf.org/html/rfc8460)
+* [DMARC](https://tools.ietf.org/html/rfc7489) **(TODO)**
+
+The reports are logged in structured denormalized form to standard
+output, the idea being that a log collection pipeline would be in
+place to get them into Elasticsearch or some other index.
+
+Things that are still to do:
+
+* Add support for DMARC reports
+* Augment records with additional information such as AS / GeoIP that
+  would be useful in aggregated views (also drop the user IP at that
+  point)
+* Maybe add support for email ingestion

browser.go

+package reportscollector
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+// Generic browser report as per https://www.w3.org/TR/reporting/.
+type report struct {
+	Type      string                 `json:"type"`
+	Age       int                    `json:"age"`
+	URL       string                 `json:"url"`
+	UserAgent string                 `json:"user_agent"`
+	Body      map[string]interface{} `json:"body"`
+}
+
+type ReportHandler struct{}
+
+func (h *ReportHandler) Parse(contentType string, req *http.Request) ([]Event, error) {
+	if contentType != "application/reports+json" {
+		return nil, ErrNoMatch
+	}
+
+	var reports []*report
+	if err := json.NewDecoder(req.Body).Decode(&reports); err != nil {
+		return nil, err
+	}
+
+	var events []Event
+	for _, r := range reports {
+		events = append(events, h.eventFromReport(r))
+	}
+	return events, nil
+}
+
+func (h *ReportHandler) eventFromReport(report *report) Event {
+	ts := time.Now().Add(time.Duration(-report.Age) * time.Second)
+
+	e := make(Event)
+	e.Set("type", report.Type)
+	e.Set("event_timestamp", ts)
+	e.Set("url", report.URL)
+	e.Set("user_agent", report.UserAgent)
+	e.Set("body", report.Body)
+	return e
+}

cmd/reports-collector/main.go

+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"time"
+
+	rc "git.autistici.org/ai3/tools/reports-collector"
+)
+
+var (
+	addr = flag.String("addr", ":4890", "address to listen on")
+)
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	collector := rc.NewCollector(
+		new(rc.LogSink),
+		new(rc.ReportHandler),
+		new(rc.TLSRPTHandler),
+	)
+
+	// Create the http.Server.
+	mux := http.NewServeMux()
+	mux.Handle("/ingest", collector)
+	server := &http.Server{
+		Addr:         *addr,
+		Handler:      mux,
+		ReadTimeout:  10 * time.Second,
+		IdleTimeout:  30 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+
+	err := server.ListenAndServe()
+	if err != nil {
+		log.Fatalf("error: %v", err)
+	}
+}

collector.go

+package reportscollector
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"mime"
+	"net/http"
+)
+
+var ErrNoMatch = errors.New("no match")
+
+type Handler interface {
+	Parse(string, *http.Request) ([]Event, error)
+}
+
+type Sink interface {
+	Send(Event)
+}
+
+type Collector struct {
+	handlers []Handler
+	sink     Sink
+}
+
+func NewCollector(sink Sink, handlers ...Handler) *Collector {
+	return &Collector{
+		handlers: handlers,
+		sink:     sink,
+	}
+}
+
+func (c *Collector) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if req.URL.Path != "/" {
+		http.NotFound(w, req)
+		return
+	}
+	if req.Method != http.MethodPost {
+		http.Error(w, "Bad method", http.StatusMethodNotAllowed)
+		return
+	}
+
+	ct, _, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Bad Content-Type: %v", err.Error()), http.StatusBadRequest)
+		log.Printf("error parsing content-type: %v", err)
+		return
+	}
+
+	// Find a handler that can successfully parse the request, and
+	// get a list of Events.
+	var events []Event
+hloop:
+	for _, h := range c.handlers {
+		var err error
+		events, err = h.Parse(ct, req)
+		switch err {
+		case ErrNoMatch:
+			continue
+		case nil:
+			w.WriteHeader(http.StatusNoContent)
+			break hloop
+		default:
+			log.Printf("error parsing report: %v", err)
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+	}
+
+	// Augment the Events with additional information obtained
+	// from the HTTP request, and send them to the forwarder.
+	ip := getRemoteIP(req)
+	for _, e := range events {
+		e.Set("ip", ip)
+		c.sink.Send(e)
+	}
+}

collector_test.go

+package reportscollector
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+type countingSink struct {
+	counter int
+}
+
+func (c *countingSink) Send(e Event) {
+	c.counter++
+}
+
+var reportsTestData = `[{
+  "type": "csp",
+  "age": 10,
+  "url": "https://example.com/vulnerable-page/",
+  "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0",
+  "body": {
+    "blocked": "https://evil.com/evil.js",
+    "directive": "script-src",
+    "policy": "script-src 'self'; object-src 'none'",
+    "status": 200,
+    "referrer": "https://evil.com/"
+  }
+}, {
+  "type": "hpkp",
+  "age": 32,
+  "url": "https://www.example.com/",
+  "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0",
+  "body": {
+    "date-time": "2014-04-06T13:00:50Z",
+    "hostname": "www.example.com",
+    "port": 443,
+    "effective-expiration-date": "2014-05-01T12:40:50Z",
+    "include-subdomains": false,
+    "served-certificate-chain": [
+      "-----BEGIN CERTIFICATE-----\nMIIEBDCCAuygAwIBAgIDAjppMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT\nHFa9llF7b1cq26KqltyMdMKVvvBulRP/F/A8rLIQjcxz++iPAsbw+zOzlTvjwsto\nWHPbqCRiOwY1nQ2pM714A5AuTHhdUDqB1O6gyHA43LL5Z/qHQF1hwFGPa4NrzQU6\nyuGnBXj8ytqU0CwIPX4WecigUCAkVDNx\n-----END CERTIFICATE-----"
+    ]
+  }
+}, {
+  "type": "nel",
+  "age": 29,
+  "url": "https://example.com/thing.js",
+  "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0",
+  "body": {
+    "referrer": "https://www.example.com/",
+    "server-ip": "234.233.232.231",
+    "protocol": "",
+    "status-code": 0,
+    "elapsed-time": 143,
+    "age": 0,
+    "type": "http.dns.name_not_resolved"
+  }
+}]`
+
+func TestBrowserReports(t *testing.T) {
+	sink := new(countingSink)
+	c := NewCollector(
+		sink,
+		new(ReportHandler),
+		new(TLSRPTHandler),
+	)
+	srv := httptest.NewServer(c)
+	defer srv.Close()
+
+	req, err := http.NewRequest("POST", srv.URL, strings.NewReader(reportsTestData))
+	if err != nil {
+		t.Fatalf("NewRequest failed: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/reports+json")
+	client := new(http.Client)
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+	if resp.StatusCode >= 300 {
+		t.Fatalf("request failed with status %d: %s", resp.StatusCode, resp.Status)
+	}
+	defer resp.Body.Close()
+
+	if sink.counter != 3 {
+		t.Fatalf("parsed %d events, expected 3", sink.counter)
+	}
+}
+
+var tlsrptTestData = `{
+  "organization-name": "Company-X",
+  "date-range": {
+    "start-datetime": "2016-04-01T00:00:00Z",
+    "end-datetime": "2016-04-01T23:59:59Z"
+  },
+  "contact-info": "sts-reporting@company-x.example",
+  "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
+  "policies": [{
+    "policy": {
+      "policy-type": "sts",
+      "policy-string": ["version: STSv1","mode: testing",
+	    "mx: *.mail.company-y.example","max_age: 86400"],
+      "policy-domain": "company-y.example",
+      "mx-host": "*.mail.company-y.example"
+    },
+    "summary": {
+      "total-successful-session-count": 5326,
+      "total-failure-session-count": 303
+    },
+    "failure-details": [{
+      "result-type": "certificate-expired",
+      "sending-mta-ip": "2001:db8:abcd:0012::1",
+      "receiving-mx-hostname": "mx1.mail.company-y.example",
+      "failed-session-count": 100
+    }, {
+      "result-type": "starttls-not-supported",
+      "sending-mta-ip": "2001:db8:abcd:0013::1",
+      "receiving-mx-hostname": "mx2.mail.company-y.example",
+      "receiving-ip": "203.0.113.56",
+      "failed-session-count": 200,
+      "additional-information": "https://reports.company-x.example/\n        report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
+    }, {
+      "result-type": "validation-failure",
+      "sending-mta-ip": "198.51.100.62",
+      "receiving-ip": "203.0.113.58",
+      "receiving-mx-hostname": "mx-backup.mail.company-y.example",
+      "failed-session-count": 3,
+      "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
+    }]
+  }]
+}`
+
+func TestTLSRPTReports(t *testing.T) {
+	sink := new(countingSink)
+	c := NewCollector(
+		sink,
+		new(ReportHandler),
+		new(TLSRPTHandler),
+	)
+	srv := httptest.NewServer(c)
+	defer srv.Close()
+
+	req, err := http.NewRequest("POST", srv.URL, strings.NewReader(tlsrptTestData))
+	if err != nil {
+		t.Fatalf("NewRequest failed: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/tlsrpt+json")
+	client := new(http.Client)
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+	if resp.StatusCode >= 300 {
+		t.Fatalf("request failed with status %d: %s", resp.StatusCode, resp.Status)
+	}
+	defer resp.Body.Close()
+
+	if sink.counter != 3 {
+		t.Fatalf("parsed %d events, expected 3", sink.counter)
+	}
+}

event.go

+package reportscollector
+
+type Event map[string]interface{}
+
+func (e Event) Set(key string, value interface{}) {
+	e[key] = value
+}

ip.go

+package reportscollector
+
+import (
+	"net"
+	"net/http"
+)
+
+func getRemoteIP(req *http.Request) string {
+	if addr := req.Header.Get("X-Forwarded-For"); addr != "" {
+		return addr
+	}
+	if addr, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
+		return addr
+	}
+	return ""
+}

sink.go

+package reportscollector
+
+import (
+	"encoding/json"
+	"log"
+)
+
+type LogSink struct {
+}
+
+func (*LogSink) Send(e Event) {
+	data, err := json.Marshal(e)
+	if err != nil {
+		return
+	}
+	log.Printf("@cee:%s", data)
+}

tlsrpt.go

+package reportscollector
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"net/http"
+	"time"
+)
+
+type tlsrptFailure struct {
+	ResultType     string `json:"result-type"`
+	SendingMTA     string `json:"sending-mta-ip"`
+	ReceivingMX    string `json:"receiving-mx-hostname"`
+	ReceivingIP    string `json:"receiving-ip"`
+	NumFailed      int    `json:"failed-session-count"`
+	ErrorCode      string `json:"failure-reason-code"`
+	AdditionalInfo string `json:"additional-information"`
+}
+
+type tlsrptPolicy struct {
+	Policy struct {
+		Type   string   `json:"policy-type"`
+		Policy []string `json:"policy-string"`
+		Domain string   `json:"policy-domain"`
+		Mx     string   `json:"mx-host"`
+	} `json:"policy"`
+	Summary struct {
+		NumSuccessful int `json:"total-successful-session-count"`
+		NumFailed     int `json:"total-failure-session-count"`
+	} `json:"summary"`
+	Failures []*tlsrptFailure `json:"failure-details"`
+}
+
+// A TLS RPT report.
+type tlsrpt struct {
+	ReportID     string `json:"report-id"`
+	Organization string `json:"organization-name"`
+	ContactInfo  string `json:"contact-info"`
+	DateRange    struct {
+		Start time.Time `json:"start-datetime"`
+		End   time.Time `json:"end-datetime"`
+	} `json:"date-range"`
+	Policies []*tlsrptPolicy `json:"policies"`
+}
+
+type TLSRPTHandler struct{}
+
+func (h *TLSRPTHandler) Parse(contentType string, req *http.Request) ([]Event, error) {
+	var r io.Reader
+	switch contentType {
+	case "application/tlsrpt+json":
+		r = req.Body
+	case "application/tlsrpt+gzip":
+		var err error
+		r, err = gzip.NewReader(req.Body)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, ErrNoMatch
+	}
+
+	var report tlsrpt
+	if err := json.NewDecoder(r).Decode(&report); err != nil {
+		return nil, err
+	}
+
+	var events []Event
+	for _, p := range report.Policies {
+		for _, f := range p.Failures {
+			events = append(events, h.eventFromFailure(&report, p, f))
+		}
+	}
+	return events, nil
+}
+
+func (h *TLSRPTHandler) eventFromFailure(report *tlsrpt, policy *tlsrptPolicy, failure *tlsrptFailure) Event {
+	e := make(Event)
+	e.Set("type", "tlsrpt")
+	e.Set("event_timestamp", report.DateRange.End)
+	e.Set("domain", policy.Policy.Domain)
+	e.Set("report_id", report.ReportID)
+	e.Set("report_organization", report.Organization)
+	e.Set("tlsrpt_type", failure.ResultType)
+	e.Set("sending_mta", failure.SendingMTA)
+	e.Set("receiving_mx", failure.ReceivingMX)
+	e.Set("receiving_ip", failure.ReceivingIP)
+	e.Set("failed_session_count", failure.NumFailed)
+	return e
+}