.github/workflows/build.yml

+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: go get -v -t -d ./...
+
+    - name: Build
+      run: go build -v .
+
+    - name: Test
+      run: go test -v ./...

.github/workflows/release.yml

+name: release
+
+on:
+  release:
+    types:
+      - published
+      - unpublished
+      - created
+      - edited
+      - prereleased
+      - released
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Unshallow
+        run: git fetch --prune --unshallow
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13.x
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v1
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -23,3 +23,5 @@ _testmain.go
 *.test
 *.prof
 rsyslog_exporter
+
+dist

.gitlab-ci.yml

-
-stages:
-  - build_src
-  - build_pkg
-  - upload
-
-build:src:
-  stage: build_src
-  image: "ai/build:stretch"
-  script: "build-dsc"
-  artifacts:
-    paths:
-      - build-deb/
-  only:
-    - master
-
-build:pkg:
-  stage: build_pkg
-  image: "ai/build:stretch"
-  script: "build-deb"
-  dependencies:
-    - build:src
-  artifacts:
-    paths:
-      - output-deb/
-  only:
-    - master
-
-upload:pkg:
-  stage: upload
-  image: "ai/pkg:base"
-  script: "upload-packages -r ai3"
-  dependencies:
-    - build:pkg
-  only:
-    - master
+include: "https://git.autistici.org/pipelines/debian/raw/master/common.yml"
 

.goreleaser.yml

+env:
+  - GO111MODULE=on
+before:
+  hooks:
+    - go mod download
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm
+      - arm64
+checksum:
+  name_template: "{{ .ProjectName }}_{{ .Version }}_checksums.txt"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"
+      - Merge pull request
+      - Merge branch
+archives:
+  - name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    replacements:
+      linux: Linux
+      amd64: x86_64
+nfpms:
+  - package_name: "rsyslog-exporter"
+    homepage: https://github.com/aleroyer/rsyslog_exporter
+    description: rsyslog-exporter for prometheus
+    maintainer: Antoine Leroyer <aleroyer@deezer.com>
+    license: Apache 2.0
+    bindir: /usr/bin
+    release: 1
+    formats:
+      - deb
+      - rpm
+    overrides:
+      deb:
+        file_name_template: '{{ replace .ProjectName "_" "-" }}_{{ .Version }}-{{ .Release }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}'
+      rpm:
+        file_name_template: "{{ .ProjectName }}-{{ .Version }}-{{ .Release }}.{{ .Arch }}"

.travis.yml

-language: go
-
-go:
-    - 1.4
-
-install:
-    - go get github.com/digitalocean/rsyslog_exporter
-
-script:
-    - make test

MAINTAINERS.md

+## Maintainers
+
+* Antoine Leroyer [@aleroyer](https://github.com/aleroyer)
+
+## Sponsor
+
+* Matthias Rampke <matthias@prometheus.io>

README.md

@@ -5,6 +5,8 @@ A [prometheus](http://prometheus.io/) exporter for [rsyslog](http://rsyslog.com)
 ## Rsyslog Configuration
 Configure rsyslog to push JSON formatted stats via omprog:
 ```
+module(load="omprog")
+
 module(
   load="impstats"
   interval="10"
@@ -17,15 +19,27 @@ ruleset(name="process_stats") {
   action(
     type="omprog"
     name="to_exporter"
-    binary="/usr/local/bin/rsyslog_exporter"
+    binary="/usr/local/bin/rsyslog_exporter [--tls.server-crt=/path/to/tls.crt --tls.server-key=/path/to/tls.key]"
   )
 }
 ```
 
 The exporter itself logs back via syslog, this cannot be configured at the moment.
 
+## Command Line Switches
+* `web.listen-address` - default `:9104` - port to listen to (NOTE: the leading
+  `:` is required for `http.ListenAndServe`)
+* `web.telemetry-path` - default `/metrics` - path from which to serve Prometheus metrics
+* `tls.server-crt` - default `""` - PEM encoded file containing the server certificate and
+  the CA certificate for use with `http.ListenAndServeTLS`
+* `tls.server-key` - default `""` - PEM encoded file containing the unencrypted
+  server key for use with `tls.server-crt`
+
+If you want the exporter to listen for TLS (`https`) you must specify both
+`tls.server-crt` and `tls.server-key`.
+
 ## Provided Metrics
-The following metrics provided by the rsyslog impstats module are tracked by rsyslog_exporter:
+The following metrics provided by the rsyslog [impstats](https://www.rsyslog.com/doc/master/configuration/modules/impstats.html) module are tracked by rsyslog_exporter:
 
 ### Actions
 Action objects describe what is to be done with a message, and are implemented via output modules.
@@ -68,4 +82,33 @@ Rsyslog tracks how it uses system resources and provides the following metrics:
 * nvcsw - number of voluntary context switches
 * nivcsw - number of involuntary context switches
 
+### Dynafile Cache
+The [omfile](https://www.rsyslog.com/rsyslog-statistic-counter-plugin-omfile/) module can generate
+file names from a template.  A cache of recent filehandles can be maintained, whose sizing can
+impact performance considerably.  The module provides the following metrics:
+
+* requests - number of requests made to obtain a dynafile
+* level0 - number of requests for the current active file
+* missed - number of cache misses
+* evicted - number of times a file needed to be evicted from cache
+* maxused - maximum number of cache entries ever used
+* closetimeouts - number of times a file was closed due to timeout settings
+
+### Dynamic Stats
+Rsyslog allows the user to define their own stats namespaces and increment counters within these
+buckets using Rainerscript function calls.
+
+These are exported as counters with the metric name identifying the bucket, and a label value
+matching the name of the counter (the label name will always be "counter").  As well as custom
+metrics, a "global" dynstats namespace is also published with some additional bookeeping counters.
+
+See the [dyn_stats](https://www.rsyslog.com/doc/master/configuration/dyn_stats.html)
+documentation for more information.
+
+### IMUDP Workerthread stats
+The [imudp](https://www.rsyslog.com/rsyslog-statistic-counter-plugin-imudp/) module can be configured
+to run on multiple worker threads and the following metrics are returned:
 
+* input_called_recvmmsg - Number of recvmmsg called
+* input_called_recvmsg -Number of recvmmsg called
+* input_received - Messages received

debian/changelog

+rsyslog-exporter (0.3) unstable; urgency=medium
+
+  * Rebased on newer upstream release
+
+ -- A/I <debian@autistici.org>  Fri, 01 Oct 2021 10:57:56 +0100
+
+rsyslog-exporter (0.2) unstable; urgency=medium
+
+  * Merged upstream changes.
+
+ -- Autistici/Inventati <debian@autistici.org>  Tue, 18 Dec 2018 07:42:22 +0000
+
 rsyslog-exporter (0.1) unstable; urgency=medium
 
   * Initial Release.

debian/compat

-10

debian/control

@@ -2,13 +2,15 @@ Source: rsyslog-exporter
 Section: admin
 Priority: optional
 Maintainer: Autistici/Inventati <debian@autistici.org>
-Build-Depends: debhelper (>=9), golang-go, dh-golang,
+Build-Depends: debhelper-compat (= 12),
+ golang-any, dh-golang,
  golang-github-prometheus-client-golang-dev
 Standards-Version: 3.9.6
 
 Package: rsyslog-exporter
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, rsyslog
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Recommends: rsyslog
+Built-Using: ${misc:Built-Using}
 Description: Rsyslog Prometheus exporter.
  Exports rsyslogd metrics to Prometheus.
-

debian/rules

@@ -10,4 +10,3 @@ export DH_GOLANG_EXCLUDES = vendor
 override_dh_install:
 	rm -fr $(CURDIR)/debian/rsyslog-exporter/usr/share/gocode
 	dh_install
-

dyn_stat_test.go

@@ -79,7 +79,7 @@ func TestDynStatToPoints(t *testing.T) {
 	}
 
 	seen := map[string]bool{}
-	for name, _ := range wants {
+	for name := range wants {
 		seen[name] = false
 	}
 

dynafile_cache.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type dfcStat struct {
+	Name          string `json:"name"`
+	Origin        string `json:"origin"`
+	Requests      int64  `json:"requests"`
+	Level0        int64  `json:"level0"`
+	Missed        int64  `json:"missed"`
+	Evicted       int64  `json:"evicted"`
+	MaxUsed       int64  `json:"maxused"`
+	CloseTimeouts int64  `json:"closetimeouts"`
+}
+
+func newDynafileCacheFromJSON(b []byte) (*dfcStat, error) {
+	var pstat dfcStat
+	err := json.Unmarshal(b, &pstat)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding dynafile cache stat `%v`: %v", string(b), err)
+	}
+	pstat.Name = strings.TrimPrefix(pstat.Name, "dynafile cache ")
+	return &pstat, nil
+}
+
+func (d *dfcStat) toPoints() []*point {
+	points := make([]*point, 6)
+
+	points[0] = &point{
+		Name:        "dynafile_cache_requests",
+		Type:        counter,
+		Value:       d.Requests,
+		Description: "number of requests made to obtain a dynafile",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+	points[1] = &point{
+		Name:        "dynafile_cache_level0",
+		Type:        counter,
+		Value:       d.Level0,
+		Description: "number of requests for the current active file",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+	points[2] = &point{
+		Name:        "dynafile_cache_missed",
+		Type:        counter,
+		Value:       d.Missed,
+		Description: "number of cache misses",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+	points[3] = &point{
+		Name:        "dynafile_cache_evicted",
+		Type:        counter,
+		Value:       d.Evicted,
+		Description: "number of times a file needed to be evicted from cache",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+	points[4] = &point{
+		Name:        "dynafile_cache_maxused",
+		Type:        counter,
+		Value:       d.MaxUsed,
+		Description: "maximum number of cache entries ever used",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+	points[5] = &point{
+		Name:        "dynafile_cache_closetimeouts",
+		Type:        counter,
+		Value:       d.CloseTimeouts,
+		Description: "number of times a file was closed due to timeout settings",
+		LabelName:   "cache",
+		LabelValue:  d.Name,
+	}
+
+	return points
+}

dynafile_cache_test.go

+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+var (
+	dynafileCacheLog = []byte(`{ "name": "dynafile cache cluster", "origin": "omfile", "requests": 1783254, "level0": 1470906, "missed": 2625, "evicted": 2525, "maxused": 100, "closetimeouts": 10 }`)
+)
+
+func TestNewDynafileCacheFromJSON(t *testing.T) {
+	logType := getStatType(dynafileCacheLog)
+	if logType != rsyslogDynafileCache {
+		t.Errorf("detected pstat type should be %d but is %d", rsyslogDynafileCache, logType)
+	}
+
+	pstat, err := newDynafileCacheFromJSON([]byte(dynafileCacheLog))
+	if err != nil {
+		t.Fatalf("expected parsing dynafile cache stat not to fail, got: %v", err)
+	}
+
+	if want, got := "cluster", pstat.Name; want != got {
+		t.Errorf("want '%s', got '%s'", want, got)
+	}
+
+	if want, got := int64(1783254), pstat.Requests; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+
+	if want, got := int64(1470906), pstat.Level0; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+
+	if want, got := int64(2625), pstat.Missed; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+
+	if want, got := int64(2525), pstat.Evicted; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+
+	if want, got := int64(100), pstat.MaxUsed; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+
+	if want, got := int64(10), pstat.CloseTimeouts; want != got {
+		t.Errorf("want '%d', got '%d'", want, got)
+	}
+}
+
+func TestDynafileCacheToPoints(t *testing.T) {
+
+	wants := map[string]point{
+		"dynafile_cache_requests": point{
+			Name:        "dynafile_cache_requests",
+			Type:        counter,
+			Value:       1783254,
+			Description: "number of requests made to obtain a dynafile",
+			LabelName:   "cache",
+			LabelValue:  "cluster",
+		},
+		"dynafile_cache_level0": point{
+			Name:        "dynafile_cache_level0",
+			Type:        counter,
+			Value:       1470906,
+			Description: "number of requests for the current active file",
+			LabelName:   "cache",
+
+			LabelValue: "cluster",
+		},
+		"dynafile_cache_missed": point{
+			Name:        "dynafile_cache_missed",
+			Type:        counter,
+			Value:       2625,
+			Description: "number of cache misses",
+			LabelName:   "cache",
+			LabelValue:  "cluster",
+		},
+		"dynafile_cache_evicted": point{
+			Name:        "dynafile_cache_evicted",
+			Type:        counter,
+			Value:       2525,
+			Description: "number of times a file needed to be evicted from cache",
+			LabelName:   "cache",
+			LabelValue:  "cluster",
+		},
+		"dynafile_cache_maxused": point{
+			Name:        "dynafile_cache_maxused",
+			Type:        counter,
+			Value:       100,
+			Description: "maximum number of cache entries ever used",
+			LabelName:   "cache",
+			LabelValue:  "cluster",
+		},
+		"dynafile_cache_closetimeouts": point{
+			Name:        "dynafile_cache_closetimeouts",
+			Type:        counter,
+			Value:       10,
+			Description: "number of times a file was closed due to timeout settings",
+			LabelName:   "cache",
+			LabelValue:  "cluster",
+		},
+	}
+
+	seen := map[string]bool{}
+	for name := range wants {
+		seen[name] = false
+	}
+
+	pstat, err := newDynafileCacheFromJSON(dynafileCacheLog)
+	if err != nil {
+		t.Fatalf("expected parsing dynafile cache stat not to fail, got: %v", err)
+	}
+
+	points := pstat.toPoints()
+	for _, got := range points {
+		want, ok := wants[got.Name]
+		if !ok {
+			t.Errorf("unexpected point, got: %+v", got)
+			continue
+		}
+
+		if !reflect.DeepEqual(want, *got) {
+			t.Errorf("expected point to be %+v, got %+v", want, got)
+		}
+
+		if seen[got.Name] {
+			t.Errorf("point seen multiple times: %+v", got)
+		}
+		seen[got.Name] = true
+	}
+
+	for name, ok := range seen {
+		if !ok {
+			t.Errorf("expected to see point with key %s, but did not", name)
+		}
+	}
+}

exporter.go

@@ -20,6 +20,10 @@ const (
 	rsyslogQueue
 	rsyslogResource
 	rsyslogDynStat
+	rsyslogDynafileCache
+	rsyslogInputIMDUP
+	rsyslogForward
+	rsyslogKubernetes
 )
 
 type rsyslogExporter struct {
@@ -68,6 +72,15 @@ func (re *rsyslogExporter) handleStatLine(rawbuf []byte) error {
 			re.set(p)
 		}
 
+	case rsyslogInputIMDUP:
+		u, err := newInputIMUDPFromJSON(buf)
+		if err != nil {
+			return err
+		}
+		for _, p := range u.toPoints() {
+			re.set(p)
+		}
+
 	case rsyslogQueue:
 		q, err := newQueueFromJSON(buf)
 		if err != nil {
@@ -93,6 +106,30 @@ func (re *rsyslogExporter) handleStatLine(rawbuf []byte) error {
 		for _, p := range s.toPoints() {
 			re.set(p)
 		}
+	case rsyslogDynafileCache:
+		d, err := newDynafileCacheFromJSON(buf)
+		if err != nil {
+			return err
+		}
+		for _, p := range d.toPoints() {
+			re.set(p)
+		}
+	case rsyslogForward:
+		f, err := newForwardFromJSON(buf)
+		if err != nil {
+			return err
+		}
+		for _, p := range f.toPoints() {
+			re.set(p)
+		}
+	case rsyslogKubernetes:
+		k, err := newKubernetesFromJSON(buf)
+		if err != nil {
+			return err
+		}
+		for _, p := range k.toPoints() {
+			re.set(p)
+		}
 
 	default:
 		return fmt.Errorf("unknown pstat type: %v", pstatType)
@@ -134,24 +171,37 @@ func (re *rsyslogExporter) Collect(ch chan<- prometheus.Metric) {
 			continue
 		}
 
+		labelValues := []string{}
+		if p.promLabelValue() != "" {
+			labelValues = []string{p.promLabelValue()}
+		}
 		metric := prometheus.MustNewConstMetric(
 			p.promDescription(),
 			p.promType(),
 			p.promValue(),
-			p.promLabelValue(),
+			labelValues...,
 		)
 
 		ch <- metric
 	}
 }
 
-func (re *rsyslogExporter) run() {
+func (re *rsyslogExporter) run(silent bool) {
+	errorPoint := &point{
+		Name:        "stats_line_errors",
+		Type:        counter,
+		Description: "Counts errors during stats line handling",
+	}
+	re.set(errorPoint)
 	for re.scanner.Scan() {
 		err := re.handleStatLine(re.scanner.Bytes())
 		if err != nil {
+			errorPoint.Value += 1
+			if !silent {
 				log.Printf("error handling stats line: %v, line was: %s", err, re.scanner.Bytes())
 			}
 		}
+	}
 	if err := re.scanner.Err(); err != nil {
 		log.Printf("error reading input: %v", err)
 	}

exporter_test.go

@@ -230,6 +230,34 @@ func TestHandleLineWithGlobal(t *testing.T) {
 	testHelper(t, log, tests)
 }
 
+func TestHandleLineWithDynafileCache(t *testing.T) {
+	tests := []*testUnit{
+		&testUnit{
+			Name:       "dynafile_cache_requests",
+			Val:        412044,
+			LabelValue: "cluster",
+		},
+		&testUnit{
+			Name:       "dynafile_cache_level0",
+			Val:        294002,
+			LabelValue: "cluster",
+		},
+		&testUnit{
+			Name:       "dynafile_cache_missed",
+			Val:        210,
+			LabelValue: "cluster",
+		},
+		&testUnit{
+			Name:       "dynafile_cache_evicted",
+			Val:        14,
+			LabelValue: "cluster",
+		},
+	}
+
+	dynafileCacheLog := []byte(`2019-07-03T17:04:01.312432+00:00 some-node.example.org rsyslogd-pstats: { "name": "dynafile cache cluster", "origin": "omfile", "requests": 412044, "level0": 294002, "missed": 210, "evicted": 14, "maxused": 100, "closetimeouts": 0 }`)
+	testHelper(t, dynafileCacheLog, tests)
+}
+
 func TestHandleUnknown(t *testing.T) {
 	unknownLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"a":"b"}`)
 

fixtures/rsyslog-stats.log

 2017-08-30T08:09:54.776051+00:00 some-node.example.org rsyslogd-pstats: { "name": "global", "origin": "dynstats", "values": { } }
+2017-08-30T08:09:54.776052+00:00 some-node.example.org rsyslogd-pstats: { "name": "global", "origin": "percentile", "values": { } }
 2017-08-30T08:09:54.776072+00:00 some-node.example.org rsyslogd-pstats: { "name": "imuxsock", "origin": "imuxsock", "submitted": 9, "ratelimit.discarded": 0, "ratelimit.numratelimiters": 0 }
 2017-08-30T08:09:54.776082+00:00 some-node.example.org rsyslogd-pstats: { "name": "action 0", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 }
 2017-08-30T08:09:54.776088+00:00 some-node.example.org rsyslogd-pstats: { "name": "to_exporter_2", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 }

forward.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type forward struct {
+	Name      string `json:"name"`
+	BytesSent int64  `json:"bytes.sent"`
+}
+
+func newForwardFromJSON(b []byte) (*forward, error) {
+	var pstat forward
+	err := json.Unmarshal(b, &pstat)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode forward stat `%v`: %v", string(b), err)
+	}
+	return &pstat, nil
+}
+
+func (f *forward) toPoints() []*point {
+	points := make([]*point, 1)
+
+	points[0] = &point{
+		Name:        "forward_bytes_total",
+		Type:        counter,
+		Value:       f.BytesSent,
+		Description: "bytes forwarded to destination",
+		LabelName:   "destination",
+		LabelValue:  f.Name,
+	}
+
+	return points
+}

forward_test.go

+package main
+
+import "testing"
+
+var (
+	forwardLog = []byte(`{ "name": "TCP-FQDN-6514", "origin": "omfwd", "bytes.sent": 666 }`)
+)
+
+func TestNewForwardFromJSON(t *testing.T) {
+	logType := getStatType(forwardLog)
+	if logType != rsyslogForward {
+		t.Errorf("detected pstat type should be %d but is %d", rsyslogForward, logType)
+	}
+
+	pstat, err := newForwardFromJSON([]byte(forwardLog))
+	if err != nil {
+		t.Fatalf("expected parsing action not to fail, got: %v", err)
+	}
+
+	if want, got := "TCP-FQDN-6514", pstat.Name; want != got {
+		t.Errorf("wanted '%s', got '%s'", want, got)
+	}
+
+	if want, got := int64(666), pstat.BytesSent; want != got {
+		t.Errorf("wanted '%d', got '%d'", want, got)
+	}
+}
+
+func TestForwardToPoints(t *testing.T) {
+	pstat, err := newForwardFromJSON([]byte(forwardLog))
+	if err != nil {
+		t.Fatalf("expected parsing action not to fail, got: %v", err)
+	}
+	points := pstat.toPoints()
+
+	point := points[0]
+	if want, got := "forward_bytes_total", point.Name; want != got {
+		t.Errorf("wanted '%s', got '%s'", want, got)
+	}
+
+	if want, got := int64(666), point.Value; want != got {
+		t.Errorf("wanted '%d', got '%d'", want, got)
+	}
+
+	if want, got := counter, point.Type; want != got {
+		t.Errorf("wanted '%d', got '%d'", want, got)
+	}
+
+	if want, got := "TCP-FQDN-6514", point.LabelValue; want != got {
+		t.Errorf("wanted '%s', got '%s'", want, got)
+	}
+}