Draft: Enable Loki as a replacement for Elasticsearch when `enable_loki` is set.

Note, it is not possible to have both Loki and Elasticsearch enabled, so enable_elasticsearch must be disabled, when enable_loki is enabled.

roles/log-collector/templates/rsyslog-collector.conf.j2

@@ -37,6 +37,78 @@ template(name="TmplFile" type="list"){
   constant(value=".log")
 }
 
+# if elasticsearch is disabled, and loki is enabled, then we setup loki templates
+# if elasticsearch is enabled, but loki is also enabled, then only ES is enabled
+{% if enable_loki %}
+module(
+  load="omhttp"
+)
+template(name="lokiTemplate"
+         type="list" ) {
+           constant(value="{")
+           constant(value="\"stream\":")
+           constant(value="{")
+           property(outname="host" name="HOSTNAME" format="jsonf")
+           constant(value=",")
+           property(outname="facility" name="syslogfacility-text" format="jsonf")
+           constant(value=",")
+           property(outname="priority" name="syslogpriority-text" caseConversion="upper" format="jsonf")
+           constant(value=",")
+           property(outname="job" name="programname" format="jsonf")
+           constant(value="},")
+           constant(value="\"values\": [[\"")
+           property(name="timegenerated" dateFormat="unixtimestamp" format="json")
+           constant(value="000000000")
+           constant(value="\",\"") property(name="msg" format="json")
+           constant(value="\"]]}")
+}
+template(name="lokiTemplateHTTP"
+         type="list" ) {
+           constant(value="{")
+           constant(value="\"stream\":")
+           constant(value="{")
+           property(outname="host" name="hostname" format="jsonf")
+           constant(value=",\"job\":\"http\"")
+           constant(value=",") property(outname="vhost" name="$!vhost" format="jsonf")
+           constant(value=",") property(outname="status" name="$!status" format="jsonf")
+           constant(value=",") property(outname="method" name="$!verb" format="jsonf")
+           constant(value="},")
+           constant(value="\"values\": [[\"")
+           property(name="timegenerated" dateFormat="unixtimestamp" format="json")
+           constant(value="000000000\",\"")
+           property(name="$!backend" format="json") constant(value=" ")
+           property(name="$!agent" format="json") constant(value=" ")
+           property(name="$!referrer" format="json") constant(value=" ")
+           property(name="$!bytes" format="json") constant(value=" ")
+           property(name="$!status"format="json") constant(value=" ")
+           property(name="$!httpversion" format="json") constant(value=" ")
+           property(name="$!request" format="json") constant(value=" ")
+           property(name="$!verb" format="json") constant(value=" ")
+           property(name="$!auth" format="json") constant(value=" ")
+           property(name="$!ident" format="json") constant(value=" ")
+           property(name="$!vhost" format="json") constant(value=" ")
+           constant(value="\"]]}")
+}
+{% endif %}
+
+template(name="outfmt" type="list" option.jsonf="on") {
+         property(outname="@timestamp"
+                  name="timereported" 
+                  dateFormat="rfc3339" format="jsonf")
+         property(outname="host"
+                  name="hostname" format="jsonf")
+         property(outname="severity"
+                  name="syslogseverity-text" caseConversion="upper" format="jsonf")
+         property(outname="facility"
+                  name="syslogfacility-text" format="jsonf")
+         property(outname="syslog-tag"
+                  name="syslogtag" format="jsonf")
+         property(outname="source"
+                  name="app-name" format="jsonf")
+         property(outname="message"
+                  name="msg" format="jsonf")
+}
+
 {% if enable_elasticsearch %}
 # Elasticsearch output support. For simplicity, it will create indexes
 # with the same naming scheme used by logstash (to simplify the Kibana
@@ -85,6 +157,8 @@ template(name="esTemplate"
   constant(value="\",")
   property(name="$!all-json" position.from="2")
 }
+
+
 template(name="esTemplateHTTP"
          type="list") {
   constant(value="{")
@@ -145,6 +219,84 @@ ruleset(name="incoming"){
   action(type="omfile" DynaFile="TmplFile")
 {% endif %}
 
+{% if enable_loki %}
+# We shouldn't send loki logs to loki
+if ($programname == "loki") then {
+  stop
+}
+if ($syslogfacility-text == "local3") then {
+    # HTTP logs from the front-end. Run it through mmnormalize to
+    # convert the standard CommonLog format into JSON, then send it to
+    # Loki.
+    action(type="mmnormalize"
+           rulebase="/etc/rsyslog-collector-lognorm/http.rb")
+    # Anonymize sso_login requests by dropping the query string.
+    if ($!request contains "/sso_login?") then {
+      set $!request = "/sso_login?";
+    }
+    action(type="omhttp"
+           server="127.0.0.1"
+           serverport="3100"
+           name="loki"
+           useHttps="off"
+           checkpath="ready"
+           httpcontenttype="application/json"
+           restpath="loki/api/v1/push"
+           template="lokiTemplateHTTP"
+           batch.format="lokirest"
+           batch="on"
+           batch.maxsize="10"
+           queue.size="10000"
+           queue.type="linkedList"
+           queue.workerthreads="3"
+           queue.workerthreadMinimumMessages="1000"
+           queue.timeoutWorkerthreadShutdown="500"
+           queue.timeoutEnqueue="10000")
+  } else {
+    # Traditional syslog message. Run it through mmnormalize to
+    # extract interesting bits of metadata according to user-defined
+    # patterns (a bit like logstash), then send the result as JSON to
+    # loki.
+
+    # Apply any blacklists first.
+{% for expr in log_collector_filter_exprs|default([]) %}
+    if ({{ expr }}) then {
+      stop
+    }
+{% endfor %}
+
+    action(type="mmnormalize"
+           rulebase="/etc/rsyslog-collector-lognorm/audit.rb")
+    action(type="mmnormalize"
+           rulebase="/etc/rsyslog-collector-lognorm/auth.rb")
+    # Drop these fields as they're just duplicating the original message.
+    unset $!originalmsg;
+    unset $!unparsed-data;
+    # Slightly silly: we have to set a variable anyway in the
+    # resulting JSON otherwise the esTemplate won't be syntactially
+    # valid and ES will refuse it.
+    set $!ignore = "1";
+    action(type="omhttp"
+           server="127.0.0.1"
+           serverport="3100"
+           name="loki"
+           useHttps="off"
+           checkpath="ready"
+           httpcontenttype="application/json"
+           restpath="loki/api/v1/push"
+           template="lokiTemplate"
+           batch.format="lokirest"
+           batch="on"
+           batch.maxsize="10"
+           queue.size="10000"
+           queue.type="linkedList"
+           queue.workerthreads="3"
+           queue.workerthreadMinimumMessages="1000"
+           queue.timeoutWorkerthreadShutdown="500"
+           queue.timeoutEnqueue="10000")
+  }
+{% endif %}
+
 {% if enable_elasticsearch %}
   # We shouldn't send Elasticsearch logs to Elasticsearch.
   if ($programname == "elasticsearch") then {