diff --git a/Dockerfile b/Dockerfile
index 65cb8fda90ccd176b2158bbb53dfa87273cc8712..f7281415451a14024da0b3d01d483fb499321b85 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,9 @@
-FROM debian:stable AS build
+FROM debian:stable-slim AS build
RUN apt-get -q update && env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git ca-certificates
RUN git clone --depth=1 https://git.schokokeks.org/git/freewvs.git /src
FROM debian:stable-slim
-RUN apt-get -q update && env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-pip python3-requests xxd ca-certificates && ln -s python3 /usr/bin/python && pip3 install backoff
+RUN apt-get -q update && env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-pip python3-requests xxd && ln -s python3 /usr/bin/python && pip3 install backoff && rm -fr /var/lib/apt/lists/*
COPY --from=build /src/freewvs /usr/bin/freewvs
COPY --from=build /src/freewvsdb/ /var/lib/freewvs
COPY --from=build /src/update-freewvsdb /usr/bin/update-freewvsdb
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..84972e9b272bf4485e9bc048133b8235a279a361
--- /dev/null
+++ b/README.md
@@ -0,0 +1,5 @@
+Container image to run *freewvs* periodically and upload the results
+to the aux-db.
+
+Since we have to resolve resource IDs (website names) from the
+filesystem paths, we need access to the homedirs.json file.
diff --git a/cron.sh b/cron.sh
index 7586bc2bbc44c0afad2c52fc59e87afd34f1a717..53c38dca535e6cb0d8cb28cbf2c236ae2e184b39 100755
--- a/cron.sh
+++ b/cron.sh
@@ -19,7 +19,7 @@ sleep $offset_secs
while true; do
echo "scanning..."
- ionice -c 3 freewvs --all --xml $scan_dir \
+ nice ionice -c 3 freewvs --all --xml $scan_dir \
| /upload.py $UPLOAD_ARGS
if [ $? -gt 0 ]; then
echo "ERROR: freewvs exited with status $?"
diff --git a/sample.xml b/sample.xml
new file mode 100644
index 0000000000000000000000000000000000000000..bb57b684bd631572a7cf57fd313f74f3273b3619
--- /dev/null
+++ b/sample.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" ?>
+<freewvs>
+ <app state="ok">
+ <appname>Wordpress-Akismet</appname>
+ <version>3.3.2</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs/wp-content/plugins/akismet</directory>
+ </app>
+ <app state="vulnerable">
+ <appname>WP Super Cache</appname>
+ <version>1.2</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs/wp-content/plugins/wp-super-memcache</directory>
+ <safeversion>1.6.9</safeversion>
+ <vulninfo>https://odd.blog/2019/07/25/wp-super-cache-1-6-9-security-update/</vulninfo>
+ </app>
+ <app state="vulnerable">
+ <appname>WP Super Cache</appname>
+ <version>1.4.9</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs/wp-content/plugins/wp-super-cache</directory>
+ <safeversion>1.6.9</safeversion>
+ <vulninfo>https://odd.blog/2019/07/25/wp-super-cache-1-6-9-security-update/</vulninfo>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress-NextGEN</appname>
+ <version>2.2.3</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs/wp-content/plugins/nextgen-gallery</directory>
+ <safeversion>3.2.11</safeversion>
+ <vulninfo>CVE-2019-14314</vulninfo>
+ </app>
+ <app state="ok">
+ <appname>Wordpress-Theme-Twentyfifteen</appname>
+ <version>1.8</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs/wp-content/themes/twentyfifteen</directory>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress</appname>
+ <version>4.8</version>
+ <directory>/home/ale/src/ai/noblogs-docker-2/noblogs</directory>
+ <safeversion>4.8.11</safeversion>
+ <vulninfo>CVE-2019-17672</vulninfo>
+ </app>
+ <app state="vulnerable">
+ <appname>WP Super Cache</appname>
+ <version>1.6.4</version>
+ <directory>/home/ale/src/ai/noblogs-wp/wp-content/plugins/wp-super-cache</directory>
+ <safeversion>1.6.9</safeversion>
+ <vulninfo>https://odd.blog/2019/07/25/wp-super-cache-1-6-9-security-update/</vulninfo>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress-NextGEN</appname>
+ <version>3.1.6</version>
+ <directory>/home/ale/src/ai/noblogs-wp/wp-content/plugins/nextgen-gallery</directory>
+ <safeversion>3.2.11</safeversion>
+ <vulninfo>CVE-2019-14314</vulninfo>
+ </app>
+ <app state="ok">
+ <appname>Wordpress-Theme-Twentyfifteen</appname>
+ <version>2.3</version>
+ <directory>/home/ale/src/ai/noblogs-wp/wp-content/themes/twentyfifteen</directory>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress</appname>
+ <version>5.1</version>
+ <directory>/home/ale/src/ai/noblogs-wp</directory>
+ <safeversion>5.1.3</safeversion>
+ <vulninfo>CVE-2019-17672</vulninfo>
+ </app>
+ <app state="ok">
+ <appname>WP Super Cache</appname>
+ <version>1.6.9</version>
+ <directory>/home/ale/src/ai3/docker/noblogs/noblogs/wp-content/plugins/wp-super-cache</directory>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress-NextGEN</appname>
+ <version>3.2.10</version>
+ <directory>/home/ale/src/ai3/docker/noblogs/noblogs/wp-content/plugins/nextgen-gallery</directory>
+ <safeversion>3.2.11</safeversion>
+ <vulninfo>CVE-2019-14314</vulninfo>
+ </app>
+ <app state="ok">
+ <appname>Wordpress-Theme-Twentyfifteen</appname>
+ <version>2.5</version>
+ <directory>/home/ale/src/ai3/docker/noblogs/noblogs/wp-content/themes/twentyfifteen</directory>
+ </app>
+ <app state="vulnerable">
+ <appname>Wordpress</appname>
+ <version>5.2.3</version>
+ <directory>/home/ale/src/ai3/docker/noblogs/noblogs</directory>
+ <safeversion>5.2.4</safeversion>
+ <vulninfo>CVE-2019-17672</vulninfo>
+ </app>
+</freewvs>
diff --git a/upload.py b/upload.py
index eceebf837e4b282988b3316f7e0222ecfd30e031..ca865cd706efdd35391cd133bf39a7d8b950151b 100755
--- a/upload.py
+++ b/upload.py
@@ -2,13 +2,29 @@
from xml.etree import ElementTree
import argparse
-import backoff
import json
import logging
+import os
import sys
-import requests
-import urllib
import time
+import requests
+
+
+DATA_TYPE = 'cms_info'
+DATA_TTL = 3 * 86400
+
+
+class HomedirMap():
+
+ def __init__(self, data):
+ self._data = data
+
+ def get_resource_id(self, key):
+ while key not in ('', '/'):
+ if key in self._data:
+ return self._data[key]
+ key = os.path.split(key)[0]
+ return None
class StatusError(Exception):
@@ -18,11 +34,7 @@ class StatusError(Exception):
resp.request.method, resp.request.url, resp.status_code, resp.text))
-class RetriableStatusError(StatusError):
- pass
-
-
-class Backend(object):
+class Backend():
def __init__(self, url, ssl_cert=None, ssl_key=None,
ssl_ca=None, timeout=60):
@@ -33,11 +45,6 @@ class Backend(object):
self._session.verify = ssl_ca
self._session.timeout = timeout
- @backoff.on_exception(backoff.expo,
- (requests.exceptions.Timeout,
- requests.exceptions.ConnectionError,
- RetriableStatusError),
- max_time=1800)
def submit(self, data):
req = self._session.prepare_request(
requests.Request(
@@ -46,21 +53,37 @@ class Backend(object):
json=data))
resp = self._session.send(req)
status = resp.status_code
- if status == 429 or status > 500:
- raise RetriableStatusError(resp)
+ if status != 200:
+ raise StatusError(resp)
return resp
-def read_input(fd, shard):
- timestamp = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+def parse_xml_app_entry(app):
+ # Make a dictionary with all the tags.
+ data = dict((x.tag, x.text) for x in app.iter())
+ # State is an attribute on the app tag.
+ data['state'] = app.attrib.get('state')
+ return data
+
+
+def read_input(fd, homedir_map):
tree = ElementTree.parse(fd)
entries = []
for app in tree.iter('app'):
- entry = dict((x.tag, x.text) for x in app.iter())
- entry['state'] = app.attrib.get('state')
- entry['shard'] = shard
- entry['timestamp'] = timestamp
- entries.append(entry)
+ entry = parse_xml_app_entry(app)
+
+ # Take the 'homedir' attr out of the entry values and make it
+ # the aux-db secondary key.
+ homedir = entry.pop('directory')
+ resource_id = homedir_map.get_resource_id(homedir)
+ if not resource_id:
+ continue
+
+ entries.append({
+ 'resource_id': resource_id,
+ 'app_key': homedir,
+ 'value_json': json.dumps(entry),
+ })
return entries
@@ -70,15 +93,23 @@ def main():
parser.add_argument('--ssl-cert', metavar='file')
parser.add_argument('--ssl-key', metavar='file')
parser.add_argument('--ssl-ca', metavar='file')
- parser.add_argument('--shard', metavar='id')
+ parser.add_argument('--homedir-map', metavar='file')
opts = parser.parse_args()
logging.basicConfig(level=logging.INFO)
+ with open(opts.homedir_map, 'rb') as fd:
+ homedir_map = HomedirMap(json.load(fd))
be = Backend(opts.report_url, opts.ssl_cert, opts.ssl_key, opts.ssl_ca)
- entries = read_input(sys.stdin, opts.shard)
+ entries = read_input(sys.stdin, homedir_map)
logging.info('submitting %d entries...', len(entries))
- resp = be.submit({'shard': opts.shard, 'entries': entries})
+ timestamp = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+ resp = be.submit({
+ 'type': DATA_TYPE,
+ 'ttl': DATA_TTL,
+ 'timestamp': timestamp,
+ 'entries': entries,
+ })
if resp.status_code != 200:
logging.error('submission failed (HTTP status %d)', resp.status_code)
sys.exit(1)