Commit 5c74a911 authored by ale's avatar ale

Refactor into split components

parent a78a90af
......@@ -33,20 +33,23 @@ API](https://python-gitlab.readthedocs.io/en/stable/), and
# Usage
There are two modes of using this software: either as a one-shot
command-line tool, or as a standalone HTTP server responding to Gitlab
Webhook requests, to integrate Docker dependencies with Gitlab CI.
The tool is split into functional components:
In both cases, the program is configured via command-line options.
* scan Gitlab and generate a dependency map (stored as a JSON file)
* manually trigger builds using the dependency map
* run a server that listens for Gitlab notifications and trigger
builds
In all cases, the program is configured via command-line options.
## Common options
The tool must be pointed at your Gitlab instance with the *--url*
command-line option,
You can pass an authentication token using the *--token* command-line
option. This is usually required in order to trigger CI pipelines: the
access token must have the *api* scope.
You can pass an authentication token using the *--token* or
*--token-file* command-line options. This is usually required in order
to trigger CI pipelines: the access token must have the *api* scope.
The tool will only examine Docker images hosted on the Docker registry
associated with the Gitlab instance. By default the registry name is
......@@ -68,7 +71,7 @@ example, it is possible to efficiently limit the scope of the tool to
a specific namespace:
```
gitlab-docker-autodep ... --match myns --filter ^myns/ ...
gitlab-docker-autodep deps --match myns --filter ^myns/ ...
```
Note that, when building the dependency tree:
......@@ -123,8 +126,9 @@ stored in */etc/gitlab_docker_token*:
```
gitlab-docker-autodep \
--url=https://my.gitlab \
--token=$(< /etc/gitlab_docker_token) \
--token-file=/etc/gitlab_docker_token \
server \
--deps=deps.json
--host=127.0.0.1 --port=14001
```
......@@ -132,6 +136,16 @@ You can then configure your project's webhooks with the URL
`http://localhost:14001/`, with the *Trigger* checkbox set only
on *Pipeline events*.
Then you should generate the *deps.json* dependency map periodically,
for instance with a cron job:
```
*/30 * * * * root gitlab-docker-autodep
--url=https://my.gitlab
--token-file=/etc/gitlab_docker_token
deps > deps.json
```
It can be useful to run the *rebuild* command from a cron job, for
instance in order to rebuild images on a periodic schedule, and
assuming all your projects share a common base image:
......@@ -139,6 +153,6 @@ assuming all your projects share a common base image:
```
50 5 * * * root gitlab-docker-autodep
--url=https://my.gitlab
--token=$(< /etc/gitlab_docker_token)
--token-file=/etc/gitlab_docker_token
rebuild $MY_BASE_IMAGE
```
import gitlab
import json
import logging
import re
import time
import sys
_from_rx = re.compile(r'^FROM\s+(.*)$')
_from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE)
def _parse_dockerfile(df):
for line in df.split('\n'):
m = _from_rx.match(line)
if m:
return m.group(1)
return _from_rx.findall(df)
def _fetch_dockerfile(gl, project, ref):
......@@ -35,7 +33,7 @@ def _remove_image_tag(name):
return name
def build_dependency_tree(gl, search_pattern=None, filter_pattern=None):
def build_docker_deps(gl, search_pattern=None, filter_pattern=None):
"""Build the project dependency map based on Dockerfiles.
This can be a fairly expensive (long) operation if the list of
......@@ -60,60 +58,43 @@ def build_dependency_tree(gl, search_pattern=None, filter_pattern=None):
projects = gl.projects.list(all=True, search=search_pattern, as_list=False)
for project in projects:
project_name = project.path_with_namespace
project_url = project_name
if filter_rx is not None and not filter_rx.search(project.path_with_namespace):
continue
if not _has_gitlab_ci(gl, project, 'master'):
continue
df = _fetch_dockerfile(gl, project, 'master')
if not df:
continue
base_image = _parse_dockerfile(df.decode('utf-8'))
if not base_image:
base_images = _parse_dockerfile(df.decode('utf-8'))
if not base_images:
logging.error('ERROR: could not find base image for %s',
project.path_with_namespace)
continue
if not _has_gitlab_ci(gl, project, 'master'):
continue
deps.setdefault(_remove_image_tag(base_image), []).append(project)
for img in base_images:
deps.setdefault(_remove_image_tag(img), []).append(project_url)
return deps
def rebuild(project, wait=False):
"""Trigger a rebuild of a project."""
pipeline = project.pipelines.create({'ref': 'master'})
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
time.sleep(3)
return pipeline
def docker_deps_to_project_deps(deps, registry_hostname):
out = {}
for image_name in deps:
if image_name.startswith(registry_hostname):
project_name = image_name[len(registry_hostname)+1:]
out[project_name] = deps[image_name]
return out
def rebuild_deps(gitlab_url, registry_hostname, gitlab_token,
search_pattern, filter_pattern, image_name,
dry_run=False, recurse=False, wait=False):
"""Rebuild dependencies of the given image."""
def dump_deps(gitlab_url, registry_hostname, gitlab_token,
deps_match, deps_filter, project_deps=True):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
deps = build_dependency_tree(gl, search_pattern)
stack = deps.get(_remove_image_tag(image_name), [])
while stack:
project = stack.pop(0)
logging.info('rebuilding %s', project.path_with_namespace)
if not dry_run:
try:
pipeline = rebuild(project, wait)
if pipeline.status not in ('success', 'pending'):
logging.error('ERROR: build failed for %s (status: %s)',
project.path_with_namespace, pipeline.status)
continue
except gitlab.exceptions.GitlabError as e:
logging.error('ERROR: gitlab error: %s: %s',
project.path_with_namespace, str(e))
continue
if recurse:
image_name = '%s/%s' % (
registry_hostname, project.path_with_namespace)
stack.extend(deps.get(image_name, []))
deps = build_docker_deps(gl, deps_match, deps_filter)
if project_deps:
deps = docker_deps_to_project_deps(deps, registry_hostname)
json.dump(deps, sys.stdout, indent=2)
from .docker_deps import _parse_dockerfile
import unittest
class TestParseDockerfile(unittest.TestCase):
def test_parse_dockerfile(self):
dockerfile = '''
FROM baseimage1 AS build
RUN build
FROM baseimage2
COPY --from=build bin /usr/bin/bin
RUN fix-perms
'''
images = _parse_dockerfile(dockerfile)
self.assertEqual(['baseimage1', 'baseimage2'], images)
......@@ -7,7 +7,8 @@ try:
except ImportError:
import urllib.parse as urlparse
from .deps import rebuild_deps
from .docker_deps import dump_deps
from .rebuild import rebuild_deps
from .server import run_app
......@@ -21,6 +22,10 @@ def main():
gitlab_opts_group = common_parser.add_argument_group('gitlab options')
gitlab_opts_group.add_argument(
'--url', metavar='URL', help='Gitlab URL')
gitlab_opts_group.add_argument(
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
help='Load Gitlab authentication token from this file')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
help='Gitlab authentication token')
......@@ -29,21 +34,35 @@ def main():
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
scope_opts_group = common_parser.add_argument_group('project scope options')
scope_opts_group.add_argument(
common_parser.add_argument('--debug', action='store_true')
# Compute deps.
deps_parser = subparsers.add_parser(
'deps',
parents=[common_parser],
help='build dependency map',
description='Generate a map of Docker-derived dependencies between '
'projects on a Gitlab instance.')
deps_parser.add_argument(
'--match',
help='Search query to filter project list on the server side')
scope_opts_group.add_argument(
deps_parser.add_argument(
'--filter',
help='Regexp to filter project list on the client side')
common_parser.add_argument('--debug', action='store_true')
# Rebuild deps.
help='Regexp to filter project list on the right-hand (dependency) side')
deps_parser.add_argument(
'--docker', action='store_true',
help='Output dependencies between Docker images, not Gitlab projects')
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
parents=[common_parser],
help='rebuild dependencies of an image',
help='rebuild dependencies of a project',
description='Rebuild all projects that depend on the specified '
'Docker image.')
'project.')
rebuild_image_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
rebuild_image_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
......@@ -52,8 +71,8 @@ def main():
help='Include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'image_name',
help='Docker image name')
'project_path',
help='Project name (relative path)')
# Server.
server_parser = subparsers.add_parser(
......@@ -63,6 +82,9 @@ def main():
description='Start a HTTP server that listens for Gitlab webhooks. '
'Configure Gitlab to send Pipeline events for your projects to this '
'server to auto-rebuild first-level dependencies.')
server_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
server_parser.add_argument(
'--port', metavar='PORT', type=int, default='5404',
dest='bind_port', help='port to listen on')
......@@ -90,25 +112,33 @@ def main():
registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
logging.error('using %s as Docker registry', registry_hostname)
if cmd == 'rebuild':
rebuild_deps(
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
if cmd == 'deps':
dump_deps(
args.url,
registry_hostname,
args.token,
gitlab_token,
args.match,
args.filter,
args.image_name,
not args.docker,
)
elif cmd == 'rebuild':
rebuild_deps(
args.url,
gitlab_token,
args.deps,
args.project_path,
args.dry_run,
args.recurse,
args.recurse,
)
elif cmd == 'server':
run_app(
args.url,
registry_hostname,
args.token,
args.match,
args.filter,
gitlab_token,
args.deps,
args.bind_host,
args.bind_port,
args.webhook_auth_token,
......
import gitlab
import json
import logging
import time
def rebuild(gl, project_path, wait=False):
"""Trigger a rebuild of a project."""
project = gl.projects.get(project_path)
if not project:
return None
pipeline = project.pipelines.create({'ref': 'master'})
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
time.sleep(3)
return pipeline
def rebuild_deps(gitlab_url, gitlab_token,
project_deps_path, project_path, dry_run, wait_and_recurse):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
with open(project_deps_path) as fd:
project_deps = json.load(fd)
stack = project_deps.get(project_path, [])
while stack:
path = stack.pop(0)
logging.info('rebuilding %s', path)
if not dry_run:
rebuild(gl, path, wait_and_recurse)
if wait_and_recurse:
stack.extend(project_deps.get(path, []))
import gitlab
import json
import logging
import os
import threading
import time
try:
import Queue
except ImportError:
import queue as Queue
from flask import Flask, request, make_response
from .deps import build_dependency_tree, rebuild
from cheroot import wsgi
from flask import Flask, request, make_response
app = Flask(__name__)
from .rebuild import rebuild
# Maintain a process-wide cache of dependencies, updated periodically
# in the background. This is protected by a mutex.
class _DepsCache(object):
class _ReloadableJSONFile(object):
update_interval = 1800
check_interval = 60
_deps_lock = threading.Lock()
_deps_loaded = threading.Event()
_deps = {}
def __init__(self, path):
self.path = path
self.lock = threading.Lock()
self._load()
t = threading.Thread(
target=self._update_thread,
name='File reload thread for %s' % path)
t.setDaemon(True)
t.start()
def wait_until_loaded(self):
self._deps_loaded.wait()
def get_contents(self):
with self.lock:
return self.data
def get_deps(self, image_name):
with self._deps_lock:
return self._deps.get(image_name, [])
def _load(self):
with self.lock:
with open(self.path) as fd:
self.data = json.load(fd)
self.stamp = os.stat(self.path).st_mtime
def update_thread(self, search_pattern, filter_pattern):
loaded = False
def _update_thread(self):
while True:
time.sleep(self.check_interval)
try:
if not loaded:
app.logger.info('scanning project dependencies...')
new_deps = build_dependency_tree(app.gl, search_pattern, filter_pattern)
with self._deps_lock:
self._deps = new_deps
if not loaded:
app.logger.info('project dependencies loaded')
loaded = True
self._deps_loaded.set()
except Exception as e:
app.logger.error('error updating project dependencies: %s' % str(e))
time.sleep(self.update_interval)
if os.stat(self.path).st_mtime > self.stamp:
self._load()
except:
pass
deps_cache = _DepsCache()
queue = Queue.Queue()
def _process_request(data):
def _process_request(gl, project_deps, data):
pipeline_status = data['object_attributes']['status']
branch = data['object_attributes']['ref']
path_with_namespace = data['project']['path_with_namespace']
action = 'none'
if pipeline_status == 'success':
# Rebuild the immediate dependencies of this image.
image_name = '%s/%s' % (app.config['REGISTRY_HOSTNAME'], path_with_namespace)
deps = project_deps.get_contents().get(path_with_namespace, [])
built_projects = []
for project in deps_cache.get_deps(image_name):
for dep_path in deps:
try:
rebuild(project)
built_projects.append(project.path_with_namespace)
p = rebuild(gl, dep_path)
logging.info('started pipeline %s', p)
except Exception as e:
app.logger.error('error rebuilding project %s: %s' % (
project.path_with_namespace, str(e)))
logging.error('error rebuilding project %s: %s' % (
path_with_namespace, str(e)))
action = 'rebuilt %s' % (', '.join(built_projects),)
app.logger.info('pipeline for %s@%s: %s, action=%s',
path_with_namespace, branch, pipeline_status, action)
logging.info('pipeline for %s@%s: %s, action=%s',
path_with_namespace, branch, pipeline_status, action)
def worker_thread():
deps_cache.wait_until_loaded()
def worker_thread(gitlab_url, gitlab_token, project_deps):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
while True:
data = queue.get()
try:
_process_request(data)
_process_request(gl, project_deps, data)
except Exception as e:
app.logger.error('error processing request: %s', str(e))
logging.error('error processing request: %s', str(e))
app = Flask(__name__)
def run_app(gitlab_url, registry_hostname, gitlab_token,
search_pattern, filter_pattern, bind_host, bind_port,
webhook_token, num_workers=2):
def run_app(gitlab_url, gitlab_token,
project_deps_path, bind_host, bind_port,
webhook_token, num_workers=3):
app.config.update({
'REGISTRY_HOSTNAME': registry_hostname,
'WEBHOOK_AUTH_TOKEN': webhook_token,
})
app.gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
app.gl.auth()
# Start the update thread that will periodically update the
# dependency map (an expensive operation).
update_t = threading.Thread(
target=deps_cache.update_thread,
args=(search_pattern, filter_pattern),
name='Dependency Update Thread')
update_t.setDaemon(True)
update_t.start()
# Start the worker threads that will process the requests.
project_deps = _ReloadableJSONFile(project_deps_path)
# Start the worker threads that will process the requests in the
# background.
for i in range(num_workers):
wt = threading.Thread(
target=worker_thread,
args=(gitlab_url, gitlab_token, project_deps),
name='Worker %d' % (i+1))
wt.setDaemon(True)
wt.start()
# Start the HTTP server to receive webhook requests.
app.run(host=bind_host, port=bind_port)
server = wsgi.Server((bind_host, bind_port), app)
server.start()
@app.route('/', methods=('POST',))
......
......@@ -4,12 +4,12 @@ from setuptools import setup, find_packages
setup(
name="gitlab-docker-autodep",
version="0.2",
version="0.3",
description="Automatically rebuild Docker images",
author="Autistici/Inventati",
author_email="info@autistici.org",
url="https://git.autistici.org/ale/gitlab-docker-autodep",
install_requires=['python-gitlab', 'Flask'],
install_requires=['python-gitlab', 'Flask', 'cheroot'],
zip_safe=True,
packages=find_packages(),
entry_points={
......
[tox]
envlist = py3
[testenv]
deps=
nose
commands=
nosetests -vv []
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment