Skip to content
Snippets Groups Projects
Commit 5c74a911 authored by ale's avatar ale
Browse files

Refactor into split components

parent a78a90af
Branches
No related tags found
No related merge requests found
......@@ -33,20 +33,23 @@ API](https://python-gitlab.readthedocs.io/en/stable/), and
# Usage
There are two modes of using this software: either as a one-shot
command-line tool, or as a standalone HTTP server responding to Gitlab
Webhook requests, to integrate Docker dependencies with Gitlab CI.
The tool is split into functional components:
In both cases, the program is configured via command-line options.
* scan Gitlab and generate a dependency map (stored as a JSON file)
* manually trigger builds using the dependency map
* run a server that listens for Gitlab notifications and trigger
builds
In all cases, the program is configured via command-line options.
## Common options
The tool must be pointed at your Gitlab instance with the *--url*
command-line option,
You can pass an authentication token using the *--token* command-line
option. This is usually required in order to trigger CI pipelines: the
access token must have the *api* scope.
You can pass an authentication token using the *--token* or
*--token-file* command-line options. This is usually required in order
to trigger CI pipelines: the access token must have the *api* scope.
The tool will only examine Docker images hosted on the Docker registry
associated with the Gitlab instance. By default the registry name is
......@@ -68,7 +71,7 @@ example, it is possible to efficiently limit the scope of the tool to
a specific namespace:
```
gitlab-docker-autodep ... --match myns --filter ^myns/ ...
gitlab-docker-autodep deps --match myns --filter ^myns/ ...
```
Note that, when building the dependency tree:
......@@ -123,8 +126,9 @@ stored in */etc/gitlab_docker_token*:
```
gitlab-docker-autodep \
--url=https://my.gitlab \
--token=$(< /etc/gitlab_docker_token) \
--token-file=/etc/gitlab_docker_token \
server \
--deps=deps.json
--host=127.0.0.1 --port=14001
```
......@@ -132,6 +136,16 @@ You can then configure your project's webhooks with the URL
`http://localhost:14001/`, with the *Trigger* checkbox set only
on *Pipeline events*.
Then you should generate the *deps.json* dependency map periodically,
for instance with a cron job:
```
*/30 * * * * root gitlab-docker-autodep
--url=https://my.gitlab
--token-file=/etc/gitlab_docker_token
deps > deps.json
```
It can be useful to run the *rebuild* command from a cron job, for
instance in order to rebuild images on a periodic schedule, and
assuming all your projects share a common base image:
......@@ -139,6 +153,6 @@ assuming all your projects share a common base image:
```
50 5 * * * root gitlab-docker-autodep
--url=https://my.gitlab
--token=$(< /etc/gitlab_docker_token)
--token-file=/etc/gitlab_docker_token
rebuild $MY_BASE_IMAGE
```
import gitlab
import json
import logging
import re
import time
import sys
_from_rx = re.compile(r'^FROM\s+(.*)$')
_from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE)
def _parse_dockerfile(df):
for line in df.split('\n'):
m = _from_rx.match(line)
if m:
return m.group(1)
return _from_rx.findall(df)
def _fetch_dockerfile(gl, project, ref):
......@@ -35,7 +33,7 @@ def _remove_image_tag(name):
return name
def build_dependency_tree(gl, search_pattern=None, filter_pattern=None):
def build_docker_deps(gl, search_pattern=None, filter_pattern=None):
"""Build the project dependency map based on Dockerfiles.
This can be a fairly expensive (long) operation if the list of
......@@ -60,60 +58,43 @@ def build_dependency_tree(gl, search_pattern=None, filter_pattern=None):
projects = gl.projects.list(all=True, search=search_pattern, as_list=False)
for project in projects:
project_name = project.path_with_namespace
project_url = project_name
if filter_rx is not None and not filter_rx.search(project.path_with_namespace):
continue
if not _has_gitlab_ci(gl, project, 'master'):
continue
df = _fetch_dockerfile(gl, project, 'master')
if not df:
continue
base_image = _parse_dockerfile(df.decode('utf-8'))
if not base_image:
base_images = _parse_dockerfile(df.decode('utf-8'))
if not base_images:
logging.error('ERROR: could not find base image for %s',
project.path_with_namespace)
continue
if not _has_gitlab_ci(gl, project, 'master'):
continue
deps.setdefault(_remove_image_tag(base_image), []).append(project)
for img in base_images:
deps.setdefault(_remove_image_tag(img), []).append(project_url)
return deps
def rebuild(project, wait=False):
"""Trigger a rebuild of a project."""
pipeline = project.pipelines.create({'ref': 'master'})
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
time.sleep(3)
return pipeline
def docker_deps_to_project_deps(deps, registry_hostname):
out = {}
for image_name in deps:
if image_name.startswith(registry_hostname):
project_name = image_name[len(registry_hostname)+1:]
out[project_name] = deps[image_name]
return out
def rebuild_deps(gitlab_url, registry_hostname, gitlab_token,
search_pattern, filter_pattern, image_name,
dry_run=False, recurse=False, wait=False):
"""Rebuild dependencies of the given image."""
def dump_deps(gitlab_url, registry_hostname, gitlab_token,
deps_match, deps_filter, project_deps=True):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
deps = build_dependency_tree(gl, search_pattern)
deps = build_docker_deps(gl, deps_match, deps_filter)
stack = deps.get(_remove_image_tag(image_name), [])
while stack:
project = stack.pop(0)
logging.info('rebuilding %s', project.path_with_namespace)
if not dry_run:
try:
pipeline = rebuild(project, wait)
if pipeline.status not in ('success', 'pending'):
logging.error('ERROR: build failed for %s (status: %s)',
project.path_with_namespace, pipeline.status)
continue
except gitlab.exceptions.GitlabError as e:
logging.error('ERROR: gitlab error: %s: %s',
project.path_with_namespace, str(e))
continue
if project_deps:
deps = docker_deps_to_project_deps(deps, registry_hostname)
if recurse:
image_name = '%s/%s' % (
registry_hostname, project.path_with_namespace)
stack.extend(deps.get(image_name, []))
json.dump(deps, sys.stdout, indent=2)
from .docker_deps import _parse_dockerfile
import unittest
class TestParseDockerfile(unittest.TestCase):
def test_parse_dockerfile(self):
dockerfile = '''
FROM baseimage1 AS build
RUN build
FROM baseimage2
COPY --from=build bin /usr/bin/bin
RUN fix-perms
'''
images = _parse_dockerfile(dockerfile)
self.assertEqual(['baseimage1', 'baseimage2'], images)
......@@ -7,7 +7,8 @@ try:
except ImportError:
import urllib.parse as urlparse
from .deps import rebuild_deps
from .docker_deps import dump_deps
from .rebuild import rebuild_deps
from .server import run_app
......@@ -21,6 +22,10 @@ def main():
gitlab_opts_group = common_parser.add_argument_group('gitlab options')
gitlab_opts_group.add_argument(
'--url', metavar='URL', help='Gitlab URL')
gitlab_opts_group.add_argument(
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
help='Load Gitlab authentication token from this file')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
help='Gitlab authentication token')
......@@ -29,21 +34,35 @@ def main():
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
scope_opts_group = common_parser.add_argument_group('project scope options')
scope_opts_group.add_argument(
common_parser.add_argument('--debug', action='store_true')
# Compute deps.
deps_parser = subparsers.add_parser(
'deps',
parents=[common_parser],
help='build dependency map',
description='Generate a map of Docker-derived dependencies between '
'projects on a Gitlab instance.')
deps_parser.add_argument(
'--match',
help='Search query to filter project list on the server side')
scope_opts_group.add_argument(
deps_parser.add_argument(
'--filter',
help='Regexp to filter project list on the client side')
common_parser.add_argument('--debug', action='store_true')
help='Regexp to filter project list on the right-hand (dependency) side')
deps_parser.add_argument(
'--docker', action='store_true',
help='Output dependencies between Docker images, not Gitlab projects')
# Rebuild deps.
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
parents=[common_parser],
help='rebuild dependencies of an image',
help='rebuild dependencies of a project',
description='Rebuild all projects that depend on the specified '
'Docker image.')
'project.')
rebuild_image_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
rebuild_image_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
......@@ -52,8 +71,8 @@ def main():
help='Include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'image_name',
help='Docker image name')
'project_path',
help='Project name (relative path)')
# Server.
server_parser = subparsers.add_parser(
......@@ -63,6 +82,9 @@ def main():
description='Start a HTTP server that listens for Gitlab webhooks. '
'Configure Gitlab to send Pipeline events for your projects to this '
'server to auto-rebuild first-level dependencies.')
server_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
server_parser.add_argument(
'--port', metavar='PORT', type=int, default='5404',
dest='bind_port', help='port to listen on')
......@@ -90,25 +112,33 @@ def main():
registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
logging.error('using %s as Docker registry', registry_hostname)
if cmd == 'rebuild':
rebuild_deps(
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
if cmd == 'deps':
dump_deps(
args.url,
registry_hostname,
args.token,
gitlab_token,
args.match,
args.filter,
args.image_name,
not args.docker,
)
elif cmd == 'rebuild':
rebuild_deps(
args.url,
gitlab_token,
args.deps,
args.project_path,
args.dry_run,
args.recurse,
args.recurse,
)
elif cmd == 'server':
run_app(
args.url,
registry_hostname,
args.token,
args.match,
args.filter,
gitlab_token,
args.deps,
args.bind_host,
args.bind_port,
args.webhook_auth_token,
......
import gitlab
import json
import logging
import time
def rebuild(gl, project_path, wait=False):
"""Trigger a rebuild of a project."""
project = gl.projects.get(project_path)
if not project:
return None
pipeline = project.pipelines.create({'ref': 'master'})
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
time.sleep(3)
return pipeline
def rebuild_deps(gitlab_url, gitlab_token,
project_deps_path, project_path, dry_run, wait_and_recurse):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
with open(project_deps_path) as fd:
project_deps = json.load(fd)
stack = project_deps.get(project_path, [])
while stack:
path = stack.pop(0)
logging.info('rebuilding %s', path)
if not dry_run:
rebuild(gl, path, wait_and_recurse)
if wait_and_recurse:
stack.extend(project_deps.get(path, []))
import gitlab
import json
import logging
import os
import threading
import time
try:
import Queue
except ImportError:
import queue as Queue
from flask import Flask, request, make_response
from .deps import build_dependency_tree, rebuild
from cheroot import wsgi
from flask import Flask, request, make_response
app = Flask(__name__)
from .rebuild import rebuild
# Maintain a process-wide cache of dependencies, updated periodically
# in the background. This is protected by a mutex.
class _DepsCache(object):
class _ReloadableJSONFile(object):
update_interval = 1800
check_interval = 60
_deps_lock = threading.Lock()
_deps_loaded = threading.Event()
_deps = {}
def __init__(self, path):
self.path = path
self.lock = threading.Lock()
self._load()
t = threading.Thread(
target=self._update_thread,
name='File reload thread for %s' % path)
t.setDaemon(True)
t.start()
def wait_until_loaded(self):
self._deps_loaded.wait()
def get_contents(self):
with self.lock:
return self.data
def get_deps(self, image_name):
with self._deps_lock:
return self._deps.get(image_name, [])
def _load(self):
with self.lock:
with open(self.path) as fd:
self.data = json.load(fd)
self.stamp = os.stat(self.path).st_mtime
def update_thread(self, search_pattern, filter_pattern):
loaded = False
def _update_thread(self):
while True:
time.sleep(self.check_interval)
try:
if not loaded:
app.logger.info('scanning project dependencies...')
new_deps = build_dependency_tree(app.gl, search_pattern, filter_pattern)
with self._deps_lock:
self._deps = new_deps
if not loaded:
app.logger.info('project dependencies loaded')
loaded = True
self._deps_loaded.set()
except Exception as e:
app.logger.error('error updating project dependencies: %s' % str(e))
time.sleep(self.update_interval)
if os.stat(self.path).st_mtime > self.stamp:
self._load()
except:
pass
deps_cache = _DepsCache()
queue = Queue.Queue()
def _process_request(data):
def _process_request(gl, project_deps, data):
pipeline_status = data['object_attributes']['status']
branch = data['object_attributes']['ref']
path_with_namespace = data['project']['path_with_namespace']
action = 'none'
if pipeline_status == 'success':
# Rebuild the immediate dependencies of this image.
image_name = '%s/%s' % (app.config['REGISTRY_HOSTNAME'], path_with_namespace)
deps = project_deps.get_contents().get(path_with_namespace, [])
built_projects = []
for project in deps_cache.get_deps(image_name):
for dep_path in deps:
try:
rebuild(project)
built_projects.append(project.path_with_namespace)
p = rebuild(gl, dep_path)
logging.info('started pipeline %s', p)
except Exception as e:
app.logger.error('error rebuilding project %s: %s' % (
project.path_with_namespace, str(e)))
logging.error('error rebuilding project %s: %s' % (
path_with_namespace, str(e)))
action = 'rebuilt %s' % (', '.join(built_projects),)
app.logger.info('pipeline for %s@%s: %s, action=%s',
logging.info('pipeline for %s@%s: %s, action=%s',
path_with_namespace, branch, pipeline_status, action)
def worker_thread():
deps_cache.wait_until_loaded()
def worker_thread(gitlab_url, gitlab_token, project_deps):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
while True:
data = queue.get()
try:
_process_request(data)
_process_request(gl, project_deps, data)
except Exception as e:
app.logger.error('error processing request: %s', str(e))
logging.error('error processing request: %s', str(e))
app = Flask(__name__)
def run_app(gitlab_url, registry_hostname, gitlab_token,
search_pattern, filter_pattern, bind_host, bind_port,
webhook_token, num_workers=2):
def run_app(gitlab_url, gitlab_token,
project_deps_path, bind_host, bind_port,
webhook_token, num_workers=3):
app.config.update({
'REGISTRY_HOSTNAME': registry_hostname,
'WEBHOOK_AUTH_TOKEN': webhook_token,
})
app.gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
app.gl.auth()
# Start the update thread that will periodically update the
# dependency map (an expensive operation).
update_t = threading.Thread(
target=deps_cache.update_thread,
args=(search_pattern, filter_pattern),
name='Dependency Update Thread')
update_t.setDaemon(True)
update_t.start()
# Start the worker threads that will process the requests.
project_deps = _ReloadableJSONFile(project_deps_path)
# Start the worker threads that will process the requests in the
# background.
for i in range(num_workers):
wt = threading.Thread(
target=worker_thread,
args=(gitlab_url, gitlab_token, project_deps),
name='Worker %d' % (i+1))
wt.setDaemon(True)
wt.start()
# Start the HTTP server to receive webhook requests.
app.run(host=bind_host, port=bind_port)
server = wsgi.Server((bind_host, bind_port), app)
server.start()
@app.route('/', methods=('POST',))
......
......@@ -4,12 +4,12 @@ from setuptools import setup, find_packages
setup(
name="gitlab-docker-autodep",
version="0.2",
version="0.3",
description="Automatically rebuild Docker images",
author="Autistici/Inventati",
author_email="info@autistici.org",
url="https://git.autistici.org/ale/gitlab-docker-autodep",
install_requires=['python-gitlab', 'Flask'],
install_requires=['python-gitlab', 'Flask', 'cheroot'],
zip_safe=True,
packages=find_packages(),
entry_points={
......
[tox]
envlist = py3
[testenv]
deps=
nose
commands=
nosetests -vv []
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment