Initial commit

.gitignore

+*.pyc
+*.egg-info
+build
+dist

README.md

+gitlab-docker-autodep
+===
+
+Automatically rebuild all the downstream dependencies of Docker-based
+projects on a Gitlab instance.
+
+It scans all repositories containing a Dockerfile looking for FROM
+lines and navigates the resulting dependency tree to find all projects
+that needs to be rebuilt when their base image (or an upstream image
+thereof) changes.
+
+By default, since it is meant to be used as a trigger as the last step
+in a CI script, it will not navigate the dependency tree recursively
+but only look at the first-level dependencies: this way, further CI
+pipelines downstream will each trigger their own deps once the image
+has been rebuilt.
+
+
+## Usage
+
+Install the tool either in a virtualenv of or system-wide with:
+
+```
+sudo python setup.py install
+```
+
+This will install the *gitlab-docker-autodep* command-line tool.
+
+Invoke the tool by pointing it at your Gitlab instance with the
+*--url* command-line option, and passing the path of the repository
+you want the dependencies of as an argument. This should be the full
+repository path including the namespace, not the full URL
+(e.g. *group/projectname*).
+
+You can pass an authentication token using the *--token* command-line
+option.
+
+The tool will only examine Docker images hosted on the Docker registry
+associated with the Gitlab instance. By default the registry name is
+automatically derived from the server URL (adding a *registry*
+prefix), but it can be changed with the *--registry* command-line
+option.
+
+The tool will print out the project names of the dependencies it
+found, and it will also trigger a rebuild if the *--rebuild* option is
+specified.

gitlab_docker_autodep/main.py

+import gitlab
+import optparse
+import re
+import sys
+import urlparse
+
+
+def parse_dockerfile(df):
+    for line in df.split('\n'):
+        if line.startswith('FROM '):
+            return line[5:].strip()
+
+
+def fetch_dockerfile(gl, project):
+    try:
+        f = project.files.get(file_path='Dockerfile', ref='master')
+        return f.decode()
+    except:
+        return None
+
+
+def repo_path_from_image(image_name, registry):
+    if not image_name.startswith(registry):
+        return None
+    repo = image_name[len(registry)+1:]
+    repo = repo.split(':')[0]
+    return repo
+
+
+def build_dependency_tree(gl, registry, match_pattern):
+    """Build a dependency map of base images for all projects.
+
+    Only includes projects matching the given regexp, having a valid
+    Dockerfile, and whose base image is hosted on the specified Docker
+    registry (usually the Gitlab-hosted one).
+
+    """
+    dtree = {}
+    match_rx = re.compile(match_pattern)
+
+    # Use a generator to scan over the full list of projects
+    # (potentially large).
+    projects = gl.projects.list(all=True, as_list=False)
+    for project in projects:
+        if not match_rx.search(project.path_with_namespace):
+            continue
+        df = fetch_dockerfile(gl, project)
+        if not df:
+            continue
+        base_image = parse_dockerfile(df)
+        if not base_image:
+            print >>sys.stderr, 'ERROR: could not find base image for %s' % (
+                project.path_with_namespace,)
+            continue
+        base_repo = repo_path_from_image(base_image, registry)
+        if not base_repo:
+            # External base.
+            continue
+
+        dtree.setdefault(base_repo, []).append(project)
+
+    return dtree
+
+
+def find_deps(dtree, repo, recurse):
+    deps = []
+    def _add_deps(x):
+        for d in dtree.get(x, []):
+            deps.append(d)
+            if recurse:
+                _add_deps(d.path_with_namespace)
+    _add_deps(repo)
+    return deps
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog [<options>] <repo_path>')
+    parser.add_option('--token', help='Authentication token')
+    parser.add_option('--registry', help='Docker registry hostname')
+    parser.add_option('--url', help='Gitlab URL')
+    parser.add_option('--rebuild', action='store_true',
+                      help='Trigger a rebuild of the dependencies')
+    parser.add_option('--recurse', action='store_true',
+                      help='Include all dependencies recursively')
+    parser.add_option('--match',
+                      default='/docker-',
+                      help='Project paths should match this regexp')
+    opts, args = parser.parse_args()
+
+    if not opts.url:
+        parser.error('Must specify --url')
+    if len(args) > 1:
+        parser.error('Too many arguments')
+
+    # If --registry is not specified, make an educated guess.
+    registry = opts.registry
+    if not registry:
+        registry = 'registry.' + urlparse.urlsplit(opts.url).netloc
+        print >>sys.stderr, 'using %s as Docker registry' % (registry,)
+
+    gl = gitlab.Gitlab(opts.url, private_token=opts.token)
+    if opts.token:
+        gl.auth()
+
+    dtree = build_dependency_tree(gl, registry, opts.match)
+    deps = find_deps(dtree, args[0], opts.recurse)
+
+    for d in deps:
+        print d.path_with_namespace
+        if opts.rebuild:
+            d.pipelines.create({'ref': 'master'})
+    
+
+if __name__ == '__main__':
+    main()

setup.py

+#!/usr/bin/python
+
+from setuptools import setup, find_packages
+
+setup(
+    name="gitlab-docker-autodep",
+    version="0.1",
+    description="Automatically rebuild Docker images",
+    author="Autistici/Inventati",
+    author_email="info@autistici.org",
+    url="https://git.autistici.org/ale/gitlab-docker-autodep",
+    install_requires=['python-gitlab'],
+    zip_safe=True,
+    packages=find_packages(),
+    entry_points={
+        "console_scripts": [
+            "gitlab-docker-autodep = gitlab_docker_autodep.main:main",
+            ],
+        },
+    )
+