.gitlab-ci.yml

-include: "https://git.autistici.org/ai3/build-deb/raw/master/ci-common.yml"
+include: "https://git.autistici.org/pipelines/debian/raw/master/common.yml"

contrib/deps-to-dot.awk

+#!/usr/bin/awk -f
+
+BEGIN {
+  print "digraph deps {"
+  print "rankdir=\"LR\";"
+}
+
+{
+  print "\"" $1 "\" -> \"" $2 "\""
+}
+
+END {
+  print "}"
+}
+

contrib/update.sh

-#!/bin/sh
+#!/bin/bash
 
 deps_file=/var/lib/gitlab-deps/deps.list
 rev_deps_file=/var/lib/gitlab-deps/reverse-deps.list
-reload=1
 ret=0
 
-if [ "x$1" = "x--no-reload" ]; then
-    reload=0
-fi
-
 [ -e /etc/default/gitlab-deps ] && . /etc/default/gitlab-deps
 
 if [ -z "$GITLAB_URL" -o -z "$GITLAB_TOKEN_FILE" ]; then
@@ -21,18 +16,18 @@ opts="--token-file=$GITLAB_TOKEN_FILE --url=$GITLAB_URL"
 tmp_file="${deps_file}.tmp.$$"
 trap "rm -f $tmp_file 2>/dev/null" EXIT
 
+set -o pipefail
 gitlab-deps list-projects $opts \
     | egrep "${PROJECT_REGEXP:-.*}" \
     | gitlab-deps deps $opts \
+        ${GITLAB_REGISTRY_HOSTNAME:+--registry=${GITLAB_REGISTRY_HOSTNAME}} \
         > $tmp_file
+if [ $? -gt 0 ]; then
+    rm -f $tmp_file
+    exit 1
+fi
 
-if ! cmp --quiet $tmp_file $deps_file; then
 mv -f $tmp_file $deps_file
 awk '{print $2, $1}' < $deps_file > $rev_deps_file
-    if [ $reload -eq 1 ]; then
-        systemctl reload-or-restart gitlab-deps
-        ret=$?
-    fi
-fi
 
-exit $ret
+exit 0

debian/compat

-13

debian/control

 Source: gitlab-deps
 Section: net
-Priority: extra
+Priority: optional
 Maintainer: Autistici/Inventati <debian@autistici.org>
-Build-Depends: debhelper (>= 11), dh-python, python3,
+Build-Depends: debhelper-compat (= 13), dh-python, python3,
 Standards-Version: 4.1.3
 
 Package: gitlab-deps

debian/gitlab-deps-update.service

@@ -5,5 +5,6 @@ After=gitlab-deps.service
 [Service]
 Type=oneshot
 ExecStart=/usr/sbin/update-gitlab-deps
+ExecStartPost=+/usr/bin/systemctl restart gitlab-deps.service
 User=gitlab-deps
 Group=gitlab-deps

debian/gitlab-deps.default

 # Set to the public URL of your Gitlab repository.
 GITLAB_URL=
 
+# Set to the registry hostname (https:// is assumed) if necessary.
+GITLAB_REGISTRY_HOSTNAME=
+
 # File with the authorization token.
 GITLAB_TOKEN_FILE=/var/lib/gitlab-deps/auth_token
 

debian/gitlab-deps.service

 [Unit]
 Description=gitlab-deps server
 After=network.target
+StartLimitIntervalSec=0
 
 [Service]
 Type=simple
 Restart=on-failure
+RestartSec=3
 EnvironmentFile=/etc/default/gitlab-deps
 ExecStart=/usr/sbin/gitlab-deps-server-wrapper
 User=gitlab-deps

debian/source/format

+3.0 (native)

gitlab_deps/deps.py

@@ -14,10 +14,9 @@ def split_project_branch(project_with_branch):
 
 def list_projects(gl, search_pattern):
     projects = gl.projects.list(
-        all=True,
         search=search_pattern,
         search_namespaces=True,
-        as_list=False,
+        iterator=True,
         simple=True,
         archived=False,
     )
@@ -29,7 +28,7 @@ def get_branches(gl, project_names):
     for path_with_namespace in project_names:
         p = gl.projects.get(path_with_namespace)
         try:
-            branches = p.branches.list()
+            branches = p.branches.list(iterator=True)
         except Exception as e:
             logging.error('error getting branches for %s: %s',
                           path_with_namespace, e)
@@ -80,12 +79,12 @@ def get_explicit_deps(gl, project_path, branch_name):
         return []
     try:
         f = p.files.get(file_path='.gitlab-deps', ref=branch_name)
-        return f.decode().decode('utf-8').split('\n')
+        return not_null(f.decode().decode('utf-8').split('\n'))
     except Exception:
         return []
 
 
-_docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:(.*))?$')
+_docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:([^@]*))?(@.*)?$')
 
 
 def docker_image_to_project(docker_image, registry_hostname):
@@ -99,13 +98,13 @@ def docker_image_to_project(docker_image, registry_hostname):
         return m[3], branch
 
 
-_url_rx = re.compile(r'^(https?://[^/]+/)([^:]+)(:.*)?$')
+_url_rx = re.compile(r'^https?://[^/]+/([^:]+)(:(.*))?$')
 
 
 def url_to_project(url, gitlab_url):
     m = _url_rx.match(url)
-    if m and m[1] == gitlab_url:
-        return m[2], m[3] or DEFAULT_BRANCH
+    if url.startswith(gitlab_url) and m:
+        return m[1], m[3] or DEFAULT_BRANCH
 
 
 def not_null(l):

gitlab_deps/hooks.py

@@ -6,14 +6,27 @@ def check_hook(gl, hook_url, webhook_token, project_path, dry_run):
     found = False
     for h in project.hooks.list():
         if h.url == hook_url and h.pipeline_events:
+            if not dry_run:
+                h.token = webhook_token
+                h.save()
             found = True
             break
     if found:
         return
     logging.info('adding pipeline_events hook to %s', project_path)
     if not dry_run:
-        project.hooks.add(
-            url=hook_url,
-            pipeline_events=True,
-            token=webhook_token,
-        )
+        project.hooks.create({
+            'url': hook_url,
+            'push_events': False,
+            'pipeline_events': True,
+            'token': webhook_token,
+        })
+
+
+def clear_hooks(gl, project_path, dry_run):
+    project = gl.projects.get(project_path)
+    for h in project.hooks.list():
+        logging.info('%s: removing hook %s', project_path, h.url)
+        if not dry_run:
+            h.delete()
+

gitlab_deps/main.py

@@ -7,7 +7,7 @@ from urllib.parse import urlsplit
 
 from .deps import get_branches, list_projects, list_deps, \
     split_project_branch, read_deps
-from .hooks import check_hook
+from .hooks import check_hook, clear_hooks
 from .rebuild import rebuild_deps
 from .server import run_app
 
@@ -105,6 +105,14 @@ from standard input if a filename is omitted or specified as '-'.
         type=argparse.FileType('r'),
         nargs='?', default=sys.stdin)
 
+    clear_hooks_parser = subparsers.add_parser(
+        'clear-hooks',
+        parents=[common_parser])
+    clear_hooks_parser.add_argument(
+        'projects_list',
+        type=argparse.FileType('r'),
+        nargs='?', default=sys.stdin)
+
     # Setup pipeline hooks on the specified projects.
     set_hooks_parser = subparsers.add_parser(
         'set-hooks',
@@ -209,9 +217,10 @@ specified as '-'.
 
     args = parser.parse_args()
     cmd = args.subparser
-
+    if not cmd:
+        parser.error('no command specified')
     if not args.url:
-        parser.error('Must specify --url')
+        parser.error('must specify --url')
 
     logging.basicConfig(
         format='%(message)s',
@@ -263,6 +272,16 @@ specified as '-'.
                 logging.error('error checking project %s: %s',
                               project_path, e)
 
+    elif cmd == 'clear-hooks':
+        projects = set(y[0] for y in (
+            split_project_branch(x.strip()) for x in args.projects_list))
+        for project_path in projects:
+            try:
+                clear_hooks(gl, project_path, args.dry_run)
+            except Exception as e:
+                logging.error('error clearing hooks for project %s: %s',
+                              project_path, e)
+
     elif cmd == 'server':
         deps = read_deps(args.dependencies_list)
         run_app(gl, deps, args.bind_host, args.bind_port,

gitlab_deps/server.py

 import logging
 import threading
-try:
-    import Queue
-except ImportError:
-    import queue as Queue
+from queue import Queue
 
 from cheroot import wsgi
 from flask import Flask, request, make_response, abort
@@ -11,7 +8,7 @@ from flask import Flask, request, make_response, abort
 from .rebuild import rebuild
 
 
-queue = Queue.Queue()
+queue = Queue()
 
 
 def _process_request(gl, project_deps, data):
@@ -61,8 +58,8 @@ def run_app(gl, project_deps, bind_host, bind_port,
         wt = threading.Thread(
             target=worker_thread,
             args=(gl, project_deps),
-            name='Worker %d' % (i+1))
-        wt.setDaemon(True)
+            name='Worker %d' % (i+1),
+            daemon=True)
         wt.start()
 
     # Start the HTTP server to receive webhook requests.
@@ -87,3 +84,8 @@ def app_webhook():
         queue.put(request.json)
 
     return make_response('{}')
+
+
+@app.route('/healthz')
+def healthz():
+    return make_response('OK')

tox.ini

-[tox]
-envlist = py3
-
-[testenv]
-deps=
-  nose
-commands=
-  nosetests -vv []
-