Update server code to support the new dependency maps

Improve help messages for the command too.

gitlab_docker_autodep/hooks.py

+import logging
 
 
-def check_hook(gl, hook_url, webhook_token, project_path):
+def check_hook(gl, hook_url, webhook_token, project_path, dry_run):
     project = gl.projects.get(project_path)
     found = False
     for h in project.hooks.list():
@@ -9,6 +10,8 @@ def check_hook(gl, hook_url, webhook_token, project_path):
             break
     if found:
         return
+    logging.info('adding pipeline_events hook to %s', project_path)
+    if not dry_run:
         project.hooks.add(
             url=hook_url,
             pipeline_events=True,

gitlab_docker_autodep/main.py

@@ -3,7 +3,7 @@ import gitlab
 import logging
 import os
 import sys
-import urllib.parse as urlparse
+from urllib.parse import urlsplit
 
 from .deps import get_branches, list_projects, list_deps, \
     split_project_branch, read_deps
@@ -18,12 +18,14 @@ def _fmtdesc(s):
 
 def main():
     parser = argparse.ArgumentParser(
-        description='Rebuild Docker images on a Gitlab instance.')
+        description='Manage Gitlab project dependencies and trigger pipelines.')
     subparsers = parser.add_subparsers(dest='subparser')
 
     # Common options.
     common_parser = argparse.ArgumentParser(add_help=False)
-    common_parser.add_argument('--debug', action='store_true')
+    common_parser.add_argument(
+        '--debug', action='store_true',
+        help='increase logging level')
     common_parser.add_argument(
         '-n', '--dry-run', action='store_true', dest='dry_run',
         help='only show what would be done')
@@ -35,16 +37,11 @@ def main():
         '--token-file', metavar='FILE',
         type=argparse.FileType('r'),
         default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
-        help='Load Gitlab authentication token from this file')
+        help='file containing the Gitlab authentication token')
     gitlab_opts_group.add_argument(
         '--token', metavar='TOKEN',
         default=os.getenv('GITLAB_AUTH_TOKEN'),
         help='Gitlab authentication token')
-    gitlab_opts_group.add_argument(
-        '--registry', metavar='NAME',
-        default=os.getenv('GITLAB_REGISTRY'),
-        help='Docker registry hostname (if empty, it will be '
-        'automatically derived from --url)')
 
     # List projects.
     list_projects_parser = subparsers.add_parser(
@@ -69,7 +66,7 @@ or
 '''))
     list_projects_parser.add_argument(
         '--search',
-        help='Search query used to filter project list on the server side')
+        help='search query used to filter project list on the server side')
 
     # Compute deps.
     deps_parser = subparsers.add_parser(
@@ -81,9 +78,9 @@ or
 Generate a map of dependencies between projects on a
 Gitlab instance.
 
-The input (on standard input) must consist of a list of projects along
-with their branches, separated by a colon, one per line. If the branch
-is unspecified, 'master' is assumed.
+The input must consist of a list of projects along with their
+branches, separated by a colon, one per line. If the branch is
+unspecified, 'master' is assumed.
 
 The output consists of pairs of project / dependency (so, these are
 'forward' dependencies), for all projects/branches specified in the
@@ -94,51 +91,108 @@ columns in the output, e.g.:
 
     gitlab-deps deps < project.list | awk '{print $2, $1}'
 
+'''), epilog=_fmtdesc('''
+Input can be read from a file (if passed as an argument), or
+from standard input if a filename is omitted or specified as '-'.
 '''))
+    deps_parser.add_argument(
+        '--registry', metavar='NAME',
+        default=os.getenv('GITLAB_REGISTRY'),
+        help='Docker registry hostname (if empty, it will be '
+        'automatically derived from --url)')
+    deps_parser.add_argument(
+        'projects_list',
+        type=argparse.FileType('r'),
+        nargs='?', default=sys.stdin)
 
     # Setup pipeline hooks on the specified projects.
     set_hooks_parser = subparsers.add_parser(
         'set-hooks',
         parents=[common_parser],
         help='set pipeline hooks on projects',
-        description='Set the pipeline hooks on the specified projects '
-        '(usually points at our own server)')
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=_fmtdesc('''
+Set a HTTP hook for pipeline_events on the specified projects.
+
+Takes a list of projects (optional branch specifiers will be ignored)
+as input. Pipeline hooks are required by 'gitlab-deps server' to
+trigger dependent builds, so a common way to use this command is to
+feed it the right-hand side of the 'gitlab-deps deps' output, e.g.:
+
+    gitlab-deps deps < project.list \\
+        | awk '{print $2}' \\
+        | gitlab-deps set-hooks --hook-url=...
+
+using --hook-url to point at the URL of 'gitlab-deps server'.
+
+'''), epilog=_fmtdesc('''
+Input can be read from a file (if passed as an argument), or
+from standard input if a filename is omitted or specified as '-'.
+'''))
     set_hooks_parser.add_argument(
         '--hook-url', metavar='URL',
         help='URL for the pipeline HTTP hook')
     set_hooks_parser.add_argument(
         '--webhook-auth-token', metavar='TOKEN',
-        help='Secret X-Gitlab-Token for request authentication')
+        help='secret X-Gitlab-Token for request authentication')
+    set_hooks_parser.add_argument(
+        'projects_list',
+        type=argparse.FileType('r'),
+        nargs='?', default=sys.stdin)
 
     # Trigger rebuilds of reverse deps.
     rebuild_image_parser = subparsers.add_parser(
         'rebuild',
         parents=[common_parser],
         help='rebuild dependencies of a project',
-        description='Rebuild all projects that depend on the specified '
-        'project.')
-    rebuild_image_parser.add_argument(
-        '--deps', metavar='FILE',
-        help='file with project dependencies')
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=_fmtdesc('''
+Rebuild all projects that depend on the specified project.
+
+Takes a single project path as argument, and triggers a rebuild of its
+direct dependencies. Useful for one-off rebuilds.
+
+If the --recurse option is provided, the tool will wait for completion
+of the pipeline and recursively trigger its dependencies too,
+navigating the entire dependency tree.
+
+'''), epilog=_fmtdesc('''
+Project dependencies can be read from a file (if passed as an
+argument), or from standard input if a filename is omitted or
+specified as '-'.
+'''))
     rebuild_image_parser.add_argument(
         '--recurse', action='store_true',
-        help='Include all dependencies recursively '
+        help='include all dependencies recursively '
         'and wait for completion of the pipelines')
     rebuild_image_parser.add_argument(
         'project_path',
-        help='Project name (relative path, with optional branch)')
+        help='project name (relative path, with optional branch)')
+    rebuild_image_parser.add_argument(
+        'dependencies_list',
+        type=argparse.FileType('r'),
+        nargs='?', default=sys.stdin)
 
     # Server.
     server_parser = subparsers.add_parser(
         'server',
         parents=[common_parser],
-        help='start a HTTP server',
-        description='Start a HTTP server that listens for Gitlab webhooks. '
-        'Configure Gitlab to send Pipeline events for your projects to this '
-        'server to auto-rebuild first-level dependencies.')
-    server_parser.add_argument(
-        '--deps', metavar='FILE',
-        help='file with project dependencies')
+        help='start the HTTP server',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=_fmtdesc('''
+Start an HTTP server that listens for Gitlab webhooks.
+
+When the server receives a pipeline event from Gitlab, it will trigger
+new builds for the direct dependencies of the project. The server is
+meant to be associated with a single Gitlab instance.
+
+You must provide the server with the list of project dependencies.
+
+'''), epilog=_fmtdesc('''
+Project dependencies can be read from a file (if passed as an
+argument), or from standard input if a filename is omitted or
+specified as '-'.
+'''))
     server_parser.add_argument(
         '--port', metavar='PORT', type=int, default='5404',
         dest='bind_port', help='port to listen on')
@@ -147,7 +201,11 @@ columns in the output, e.g.:
         dest='bind_host', help='address to listen on')
     server_parser.add_argument(
         '--webhook-auth-token', metavar='TOKEN',
-        help='Secret X-Gitlab-Token for request authentication')
+        help='secret X-Gitlab-Token for request authentication')
+    server_parser.add_argument(
+        'dependencies_list',
+        type=argparse.FileType('r'),
+        nargs='?', default=sys.stdin)
 
     args = parser.parse_args()
     cmd = args.subparser
@@ -160,13 +218,7 @@ columns in the output, e.g.:
         level=logging.DEBUG if args.debug else logging.INFO,
     )
 
-    # If --registry is not specified, make an educated guess.
-    registry_hostname = args.registry
-    if not registry_hostname:
-        registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
-        logging.warning('guessed %s for the Docker registry hostname',
-                        registry_hostname)
-
+    # Connect to the Gitlab API.
     gitlab_token = args.token
     if not gitlab_token and args.token_file:
         gitlab_token = args.token_file.read().strip().encode('utf-8')
@@ -179,25 +231,36 @@ columns in the output, e.g.:
     if cmd == 'list-projects':
         for p, b in get_branches(gl, list_projects(gl, args.search)):
             print(f'{p}:{b}')
-    if cmd == 'deps':
-        projects = [split_project_branch(x.strip()) for x in sys.stdin]
+
+    elif cmd == 'deps':
+        # If --registry is not specified, make an educated guess.
+        registry_hostname = args.registry
+        if not registry_hostname:
+            registry_hostname = 'registry.' + urlsplit(args.url).netloc
+        logging.warning('guessed %s for the Docker registry hostname',
+                        registry_hostname)
+        projects = [split_project_branch(x.strip())
+                    for x in args.projects_list]
         list_deps(gl, args.url, registry_hostname, projects)
+
     elif cmd == 'rebuild':
-        deps = read_deps(sys.stdin)
+        deps = read_deps(args.dependencies_list)
         project_path, branch_name = split_project_branch(args.project_path)
         rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
                      args.recurse)
+
     elif cmd == 'set-hooks':
         if not args.hook_url:
             parser.error('Must specify --hook-url')
         # Need a project list on input, ignore branches.
         projects = set(y[0] for y in (
-            split_project_branch(x.strip()) for x in sys.stdin))
+            split_project_branch(x.strip()) for x in args.projects_list))
         for project_path in projects:
             check_hook(gl, args.hook_url, args.webhook_auth_token,
-                       project_path)
+                       project_path, args.dry_run)
+
     elif cmd == 'server':
-        deps = read_deps(sys.stdin)
+        deps = read_deps(args.dependencies_list)
         run_app(gl, deps, args.bind_host, args.bind_port,
                 args.webhook_auth_token)
 

gitlab_docker_autodep/rebuild.py

@@ -2,13 +2,14 @@ import logging
 import time
 
 
-def rebuild(gl, project_path, wait=False):
+def rebuild(gl, project_path, branch_name, wait=False):
     """Trigger a rebuild of a project."""
     project = gl.projects.get(project_path)
     if not project:
         return None
 
-    pipeline = project.pipelines.create({'ref': 'master'})
+    pipeline = project.pipelines.create({'ref': branch_name})
+    logging.info('started pipeline %s', pipeline.web_url)
     if wait:
         while pipeline.finished_at is None:
             pipeline.refresh()
@@ -20,9 +21,9 @@ def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
                  wait_and_recurse):
     stack = project_deps.get((project_path, branch_name), [])
     while stack:
-        path = stack.pop(0)
-        logging.info('rebuilding %s', path)
+        path, branch = stack.pop(0)
+        logging.info('rebuilding %s:%s', path, branch)
         if not dry_run:
-            rebuild(gl, path, wait_and_recurse)
+            rebuild(gl, path, branch, wait_and_recurse)
         if wait_and_recurse:
-            stack.extend(project_deps.get(path, []))
+            stack.extend(project_deps.get((path, branch), []))

gitlab_docker_autodep/server.py

@@ -16,25 +16,25 @@ queue = Queue.Queue()
 
 def _process_request(gl, project_deps, data):
     pipeline_status = data['object_attributes']['status']
-    branch = data['object_attributes']['ref']
-    path_with_namespace = data['project']['path_with_namespace']
+    branch_name = data['object_attributes']['ref']
+    project_path = data['project']['path_with_namespace']
 
     action = 'none'
     if pipeline_status == 'success':
-        deps = project_deps.get_contents().get(path_with_namespace, [])
+        deps = project_deps.get((project_path, branch_name), [])
 
         built_projects = []
-        for dep_path in deps:
+        for dep_path, dep_branch in deps:
             try:
-                p = rebuild(gl, dep_path)
-                logging.info('started pipeline %s', p)
+                rebuild(gl, dep_path, dep_branch)
+                built_projects.append(f'{dep_path}:{dep_branch}')
             except Exception as e:
-                logging.error('error rebuilding project %s: %s' % (
-                    path_with_namespace, str(e)))
+                logging.error('error rebuilding project %s:%s: %s' % (
+                    dep_path, dep_branch, str(e)))
         action = 'rebuilt %s' % (', '.join(built_projects),)
 
     logging.info('pipeline for %s@%s: %s, action=%s',
-                 path_with_namespace, branch, pipeline_status, action)
+                 project_path, branch_name, pipeline_status, action)
 
 
 def worker_thread(gl, project_deps):