diff --git a/gitlab_docker_autodep/hooks.py b/gitlab_docker_autodep/hooks.py
index dc99626b786ef3e1dc7353514b05149dc13e2194..5f53b2113e83473cc8a4f24da4e75ea9f2421c97 100644
--- a/gitlab_docker_autodep/hooks.py
+++ b/gitlab_docker_autodep/hooks.py
@@ -1,6 +1,7 @@
+import logging
-def check_hook(gl, hook_url, webhook_token, project_path):
+def check_hook(gl, hook_url, webhook_token, project_path, dry_run):
project = gl.projects.get(project_path)
found = False
for h in project.hooks.list():
@@ -9,8 +10,10 @@ def check_hook(gl, hook_url, webhook_token, project_path):
break
if found:
return
- project.hooks.add(
- url=hook_url,
- pipeline_events=True,
- token=webhook_token,
- )
+ logging.info('adding pipeline_events hook to %s', project_path)
+ if not dry_run:
+ project.hooks.add(
+ url=hook_url,
+ pipeline_events=True,
+ token=webhook_token,
+ )
diff --git a/gitlab_docker_autodep/main.py b/gitlab_docker_autodep/main.py
index 9a14950473c516205d22a042177e2b5eb48c2c5a..6fec413f3b57d880d1fdd05b26d6428351193b63 100644
--- a/gitlab_docker_autodep/main.py
+++ b/gitlab_docker_autodep/main.py
@@ -3,7 +3,7 @@ import gitlab
import logging
import os
import sys
-import urllib.parse as urlparse
+from urllib.parse import urlsplit
from .deps import get_branches, list_projects, list_deps, \
split_project_branch, read_deps
@@ -18,12 +18,14 @@ def _fmtdesc(s):
def main():
parser = argparse.ArgumentParser(
- description='Rebuild Docker images on a Gitlab instance.')
+ description='Manage Gitlab project dependencies and trigger pipelines.')
subparsers = parser.add_subparsers(dest='subparser')
# Common options.
common_parser = argparse.ArgumentParser(add_help=False)
- common_parser.add_argument('--debug', action='store_true')
+ common_parser.add_argument(
+ '--debug', action='store_true',
+ help='increase logging level')
common_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
@@ -35,16 +37,11 @@ def main():
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
- help='Load Gitlab authentication token from this file')
+ help='file containing the Gitlab authentication token')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
default=os.getenv('GITLAB_AUTH_TOKEN'),
help='Gitlab authentication token')
- gitlab_opts_group.add_argument(
- '--registry', metavar='NAME',
- default=os.getenv('GITLAB_REGISTRY'),
- help='Docker registry hostname (if empty, it will be '
- 'automatically derived from --url)')
# List projects.
list_projects_parser = subparsers.add_parser(
@@ -69,7 +66,7 @@ or
'''))
list_projects_parser.add_argument(
'--search',
- help='Search query used to filter project list on the server side')
+ help='search query used to filter project list on the server side')
# Compute deps.
deps_parser = subparsers.add_parser(
@@ -81,9 +78,9 @@ or
Generate a map of dependencies between projects on a
Gitlab instance.
-The input (on standard input) must consist of a list of projects along
-with their branches, separated by a colon, one per line. If the branch
-is unspecified, 'master' is assumed.
+The input must consist of a list of projects along with their
+branches, separated by a colon, one per line. If the branch is
+unspecified, 'master' is assumed.
The output consists of pairs of project / dependency (so, these are
'forward' dependencies), for all projects/branches specified in the
@@ -94,51 +91,108 @@ columns in the output, e.g.:
gitlab-deps deps < project.list | awk '{print $2, $1}'
+'''), epilog=_fmtdesc('''
+Input can be read from a file (if passed as an argument), or
+from standard input if a filename is omitted or specified as '-'.
'''))
+ deps_parser.add_argument(
+ '--registry', metavar='NAME',
+ default=os.getenv('GITLAB_REGISTRY'),
+ help='Docker registry hostname (if empty, it will be '
+ 'automatically derived from --url)')
+ deps_parser.add_argument(
+ 'projects_list',
+ type=argparse.FileType('r'),
+ nargs='?', default=sys.stdin)
# Setup pipeline hooks on the specified projects.
set_hooks_parser = subparsers.add_parser(
'set-hooks',
parents=[common_parser],
help='set pipeline hooks on projects',
- description='Set the pipeline hooks on the specified projects '
- '(usually points at our own server)')
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=_fmtdesc('''
+Set a HTTP hook for pipeline_events on the specified projects.
+
+Takes a list of projects (optional branch specifiers will be ignored)
+as input. Pipeline hooks are required by 'gitlab-deps server' to
+trigger dependent builds, so a common way to use this command is to
+feed it the right-hand side of the 'gitlab-deps deps' output, e.g.:
+
+ gitlab-deps deps < project.list \\
+ | awk '{print $2}' \\
+ | gitlab-deps set-hooks --hook-url=...
+
+using --hook-url to point at the URL of 'gitlab-deps server'.
+
+'''), epilog=_fmtdesc('''
+Input can be read from a file (if passed as an argument), or
+from standard input if a filename is omitted or specified as '-'.
+'''))
set_hooks_parser.add_argument(
'--hook-url', metavar='URL',
help='URL for the pipeline HTTP hook')
set_hooks_parser.add_argument(
'--webhook-auth-token', metavar='TOKEN',
- help='Secret X-Gitlab-Token for request authentication')
+ help='secret X-Gitlab-Token for request authentication')
+ set_hooks_parser.add_argument(
+ 'projects_list',
+ type=argparse.FileType('r'),
+ nargs='?', default=sys.stdin)
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
parents=[common_parser],
help='rebuild dependencies of a project',
- description='Rebuild all projects that depend on the specified '
- 'project.')
- rebuild_image_parser.add_argument(
- '--deps', metavar='FILE',
- help='file with project dependencies')
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=_fmtdesc('''
+Rebuild all projects that depend on the specified project.
+
+Takes a single project path as argument, and triggers a rebuild of its
+direct dependencies. Useful for one-off rebuilds.
+
+If the --recurse option is provided, the tool will wait for completion
+of the pipeline and recursively trigger its dependencies too,
+navigating the entire dependency tree.
+
+'''), epilog=_fmtdesc('''
+Project dependencies can be read from a file (if passed as an
+argument), or from standard input if a filename is omitted or
+specified as '-'.
+'''))
rebuild_image_parser.add_argument(
'--recurse', action='store_true',
- help='Include all dependencies recursively '
+ help='include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'project_path',
- help='Project name (relative path, with optional branch)')
+ help='project name (relative path, with optional branch)')
+ rebuild_image_parser.add_argument(
+ 'dependencies_list',
+ type=argparse.FileType('r'),
+ nargs='?', default=sys.stdin)
# Server.
server_parser = subparsers.add_parser(
'server',
parents=[common_parser],
- help='start a HTTP server',
- description='Start a HTTP server that listens for Gitlab webhooks. '
- 'Configure Gitlab to send Pipeline events for your projects to this '
- 'server to auto-rebuild first-level dependencies.')
- server_parser.add_argument(
- '--deps', metavar='FILE',
- help='file with project dependencies')
+ help='start the HTTP server',
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=_fmtdesc('''
+Start an HTTP server that listens for Gitlab webhooks.
+
+When the server receives a pipeline event from Gitlab, it will trigger
+new builds for the direct dependencies of the project. The server is
+meant to be associated with a single Gitlab instance.
+
+You must provide the server with the list of project dependencies.
+
+'''), epilog=_fmtdesc('''
+Project dependencies can be read from a file (if passed as an
+argument), or from standard input if a filename is omitted or
+specified as '-'.
+'''))
server_parser.add_argument(
'--port', metavar='PORT', type=int, default='5404',
dest='bind_port', help='port to listen on')
@@ -147,7 +201,11 @@ columns in the output, e.g.:
dest='bind_host', help='address to listen on')
server_parser.add_argument(
'--webhook-auth-token', metavar='TOKEN',
- help='Secret X-Gitlab-Token for request authentication')
+ help='secret X-Gitlab-Token for request authentication')
+ server_parser.add_argument(
+ 'dependencies_list',
+ type=argparse.FileType('r'),
+ nargs='?', default=sys.stdin)
args = parser.parse_args()
cmd = args.subparser
@@ -160,13 +218,7 @@ columns in the output, e.g.:
level=logging.DEBUG if args.debug else logging.INFO,
)
- # If --registry is not specified, make an educated guess.
- registry_hostname = args.registry
- if not registry_hostname:
- registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
- logging.warning('guessed %s for the Docker registry hostname',
- registry_hostname)
-
+ # Connect to the Gitlab API.
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
@@ -179,25 +231,36 @@ columns in the output, e.g.:
if cmd == 'list-projects':
for p, b in get_branches(gl, list_projects(gl, args.search)):
print(f'{p}:{b}')
- if cmd == 'deps':
- projects = [split_project_branch(x.strip()) for x in sys.stdin]
+
+ elif cmd == 'deps':
+ # If --registry is not specified, make an educated guess.
+ registry_hostname = args.registry
+ if not registry_hostname:
+ registry_hostname = 'registry.' + urlsplit(args.url).netloc
+ logging.warning('guessed %s for the Docker registry hostname',
+ registry_hostname)
+ projects = [split_project_branch(x.strip())
+ for x in args.projects_list]
list_deps(gl, args.url, registry_hostname, projects)
+
elif cmd == 'rebuild':
- deps = read_deps(sys.stdin)
+ deps = read_deps(args.dependencies_list)
project_path, branch_name = split_project_branch(args.project_path)
rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
args.recurse)
+
elif cmd == 'set-hooks':
if not args.hook_url:
parser.error('Must specify --hook-url')
# Need a project list on input, ignore branches.
projects = set(y[0] for y in (
- split_project_branch(x.strip()) for x in sys.stdin))
+ split_project_branch(x.strip()) for x in args.projects_list))
for project_path in projects:
check_hook(gl, args.hook_url, args.webhook_auth_token,
- project_path)
+ project_path, args.dry_run)
+
elif cmd == 'server':
- deps = read_deps(sys.stdin)
+ deps = read_deps(args.dependencies_list)
run_app(gl, deps, args.bind_host, args.bind_port,
args.webhook_auth_token)
diff --git a/gitlab_docker_autodep/rebuild.py b/gitlab_docker_autodep/rebuild.py
index b9cbf4cb40b10251caacd20440d8e643c04fab45..96ebc69d0b2d3b20fc3e3a5a5acaeb4aac644c12 100644
--- a/gitlab_docker_autodep/rebuild.py
+++ b/gitlab_docker_autodep/rebuild.py
@@ -2,13 +2,14 @@ import logging
import time
-def rebuild(gl, project_path, wait=False):
+def rebuild(gl, project_path, branch_name, wait=False):
"""Trigger a rebuild of a project."""
project = gl.projects.get(project_path)
if not project:
return None
- pipeline = project.pipelines.create({'ref': 'master'})
+ pipeline = project.pipelines.create({'ref': branch_name})
+ logging.info('started pipeline %s', pipeline.web_url)
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
@@ -20,9 +21,9 @@ def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
wait_and_recurse):
stack = project_deps.get((project_path, branch_name), [])
while stack:
- path = stack.pop(0)
- logging.info('rebuilding %s', path)
+ path, branch = stack.pop(0)
+ logging.info('rebuilding %s:%s', path, branch)
if not dry_run:
- rebuild(gl, path, wait_and_recurse)
+ rebuild(gl, path, branch, wait_and_recurse)
if wait_and_recurse:
- stack.extend(project_deps.get(path, []))
+ stack.extend(project_deps.get((path, branch), []))
diff --git a/gitlab_docker_autodep/server.py b/gitlab_docker_autodep/server.py
index ffc0351d307dfad601472bb535d97b04e8202de4..2f5ae406eaf239aba236ca0e2217df1e0251a850 100644
--- a/gitlab_docker_autodep/server.py
+++ b/gitlab_docker_autodep/server.py
@@ -16,25 +16,25 @@ queue = Queue.Queue()
def _process_request(gl, project_deps, data):
pipeline_status = data['object_attributes']['status']
- branch = data['object_attributes']['ref']
- path_with_namespace = data['project']['path_with_namespace']
+ branch_name = data['object_attributes']['ref']
+ project_path = data['project']['path_with_namespace']
action = 'none'
if pipeline_status == 'success':
- deps = project_deps.get_contents().get(path_with_namespace, [])
+ deps = project_deps.get((project_path, branch_name), [])
built_projects = []
- for dep_path in deps:
+ for dep_path, dep_branch in deps:
try:
- p = rebuild(gl, dep_path)
- logging.info('started pipeline %s', p)
+ rebuild(gl, dep_path, dep_branch)
+ built_projects.append(f'{dep_path}:{dep_branch}')
except Exception as e:
- logging.error('error rebuilding project %s: %s' % (
- path_with_namespace, str(e)))
+ logging.error('error rebuilding project %s:%s: %s' % (
+ dep_path, dep_branch, str(e)))
action = 'rebuilt %s' % (', '.join(built_projects),)
logging.info('pipeline for %s@%s: %s, action=%s',
- path_with_namespace, branch, pipeline_status, action)
+ project_path, branch_name, pipeline_status, action)
def worker_thread(gl, project_deps):