Commit 3c7b79cb authored by ale's avatar ale
Browse files

Update server code to support the new dependency maps

Improve help messages for the command too.
parent cadc1cf7
import logging
def check_hook(gl, hook_url, webhook_token, project_path):
def check_hook(gl, hook_url, webhook_token, project_path, dry_run):
project = gl.projects.get(project_path)
found = False
for h in project.hooks.list():
......@@ -9,8 +10,10 @@ def check_hook(gl, hook_url, webhook_token, project_path):
break
if found:
return
project.hooks.add(
url=hook_url,
pipeline_events=True,
token=webhook_token,
)
logging.info('adding pipeline_events hook to %s', project_path)
if not dry_run:
project.hooks.add(
url=hook_url,
pipeline_events=True,
token=webhook_token,
)
......@@ -3,7 +3,7 @@ import gitlab
import logging
import os
import sys
import urllib.parse as urlparse
from urllib.parse import urlsplit
from .deps import get_branches, list_projects, list_deps, \
split_project_branch, read_deps
......@@ -18,12 +18,14 @@ def _fmtdesc(s):
def main():
parser = argparse.ArgumentParser(
description='Rebuild Docker images on a Gitlab instance.')
description='Manage Gitlab project dependencies and trigger pipelines.')
subparsers = parser.add_subparsers(dest='subparser')
# Common options.
common_parser = argparse.ArgumentParser(add_help=False)
common_parser.add_argument('--debug', action='store_true')
common_parser.add_argument(
'--debug', action='store_true',
help='increase logging level')
common_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
......@@ -35,16 +37,11 @@ def main():
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
help='Load Gitlab authentication token from this file')
help='file containing the Gitlab authentication token')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
default=os.getenv('GITLAB_AUTH_TOKEN'),
help='Gitlab authentication token')
gitlab_opts_group.add_argument(
'--registry', metavar='NAME',
default=os.getenv('GITLAB_REGISTRY'),
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
# List projects.
list_projects_parser = subparsers.add_parser(
......@@ -69,7 +66,7 @@ or
'''))
list_projects_parser.add_argument(
'--search',
help='Search query used to filter project list on the server side')
help='search query used to filter project list on the server side')
# Compute deps.
deps_parser = subparsers.add_parser(
......@@ -81,9 +78,9 @@ or
Generate a map of dependencies between projects on a
Gitlab instance.
The input (on standard input) must consist of a list of projects along
with their branches, separated by a colon, one per line. If the branch
is unspecified, 'master' is assumed.
The input must consist of a list of projects along with their
branches, separated by a colon, one per line. If the branch is
unspecified, 'master' is assumed.
The output consists of pairs of project / dependency (so, these are
'forward' dependencies), for all projects/branches specified in the
......@@ -94,51 +91,108 @@ columns in the output, e.g.:
gitlab-deps deps < project.list | awk '{print $2, $1}'
'''), epilog=_fmtdesc('''
Input can be read from a file (if passed as an argument), or
from standard input if a filename is omitted or specified as '-'.
'''))
deps_parser.add_argument(
'--registry', metavar='NAME',
default=os.getenv('GITLAB_REGISTRY'),
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
deps_parser.add_argument(
'projects_list',
type=argparse.FileType('r'),
nargs='?', default=sys.stdin)
# Setup pipeline hooks on the specified projects.
set_hooks_parser = subparsers.add_parser(
'set-hooks',
parents=[common_parser],
help='set pipeline hooks on projects',
description='Set the pipeline hooks on the specified projects '
'(usually points at our own server)')
formatter_class=argparse.RawDescriptionHelpFormatter,
description=_fmtdesc('''
Set a HTTP hook for pipeline_events on the specified projects.
Takes a list of projects (optional branch specifiers will be ignored)
as input. Pipeline hooks are required by 'gitlab-deps server' to
trigger dependent builds, so a common way to use this command is to
feed it the right-hand side of the 'gitlab-deps deps' output, e.g.:
gitlab-deps deps < project.list \\
| awk '{print $2}' \\
| gitlab-deps set-hooks --hook-url=...
using --hook-url to point at the URL of 'gitlab-deps server'.
'''), epilog=_fmtdesc('''
Input can be read from a file (if passed as an argument), or
from standard input if a filename is omitted or specified as '-'.
'''))
set_hooks_parser.add_argument(
'--hook-url', metavar='URL',
help='URL for the pipeline HTTP hook')
set_hooks_parser.add_argument(
'--webhook-auth-token', metavar='TOKEN',
help='Secret X-Gitlab-Token for request authentication')
help='secret X-Gitlab-Token for request authentication')
set_hooks_parser.add_argument(
'projects_list',
type=argparse.FileType('r'),
nargs='?', default=sys.stdin)
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
parents=[common_parser],
help='rebuild dependencies of a project',
description='Rebuild all projects that depend on the specified '
'project.')
rebuild_image_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
formatter_class=argparse.RawDescriptionHelpFormatter,
description=_fmtdesc('''
Rebuild all projects that depend on the specified project.
Takes a single project path as argument, and triggers a rebuild of its
direct dependencies. Useful for one-off rebuilds.
If the --recurse option is provided, the tool will wait for completion
of the pipeline and recursively trigger its dependencies too,
navigating the entire dependency tree.
'''), epilog=_fmtdesc('''
Project dependencies can be read from a file (if passed as an
argument), or from standard input if a filename is omitted or
specified as '-'.
'''))
rebuild_image_parser.add_argument(
'--recurse', action='store_true',
help='Include all dependencies recursively '
help='include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'project_path',
help='Project name (relative path, with optional branch)')
help='project name (relative path, with optional branch)')
rebuild_image_parser.add_argument(
'dependencies_list',
type=argparse.FileType('r'),
nargs='?', default=sys.stdin)
# Server.
server_parser = subparsers.add_parser(
'server',
parents=[common_parser],
help='start a HTTP server',
description='Start a HTTP server that listens for Gitlab webhooks. '
'Configure Gitlab to send Pipeline events for your projects to this '
'server to auto-rebuild first-level dependencies.')
server_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
help='start the HTTP server',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=_fmtdesc('''
Start an HTTP server that listens for Gitlab webhooks.
When the server receives a pipeline event from Gitlab, it will trigger
new builds for the direct dependencies of the project. The server is
meant to be associated with a single Gitlab instance.
You must provide the server with the list of project dependencies.
'''), epilog=_fmtdesc('''
Project dependencies can be read from a file (if passed as an
argument), or from standard input if a filename is omitted or
specified as '-'.
'''))
server_parser.add_argument(
'--port', metavar='PORT', type=int, default='5404',
dest='bind_port', help='port to listen on')
......@@ -147,7 +201,11 @@ columns in the output, e.g.:
dest='bind_host', help='address to listen on')
server_parser.add_argument(
'--webhook-auth-token', metavar='TOKEN',
help='Secret X-Gitlab-Token for request authentication')
help='secret X-Gitlab-Token for request authentication')
server_parser.add_argument(
'dependencies_list',
type=argparse.FileType('r'),
nargs='?', default=sys.stdin)
args = parser.parse_args()
cmd = args.subparser
......@@ -160,13 +218,7 @@ columns in the output, e.g.:
level=logging.DEBUG if args.debug else logging.INFO,
)
# If --registry is not specified, make an educated guess.
registry_hostname = args.registry
if not registry_hostname:
registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
logging.warning('guessed %s for the Docker registry hostname',
registry_hostname)
# Connect to the Gitlab API.
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
......@@ -179,25 +231,36 @@ columns in the output, e.g.:
if cmd == 'list-projects':
for p, b in get_branches(gl, list_projects(gl, args.search)):
print(f'{p}:{b}')
if cmd == 'deps':
projects = [split_project_branch(x.strip()) for x in sys.stdin]
elif cmd == 'deps':
# If --registry is not specified, make an educated guess.
registry_hostname = args.registry
if not registry_hostname:
registry_hostname = 'registry.' + urlsplit(args.url).netloc
logging.warning('guessed %s for the Docker registry hostname',
registry_hostname)
projects = [split_project_branch(x.strip())
for x in args.projects_list]
list_deps(gl, args.url, registry_hostname, projects)
elif cmd == 'rebuild':
deps = read_deps(sys.stdin)
deps = read_deps(args.dependencies_list)
project_path, branch_name = split_project_branch(args.project_path)
rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
args.recurse)
elif cmd == 'set-hooks':
if not args.hook_url:
parser.error('Must specify --hook-url')
# Need a project list on input, ignore branches.
projects = set(y[0] for y in (
split_project_branch(x.strip()) for x in sys.stdin))
split_project_branch(x.strip()) for x in args.projects_list))
for project_path in projects:
check_hook(gl, args.hook_url, args.webhook_auth_token,
project_path)
project_path, args.dry_run)
elif cmd == 'server':
deps = read_deps(sys.stdin)
deps = read_deps(args.dependencies_list)
run_app(gl, deps, args.bind_host, args.bind_port,
args.webhook_auth_token)
......
......@@ -2,13 +2,14 @@ import logging
import time
def rebuild(gl, project_path, wait=False):
def rebuild(gl, project_path, branch_name, wait=False):
"""Trigger a rebuild of a project."""
project = gl.projects.get(project_path)
if not project:
return None
pipeline = project.pipelines.create({'ref': 'master'})
pipeline = project.pipelines.create({'ref': branch_name})
logging.info('started pipeline %s', pipeline.web_url)
if wait:
while pipeline.finished_at is None:
pipeline.refresh()
......@@ -20,9 +21,9 @@ def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
wait_and_recurse):
stack = project_deps.get((project_path, branch_name), [])
while stack:
path = stack.pop(0)
logging.info('rebuilding %s', path)
path, branch = stack.pop(0)
logging.info('rebuilding %s:%s', path, branch)
if not dry_run:
rebuild(gl, path, wait_and_recurse)
rebuild(gl, path, branch, wait_and_recurse)
if wait_and_recurse:
stack.extend(project_deps.get(path, []))
stack.extend(project_deps.get((path, branch), []))
......@@ -16,25 +16,25 @@ queue = Queue.Queue()
def _process_request(gl, project_deps, data):
pipeline_status = data['object_attributes']['status']
branch = data['object_attributes']['ref']
path_with_namespace = data['project']['path_with_namespace']
branch_name = data['object_attributes']['ref']
project_path = data['project']['path_with_namespace']
action = 'none'
if pipeline_status == 'success':
deps = project_deps.get_contents().get(path_with_namespace, [])
deps = project_deps.get((project_path, branch_name), [])
built_projects = []
for dep_path in deps:
for dep_path, dep_branch in deps:
try:
p = rebuild(gl, dep_path)
logging.info('started pipeline %s', p)
rebuild(gl, dep_path, dep_branch)
built_projects.append(f'{dep_path}:{dep_branch}')
except Exception as e:
logging.error('error rebuilding project %s: %s' % (
path_with_namespace, str(e)))
logging.error('error rebuilding project %s:%s: %s' % (
dep_path, dep_branch, str(e)))
action = 'rebuilt %s' % (', '.join(built_projects),)
logging.info('pipeline for %s@%s: %s, action=%s',
path_with_namespace, branch, pipeline_status, action)
project_path, branch_name, pipeline_status, action)
def worker_thread(gl, project_deps):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment