import argparse import gitlab import logging import os import sys from urllib.parse import urlsplit from .deps import get_branches, list_projects, list_deps, \ split_project_branch, read_deps from .hooks import check_hook from .rebuild import rebuild_deps from .server import run_app def _fmtdesc(s): return s.strip() def main(): parser = argparse.ArgumentParser( description='Manage Gitlab project dependencies and trigger pipelines.') subparsers = parser.add_subparsers(dest='subparser') # Common options. common_parser = argparse.ArgumentParser(add_help=False) common_parser.add_argument( '--debug', action='store_true', help='increase logging level') common_parser.add_argument( '-n', '--dry-run', action='store_true', dest='dry_run', help='only show what would be done') gitlab_opts_group = common_parser.add_argument_group('gitlab options') gitlab_opts_group.add_argument( '--url', metavar='URL', help='Gitlab URL', default=os.getenv('GITLAB_URL')) gitlab_opts_group.add_argument( '--token-file', metavar='FILE', type=argparse.FileType('r'), default=os.getenv('GITLAB_AUTH_TOKEN_FILE'), help='file containing the Gitlab authentication token') gitlab_opts_group.add_argument( '--token', metavar='TOKEN', default=os.getenv('GITLAB_AUTH_TOKEN'), help='Gitlab authentication token') # List projects. list_projects_parser = subparsers.add_parser( 'list-projects', parents=[common_parser], help='list projects', formatter_class=argparse.RawDescriptionHelpFormatter, description=_fmtdesc(''' List all projects and their branches on the Gitlab instance. The output is a list of project paths with all their branches, separated by a colon, one per line. Since the Gitlab 'search' API is quite coarse, you can then filter the output for specific projects or branches using 'grep', e.g.: gitlab-deps list-projects | grep ^path/to/my/group/ or gitlab-deps list-projects | grep ':master$' ''')) list_projects_parser.add_argument( '--search', help='search query used to filter project list on the server side') # Compute deps. deps_parser = subparsers.add_parser( 'deps', parents=[common_parser], help='build dependency map', formatter_class=argparse.RawDescriptionHelpFormatter, description=_fmtdesc(''' Generate a map of dependencies between projects on a Gitlab instance. The input must consist of a list of projects along with their branches, separated by a colon, one per line. If the branch is unspecified, 'master' is assumed. The output consists of pairs of project / dependency (so, these are 'forward' dependencies), for all projects/branches specified in the input. To obtain a list of reverse dependencies, one can simply swap the columns in the output, e.g.: gitlab-deps deps < project.list | awk '{print $2, $1}' '''), epilog=_fmtdesc(''' Input can be read from a file (if passed as an argument), or from standard input if a filename is omitted or specified as '-'. ''')) deps_parser.add_argument( '--registry', metavar='NAME', default=os.getenv('GITLAB_REGISTRY'), help='Docker registry hostname (if empty, it will be ' 'automatically derived from --url)') deps_parser.add_argument( 'projects_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin) # Setup pipeline hooks on the specified projects. set_hooks_parser = subparsers.add_parser( 'set-hooks', parents=[common_parser], help='set pipeline hooks on projects', formatter_class=argparse.RawDescriptionHelpFormatter, description=_fmtdesc(''' Set a HTTP hook for pipeline_events on the specified projects. Takes a list of projects (optional branch specifiers will be ignored) as input. Pipeline hooks are required by 'gitlab-deps server' to trigger dependent builds, so a common way to use this command is to feed it the right-hand side of the 'gitlab-deps deps' output, e.g.: gitlab-deps deps < project.list \\ | awk '{print $2}' \\ | gitlab-deps set-hooks --hook-url=... using --hook-url to point at the URL of 'gitlab-deps server'. '''), epilog=_fmtdesc(''' Input can be read from a file (if passed as an argument), or from standard input if a filename is omitted or specified as '-'. ''')) set_hooks_parser.add_argument( '--hook-url', metavar='URL', help='URL for the pipeline HTTP hook') set_hooks_parser.add_argument( '--webhook-auth-token', metavar='TOKEN', help='secret X-Gitlab-Token for request authentication') set_hooks_parser.add_argument( 'projects_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin) # Trigger rebuilds of reverse deps. rebuild_image_parser = subparsers.add_parser( 'rebuild', parents=[common_parser], help='rebuild dependencies of a project', formatter_class=argparse.RawDescriptionHelpFormatter, description=_fmtdesc(''' Rebuild all projects that depend on the specified project. Takes a single project path as argument, and triggers a rebuild of its direct dependencies. Useful for one-off rebuilds. If the --recurse option is provided, the tool will wait for completion of the pipeline and recursively trigger its dependencies too, navigating the entire dependency tree. '''), epilog=_fmtdesc(''' Project dependencies can be read from a file (if passed as an argument), or from standard input if a filename is omitted or specified as '-'. ''')) rebuild_image_parser.add_argument( '--recurse', action='store_true', help='include all dependencies recursively ' 'and wait for completion of the pipelines') rebuild_image_parser.add_argument( 'project_path', help='project name (relative path, with optional branch)') rebuild_image_parser.add_argument( 'dependencies_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin) # Server. server_parser = subparsers.add_parser( 'server', parents=[common_parser], help='start the HTTP server', formatter_class=argparse.RawDescriptionHelpFormatter, description=_fmtdesc(''' Start an HTTP server that listens for Gitlab webhooks. When the server receives a pipeline event from Gitlab, it will trigger new builds for the direct dependencies of the project. The server is meant to be associated with a single Gitlab instance. You must provide the server with the list of project dependencies. '''), epilog=_fmtdesc(''' Project dependencies can be read from a file (if passed as an argument), or from standard input if a filename is omitted or specified as '-'. ''')) server_parser.add_argument( '--port', metavar='PORT', type=int, default='5404', dest='bind_port', help='port to listen on') server_parser.add_argument( '--addr', metavar='IP', default='127.0.0.1', dest='bind_host', help='address to listen on') server_parser.add_argument( '--webhook-auth-token', metavar='TOKEN', help='secret X-Gitlab-Token for request authentication') server_parser.add_argument( 'dependencies_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin) args = parser.parse_args() cmd = args.subparser if not args.url: parser.error('Must specify --url') logging.basicConfig( format='%(message)s', level=logging.DEBUG if args.debug else logging.INFO, ) # Connect to the Gitlab API. gitlab_token = args.token if not gitlab_token and args.token_file: gitlab_token = args.token_file.read().strip().encode('utf-8') gl = gitlab.Gitlab(args.url, private_token=gitlab_token) if gitlab_token: gl.auth() # Dispatch to the command executor. if cmd == 'list-projects': for p, b in get_branches(gl, list_projects(gl, args.search)): print(f'{p}:{b}') elif cmd == 'deps': # If --registry is not specified, make an educated guess. registry_hostname = args.registry if not registry_hostname: registry_hostname = 'registry.' + urlsplit(args.url).netloc logging.warning('guessed %s for the Docker registry hostname', registry_hostname) projects = [split_project_branch(x.strip()) for x in args.projects_list] list_deps(gl, args.url, registry_hostname, projects) elif cmd == 'rebuild': deps = read_deps(args.dependencies_list) project_path, branch_name = split_project_branch(args.project_path) rebuild_deps(gl, deps, project_path, branch_name, args.dry_run, args.recurse) elif cmd == 'set-hooks': if not args.hook_url: parser.error('Must specify --hook-url') # Need a project list on input, ignore branches. projects = set(y[0] for y in ( split_project_branch(x.strip()) for x in args.projects_list)) for project_path in projects: check_hook(gl, args.hook_url, args.webhook_auth_token, project_path, args.dry_run) elif cmd == 'server': deps = read_deps(args.dependencies_list) run_app(gl, deps, args.bind_host, args.bind_port, args.webhook_auth_token) if __name__ == '__main__': main()