diff --git a/gitlab_docker_autodep/deps.py b/gitlab_docker_autodep/deps.py
new file mode 100644
index 0000000000000000000000000000000000000000..531a9c218f2ca68fa235133c8ee7f777726b5aa3
--- /dev/null
+++ b/gitlab_docker_autodep/deps.py
@@ -0,0 +1,118 @@
+import re
+
+
+DEFAULT_BRANCH = 'master'
+
+
+def split_project_branch(project_with_branch):
+ if ':' in project_with_branch:
+ p, b = project_with_branch.split(':')
+ return p, b
+ return project_with_branch, DEFAULT_BRANCH
+
+
+def list_projects(gl, search_pattern):
+ projects = gl.projects.list(
+ all=True,
+ search=search_pattern,
+ search_namespaces=True,
+ as_list=False,
+ simple=True,
+ )
+ for p in projects:
+ yield p.path_with_namespace
+
+
+def get_branches(gl, project_names):
+ for path_with_namespace in project_names:
+ p = gl.projects.get(path_with_namespace)
+ for b in p.branches.list():
+ yield (path_with_namespace, b.name)
+
+
+def has_ci(gl, project_path, branch_name):
+ p = gl.projects.get(project_path)
+ try:
+ p.files.get(file_path='.gitlab-ci.yml', ref=branch_name)
+ return True
+ except Exception:
+ return False
+
+
+_from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE)
+
+
+def get_docker_deps(gl, project_path, branch_name):
+ p = gl.projects.get(project_path)
+ try:
+ f = p.files.get(file_path='Dockerfile', ref=branch_name)
+ return _from_rx.findall(f.decode().decode('utf-8'))
+ except Exception:
+ return []
+
+
+def get_explicit_deps(gl, project_path, branch_name):
+ p = gl.projects.get(project_path)
+ try:
+ f = p.files.get(file_path='.gitlab-deps', ref=branch_name)
+ return f.decode().decode('utf-8').split('\n')
+ except Exception:
+ return []
+
+
+_docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:(.*))?$')
+
+
+def docker_image_to_project(docker_image, registry_hostname):
+ m = _docker_image_rx.match(docker_image)
+ if m and m[1] == registry_hostname:
+ # The branch is the tag, except for 'latest'
+ if not m[5] or m[5] == 'latest':
+ branch = DEFAULT_BRANCH
+ else:
+ branch = m[5]
+ return m[3], branch
+ return None, None
+
+
+_url_rx = re.compile(r'^(https?://[^/]+/)([^:]+)(:.*)?$')
+
+
+def url_to_project(url, gitlab_url):
+ m = _url_rx.match(url)
+ if m and m[1] == gitlab_url:
+ return m[2], m[3] or DEFAULT_BRANCH
+
+
+def not_null(l):
+ return filter(None, l)
+
+
+def get_deps(gl, gitlab_url, registry_hostname, project_path, branch_name):
+ deps = []
+ deps.extend(not_null(
+ url_to_project(url, gitlab_url)
+ for url in get_explicit_deps(gl, project_path, branch_name)))
+ deps.extend(not_null(
+ docker_image_to_project(img, registry_hostname)
+ for img in get_docker_deps(gl, project_path, branch_name)))
+ return deps
+
+
+def list_deps(gl, gitlab_url, registry_hostname, projects):
+ for project_path, branch_name in projects:
+ deps = get_deps(gl, gitlab_url, registry_hostname,
+ project_path, branch_name)
+ for dep_path, dep_branch in deps:
+ print(f'{project_path}:{branch_name} {dep_path}:{dep_branch}')
+
+
+def read_deps(fd):
+ deps = {}
+ for line in fd:
+ src, dst = line.strip().split()
+ src_project, src_branch = split_project_branch(src)
+ dst_project, dst_branch = split_project_branch(dst)
+ deps.setdefault((src_project, src_branch), []).append(
+ (dst_project, dst_branch))
+ return deps
diff --git a/gitlab_docker_autodep/main.py b/gitlab_docker_autodep/main.py
index 041f2cd1b74505c9e72fea286938359840d2f77d..eb3620201a841cafce92d17957eb107960bea319 100644
--- a/gitlab_docker_autodep/main.py
+++ b/gitlab_docker_autodep/main.py
@@ -1,13 +1,12 @@
import argparse
+import gitlab
import logging
import os
-import time
-try:
- import urlparse
-except ImportError:
- import urllib.parse as urlparse
+import sys
+import urllib.parse as urlparse
-from .docker_deps import dump_deps
+from .deps import get_branches, list_projects, list_deps, \
+ split_project_branch, read_deps
from .rebuild import rebuild_deps
from .server import run_app
@@ -19,40 +18,51 @@ def main():
# Common options.
common_parser = argparse.ArgumentParser(add_help=False)
+ common_parser.add_argument('--debug', action='store_true')
+ common_parser.add_argument(
+ '-n', '--dry-run', action='store_true', dest='dry_run',
+ help='only show what would be done')
gitlab_opts_group = common_parser.add_argument_group('gitlab options')
gitlab_opts_group.add_argument(
- '--url', metavar='URL', help='Gitlab URL')
+ '--url', metavar='URL', help='Gitlab URL',
+ default=os.getenv('GITLAB_URL'))
gitlab_opts_group.add_argument(
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
+ default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
help='Load Gitlab authentication token from this file')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
+ default=os.getenv('GITLAB_AUTH_TOKEN'),
help='Gitlab authentication token')
gitlab_opts_group.add_argument(
'--registry', metavar='NAME',
+ default=os.getenv('GITLAB_REGISTRY'),
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
- scope_opts_group = common_parser.add_argument_group('project scope options')
- common_parser.add_argument('--debug', action='store_true')
+
+ # List projects.
+ list_projects_parser = subparsers.add_parser(
+ 'list-projects',
+ parents=[common_parser],
+ help='list projects',
+ description='List all projects and their branches on the Gitlab '
+ 'instance.')
+ list_projects_parser.add_argument(
+ '--search',
+ help='Search query used to filter project list on the server side')
# Compute deps.
deps_parser = subparsers.add_parser(
'deps',
parents=[common_parser],
help='build dependency map',
- description='Generate a map of Docker-derived dependencies between '
- 'projects on a Gitlab instance.')
- deps_parser.add_argument(
- '--match',
- help='Search query to filter project list on the server side')
- deps_parser.add_argument(
- '--filter',
- help='Regexp to filter project list on the right-hand (dependency) side')
+ description='Generate a map of dependencies between projects on a '
+ 'Gitlab instance.')
deps_parser.add_argument(
'--docker', action='store_true',
help='Output dependencies between Docker images, not Gitlab projects')
-
+
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
@@ -63,16 +73,13 @@ def main():
rebuild_image_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
- rebuild_image_parser.add_argument(
- '-n', '--dry-run', action='store_true', dest='dry_run',
- help='only show what would be done')
rebuild_image_parser.add_argument(
'--recurse', action='store_true',
help='Include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'project_path',
- help='Project name (relative path)')
+ help='Project name (relative path, with optional branch)')
# Server.
server_parser = subparsers.add_parser(
@@ -110,31 +117,31 @@ def main():
registry_hostname = args.registry
if not registry_hostname:
registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
- logging.error('using %s as Docker registry', registry_hostname)
+ logging.warning('guessed %s for the Docker registry hostname',
+ registry_hostname)
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
+ gl = gitlab.Gitlab(args.url, private_token=gitlab_token)
+ if gitlab_token:
+ gl.auth()
+
+ # Dispatch to the command executor.
+ if cmd == 'list-projects':
+ for p, b in get_branches(gl, list_projects(gl, args.search)):
+ print(f'{p}:{b}')
if cmd == 'deps':
- dump_deps(
- args.url,
- registry_hostname,
- gitlab_token,
- args.match,
- args.filter,
- not args.docker,
- )
+ projects = [split_project_branch(x.strip()) for x in sys.stdin]
+ list_deps(gl, args.url, registry_hostname, projects)
elif cmd == 'rebuild':
- rebuild_deps(
- args.url,
- gitlab_token,
- args.deps,
- args.project_path,
- args.dry_run,
- args.recurse,
- )
+ deps = read_deps(sys.stdin)
+ project_path, branch_name = split_project_branch(args.project_path)
+ rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
+ args.recurse)
elif cmd == 'server':
+ # TODO
run_app(
args.url,
gitlab_token,
diff --git a/gitlab_docker_autodep/rebuild.py b/gitlab_docker_autodep/rebuild.py
index 84ac48ca2bf165ef496a46bcf3912a36ac3b2dbc..b7b70a3345efa42818b1be6c256d355157fa0dec 100644
--- a/gitlab_docker_autodep/rebuild.py
+++ b/gitlab_docker_autodep/rebuild.py
@@ -1,5 +1,3 @@
-import gitlab
-import json
import logging
import time
@@ -18,16 +16,9 @@ def rebuild(gl, project_path, wait=False):
return pipeline
-def rebuild_deps(gitlab_url, gitlab_token,
- project_deps_path, project_path, dry_run, wait_and_recurse):
- gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
- if gitlab_token:
- gl.auth()
-
- with open(project_deps_path) as fd:
- project_deps = json.load(fd)
-
- stack = project_deps.get(project_path, [])
+def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
+ wait_and_recurse):
+ stack = project_deps.get((project_path, branch_name), [])
while stack:
path = stack.pop(0)
logging.info('rebuilding %s', path)