Refactor the tool into smaller composible modules

The idea is that all the filtering capability should be replaced by
"grep" and shell pipes.

Part one (server still TODO).

gitlab_docker_autodep/deps.py

+import re
+
+
+DEFAULT_BRANCH = 'master'
+
+
+def split_project_branch(project_with_branch):
+    if ':' in project_with_branch:
+        p, b = project_with_branch.split(':')
+        return p, b
+    return project_with_branch, DEFAULT_BRANCH
+
+
+def list_projects(gl, search_pattern):
+    projects = gl.projects.list(
+        all=True,
+        search=search_pattern,
+        search_namespaces=True,
+        as_list=False,
+        simple=True,
+    )
+    for p in projects:
+        yield p.path_with_namespace
+
+
+def get_branches(gl, project_names):
+    for path_with_namespace in project_names:
+        p = gl.projects.get(path_with_namespace)
+        for b in p.branches.list():
+            yield (path_with_namespace, b.name)
+
+
+def has_ci(gl, project_path, branch_name):
+    p = gl.projects.get(project_path)
+    try:
+        p.files.get(file_path='.gitlab-ci.yml', ref=branch_name)
+        return True
+    except Exception:
+        return False
+
+
+_from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE)
+
+
+def get_docker_deps(gl, project_path, branch_name):
+    p = gl.projects.get(project_path)
+    try:
+        f = p.files.get(file_path='Dockerfile', ref=branch_name)
+        return _from_rx.findall(f.decode().decode('utf-8'))
+    except Exception:
+        return []
+
+
+def get_explicit_deps(gl, project_path, branch_name):
+    p = gl.projects.get(project_path)
+    try:
+        f = p.files.get(file_path='.gitlab-deps', ref=branch_name)
+        return f.decode().decode('utf-8').split('\n')
+    except Exception:
+        return []
+
+
+_docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:(.*))?$')
+
+
+def docker_image_to_project(docker_image, registry_hostname):
+    m = _docker_image_rx.match(docker_image)
+    if m and m[1] == registry_hostname:
+        # The branch is the tag, except for 'latest'
+        if not m[5] or m[5] == 'latest':
+            branch = DEFAULT_BRANCH
+        else:
+            branch = m[5]
+        return m[3], branch
+    return None, None
+
+
+_url_rx = re.compile(r'^(https?://[^/]+/)([^:]+)(:.*)?$')
+
+
+def url_to_project(url, gitlab_url):
+    m = _url_rx.match(url)
+    if m and m[1] == gitlab_url:
+        return m[2], m[3] or DEFAULT_BRANCH
+
+
+def not_null(l):
+    return filter(None, l)
+
+
+def get_deps(gl, gitlab_url, registry_hostname, project_path, branch_name):
+    deps = []
+    deps.extend(not_null(
+        url_to_project(url, gitlab_url)
+        for url in get_explicit_deps(gl, project_path, branch_name)))
+    deps.extend(not_null(
+        docker_image_to_project(img, registry_hostname)
+        for img in get_docker_deps(gl, project_path, branch_name)))
+    return deps
+
+
+def list_deps(gl, gitlab_url, registry_hostname, projects):
+    for project_path, branch_name in projects:
+        deps = get_deps(gl, gitlab_url, registry_hostname,
+                        project_path, branch_name)
+        for dep_path, dep_branch in deps:
+            print(f'{project_path}:{branch_name} {dep_path}:{dep_branch}')
+
+
+def read_deps(fd):
+    deps = {}
+    for line in fd:
+        src, dst = line.strip().split()
+        src_project, src_branch = split_project_branch(src)
+        dst_project, dst_branch = split_project_branch(dst)
+        deps.setdefault((src_project, src_branch), []).append(
+            (dst_project, dst_branch))
+    return deps

gitlab_docker_autodep/main.py

 import argparse
+import gitlab
 import logging
 import os
-import time
-try:
-    import urlparse
-except ImportError:
+import sys
 import urllib.parse as urlparse
 
-from .docker_deps import dump_deps
+from .deps import get_branches, list_projects, list_deps, \
+    split_project_branch, read_deps
 from .rebuild import rebuild_deps
 from .server import run_app
 
@@ -19,36 +18,47 @@ def main():
 
     # Common options.
     common_parser = argparse.ArgumentParser(add_help=False)
+    common_parser.add_argument('--debug', action='store_true')
+    common_parser.add_argument(
+        '-n', '--dry-run', action='store_true', dest='dry_run',
+        help='only show what would be done')
     gitlab_opts_group = common_parser.add_argument_group('gitlab options')
     gitlab_opts_group.add_argument(
-        '--url', metavar='URL', help='Gitlab URL')
+        '--url', metavar='URL', help='Gitlab URL',
+        default=os.getenv('GITLAB_URL'))
     gitlab_opts_group.add_argument(
         '--token-file', metavar='FILE',
         type=argparse.FileType('r'),
+        default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
         help='Load Gitlab authentication token from this file')
     gitlab_opts_group.add_argument(
         '--token', metavar='TOKEN',
+        default=os.getenv('GITLAB_AUTH_TOKEN'),
         help='Gitlab authentication token')
     gitlab_opts_group.add_argument(
         '--registry', metavar='NAME',
+        default=os.getenv('GITLAB_REGISTRY'),
         help='Docker registry hostname (if empty, it will be '
         'automatically derived from --url)')
-    scope_opts_group = common_parser.add_argument_group('project scope options')
-    common_parser.add_argument('--debug', action='store_true')
+
+    # List projects.
+    list_projects_parser = subparsers.add_parser(
+        'list-projects',
+        parents=[common_parser],
+        help='list projects',
+        description='List all projects and their branches on the Gitlab '
+        'instance.')
+    list_projects_parser.add_argument(
+        '--search',
+        help='Search query used to filter project list on the server side')
 
     # Compute deps.
     deps_parser = subparsers.add_parser(
         'deps',
         parents=[common_parser],
         help='build dependency map',
-        description='Generate a map of Docker-derived dependencies between '
-        'projects on a Gitlab instance.')
-    deps_parser.add_argument(
-        '--match',
-        help='Search query to filter project list on the server side')
-    deps_parser.add_argument(
-        '--filter',
-        help='Regexp to filter project list on the right-hand (dependency) side')
+        description='Generate a map of dependencies between projects on a '
+        'Gitlab instance.')
     deps_parser.add_argument(
         '--docker', action='store_true',
         help='Output dependencies between Docker images, not Gitlab projects')
@@ -63,16 +73,13 @@ def main():
     rebuild_image_parser.add_argument(
         '--deps', metavar='FILE',
         help='file with project dependencies')
-    rebuild_image_parser.add_argument(
-        '-n', '--dry-run', action='store_true', dest='dry_run',
-        help='only show what would be done')
     rebuild_image_parser.add_argument(
         '--recurse', action='store_true',
         help='Include all dependencies recursively '
         'and wait for completion of the pipelines')
     rebuild_image_parser.add_argument(
         'project_path',
-        help='Project name (relative path)')
+        help='Project name (relative path, with optional branch)')
 
     # Server.
     server_parser = subparsers.add_parser(
@@ -110,31 +117,31 @@ def main():
     registry_hostname = args.registry
     if not registry_hostname:
         registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
-        logging.error('using %s as Docker registry', registry_hostname)
+        logging.warning('guessed %s for the Docker registry hostname',
+                        registry_hostname)
 
     gitlab_token = args.token
     if not gitlab_token and args.token_file:
         gitlab_token = args.token_file.read().strip().encode('utf-8')
 
+    gl = gitlab.Gitlab(args.url, private_token=gitlab_token)
+    if gitlab_token:
+        gl.auth()
+
+    # Dispatch to the command executor.
+    if cmd == 'list-projects':
+        for p, b in get_branches(gl, list_projects(gl, args.search)):
+            print(f'{p}:{b}')
     if cmd == 'deps':
-        dump_deps(
-            args.url,
-            registry_hostname,
-            gitlab_token,
-            args.match,
-            args.filter,
-            not args.docker,
-        )
+        projects = [split_project_branch(x.strip()) for x in sys.stdin]
+        list_deps(gl, args.url, registry_hostname, projects)
     elif cmd == 'rebuild':
-        rebuild_deps(
-            args.url,
-            gitlab_token,
-            args.deps,
-            args.project_path,
-            args.dry_run,
-            args.recurse,
-        )
+        deps = read_deps(sys.stdin)
+        project_path, branch_name = split_project_branch(args.project_path)
+        rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
+                     args.recurse)
     elif cmd == 'server':
+        # TODO
         run_app(
             args.url,
             gitlab_token,

gitlab_docker_autodep/rebuild.py

-import gitlab
-import json
 import logging
 import time
 
@@ -18,16 +16,9 @@ def rebuild(gl, project_path, wait=False):
     return pipeline
 
 
-def rebuild_deps(gitlab_url, gitlab_token,
-                 project_deps_path, project_path, dry_run, wait_and_recurse):
-    gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
-    if gitlab_token:
-        gl.auth()
-
-    with open(project_deps_path) as fd:
-        project_deps = json.load(fd)
-
-    stack = project_deps.get(project_path, [])
+def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
+                 wait_and_recurse):
+    stack = project_deps.get((project_path, branch_name), [])
     while stack:
         path = stack.pop(0)
         logging.info('rebuilding %s', path)