Commit d4175d13 authored by ale's avatar ale

Refactor the tool into smaller composible modules

The idea is that all the filtering capability should be replaced by
"grep" and shell pipes.

Part one (server still TODO).
parent 51747af8
import re
DEFAULT_BRANCH = 'master'
def split_project_branch(project_with_branch):
if ':' in project_with_branch:
p, b = project_with_branch.split(':')
return p, b
return project_with_branch, DEFAULT_BRANCH
def list_projects(gl, search_pattern):
projects = gl.projects.list(
all=True,
search=search_pattern,
search_namespaces=True,
as_list=False,
simple=True,
)
for p in projects:
yield p.path_with_namespace
def get_branches(gl, project_names):
for path_with_namespace in project_names:
p = gl.projects.get(path_with_namespace)
for b in p.branches.list():
yield (path_with_namespace, b.name)
def has_ci(gl, project_path, branch_name):
p = gl.projects.get(project_path)
try:
p.files.get(file_path='.gitlab-ci.yml', ref=branch_name)
return True
except Exception:
return False
_from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE)
def get_docker_deps(gl, project_path, branch_name):
p = gl.projects.get(project_path)
try:
f = p.files.get(file_path='Dockerfile', ref=branch_name)
return _from_rx.findall(f.decode().decode('utf-8'))
except Exception:
return []
def get_explicit_deps(gl, project_path, branch_name):
p = gl.projects.get(project_path)
try:
f = p.files.get(file_path='.gitlab-deps', ref=branch_name)
return f.decode().decode('utf-8').split('\n')
except Exception:
return []
_docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:(.*))?$')
def docker_image_to_project(docker_image, registry_hostname):
m = _docker_image_rx.match(docker_image)
if m and m[1] == registry_hostname:
# The branch is the tag, except for 'latest'
if not m[5] or m[5] == 'latest':
branch = DEFAULT_BRANCH
else:
branch = m[5]
return m[3], branch
return None, None
_url_rx = re.compile(r'^(https?://[^/]+/)([^:]+)(:.*)?$')
def url_to_project(url, gitlab_url):
m = _url_rx.match(url)
if m and m[1] == gitlab_url:
return m[2], m[3] or DEFAULT_BRANCH
def not_null(l):
return filter(None, l)
def get_deps(gl, gitlab_url, registry_hostname, project_path, branch_name):
deps = []
deps.extend(not_null(
url_to_project(url, gitlab_url)
for url in get_explicit_deps(gl, project_path, branch_name)))
deps.extend(not_null(
docker_image_to_project(img, registry_hostname)
for img in get_docker_deps(gl, project_path, branch_name)))
return deps
def list_deps(gl, gitlab_url, registry_hostname, projects):
for project_path, branch_name in projects:
deps = get_deps(gl, gitlab_url, registry_hostname,
project_path, branch_name)
for dep_path, dep_branch in deps:
print(f'{project_path}:{branch_name} {dep_path}:{dep_branch}')
def read_deps(fd):
deps = {}
for line in fd:
src, dst = line.strip().split()
src_project, src_branch = split_project_branch(src)
dst_project, dst_branch = split_project_branch(dst)
deps.setdefault((src_project, src_branch), []).append(
(dst_project, dst_branch))
return deps
import argparse
import gitlab
import logging
import os
import time
try:
import urlparse
except ImportError:
import urllib.parse as urlparse
import sys
import urllib.parse as urlparse
from .docker_deps import dump_deps
from .deps import get_branches, list_projects, list_deps, \
split_project_branch, read_deps
from .rebuild import rebuild_deps
from .server import run_app
......@@ -19,40 +18,51 @@ def main():
# Common options.
common_parser = argparse.ArgumentParser(add_help=False)
common_parser.add_argument('--debug', action='store_true')
common_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
gitlab_opts_group = common_parser.add_argument_group('gitlab options')
gitlab_opts_group.add_argument(
'--url', metavar='URL', help='Gitlab URL')
'--url', metavar='URL', help='Gitlab URL',
default=os.getenv('GITLAB_URL'))
gitlab_opts_group.add_argument(
'--token-file', metavar='FILE',
type=argparse.FileType('r'),
default=os.getenv('GITLAB_AUTH_TOKEN_FILE'),
help='Load Gitlab authentication token from this file')
gitlab_opts_group.add_argument(
'--token', metavar='TOKEN',
default=os.getenv('GITLAB_AUTH_TOKEN'),
help='Gitlab authentication token')
gitlab_opts_group.add_argument(
'--registry', metavar='NAME',
default=os.getenv('GITLAB_REGISTRY'),
help='Docker registry hostname (if empty, it will be '
'automatically derived from --url)')
scope_opts_group = common_parser.add_argument_group('project scope options')
common_parser.add_argument('--debug', action='store_true')
# List projects.
list_projects_parser = subparsers.add_parser(
'list-projects',
parents=[common_parser],
help='list projects',
description='List all projects and their branches on the Gitlab '
'instance.')
list_projects_parser.add_argument(
'--search',
help='Search query used to filter project list on the server side')
# Compute deps.
deps_parser = subparsers.add_parser(
'deps',
parents=[common_parser],
help='build dependency map',
description='Generate a map of Docker-derived dependencies between '
'projects on a Gitlab instance.')
deps_parser.add_argument(
'--match',
help='Search query to filter project list on the server side')
deps_parser.add_argument(
'--filter',
help='Regexp to filter project list on the right-hand (dependency) side')
description='Generate a map of dependencies between projects on a '
'Gitlab instance.')
deps_parser.add_argument(
'--docker', action='store_true',
help='Output dependencies between Docker images, not Gitlab projects')
# Trigger rebuilds of reverse deps.
rebuild_image_parser = subparsers.add_parser(
'rebuild',
......@@ -63,16 +73,13 @@ def main():
rebuild_image_parser.add_argument(
'--deps', metavar='FILE',
help='file with project dependencies')
rebuild_image_parser.add_argument(
'-n', '--dry-run', action='store_true', dest='dry_run',
help='only show what would be done')
rebuild_image_parser.add_argument(
'--recurse', action='store_true',
help='Include all dependencies recursively '
'and wait for completion of the pipelines')
rebuild_image_parser.add_argument(
'project_path',
help='Project name (relative path)')
help='Project name (relative path, with optional branch)')
# Server.
server_parser = subparsers.add_parser(
......@@ -110,31 +117,31 @@ def main():
registry_hostname = args.registry
if not registry_hostname:
registry_hostname = 'registry.' + urlparse.urlsplit(args.url).netloc
logging.error('using %s as Docker registry', registry_hostname)
logging.warning('guessed %s for the Docker registry hostname',
registry_hostname)
gitlab_token = args.token
if not gitlab_token and args.token_file:
gitlab_token = args.token_file.read().strip().encode('utf-8')
gl = gitlab.Gitlab(args.url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
# Dispatch to the command executor.
if cmd == 'list-projects':
for p, b in get_branches(gl, list_projects(gl, args.search)):
print(f'{p}:{b}')
if cmd == 'deps':
dump_deps(
args.url,
registry_hostname,
gitlab_token,
args.match,
args.filter,
not args.docker,
)
projects = [split_project_branch(x.strip()) for x in sys.stdin]
list_deps(gl, args.url, registry_hostname, projects)
elif cmd == 'rebuild':
rebuild_deps(
args.url,
gitlab_token,
args.deps,
args.project_path,
args.dry_run,
args.recurse,
)
deps = read_deps(sys.stdin)
project_path, branch_name = split_project_branch(args.project_path)
rebuild_deps(gl, deps, project_path, branch_name, args.dry_run,
args.recurse)
elif cmd == 'server':
# TODO
run_app(
args.url,
gitlab_token,
......
import gitlab
import json
import logging
import time
......@@ -18,16 +16,9 @@ def rebuild(gl, project_path, wait=False):
return pipeline
def rebuild_deps(gitlab_url, gitlab_token,
project_deps_path, project_path, dry_run, wait_and_recurse):
gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
if gitlab_token:
gl.auth()
with open(project_deps_path) as fd:
project_deps = json.load(fd)
stack = project_deps.get(project_path, [])
def rebuild_deps(gl, project_deps, project_path, branch_name, dry_run,
wait_and_recurse):
stack = project_deps.get((project_path, branch_name), [])
while stack:
path = stack.pop(0)
logging.info('rebuilding %s', path)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment