import re DEFAULT_BRANCH = 'master' def split_project_branch(project_with_branch): if ':' in project_with_branch: p, b = project_with_branch.split(':') return p, b return project_with_branch, DEFAULT_BRANCH def list_projects(gl, search_pattern): projects = gl.projects.list( all=True, search=search_pattern, search_namespaces=True, as_list=False, simple=True, ) for p in projects: yield p.path_with_namespace def get_branches(gl, project_names): for path_with_namespace in project_names: p = gl.projects.get(path_with_namespace) for b in p.branches.list(): yield (path_with_namespace, b.name) def has_ci(gl, project_path, branch_name): p = gl.projects.get(project_path) try: p.files.get(file_path='.gitlab-ci.yml', ref=branch_name) return True except Exception: return False _from_rx = re.compile(r'^FROM\s+(\S+).*$', re.MULTILINE) def get_docker_deps(gl, project_path, branch_name): p = gl.projects.get(project_path) try: f = p.files.get(file_path='Dockerfile', ref=branch_name) return _from_rx.findall(f.decode().decode('utf-8')) except Exception: return [] def get_explicit_deps(gl, project_path, branch_name): p = gl.projects.get(project_path) try: f = p.files.get(file_path='.gitlab-deps', ref=branch_name) return f.decode().decode('utf-8').split('\n') except Exception: return [] _docker_image_rx = re.compile(r'^([^/]*)(/([^:]*))?(:(.*))?$') def docker_image_to_project(docker_image, registry_hostname): m = _docker_image_rx.match(docker_image) if m and m[1] == registry_hostname: # The branch is the tag, except for 'latest' if not m[5] or m[5] == 'latest': branch = DEFAULT_BRANCH else: branch = m[5] return m[3], branch return None, None _url_rx = re.compile(r'^(https?://[^/]+/)([^:]+)(:.*)?$') def url_to_project(url, gitlab_url): m = _url_rx.match(url) if m and m[1] == gitlab_url: return m[2], m[3] or DEFAULT_BRANCH def not_null(l): return filter(None, l) def get_deps(gl, gitlab_url, registry_hostname, project_path, branch_name): deps = [] deps.extend(not_null( url_to_project(url, gitlab_url) for url in get_explicit_deps(gl, project_path, branch_name))) deps.extend(not_null( docker_image_to_project(img, registry_hostname) for img in get_docker_deps(gl, project_path, branch_name))) return deps def list_deps(gl, gitlab_url, registry_hostname, projects): for project_path, branch_name in projects: deps = get_deps(gl, gitlab_url, registry_hostname, project_path, branch_name) for dep_path, dep_branch in deps: print(f'{project_path}:{branch_name} {dep_path}:{dep_branch}') def read_deps(fd): deps = {} for line in fd: src, dst = line.strip().split() src_project, src_branch = split_project_branch(src) dst_project, dst_branch = split_project_branch(dst) deps.setdefault((src_project, src_branch), []).append( (dst_project, dst_branch)) return deps