import gitlab import logging import re import time _from_rx = re.compile(r'^FROM\s+(.*)$') def _parse_dockerfile(df): for line in df.split('\n'): m = _from_rx.match(line) if m: return m.group(1) def _fetch_dockerfile(gl, project, ref): try: f = project.files.get(file_path='Dockerfile', ref=ref) return f.decode() except: return None def _has_gitlab_ci(gl, project, ref): try: project.files.get(file_path='.gitlab-ci.yml', ref=ref) return True except: return False def _remove_image_tag(name): if ':' in name: return name.split(':')[0] return name def build_dependency_tree(gl, search_pattern=None, filter_pattern=None): """Build the project dependency map based on Dockerfiles. This can be a fairly expensive (long) operation if the list of projects is large. The 'search_pattern' argument allows for filtering on the server side, using Gitlab search query syntax. On the client side, the project list can be filtered with a regular expression using the 'filter_pattern' argument, which will be applied to the project's path_with_namespace. Returns an {image_name: [projects]}, where 'projects' is the list of projects that have 'image_name' as their base Docker image. These are gitlab.Project instances. We only examine Dockerfiles in the master branch of repositories. """ deps = {} filter_rx = None if filter_pattern: filter_rx = re.compile(filter_pattern) projects = gl.projects.list(all=True, search=search_pattern, as_list=False) for project in projects: if filter_rx is not None and not filter_rx.search(project.path_with_namespace): continue df = _fetch_dockerfile(gl, project, 'master') if not df: continue base_image = _parse_dockerfile(df.decode('utf-8')) if not base_image: logging.error('ERROR: could not find base image for %s', project.path_with_namespace) continue if not _has_gitlab_ci(gl, project, 'master'): continue deps.setdefault(_remove_image_tag(base_image), []).append(project) return deps def rebuild(project, wait=False): """Trigger a rebuild of a project.""" pipeline = project.pipelines.create({'ref': 'master'}) if wait: while pipeline.finished_at is None: pipeline.refresh() time.sleep(3) return pipeline def rebuild_deps(gitlab_url, registry_hostname, gitlab_token, search_pattern, filter_pattern, image_name, dry_run=False, recurse=False, wait=False): """Rebuild dependencies of the given image.""" gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token) if gitlab_token: gl.auth() deps = build_dependency_tree(gl, search_pattern) stack = deps.get(_remove_image_tag(image_name), []) while stack: project = stack.pop(0) logging.info('rebuilding %s', project.path_with_namespace) if not dry_run: try: pipeline = rebuild(project, wait) if pipeline.status not in ('success', 'pending'): logging.error('ERROR: build failed for %s (status: %s)', project.path_with_namespace, pipeline.status) continue except gitlab.exceptions.GitlabError as e: logging.error('ERROR: gitlab error: %s: %s', project.path_with_namespace, str(e)) continue if recurse: image_name = '%s/%s' % ( registry_hostname, project.path_with_namespace) stack.extend(deps.get(image_name, []))