Feat/digest check

.gitlab-ci.yml

@@ -5,17 +5,6 @@ stages:
   - deploy
 
 # lint
-pylint:
-  stage: lint
-  image: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/python:latest
-  before_script:
-    - ci/dependencies/install-pymodules.sh ci/dependencies/requirements.pip > pymodules-install.log
-  script:
-    - pylint ./ci/src
-  allow_failure: true
-  except:
-    - schedules
-
 yamllint:
   stage: lint
   image:
@@ -23,25 +12,16 @@ yamllint:
     entrypoint: ["/bin/ash", "-c"]
   script:
     - yamllint -c ci/config/yamllint_config.yaml .
-  allow_failure: false
   except:
     - schedules
 
 # deploy
-update-registry-images:
-  stage: deploy
-  image: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/debian:latest
-  variables:
-    PUSHGATEWAY_URL: "http://monitoring1.priv.cloud.muni.cz:9091"
+.sshuttle:  # Required to access prometheus push gateway
+  variables: &sshuttle_vars
     SSH_EXTRA_OPTS: "-o StrictHostKeyChecking=no"
     SSHUTTLE_ROUTED_SUBNETS: "10.16.0.0/16 147.251.62.0/24 147.251.63.0/24"
     SSHUTTLE_EXCLUDED_SUBNETS: "147.251.62.9/32"
-  before_script:
-    - ci/dependencies/install-pkgs.sh ci/dependencies/requirements.apt > pkgs-install.log
-    - ci/dependencies/install-pymodules.sh ci/dependencies/requirements.pip > pymodules-install.log
-    - service docker start
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
+  before_script: &sshuttle_before_script
     - mkdir -p "${HOME}/.ssh"
     - ssh_private_key_file="SSH_PRIVATE_KEY_PRODUCTION"
     - cp -f "${!ssh_private_key_file}" "${HOME}/.ssh/id_rsa"
@@ -49,8 +29,31 @@ update-registry-images:
     - chmod 600 "${HOME}/.ssh/id_rsa"
     - ssh "${SSH_EXTRA_OPTS}" root@jump.cloud.muni.cz 'uname -a'
     - sshuttle --remote root@jump.cloud.muni.cz --listen 0.0.0.0 --exclude ${SSHUTTLE_EXCLUDED_SUBNETS} --daemon --ssh-cmd "ssh ${SSH_EXTRA_OPTS}" ${SSHUTTLE_ROUTED_SUBNETS}
+
+.update-registry-images: &update-registry-images
+  stage: deploy
+  image: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/debian:latest
+  variables:
+    PUSHGATEWAY_URL: "http://monitoring1.priv.cloud.muni.cz:9091"
+    <<: *sshuttle_vars
+  before_script:
+    - ci/dependencies/install-pkgs.sh ci/dependencies/requirements.apt > pkgs-install.log
+    - ci/dependencies/install-pymodules.sh ci/dependencies/requirements.pip > pymodules-install.log
+    - service docker start
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - *sshuttle_before_script
   script:
     - ./ci/src/push_metric.py --metric-label app=cloud/g2/container-registry.git --metric-name job_last_run_timestamp --metric-help "Last container registry update execution timestamp." --metric-value `date "+%s"` --pushgateway-url "${PUSHGATEWAY_URL}"
-    - python3 ci/src/store_images_locally.py --input-path image-description.yaml --container-exec docker
+    - python3 ci/src/registry_synchronizer.py --input-path image-description.yaml --container-exec docker --log-level info
     - ./ci/src/push_metric.py --metric-label app=cloud/g2/container-registry.git --metric-name job_last_successful_run_timestamp --metric-help "Last container registry update successful job execution timestamp." --metric-value `date "+%s"` --pushgateway-url "${PUSHGATEWAY_URL}"
-  allow_failure: false
+
+update-on-branch:
+  <<: *update-registry-images
+  except:
+    - master
+    - schedules
+
+scheduled-update:
+  <<: *update-registry-images
+  only:
+    - schedules

README.md

@@ -7,12 +7,16 @@ Description of docker container images used in g2-openstack is described in imag
 Add a new container description into ```image-description.yaml```, keep the same syntax. Automatic pipeline will take care of the rest.
 
 Because gitlab doesn't allow more than 1 ```/``` in registry names, it is replaced with 2 of ```_```. Refer to the example below. \
-If you want to download the image with every pipeline run (weekly), use ```overwrite-existing: true```. This is recommended for tag ```latest```, but fairly pointless otherwise.
+Also write the image with its source. For example DO ```docker.io/debian:latest``` and NOT just ```debian:latest```. Podman requires source. \
+If you want to update the image with every pipeline run (weekly), use ```overwrite-existing: true```. This is recommended.
 
 Example:
 ```
-  - upstream-image: docker.io/library/rabbitmq:3.9.0-management
+  - upstream-image: docker.itera.io/openstackhelm/mariadb:ubuntu_xenial-20200303
     downstream-images:
-      - docker.io__library__rabbitmq:3.9.0-management
-    overwrite-existing: false
+      - docker.itera.io__openstackhelm__mariadb:ubuntu_xenial-20200303
+    overwrite-existing: true
 ```
+
+## Note
+If pipeline passes with a warning, outdated image exists in registry, but has ```overwrite-existing: false``` set.
\ No newline at end of file

ci/dependencies/requirements.apt

 # assuming Debian 11 with python3 (should be embedded)
 
 docker.io
+podman
 sshuttle
 python3-pip
+skopeo
\ No newline at end of file

ci/dependencies/requirements.pip

-pylint
 pyyaml
 prometheus-client
-tenacity

ci/src/push_metric.py

@@ -5,53 +5,49 @@ push-metric.py prepares and pushes customized metrics into Prometheus Pushgatewa
 
 
 Usage:
-    python3 push-metric.py  --metric-type ["Gauge","Histogram","Counter"] 
-                            --metric-label [job=<job_name>|app=<app_name>]
-                              (- Either job or app name label must be specified)
-                            --metric-label <label_name>=<label_description>
-                            --pushgateway-url <pushgateway_url>
-                            --metric-name <metric_name>
-                            --metric-help <metric_help>
-                            --metric-value <metric_value>
+python3 push-metric.py  --metric-type ["Gauge","Histogram","Counter"]
+                        --metric-label [job=<job_name>|app=<app_name>]
+                          (- Either job or app name label must be specified)
+                        --metric-label <label_name>=<label_description>
+                        --pushgateway-url <pushgateway_url>
+                        --metric-name <metric_name>
+                        --metric-help <metric_help>
+                        --metric-value <metric_value>
 
 Example of usage:
  * Push metric job_last_run_timestamp{app="image-rotation"} <timestamp>
    into Prometheus Pushgateway at pgw.stage
-    * $ ./push-metric.py    --metric-label job=image-rotation 
-                            --pushgateway-url 'http://monitoring1.stage.priv.cloud.muni.cz:9091'
-                              (- port must be specified)
-                            --metric-name job_last_successful_run_timestamp
-                            --metric-help 'Last image rotation successful job execution timestamp.'
-                            --metric-type Gauge                           
-                            --metric-value `date "+%s"`
-    
-    * This command executes script for job named "image-rotation".
-      It creates Gauge metric named "job_last_successful_run_timestamp"/
-    * Then it assigns timestamp value and help metric description
-      and it pushes metric to pushgateway-url http://monitoring1.stage.priv.cloud.muni.cz:9091
+   * $ ./push-metric.py    --metric-label job=image-rotation
+                           --pushgateway-url 'http://monitoring1.stage.priv.cloud.muni.cz:9091'
+                             (- port must be specified)
+                           --metric-name job_last_successful_run_timestamp
+                           --metric-help 'Last image rotation successful job execution timestamp.'
+                           --metric-type Gauge
+                           --metric-value `date "+%s"`
+
+   * This command executes script for job named "image-rotation".
+     It creates Gauge metric named "job_last_successful_run_timestamp"/
+   * Then it assigns timestamp value and help metric description
+     and it pushes metric to pushgateway-url http://monitoring1.stage.priv.cloud.muni.cz:9091
 """
 
 import argparse
 import copy
 
 import prometheus_client
-import tenacity
 
 class ArgKeyValuePair(argparse.Action):
     """parse `<key>=<value>` argument value string into dict"""
-
     def __call__(self, _, namespace, values, option_string=None):
         items = copy.copy(getattr(namespace, self.dest, {}))
         option_name, option_value = values.split("=", 1)
         items[option_name.strip()] = option_value
         setattr(namespace, self.dest, items)
 
-@tenacity.retry(stop=tenacity.stop_after_attempt(5))
 def push_metric(gateway_url, job, registry):
     """Push of prepared metrics via Pushgateway"""
     prometheus_client.pushadd_to_gateway(gateway_url, job=job, registry=registry)
 
-
 def prepare_metric(args, registry):
     """Processing and preparation of metric"""
     metric_type = getattr(prometheus_client, getattr(args, "metric_type"))
@@ -63,7 +59,6 @@ def prepare_metric(args, registry):
     )
     metric.labels(**args.metric_labels).set(args.metric_value)
 
-
 def get_job_name(args):
     """Get job name"""
     if "job" in args.metric_labels:
@@ -72,7 +67,6 @@ def get_job_name(args):
         return args.metric_labels["app"]
     return None
 
-
 if __name__ == "__main__":
     AP = argparse.ArgumentParser(epilog=globals().get("__doc__"))
 

ci/src/registry_synchronizer.py

+#!/usr/bin/env python3
+
+'''
+Script reads .yaml file, containing a description of container images,
+and synchronizes their local versions with those on hub.
+
+usage: registry_synchronizer.py [-h] --input-path INPUT_PATH
+                                [--container-exec CONTAINER_EXEC]
+                                [--log-level LOG_LEVEL]
+
+Descriptive file syntax example (in downstream, '/' is replaced with 2 of '_'):
+
+---
+downstream_container_registry:
+  address: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/
+
+images:
+  - upstream-image: docker.io/cytopia/yamllint:latest
+    downstream-images:
+      - docker.io__cytopia__yamllint:latest
+    overwrite-existing: true
+
+  - upstream-image: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
+    downstream-images:
+      - docker.io__openstackhelm__mariadb:ubuntu_focal-20210415
+    overwrite-existing: true
+'''
+
+import argparse
+import logging
+import subprocess # Run bash commands
+from ast import literal_eval # Interpret dictionary_string as dictionary
+import yaml # For reading descriptive file
+
+LOG_LEVELS = {
+    'debug':logging.DEBUG,
+    'info':logging.INFO,
+    'warning':logging.WARNING,
+    'error':logging.ERROR,
+    'critical':logging.CRITICAL}
+
+class RegistrySynchronizer:
+    ''' Synchronizes local image registry with docker hub '''
+    def __init__(self, args):
+        self.input_path = args.input_path
+        self.container_exec = args.container_exec
+        self.updated_images = 0
+        self.new_images = 0
+        self.outdated_images = set()
+        self.images_without_digest = set()
+
+    def print_summary(self):
+        ''' Print action summary at the end of synchronization '''
+        truly_updated = self.updated_images - self.new_images
+        logging.info("%d updated, %d newly stored", truly_updated, self.new_images)
+
+        known_outdated = self.outdated_images - self.images_without_digest
+        if known_outdated:
+            logging.info("Outdated: %s", known_outdated)
+
+    def run_command(self, command):
+        ''' Wrapper for running bash commands + exception handling '''
+        try:
+            output = subprocess.run(command, check=True, capture_output=True)
+            logging.debug("%s returned: %s", command, output)
+            return (output.stdout).decode('utf-8')
+        except subprocess.CalledProcessError as ex:
+            logging.info((ex.stderr).decode('utf-8'))
+            raise
+
+    def update(self, image, local_image):
+        ''' Pull image from remote repository, tag it and push to local repo '''
+        self.run_command([self.container_exec, "pull", image])
+        self.run_command([self.container_exec, "tag", image, local_image])
+        self.run_command([self.container_exec, "push", local_image])
+
+        new_digest = self.get_local_digest(local_image)
+        logging.info("Updated: %s | new local digest: %s", image, new_digest)
+        self.updated_images += 1
+
+    def get_manifest(self, image):
+        ''' Returns image manifest '''
+        command = ["podman", "manifest", "inspect", image]
+        output = subprocess.run(command, check=True, capture_output=True)
+        logging.debug("Manifest inspect output: %s", output)
+        return literal_eval((output.stdout).decode('utf-8'))
+
+    def get_hub_digest(self, image):
+        ''' Returns image digest, handles failures '''
+        try:
+            hub_manifest = self.get_manifest(image)
+            return hub_manifest['config']['digest']
+        except KeyError: # List of manifests was returned, image manifest is needed
+            logging.debug("Received list of manifests for different architectures.")
+            for manifest in hub_manifest['manifests']:
+                if manifest['platform']['architecture'] == "amd64":
+                    manifest_digest = manifest['digest']
+                    image = image.split(':')[0] + '@' + manifest_digest
+                    logging.debug("Trying amd64 version: %s", image)
+                    return self.get_hub_digest(image)
+            raise
+        except subprocess.CalledProcessError as ex: # Can't access hub digest, but image exists
+            if (('unsupported manifest media type and no default available' in str(ex.stderr) or
+                 'error parsing manifest blob' in str(ex.stderr))
+                and self.run_command(["skopeo", "inspect", f"docker://{image}"])):
+                return False
+
+            logging.info((ex.stderr).decode('utf-8'))
+            raise
+
+    def get_local_digest(self, local_image):
+        ''' Check local image manifest, returns manifest or False '''
+        try:
+            return (self.get_manifest(local_image))['config']['digest']
+        except subprocess.CalledProcessError as ex: # No manifest in local registry is not an error
+            if 'no such manifest' in str(ex.stderr) or 'manifest unknown' in str(ex.stderr):
+                return False
+
+            logging.info((ex.stderr).decode('utf-8'))
+            raise
+
+    def up_to_date(self, image, local_image):
+        ''' Compare digests of local registry image and docker hub image '''
+        local_digest = self.get_local_digest(local_image)
+        logging.debug("Local registry image digest: %s", local_digest)
+        if not local_digest:
+            logging.info("Not found locally: %s", image)
+            self.new_images += 1
+            return False
+
+        hub_digest = self.get_hub_digest(image)
+        logging.debug("Hub image digest: %s", hub_digest)
+        if not hub_digest:
+            logging.info("Unsupported hub manifest type, but image exists locally: %s", image)
+            self.images_without_digest.add(image)
+            return False
+
+        if local_digest == hub_digest:
+            logging.info("Up to date: %s | digest: %s", image, hub_digest)
+            return True
+
+        logging.info("Outdated image: %s | hub digest: %s, local digest: %s",
+                     image, hub_digest, local_digest)
+        return False
+
+    def synchronize(self):
+        ''' Read descriptive file, check local and hub digests, update outdated '''
+        with open(self.input_path, 'r', encoding="utf-8") as desc_file:
+            image_description = yaml.safe_load(desc_file)
+        logging.debug("Opened descriptive file: %s", self.input_path)
+
+        registry_url = image_description['downstream_container_registry']['address']
+        logging.debug("Registry URL: %s", registry_url)
+
+        for image in image_description['images']:
+            hub_image = image['upstream-image']
+            logging.debug("Working with image: %s", hub_image)
+
+            for local_image in image['downstream-images']:
+                # Add registry URL to name, if it is not part of downstream image name
+                if not registry_url in local_image:
+                    local_image = registry_url + local_image
+                logging.debug("Downstream image: %s", local_image)
+
+                # Check if registry image is up to date with hub image
+                if not self.up_to_date(hub_image,local_image):
+                    if image['overwrite-existing']:
+                        self.update(hub_image,local_image)
+                    else:
+                        self.outdated_images.add(hub_image)
+
+        self.print_summary()
+
+def parse_arguments():
+    ''' Returns parsed command line arguments '''
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--input-path', '-f', required=True, type=str,
+                        help='Path to descriptive YAML file.')
+    parser.add_argument('--container-exec', default="docker", type=str,
+                        help='Software for managing containers. (default: %(default)s)')
+    parser.add_argument('--log-level', default="info", type=str,
+                        help='Level of displayed logs. (default: %(default)s)')
+
+    return parser.parse_args()
+
+def main():
+    """ Main entry point """
+    args = parse_arguments()
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=LOG_LEVELS[args.log_level])
+    registry_synchronizer = RegistrySynchronizer(args)
+    registry_synchronizer.synchronize()
+
+if __name__ == "__main__":
+    main()

ci/src/store_images_locally.py

-#!/usr/bin/env python3
-
-'''
-Script reads .yaml file, containing a description of docker images,
-and pulls them to local a repository.
-
-Usage Examples:
-* Run manually
-	python3 store_images_locally.py --input-path INPUT_PATH
-                               [--container-exec CONTAINER_EXEC]
-	You need to specify path, container executor is set to default "docker"
-* Include in CI/CD
-	Similar to above. You need a container with both docker and python.
-
-Descriptive file syntax example:
-
----
-downstream_container_registry:
-  address: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/
-
-images:
-  - upstream-image: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
-    downstream-images:
-      - docker.io__openstackhelm__mariadb:ubuntu_focal-20210415
-    overwrite-existing: false
-
-  - upstream-image: docker.itera.io/ceph/daemon:ubuntu_bionic-20200521
-    downstream-images:
-      - docker.itera.io__ceph__daemon:ubuntu_bionic-20200521
-    overwrite-existing: false
-'''
-
-import subprocess
-import argparse
-import yaml
-
-def run_command(command):
-    ''' Wrapper for running bash commands + exception handling'''
-    try:
-        subprocess.run(command, check=True, capture_output=True)
-        print(command)
-        return 0
-    except subprocess.CalledProcessError as ex:
-        # Missing manifest is not an error and is handled
-        if command[1] == "manifest":
-            return ex.returncode
-
-        print('\033[1;31m')
-        print(ex)
-        print(ex.stderr)
-        print('\033[0m')
-        raise
-
-def pull_tag_push(image, image_name, registry_url, container_exec):
-    ''' Pull image from remote repository, tag it and push to local repo. '''
-    run_command([container_exec,"pull", image_name])
-
-    # Tag images based on descriptive file and push
-    for local_image in image['downstream-images']:
-        # Add registry URL to name, if it is not part of downstream image name
-        if not registry_url in local_image:
-            local_image = registry_url + local_image
-
-        run_command([container_exec,"tag", image_name, local_image])
-        run_command([container_exec,"push", local_image])
-
-def parse_arguments():
-    ''' Parse command line arguments and pass them to main() '''
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('--input-path', required=True, type=str,
-                    help='Path to descriptive YAML file.')
-    parser.add_argument('--container-exec', default="docker", type=str,
-                    help='Software for managing containers.')
-
-    return parser.parse_args()
-
-def main(args):
-    ''' Main '''
-    # Open descriptive file
-    with open(args.input_path, 'r', encoding="utf-8") as file:
-        image_description = yaml.safe_load(file)
-
-    registry_url = image_description['downstream_container_registry']['address']
-
-    for image in image_description['images']:
-        image_name = image['upstream-image']
-
-        # Checks if image exists in registry or overwrite is required and skips actions otherwise
-        if run_command([args.container_exec,"manifest","inspect",
-                        registry_url + image_name.replace("/","__")]) != 0:
-            pull_tag_push(image, image_name, registry_url, args.container_exec)
-        elif image['overwrite-existing']:
-            pull_tag_push(image, image_name, registry_url, args.container_exec)
-
-if __name__ == "__main__":
-    main(parse_arguments())

image-description.yaml

@@ -4,45 +4,40 @@ downstream_container_registry:
   address: registry.gitlab.ics.muni.cz:443/cloud/g2/container-registry/
 
 images:
-  - upstream-image: nginx:latest
+  - upstream-image: docker.io/nginx:latest
     downstream-images:
-      - nginx:latest
+      - docker.io__nginx:latest
     overwrite-existing: true
 
-  - upstream-image: debian:latest
+  - upstream-image: docker.io/debian:latest
     downstream-images:
-      - debian:latest
+      - docker.io__debian:latest
     overwrite-existing: true
 
-  - upstream-image: python:latest
+  - upstream-image: docker.io/cytopia/yamllint:latest
     downstream-images:
-      - python:latest
-    overwrite-existing: true
-
-  - upstream-image: cytopia/yamllint:latest
-    downstream-images:
-      - cytopia__yamllint:latest
+      - docker.io__cytopia__yamllint:latest
     overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415
     downstream-images:
       - docker.io__openstackhelm__mariadb:ubuntu_focal-20210415
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/ceph/daemon:ubuntu_bionic-20200521
     downstream-images:
       - docker.itera.io__ceph__daemon:ubuntu_bionic-20200521
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/mariadb:ubuntu_xenial-20200303
     downstream-images:
       - docker.itera.io__openstackhelm__mariadb:ubuntu_xenial-20200303
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
     downstream-images:
       - docker.io__openstackhelm__neutron:stein-ubuntu_bionic
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/prom/memcached-exporter:v0.4.1
     downstream-images:
@@ -52,42 +47,42 @@ images:
   - upstream-image: docker.itera.io/stackanetes/kubernetes-entrypoint:v0.3.1
     downstream-images:
       - docker.itera.io__stackanetes__kubernetes-entrypoint:v0.3.1
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: k8s.gcr.io/ingress-nginx/controller:v1.1.3
     downstream-images:
       - k8s.gcr.io__ingress-nginx__controller:v1.1.3
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/library/rabbitmq:3.9.0-management
     downstream-images:
       - docker.io__library__rabbitmq:3.9.0-management
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/xrally/xrally-openstack:2.0.0
     downstream-images:
       - docker.io__xrally__xrally-openstack:2.0.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/osixia/keepalived:1.4.5
     downstream-images:
       - docker.io__osixia__keepalived:1.4.5
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/rabbitmq:3.9-management
     downstream-images:
       - docker.io__rabbitmq:3.9-management
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/nova:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__nova:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/external_storage/rbd-provisioner:ubuntu_bionic-20200521
     downstream-images:
       - docker.itera.io__external_storage__rbd-provisioner:ubuntu_bionic-20200521
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
     downstream-images:
@@ -97,32 +92,32 @@ images:
   - upstream-image: docker.io/openstackhelm/glance:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__glance:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/library/memcached:1.5.5
     downstream-images:
       - docker.io__library__memcached:1.5.5
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113
     downstream-images:
       - docker.io__openstackhelm__ceph-daemon:change_770201_ubuntu_bionic-20210113
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/xrally/xrally-openstack:2.0.0
     downstream-images:
       - docker.itera.io__xrally__xrally-openstack:2.0.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: gcr.io/google_containers/hyperkube-amd64:v1.11.6
     downstream-images:
       - gcr.io__google_containers__hyperkube-amd64:v1.11.6
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/docker:17.07.0
     downstream-images:
       - docker.itera.io__docker:17.07.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ospurge:latest
     downstream-images:
@@ -132,7 +127,7 @@ images:
   - upstream-image: docker.io/library/rabbitmq:3.9.0
     downstream-images:
       - docker.io__library__rabbitmq:3.9.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/mariadb:latest-ubuntu_focal
     downstream-images:
@@ -144,40 +139,40 @@ images:
       - docker.io__openstackhelm__nova-ssh:latest-ubuntu_focal
     overwrite-existing: true
 
-  - upstream-image: grafana/grafana:7.3.7-ubuntu
+  - upstream-image: docker.io/grafana/grafana:7.3.7-ubuntu
     downstream-images:
-      - grafana__grafana:7.3.7-ubuntu
-    overwrite-existing: false
+      - docker.io__grafana__grafana:7.3.7-ubuntu
+    overwrite-existing: true
 
   - upstream-image: docker.io/library/docker:17.07.0
     downstream-images:
       - docker.io__library__docker:17.07.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/prom/mysqld-exporter:v0.12.1
     downstream-images:
       - docker.io__prom__mysqld-exporter:v0.12.1
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/library/mariadb:10.5.9-focal
     downstream-images:
       - docker.io__library__mariadb:10.5.9-focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/keystone:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__keystone:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/mariadb:ubuntu_xenial-20191031
     downstream-images:
       - docker.itera.io__openstackhelm__mariadb:ubuntu_xenial-20191031
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/cinder:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__cinder:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/prometheus-openstack-exporter:latest-ubuntu_bionic
     downstream-images:
@@ -187,32 +182,32 @@ images:
   - upstream-image: docker.itera.io/prom/alertmanager:v0.11.0
     downstream-images:
       - docker.itera.io__prom__alertmanager:v0.11.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/external_storage/cephfs-provisioner:ubuntu_bionic-20200521
     downstream-images:
       - docker.itera.io__external_storage__cephfs-provisioner:ubuntu_bionic-20200521
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/prom/node-exporter:v1.1.2
     downstream-images:
       - docker.itera.io__prom__node-exporter:v1.1.2
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/prom/mysqld-exporter:v0.11.0
     downstream-images:
       - docker.itera.io__prom__mysqld-exporter:v0.11.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: k8s.gcr.io/defaultbackend:1.4
     downstream-images:
       - k8s.gcr.io__defaultbackend:1.4
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/placement:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__placement:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic
     downstream-images:
@@ -227,17 +222,17 @@ images:
   - upstream-image: docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113
     downstream-images:
       - docker.io__openstackhelm__ceph-config-helper:change_770201_ubuntu_bionic-20210113
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/httpd:2.4
     downstream-images:
       - docker.itera.io__httpd:2.4
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/kolla/ubuntu-binary-nova-libvirt:ussuri-updated
     downstream-images:
       - docker.itera.io__kolla__ubuntu-binary-nova-libvirt:ussuri-updated
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     downstream-images:
@@ -247,7 +242,7 @@ images:
   - upstream-image: docker.itera.io/osixia/openldap:1.2.0
     downstream-images:
       - docker.itera.io__osixia__openldap:1.2.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/barbican:latest-victoria-ubuntu_bionic
     downstream-images:
@@ -257,37 +252,37 @@ images:
   - upstream-image: docker.itera.io/osixia/keepalived:1.4.5
     downstream-images:
       - docker.itera.io__osixia__keepalived:1.4.5
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/kbudde/rabbitmq-exporter:v1.0.0-RC7.1
     downstream-images:
       - docker.io__kbudde__rabbitmq-exporter:v1.0.0-RC7.1
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/stackanetes/kubernetes-entrypoint:v1.0.0
     downstream-images:
       - docker.itera.io__stackanetes__kubernetes-entrypoint:v1.0.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__heat:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/port/ceph-config-helper:ubuntu_bionic-20200521
     downstream-images:
       - docker.itera.io__port__ceph-config-helper:ubuntu_bionic-20200521
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/openvswitch:ubuntu_bionic-20191031
     downstream-images:
       - docker.itera.io__openstackhelm__openvswitch:ubuntu_bionic-20191031
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/heat:newton-ubuntu_xenial
     downstream-images:
       - docker.itera.io__openstackhelm__heat:newton-ubuntu_xenial
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/heat:latest-wallaby-ubuntu_bionic
     downstream-images:
@@ -297,7 +292,7 @@ images:
   - upstream-image: k8s.gcr.io/ingress-nginx/controller:v1.1.2
     downstream-images:
       - k8s.gcr.io__ingress-nginx__controller:v1.1.2
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/horizon:latest-victoria-ubuntu_bionic
     downstream-images:
@@ -307,12 +302,12 @@ images:
   - upstream-image: docker.itera.io/prom/prometheus:v2.25.2
     downstream-images:
       - docker.itera.io__prom__prometheus:v2.25.2
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/neutron:wallaby-ubuntu_focal
     downstream-images:
       - docker.io__openstackhelm__neutron:wallaby-ubuntu_focal
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/stackanetes/porthole-mysqlclient-utility:latest-ubuntu_bionic
     downstream-images:
@@ -322,17 +317,17 @@ images:
   - upstream-image: docker.io/docker:17.07.0
     downstream-images:
       - docker.io__docker:17.07.0
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/kolla/ubuntu-source-nova-compute-ironic:ussuri
     downstream-images:
       - docker.io__kolla__ubuntu-source-nova-compute-ironic:ussuri
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/kolla/ubuntu-source-nova-novncproxy:ussuri
     downstream-images:
       - docker.io__kolla__ubuntu-source-nova-novncproxy:ussuri
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/neutron:latest-ussuri-ubuntu_bionic
     downstream-images:
@@ -342,12 +337,12 @@ images:
   - upstream-image: docker.io/openstackhelm/heat:stein-ubuntu_bionic
     downstream-images:
       - docker.io__openstackhelm__heat:stein-ubuntu_bionic
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/google_containers/defaultbackend:1.4
     downstream-images:
       - docker.itera.io__google_containers__defaultbackend:1.4
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.itera.io/openstackhelm/neutron:latest-wallaby-ubuntu_bionic
     downstream-images:
@@ -362,84 +357,114 @@ images:
   - upstream-image: docker.itera.io/mariadb:10.2.31
     downstream-images:
       - docker.itera.io__mariadb:10.2.31
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/neutron:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__neutron:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/glance:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__glance:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20220802
     downstream-images:
       - docker.io__openstackhelm__ceph-config-helper:ubuntu_bionic-20220802
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/keystone:xena-ubuntu_focal-20221119
     downstream-images:
       - docker.io__openstackhelm__keystone:xena-ubuntu_focal-20221119
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/barbican:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__barbican:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/cinder:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__cinder:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/nova:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__nova:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/mariadb:ubuntu_focal-20221119
     downstream-images:
       - docker.io__openstackhelm__mariadb:ubuntu_focal-20221119
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal-20221212
     downstream-images:
       - docker.io__openstackhelm__ceph-config-helper:ubuntu_focal-20221212
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/placement:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__placement:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/horizon:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__horizon:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/libvirt:periodic-weekly_yoga-focal-20230204
     downstream-images:
       - docker.io__openstackhelm__libvirt:periodic-weekly_yoga-focal-20230204
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/heat:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__heat:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-daemon:ubuntu_focal-20221212
     downstream-images:
       - docker.io__openstackhelm__ceph-daemon:ubuntu_focal-20221212
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20220202
     downstream-images:
       - docker.io__openstackhelm__ceph-daemon:ubuntu_bionic-20220202
-    overwrite-existing: false
+    overwrite-existing: true
 
   - upstream-image: docker.io/openstackhelm/keystone:periodic_yoga-ubuntu_focal-20220729
     downstream-images:
       - docker.io__openstackhelm__keystone:periodic_yoga-ubuntu_focal-20220729
-    overwrite-existing: false
+    overwrite-existing: true
+
+  - upstream-image: docker.io/kbudde/rabbitmq-exporter:v1.0.0-RC11
+    downstream-images:
+      - docker.io__kbudde__rabbitmq-exporter:v1.0.0-RC11
+    overwrite-existing: true
+
+  - upstream-image: docker.io/library/memcached:1.5.22
+    downstream-images:
+      - docker.io__library__memcached:1.5.22
+    overwrite-existing: true
+
+  - upstream-image: docker.io/library/rabbitmq:3.9.28
+    downstream-images:
+      - docker.io__library__rabbitmq:3.9.28
+    overwrite-existing: true
+
+  - upstream-image: docker.io/openstackhelm/openvswitch:periodic-weekly_ubuntu_focal-20230218
+    downstream-images:
+      - docker.io__openstackhelm__openvswitch:periodic-weekly_ubuntu_focal-20230218
+    overwrite-existing: true
+
+  - upstream-image: docker.io/library/rabbitmq:3.9.28-management
+    downstream-images:
+      - docker.io__library__rabbitmq:3.9.28-management
+    overwrite-existing: true
+
+  - upstream-image: docker.io/prom/memcached-exporter:v0.11.1
+    downstream-images:
+      - docker.io__prom__memcached-exporter:v0.11.1
+    overwrite-existing: true