Remove image-registry doc support in download-requirements (#3260) (#…

…3269)

* Add `--full-download` flag to download-requirements (#3260)

* Remove image-registry doc support

ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py

@@ -62,6 +62,8 @@ def __log_info_summary(self):
 
         if self.dest_manifest:
             lines.append(f'Manifest used: {str(self.dest_manifest.absolute())}')
+        else:
+            lines.append('Manifest not used, downloading all available requirements...')
 
         if self.is_log_file_enabled:
             lines.append(f'Log file location: {str(self.log_file.absolute())}')
@@ -207,17 +209,17 @@ def __add_args(self, argv: List[str]):
         self.distro_subdir = self.family_subdir / self.os_type.os_name
 
         # add optional arguments
+        self.dest_manifest = args['manifest'] or None
+        self.is_log_file_enabled = False if args['no_logfile'] else True
         self.repos_backup_file = Path(args['repos_backup_file'])
         self.retries = args['retries']
-        self.is_log_file_enabled = False if args['no_logfile'] else True
-        self.dest_manifest = args['manifest'] or None
         self.verbose_mode = True if self.log_level == logging.DEBUG else args['verbose']
 
         # offline mode
         self.rerun = args['rerun']
         self.pyyaml_installed = args['pyyaml_installed']
 
-    def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: Dict[str, Any]):
+    def __get_parsed_manifest_data_output(self, manifest: Dict[str, Any]) -> str:
         lines: List[str] = ['Manifest summary:']
 
         lines.append('-' * self.__LINE_SIZE)
@@ -232,6 +234,11 @@ def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: D
         for feature in manifest['requested-features']:
             lines.append(f'- {feature}')
 
+        return '\n'.join(lines)
+
+    def __get_requirements_output(self, requirements: Dict[str, Any]) -> str:
+        lines: List[str] = []
+
         for reqs in (('files', 'Files'),
                      ('grafana-dashboards', 'Dashboards')):
             reqs_to_download = sorted(requirements[reqs[0]])
@@ -250,12 +257,12 @@ def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: D
         if images_to_print:
             lines.append('')
             lines.append('Images to download:')
-            for image in sorted(images_to_print):
+            for image in sorted(set(images_to_print)):
                 lines.append(f'- {image}')
 
-        lines.append('-' * self.__LINE_SIZE)
+        lines.append('')
 
-        logging.info('\n'.join(lines))
+        return '\n'.join(lines)
 
     def __filter_files(self, requirements: Dict[str, Any],
                              manifest: Dict[str, Any]):
@@ -283,19 +290,12 @@ def __filter_images(self, requirements: Dict[str, Any], manifest: Dict[str, Any]
         for image_group in images:
             images_to_download[image_group] = {}
 
-        if len(manifest['requested-images']):  # if image-registry document used:
-            for image_group in images:
+        for image_group in images:
+            if image_group in manifest['requested-features']:
                 for image, data in images[image_group].items():
-                    if image in manifest['requested-images'] and image not in selected_images:
+                    if image not in selected_images:
                         images_to_download[image_group][image] = data
                         selected_images.add(image)
-        else:                                  # otherwise check features used:
-            for image_group in images:
-                if image_group in manifest['requested-features']:
-                    for image, data in images[image_group].items():
-                        if image not in selected_images:
-                            images_to_download[image_group][image] = data
-                            selected_images.add(image)
 
         if images_to_download:
             requirements['images'] = images_to_download
@@ -322,18 +322,21 @@ def read_manifest(self, requirements: Dict[str, Any]):
         :param requirements: parsed requirements which will be filtered based on the manifest output
         """
         if not self.dest_manifest:
+            logging.info(self.__get_requirements_output(requirements))
             return
 
         # Needs to be imported here as the libyaml might be missing on the OS,
         # this could cause crash on config.py import.
         from src.config.manifest_reader import ManifestReader
 
-        mreader = ManifestReader(self.dest_manifest, self.os_arch)
+        mreader = ManifestReader(self.dest_manifest)
         try:
             manifest = mreader.parse_manifest()
             self.__filter_manifest(requirements, manifest)
 
             if self.verbose_mode:
-                self.__print_parsed_manifest_data(requirements, manifest)
+                logging.info(f'{self.__get_parsed_manifest_data_output(manifest)}\n'
+                             f'{self.__get_requirements_output(requirements)}'
+                             f'{"-" * self.__LINE_SIZE}')
         except OldManifestVersion:
             pass  # old manifest used, cannot optimize download time

ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py

@@ -28,15 +28,13 @@ class ManifestReader:
     Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output.
     """
 
-    def __init__(self, dest_manifest: Path, arch: OSArch):
+    def __init__(self, dest_manifest: Path):
         self.__dest_manifest = dest_manifest
-        self.__os_arch: str = arch.value
 
         self.__k8s_as_cloud_service: bool = False
 
         self.__requested_components: Set = set()
         self.__requested_features: Set = set()
-        self.__requested_images: Set = set()
 
     def __parse_cluster_doc(self, cluster_doc: Dict):
         """
@@ -73,30 +71,14 @@ def __parse_feature_mappings_doc(self, feature_mappings_doc: Dict):
         if self.__k8s_as_cloud_service:
             self.__requested_features.add('k8s-as-cloud-service')
 
-    def __parse_image_registry_doc(self, image_registry_doc: Dict):
-        """
-        Parse `configuration/image-registry` document and extract only used images.
-
-        :param image_registry_doc: handler to a `configuration/image-registry` document
-        """
-        self.__requested_images.add(image_registry_doc['specification']['registry_image']['name'])
-
-        target_arch_images = image_registry_doc['specification']['images_to_load'][self.__os_arch]
-        for target_images in target_arch_images:
-            features = target_arch_images[target_images]
-            for feature in features:
-                for image in features[feature]:
-                    self.__requested_images.add(image['name'])
-
     def parse_manifest(self) -> Dict[str, Any]:
         """
         Load the manifest file, call parsers on required docs and return formatted output.
         """
         required_docs: Set[str] = {'epiphany-cluster', 'configuration/feature-mappings'}
         parse_doc: Dict[str, Callable] = {
             'epiphany-cluster':               self.__parse_cluster_doc,
-            'configuration/feature-mappings': self.__parse_feature_mappings_doc,
-            'configuration/image-registry':   self.__parse_image_registry_doc
+            'configuration/feature-mappings': self.__parse_feature_mappings_doc
         }
 
         parsed_docs: Set[str] = set()
@@ -112,5 +94,4 @@ def parse_manifest(self) -> Dict[str, Any]:
             raise CriticalError(f'ManifestReader - could not find document(s): {parsed_docs ^ required_docs}')
 
         return {'requested-components': sorted(list(self.__requested_components)),
-                'requested-features': sorted(list(self.__requested_features)),
-                'requested-images': sorted(list(self.__requested_images))}
+                'requested-features': sorted(list(self.__requested_features))}

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py

@@ -6,11 +6,12 @@
 
 from src.config.config import Config
 from tests.data.config import (
+    ALL_REQUIREMENTS,
     DASHBOARD_REQUIREMENTS,
+    EXPECTED_FULL_DOWNLOAD_OUTPUT,
     EXPECTED_VERBOSE_DASHBOARD_OUTPUT,
     EXPECTED_VERBOSE_FILE_OUTPUT,
     EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT,
-    EXPECTED_VERBOSE_IMAGE_OUTPUT,
     EXPECTED_VERBOSE_K8S_AS_CLOUD_SERVICE_OUTPUT,
     EXPECTED_VERBOSE_OUTPUT,
     FILE_REQUIREMENTS,
@@ -20,7 +21,6 @@
     INPUT_MANIFEST_FEATURE_MAPPINGS,
     INPUT_MANIFEST_IMAGES_NO_DOCUMENT,
     INPUT_MANIFEST_WITH_DASHBOARDS,
-    INPUT_MANIFEST_WITH_IMAGES,
     INPUT_MANIFEST_WITH_K8S_AS_CLOUD_SERVICE
 )
 from src.config.os_type import OSArch
@@ -31,7 +31,6 @@
                           (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS),
                           (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS),
                           (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS),
-                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_VERBOSE_IMAGE_OUTPUT, IMAGE_REQUIREMENTS),
                           (INPUT_MANIFEST_IMAGES_NO_DOCUMENT, EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT, IMAGE_REQUIREMENTS),
                           (INPUT_MANIFEST_WITH_K8S_AS_CLOUD_SERVICE, EXPECTED_VERBOSE_K8S_AS_CLOUD_SERVICE_OUTPUT, FILE_REQUIREMENTS)
                          ])
@@ -73,3 +72,28 @@ def test_manifest_verbose_output(INPUT_DOC: str,
     log_output = f'\n{"".join(caplog.messages)}\n'
 
     assert log_output == EXPECTED_OUTPUT_DOC
+
+
+def test_verbose_full_download_output(caplog):
+    """
+    Check output produced when running download-requirements script with the `-v|--verbose` flag and without `-m|--manifest`
+    provided
+    """
+    caplog.set_level(logging.INFO)
+
+    # mock Config's init methods:
+    Config._Config__add_args = lambda *args: None
+    Config._Config__log_info_summary = lambda *args: None
+
+    config = Config([])
+
+    # mock required config data:
+    config.dest_manifest = None
+    config.os_arch = OSArch.X86_64
+    config.verbose_mode = True
+
+    config.read_manifest(ALL_REQUIREMENTS)
+
+    log_output = f'\n{"".join(caplog.messages)}\n'
+
+    assert log_output == EXPECTED_FULL_DOWNLOAD_OUTPUT

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py

@@ -4,23 +4,17 @@
 import yaml
 
 from src.config.manifest_reader import ManifestReader
-from src.config.os_type import OSArch
 from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS,
                                         EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS,
-                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64,
-                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64,
                                         INPUT_MANIFEST_FEATURE_MAPPINGS,
-                                        INPUT_MANIFEST_WITH_DASHBOARDS,
-                                        INPUT_MANIFEST_WITH_IMAGES)
+                                        INPUT_MANIFEST_WITH_DASHBOARDS)
 
-@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH',
-                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS, OSArch.X86_64),
-                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, OSArch.X86_64),
-                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64, OSArch.X86_64),
-                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64, OSArch.ARM64)])
-def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH, mocker):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
+                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS)])
+def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker):
     ''' Check manifest file parsing '''
     mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
 
-    mreader = ManifestReader(Path('/some/path'), OS_ARCH)
+    mreader = ManifestReader(Path('/some/path'))
     assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC

ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py

@@ -1,3 +1,6 @@
+from typing import Dict
+
+
 FILE_REQUIREMENTS = """
 files:
   # --- Exporters ---
@@ -345,6 +348,61 @@
 """
 
 
+ALL_REQUIREMENTS: Dict[str, Dict] = {
+    'files': {
+        'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz': {
+            'sha256': 'ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0',
+            'deps': ['kafka-exporter']
+        },
+        'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': {
+            'sha256': '0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e',
+            'deps': ['kafka']
+        },
+        'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz': {
+            'sha256': '68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949',
+            'deps': ['node-exporter']
+        },
+        'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz': {
+            'sha256': '1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1',
+            'deps': ['postgres-exporter']
+        }
+    },
+    'grafana-dashboards': {
+        'grafana_dashboard_7249': {
+            'url': 'https://grafana.com/api/dashboards/7249/revisions/1/download',
+            'sha256': '41cc2794b1cc9fc537baf045fee12d086d23632b4c8b2e88985274bb9862e731'
+        },
+        'grafana_dashboard_315': {
+            'url': 'https://grafana.com/api/dashboards/315/revisions/3/download',
+            'sha256': 'ee46dd6e68a9950aa78e8c88ae5e565c8ebde6cbdbe08972a70f06c5486618fb'
+        },
+        'grafana_dashboard_11074': {
+            'url': 'https://grafana.com/api/dashboards/11074/revisions/9/download',
+            'sha256': '151b23305da46eab84930e99175e1c07e375af73dbbb4b8f501ca25f5ac62785'
+        }
+    },
+    'images': {
+        'haproxy': {
+            'haproxy:2.2.2-alpine': {
+                'sha1': 'dff8993b065b7f7846adb553548bcdcfcd1b6e8e'
+            }
+        },
+        'image-registry': {
+            'registry:2.8.0': {
+                'sha1': '89795c17099199c752d02ad8797c1d4565a08aff',
+                'allow_mismatch': True
+            }
+        },
+        'applications': {
+            'bitnami/pgpool:4.2.4': {
+                'sha1': '66741f3cf4a508bd1f80e2965b0086a4c0fc3580'
+            }
+        }
+    },
+    'packages': {}
+}
+
+
 EXPECTED_VERBOSE_OUTPUT = """
 Manifest summary:
 --------------------------------------------------
@@ -428,40 +486,6 @@
 """
 
 
-EXPECTED_VERBOSE_IMAGE_OUTPUT = """
-Manifest summary:
---------------------------------------------------
-Components requested:
-- repository
-
-Features requested:
-- filebeat
-- firewall
-- image-registry
-- node-exporter
-- repository
-
-Images to download:
-- bitnami/pgpool:4.2.4
-- epiphanyplatform/keycloak:14.0.0
-- haproxy:2.2.2-alpine
-- k8s.gcr.io/coredns/coredns:v1.8.0
-- k8s.gcr.io/etcd:3.4.13-0
-- k8s.gcr.io/kube-apiserver:v1.21.7
-- k8s.gcr.io/kube-apiserver:v1.22.4
-- k8s.gcr.io/kube-controller-manager:v1.21.7
-- k8s.gcr.io/kube-controller-manager:v1.22.4
-- k8s.gcr.io/kube-proxy:v1.21.7
-- k8s.gcr.io/kube-scheduler:v1.21.7
-- k8s.gcr.io/pause:3.4.1
-- kubernetesui/dashboard:v2.3.1
-- kubernetesui/metrics-scraper:v1.0.7
-- rabbitmq:3.8.9
-- registry:2.8.0
---------------------------------------------------
-"""
-
-
 EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT = """
 Manifest summary:
 --------------------------------------------------
@@ -508,3 +532,24 @@
 - node-exporter-chart
 --------------------------------------------------
 """
+
+
+EXPECTED_FULL_DOWNLOAD_OUTPUT = """
+
+Files to download:
+- https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz
+- https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz
+- https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
+- https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
+
+Dashboards to download:
+- grafana_dashboard_11074
+- grafana_dashboard_315
+- grafana_dashboard_7249
+
+Images to download:
+- bitnami/pgpool:4.2.4
+- haproxy:2.2.2-alpine
+- registry:2.8.0
+
+"""