From bf50d7f103b91ab03bb4dbba616a0306c7cfa746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lipovsk=C3=BD?= Date: Wed, 22 Nov 2023 14:49:22 +0100 Subject: [PATCH] Adding parameter ignore_bundle_ocp_version to merge-index API - If ignore_bundle_ocp_version parameter is set to true while calling merge-index API and target_index image is listed in iib_no_ocp_label_allow_list in config file then any bundle without "com.redhat.openshift.versions" set will be added to target_index. [CLOUDDST-20869] --- iib/web/api_v1.py | 1 + iib/web/iib_static_types.py | 1 + ...83d0ab_adding_ignore_bundle_ocp_version.py | 26 +++++++++++ iib/web/models.py | 2 + iib/web/static/api_v1.yaml | 15 +++++++ iib/workers/tasks/build_merge_index_image.py | 20 +++++++-- tests/test_web/test_api_v1.py | 2 + .../test_build_merge_index_image.py | 45 +++++++++++++++---- 8 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 iib/web/migrations/versions/1920ad83d0ab_adding_ignore_bundle_ocp_version.py diff --git a/iib/web/api_v1.py b/iib/web/api_v1.py index 4293e260..c02b8362 100644 --- a/iib/web/api_v1.py +++ b/iib/web/api_v1.py @@ -1091,6 +1091,7 @@ def merge_index_image() -> Tuple[flask.Response, int]: flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], payload.get('build_tags', []), payload.get('graph_update_mode'), + payload.get('ignore_bundle_ocp_version'), ] safe_args = _get_safe_args(args, payload) diff --git a/iib/web/iib_static_types.py b/iib/web/iib_static_types.py index 67514da7..b72520d8 100644 --- a/iib/web/iib_static_types.py +++ b/iib/web/iib_static_types.py @@ -391,6 +391,7 @@ class MergeIndexImageRequestResponse(APIPartImageBuildRequestResponse): deprecation_list: List[str] distribution_scope: str graph_update_mode: GRAPH_MODE_LITERAL + ignore_bundle_ocp_version: Optional[bool] index_image: Optional[str] source_from_index: str source_from_index_resolved: Optional[str] diff --git a/iib/web/migrations/versions/1920ad83d0ab_adding_ignore_bundle_ocp_version.py b/iib/web/migrations/versions/1920ad83d0ab_adding_ignore_bundle_ocp_version.py new file mode 100644 index 00000000..25da7788 --- /dev/null +++ b/iib/web/migrations/versions/1920ad83d0ab_adding_ignore_bundle_ocp_version.py @@ -0,0 +1,26 @@ +"""Adding ignore_bundle_ocp_version. + +Revision ID: 1920ad83d0ab +Revises: 9e9d4f9730c8 +Create Date: 2023-11-22 12:03:50.711489 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1920ad83d0ab' +down_revision = '9e9d4f9730c8' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('request_merge_index_image', schema=None) as batch_op: + batch_op.add_column(sa.Column('ignore_bundle_ocp_version', sa.Boolean(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table('request_merge_index_image', schema=None) as batch_op: + batch_op.drop_column('ignore_bundle_ocp_version') diff --git a/iib/web/models.py b/iib/web/models.py index a8be1607..b6d36b37 100644 --- a/iib/web/models.py +++ b/iib/web/models.py @@ -1534,6 +1534,7 @@ class RequestMergeIndexImage(Request): ) distribution_scope: Mapped[Optional[str]] graph_update_mode: Mapped[Optional[str]] + ignore_bundle_ocp_version: Mapped[Optional[bool]] __mapper_args__ = { 'polymorphic_identity': RequestTypeMapping.__members__['merge_index_image'].value @@ -1663,6 +1664,7 @@ def to_json(self, verbose: Optional[bool] = True) -> MergeIndexImageRequestRespo ) rv['deprecation_list'] = [bundle.pull_specification for bundle in self.deprecation_list] rv['graph_update_mode'] = self.graph_update_mode + rv['ignore_bundle_ocp_version'] = self.ignore_bundle_ocp_version rv['index_image'] = getattr(self.index_image, 'pull_specification', None) rv['source_from_index'] = self.source_from_index.pull_specification rv['source_from_index_resolved'] = getattr( diff --git a/iib/web/static/api_v1.yaml b/iib/web/static/api_v1.yaml index f62c6fb2..e6d0fea4 100644 --- a/iib/web/static/api_v1.yaml +++ b/iib/web/static/api_v1.yaml @@ -1287,6 +1287,13 @@ components: items: type: string example: ["v4.5-10-08-2021"] + ignore_bundle_ocp_version: + type: boolean + description: > + When `ignore_bundle_ocp_version` is set to `true` and image set as target_index + is listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. + example: 'true' required: - source_from_index MergeIndexImageResponse: @@ -1347,6 +1354,14 @@ components: image pull specs configured in IIB_GRAPH_MODE_INDEX_ALLOW_LIST in the IIB API Config. If not specified, "--mode" will not be added to OPM commands to add the bundle(s) to the index. default: None + ignore_bundle_ocp_version: + type: boolean + description: > + When `ignore_bundle_ocp_version` is set to `true` and image set as target_index + is listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. + example: 'true' + default: False RequestCreateEmptyIndex: type: object properties: diff --git a/iib/workers/tasks/build_merge_index_image.py b/iib/workers/tasks/build_merge_index_image.py index 19f2f558..477a2307 100644 --- a/iib/workers/tasks/build_merge_index_image.py +++ b/iib/workers/tasks/build_merge_index_image.py @@ -63,6 +63,7 @@ def _add_bundles_missing_in_source( graph_update_mode: Optional[str] = None, target_index=None, overwrite_target_index_token: Optional[str] = None, + ignore_bundle_ocp_version: Optional[bool] = False, ) -> Tuple[List[BundleImage], List[BundleImage]]: """ Rebuild index image with bundles missing from source image but present in target image. @@ -84,6 +85,9 @@ def _add_bundles_missing_in_source( :param str overwrite_target_index_token: the token used for overwriting the input ``source_from_index`` image. This is required to use ``overwrite_target_index``. The format of the token must be in the format "user:password". + :param bool ignore_bundle_ocp_version: When set to `true` and image set as target_index is + listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. :return: tuple where the first value is a list of bundles which were added to the index image and the second value is a list of bundles in the new index whose ocp_version range does not satisfy the ocp_version value of the target index. @@ -124,10 +128,13 @@ def _add_bundles_missing_in_source( missing_bundles.append(bundle) missing_bundle_paths.append(bundle['bundlePath']) - allow_no_ocp_version = any( - target_index.startswith(index) - for index in get_worker_config()['iib_no_ocp_label_allow_list'] - ) + if ignore_bundle_ocp_version and target_index is not None: + allow_no_ocp_version = any( + target_index.startswith(index) + for index in get_worker_config()['iib_no_ocp_label_allow_list'] + ) + else: + allow_no_ocp_version = False for bundle in itertools.chain(missing_bundles, source_index_bundles): if not is_bundle_version_valid(bundle['bundlePath'], ocp_version, allow_no_ocp_version): @@ -190,6 +197,7 @@ def handle_merge_request( binary_image_config: Optional[str] = None, build_tags: Optional[List[str]] = None, graph_update_mode: Optional[str] = None, + ignore_bundle_ocp_version: Optional[bool] = False, ) -> None: """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. @@ -212,6 +220,9 @@ def handle_merge_request( :param build_tags: list of extra tag to use for intermetdiate index image :param str graph_update_mode: Graph update mode that defines how channel graphs are updated in the index. + :param bool ignore_bundle_ocp_version: When set to `true` and image set as target_index is + listed in `iib_no_ocp_label_allow_list` config then bundles without + "com.redhat.openshift.versions" label set will be added in the result `index_image`. :raises IIBError: if the index image merge fails. """ _cleanup() @@ -282,6 +293,7 @@ def handle_merge_request( target_index=target_index, overwrite_target_index_token=overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], + ignore_bundle_ocp_version=ignore_bundle_ocp_version, ) missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] diff --git a/tests/test_web/test_api_v1.py b/tests/test_web/test_api_v1.py index 2c840be3..ebcaa3bb 100644 --- a/tests/test_web/test_api_v1.py +++ b/tests/test_web/test_api_v1.py @@ -1963,6 +1963,7 @@ def test_merge_index_image_success( 'distribution_scope': distribution_scope, 'graph_update_mode': 'semver', 'id': 1, + 'ignore_bundle_ocp_version': None, 'index_image': None, 'logs': { 'expiration': '2020-02-15T17:03:00Z', @@ -2067,6 +2068,7 @@ def test_merge_index_image_custom_user_queue( 'source_from_index': 'source_index:image', 'target_index': 'target_index:image', 'graph_update_mode': 'replaces', + 'ignore_bundle_ocp_version': True, } if overwrite_from_index: data['overwrite_target_index'] = True diff --git a/tests/test_workers/test_tasks/test_build_merge_index_image.py b/tests/test_workers/test_tasks/test_build_merge_index_image.py index a49e501a..ef0aeed3 100644 --- a/tests/test_workers/test_tasks/test_build_merge_index_image.py +++ b/tests/test_workers/test_tasks/test_build_merge_index_image.py @@ -331,6 +331,7 @@ def test_handle_merge_request_no_deprecate( mock_run_cmd.assert_not_called() +@mock.patch('iib.workers.config.get_worker_config') @mock.patch('iib.workers.tasks.build_merge_index_image.is_image_fbc') @mock.patch('iib.workers.tasks.build_merge_index_image.get_image_label') @mock.patch('iib.workers.tasks.build_merge_index_image._create_and_push_manifest_list') @@ -340,7 +341,7 @@ def test_handle_merge_request_no_deprecate( @mock.patch('iib.workers.tasks.build_merge_index_image._opm_index_add') @mock.patch('iib.workers.tasks.build_merge_index_image.set_request_state') def test_add_bundles_missing_in_source( - mock_srs, mock_oia, mock_aolti, mock_bi, mock_pi, mock_capml, mock_gil, mock_iifbc + mock_srs, mock_oia, mock_aolti, mock_bi, mock_pi, mock_capml, mock_gil, mock_iifbc, mock_gwc ): source_bundles = [ { @@ -387,8 +388,19 @@ def test_add_bundles_missing_in_source( 'bundlePath': 'quay.io/bundle4@sha256:569854', 'csvName': 'bundle5-5.0', }, + { + 'packageName': 'bundle6-ignore-ocp-failed', + 'version': '14.0', + 'bundlePath': 'quay.io/ignore-ocp-failed-bundle4@sha256:567890', + 'csvName': 'bundle6-ignore-ocp-failed-14.0', + }, ] - mock_gil.side_effect = ['=v4.5', '=v4.6', 'v4.7', 'v4.5-v4.7', 'v4.5,v4.6'] + + mock_gwc.iib_api_url.return_value = { + 'iib_no_ocp_label_allow_list': ['quay.io/bundle'], + } + + mock_gil.side_effect = ['=v4.5', '=v4.6', 'v4.7', 'v4.5-v4.7', 'v4.5,v4.6', ''] mock_iifbc.return_value = False missing_bundles, invalid_bundles = build_merge_index_image._add_bundles_missing_in_source( source_bundles, @@ -401,6 +413,7 @@ def test_add_bundles_missing_in_source( '4.6', 'dev', 'replaces', + ignore_bundle_ocp_version=True, ) assert missing_bundles == [ { @@ -415,6 +428,12 @@ def test_add_bundles_missing_in_source( 'bundlePath': 'quay.io/bundle4@sha256:567890', 'csvName': 'bundle4-4.0', }, + { + 'bundlePath': 'quay.io/ignore-ocp-failed-bundle4@sha256:567890', + 'csvName': 'bundle6-ignore-ocp-failed-14.0', + 'packageName': 'bundle6-ignore-ocp-failed', + 'version': '14.0', + }, ] assert invalid_bundles == [ { @@ -424,22 +443,32 @@ def test_add_bundles_missing_in_source( 'csvName': 'bundle3-3.0', }, { - 'packageName': 'bundle1', - 'version': '1.0', - 'bundlePath': 'quay.io/bundle1@sha256:123456', - 'csvName': 'bundle1-1.0', + 'packageName': 'bundle6-ignore-ocp-failed', + 'version': '14.0', + 'bundlePath': 'quay.io/ignore-ocp-failed-bundle4@sha256:567890', + 'csvName': 'bundle6-ignore-ocp-failed-14.0', + }, + { + 'packageName': 'bundle5', + 'version': '5.0-2', + 'bundlePath': 'quay.io/bundle2@sha256:456132', + 'csvName': 'bundle5-5.0', }, ] mock_srs.assert_called_once() mock_oia.assert_called_once_with( base_dir='some_dir', - bundles=['quay.io/bundle3@sha256:456789', 'quay.io/bundle4@sha256:567890'], + bundles=[ + 'quay.io/bundle3@sha256:456789', + 'quay.io/bundle4@sha256:567890', + 'quay.io/ignore-ocp-failed-bundle4@sha256:567890', + ], binary_image='binary-image:4.5', from_index='index-image:4.6', container_tool='podman', graph_update_mode='replaces', ) - assert mock_gil.call_count == 5 + assert mock_gil.call_count == 6 assert mock_aolti.call_count == 2 mock_bi.assert_called_once() mock_pi.assert_called_once()