diff --git a/iib/web/api_v1.py b/iib/web/api_v1.py index b8b706661..52bb376c4 100644 --- a/iib/web/api_v1.py +++ b/iib/web/api_v1.py @@ -37,6 +37,8 @@ RequestTypeMapping, RequestCreateEmptyIndex, User, + RequestAddDeprecationsDeprecationSchema, + DeprecationSchema, ) from iib.web.s3_utils import get_object_from_s3_bucket from botocore.response import StreamingBody @@ -45,6 +47,7 @@ handle_add_request, handle_rm_request, ) +from iib.workers.tasks.build_add_deprecations import handle_add_deprecations_request from iib.workers.tasks.build_fbc_operations import handle_fbc_operation_request from iib.workers.tasks.build_recursive_related_bundles import ( handle_recursive_related_bundles_request, @@ -1345,5 +1348,65 @@ def add_deprecations() -> Tuple[flask.Response, int]: db.session.add(request) db.session.commit() - flask.current_app.logger.debug('Successfully validated request %d', request.id) - return flask.jsonify({'msg': 'This API endpoint hasn not been implemented yet'}), 501 + messaging.send_message_for_state_change(request, new_batch_msg=True) + + overwrite_from_index = payload.get('overwrite_from_index', False) + from_index_pull_spec = request.from_index.pull_specification + celery_queue = _get_user_queue( + serial=overwrite_from_index, from_index_pull_spec=from_index_pull_spec + ) + + args = [ + request.id, + payload['operator_package'], + payload['deprecation_schema'], + payload['from_index'], + payload.get('binary_image'), + payload.get('build_tags'), + payload.get('overwrite_from_index'), + payload.get('overwrite_from_index_token'), + flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + ] + safe_args = _get_safe_args(args, payload) + error_callback = failed_request_callback.s(request.id) + try: + handle_add_deprecations_request.apply_async( + args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue + ) + except kombu.exceptions.OperationalError: + handle_broker_error(request) + + flask.current_app.logger.debug('Successfully scheduled request %d', request.id) + return flask.jsonify(request.to_json()), 201 + + +@api_v1.route('/builds//deprecation-schema') +@instrument_tracing(span_name="web.api_v1.get_deprecation_schema") +def get_deprecation_schema(request_id: int) -> flask.Response: + """ + Retrieve the deprecation-schema for add-deprecations request. + + :param int request_id: the request ID that was passed in through the URL. + :rtype: flask.Response + :raise NotFound: if the request is not found or there are no deprecation_schema for the request + :raise ValidationError: if the request is of invalid type or is not completed yet + """ + request = Request.query.get_or_404(request_id) + if request.type != RequestTypeMapping.add_deprecations.value: + raise ValidationError( + f'The request {request_id} is of type {request.type_name}. ' + 'This endpoint is only valid for requests of type add-deprecations.' + ) + + try: + deprecation_schema_for_request = ( + DeprecationSchema.query.join(RequestAddDeprecationsDeprecationSchema) + .filter( + RequestAddDeprecationsDeprecationSchema.request_add_deprecations_id == request_id + ) + .first() + ) + except Exception as e: + raise e + + return flask.Response(deprecation_schema_for_request.schema, mimetype='application/json') diff --git a/iib/web/iib_static_types.py b/iib/web/iib_static_types.py index f27c74fa7..8be7deace 100644 --- a/iib/web/iib_static_types.py +++ b/iib/web/iib_static_types.py @@ -90,6 +90,7 @@ class AddDeprecationRequestPayload(TypedDict): """Data structure of the request to /builds/add-deprecations API endpoint.""" binary_image: NotRequired[str] + build_tags: NotRequired[List[str]] deprecation_schema: str from_index: str operator_package: str @@ -447,4 +448,11 @@ class FbcOperationRequestResponse(BaseClassRequestResponse): fbc_fragment_resolved: Optional[str] +class AddDeprecationsRequestResponse(BaseClassRequestResponse): + """Datastructure of the response to request from /builds/add-deprecations API point.""" + + operator_package: str + deprecation_schema_url: str + + # End of the RequestResponses Part diff --git a/iib/web/models.py b/iib/web/models.py index 50e6d28da..6bb1f97e8 100644 --- a/iib/web/models.py +++ b/iib/web/models.py @@ -43,6 +43,7 @@ RmRequestPayload, FbcOperationRequestPayload, FbcOperationRequestResponse, + AddDeprecationsRequestResponse, ) @@ -2344,8 +2345,13 @@ def from_json( # type: ignore[override] # noqa: F821 deprecation_schema=_deprecation_schema ) + build_tags = request_kwargs.pop('build_tags', []) request = cls(**request_kwargs) - request.add_state('failed', 'The API endpoint has not been implemented yet') + + for bt in build_tags: + request.add_build_tag(bt) + + request.add_state('in_progress', 'The request was initiated') return request def get_mutable_keys(self) -> Set[str]: @@ -2358,3 +2364,28 @@ def get_mutable_keys(self) -> Set[str]: rv = super().get_mutable_keys() rv.update(self.get_index_image_mutable_keys()) return rv + + def to_json(self, verbose: Optional[bool] = True) -> AddDeprecationsRequestResponse: + """ + Provide the JSON representation of a "add-deprecations" build request. + + :param bool verbose: determines if the JSON output should be verbose + :return: a dictionary representing the JSON of the build request + :rtype: dict + """ + # cast to result type, super-type returns Union + rv = cast(AddDeprecationsRequestResponse, super().to_json(verbose=verbose)) + rv.update(self.get_common_index_image_json()) # type: ignore + rv['operator_package'] = self.operator_package.name + rv['deprecation_schema_url'] = url_for( + '.get_deprecation_schema', request_id=self.id, _external=True + ) + + rv.pop('bundles') + rv.pop('bundle_mapping') + rv.pop('deprecation_list') + rv.pop('distribution_scope') + rv.pop('organization') + rv.pop('removed_operators') + + return rv diff --git a/iib/web/static/api_v1.yaml b/iib/web/static/api_v1.yaml index e6d0fea42..4ea01f14b 100644 --- a/iib/web/static/api_v1.yaml +++ b/iib/web/static/api_v1.yaml @@ -616,6 +616,79 @@ paths: error: type: string example: You must be authenticated to perform this action + /builds/add-deprecations: + post: + description: > + Submit a request to add deprecation information to an index image + requestBody: + description: The request to add deprecation schema + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AddDeprecationsRequest' + security: + - Kerberos Authentication: [] + responses: + '201': + description: The build request was initiated + content: + application/json: + schema: + $ref: '#/components/schemas/AddDeprecationsResponseVerbose' + '400': + description: The input is invalid + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: '"operator_package" should be a non-empty string' + '401': + description: > + The user is not allowed to create a request with this input + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: You must be authenticated to perform this action + '/builds/{id}/deprecation-schema': + get: + description: Return deprecation schema of the build request + parameters: + - name: id + in: path + required: true + description: The ID of the build request to retrieve the related bundles for + schema: + type: integer + responses: + '200': + description: The deprecation schema for the build request + content: + application/json: + schema: + type: array + items: + type: string + example: + - '{"schema":"olm.deprecations","package":"test-operator"}' + '404': + description: > + The deprecation schema is not available for this request. + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: The requested resource was not found. /builds/merge-index-image: post: description: Submit a build request to merge index images @@ -1601,6 +1674,109 @@ components: fbc_fragment_resolved: type: string example: "quay.io/iib/fbc-fragment@sha256:7d8e5dddad0275bc903b6ef17840f98441b15b8b999609af2c9579960e52080e" + AddDeprecationsRequest: + type: object + properties: + binary_image: + type: string + description: > + The pull specification of the container image where the opm binary gets copied from. + example: 'quay.io/operator-framework/upstream-registry-builder:v1.26.3' + operator_package: + type: string + description: > + operator_package is required. + example: 'test-operator' + deprecation_schema: + type: string + description: > + Jsonified string of deprecation scheme to be added + example: '{"schema":"olm.deprecations","package":"test-operator"}' + from_index: + type: string + description: > + from_index is required. + example: 'quay.io/iib-stage/iib:4' + overwrite_from_index: + type: boolean + description: > + Overwrites the input from_index image with the built index image. This can only be + performed when overwrite_from_index_token is provided. + default: false + overwrite_from_index_token: + type: string + description: > + The token used for reading and overwriting the input from_index image. This is required + to use overwrite_from_index. The format of the token is in the format "user:password". + example: token + build_tags: + description: > + Extra tags applied to intermediate index image + type: array + items: + type: string + example: ["v4.5-10-08-2021"] + required: + - operator_package + - deprecation_schema + - from_index + AddDeprecationsResponse: + allOf: + - $ref: '#/components/schemas/BaseResponse' + - type: object + properties: + binary_image: + type: string + example: 'quay.io/operator-framework/upstream-registry-builder:v1.5.9' + binary_image_resolved: + type: string + example: >- + quay.io/operator-framework/upstream-registry-builder@sha256:7d8e5dddad0275bc903b6ef17840f98441b15b8b999609af2c9579960e52080e + from_index: + type: string + example: 'quay.io/iib-stage/iib:4' + from_index_resolved: + type: string + example: >- + quay.io/iib-stage/iib@sha256:7d8e5dddad0275bc903b6ef17840f98441b15b8b999609af2c9579960e52080e + index_image: + type: string + example: 'quay.io/iib-stage/iib:5' + index_image_resolved: + type: string + example: 'quay.io/iib-stage/iib@sha256:abcdef012356789' + internal_index_image_copy: + description: > + The pullspec of the internal copy of the index image built by IIB. + It will have the same value as index_image when overwrite_from_index + is not provided. + type: string + example: 'quay.io/iib-stage/iib:5' + internal_index_image_copy_resolved: + description: > + The resolved pullspec of the internal copy of the index image built by IIB. + It will have the same value as index_image_resolved when overwrite_from_index + is not provided. + type: string + example: 'quay.io/iib-stage/iib@sha256:abcdef012356789' + build_tags: + description: > + Extra tags applied to intermediate index image + type: array + items: + type: string + example: ["v4.5-10-08-2021"] + request_type: + type: string + example: add-deprecations + operator_package: + type: string + description: operator_package of deprecation information. + example: 'test-operator' + deprecation_schema_url: + type: string + description: url to access deprecation schema + example: https://iib.domain.local/api/v1/get_deprecation_schema RequestUpdate: type: object properties: @@ -1676,6 +1852,10 @@ components: allOf: - $ref: '#/components/schemas/BaseResponseVerbose' - $ref: '#/components/schemas/FbcResponse' + AddDeprecationsResponseVerbose: + allOf: + - $ref: '#/components/schemas/BaseResponseVerbose' + - $ref: '#/components/schemas/AddDeprecationsResponse' StateHistory: type: object properties: diff --git a/iib/workers/config.py b/iib/workers/config.py index f6379dae0..4a072c9e2 100644 --- a/iib/workers/config.py +++ b/iib/workers/config.py @@ -83,6 +83,7 @@ class Config(object): 'iib.workers.tasks.build_regenerate_bundle', 'iib.workers.tasks.build_create_empty_index', 'iib.workers.tasks.build_fbc_operations', + 'iib.workers.tasks.build_add_deprecations', 'iib.workers.tasks.general', ] # Path to hidden location of SQLite database @@ -90,8 +91,7 @@ class Config(object): # path where catalog resides in fbc_fragment # might need to be changed, currently based on test fbc-fragment fbc_fragment_catalog_path: str = '/configs' - # path where operator deprecations will be stored - # it's a sub-directory of fbc_fragment_catalog_path + # sub-directory under fbc_fragment_catalog_path where operator deprecations will be stored operator_deprecations_dir: str = '_operator-deprecations-content' # The task messages will be acknowledged after the task has been executed, # instead of just before diff --git a/iib/workers/tasks/build_add_deprecations.py b/iib/workers/tasks/build_add_deprecations.py new file mode 100644 index 000000000..2157975fa --- /dev/null +++ b/iib/workers/tasks/build_add_deprecations.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import os +import json +import logging +import tempfile +from typing import Dict, Optional, Set + +from iib.common.common_utils import get_binary_versions +from iib.common.tracing import instrument_tracing +from iib.workers.api_utils import set_request_state +from iib.workers.tasks.build import ( + _add_label_to_index, + _build_image, + _cleanup, + _create_and_push_manifest_list, + _push_image, + _update_index_image_build_state, + _update_index_image_pull_spec, +) +from iib.workers.tasks.celery import app +from iib.workers.tasks.fbc_utils import get_catalog_dir +from iib.workers.tasks.opm_operations import ( + Opm, + create_dockerfile, + generate_cache_locally, + opm_validate, + verify_operators_exists, +) +from iib.workers.tasks.utils import ( + prepare_request_for_build, + request_logger, + RequestConfigAddDeprecations, + IIBError, + get_worker_config, +) + +__all__ = ['handle_add_deprecations_request'] + +log = logging.getLogger(__name__) + + +@app.task +@request_logger +@instrument_tracing( + span_name="workers.tasks.build.handle_add_deprecations_request", + attributes=get_binary_versions(), +) +def handle_add_deprecations_request( + request_id: int, + operator_package: str, + deprecation_schema: str, + from_index: str, + overwrite_from_index: bool = False, + overwrite_from_index_token: Optional[str] = None, + binary_image: Optional[str] = None, + build_tags: Optional[Set[str]] = None, + binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, +) -> None: + """ + Add a deprecation schema to index image. + + :param int request_id: the ID of the IIB build request. + :param str operator_package: Operator package of deprecation schema. + :param str deprecation_schema: deprecation_schema to be added to index image. + :param str from_index: the pull specification of the container image containing the index that + the index image build will be based from. + :param str binary_image: the pull specification of the container image where the opm binary + gets copied from. + :param bool overwrite_from_index: if True, overwrite the input ``from_index`` with the built + index image. + :param str overwrite_from_index_token: the token used for overwriting the input + ``from_index`` image. This is required to use ``overwrite_from_index``. + The format of the token must be in the format "user:password". + :param list build_tags: List of tags which will be applied to intermediate index images. + :param dict binary_image_config: the dict of config required to identify the appropriate + ``binary_image`` to use. + """ + _cleanup() + set_request_state(request_id, 'in_progress', 'Resolving the index images') + + prebuild_info = prepare_request_for_build( + request_id, + RequestConfigAddDeprecations( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=overwrite_from_index_token, + operator_package=operator_package, + deprecation_schema=deprecation_schema, + binary_image_config=binary_image_config, + ), + ) + + from_index_resolved = prebuild_info['from_index_resolved'] + binary_image_resolved = prebuild_info['binary_image_resolved'] + Opm.set_opm_version(from_index_resolved) + + _update_index_image_build_state(request_id, prebuild_info) + + with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: + operators_in_db, index_db_path = verify_operators_exists( + from_index_resolved, + temp_dir, + [operator_package], + overwrite_from_index_token, + ) + if not operators_in_db: + err_msg = ( + f'Cannot add deprecations for {operator_package},' + f' It is either not present in index or opted in fbc' + ) + log.error(err_msg) + raise IIBError(err_msg) + + add_deprecations_to_index( + request_id, + temp_dir, + from_index_resolved, + operator_package, + deprecation_schema, + binary_image_resolved, + index_db_path, + ) + + _add_label_to_index( + 'com.redhat.index.delivery.version', + prebuild_info['ocp_version'], + temp_dir, + 'index.Dockerfile', + ) + + _add_label_to_index( + 'com.redhat.index.delivery.distribution_scope', + prebuild_info['distribution_scope'], + temp_dir, + 'index.Dockerfile', + ) + + arches = prebuild_info['arches'] + for arch in sorted(arches): + _build_image(temp_dir, 'index.Dockerfile', request_id, arch) + _push_image(request_id, arch) + + set_request_state(request_id, 'in_progress', 'Creating the manifest list') + output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags) + + _update_index_image_pull_spec( + output_pull_spec, + request_id, + arches, + from_index, + overwrite_from_index, + overwrite_from_index_token, + from_index_resolved, + add_or_rm=True, + ) + _cleanup() + set_request_state( + request_id, 'complete', 'The deprecation schema was successfully added to the index image' + ) + + +def add_deprecations_to_index( + request_id, + temp_dir, + from_index_resolved, + operator_package, + deprecation_schema, + binary_image_resolved, + index_db_path, +) -> None: + """ + Add deprecation schema for a package in operator-deprecations sub-directory. + + :param int request_id: the ID of the IIB build request. + :param str temp_dir: the base directory to generate the database and index.Dockerfile in. + :param str from_index_resolved: the resolved pull specification of the container image. + containing the index that the index image build will be based from. + :param str operator_package: the operator package for which deprecations need to be added. + :param str deprecation_schema: deprecation_schema to be added to index image. + :param str binary_image_resolved: the pull specification of the image where the opm binary + gets copied from. + :param str index_db: path to locally stored index.db. + + """ + set_request_state(request_id, 'in_progress', 'Getting all deprecations present in index image') + + from_index_configs_dir = get_catalog_dir(from_index=from_index_resolved, base_dir=temp_dir) + conf = get_worker_config() + from_index_configs_deprecations_dir = os.path.join( + from_index_configs_dir, conf['operator_deprecations_dir'] + ) + + operator_dir = os.path.join(from_index_configs_deprecations_dir, operator_package) + if not os.path.exists(operator_dir): + os.makedirs(operator_dir) + + operator_deprecations_file = os.path.join(operator_dir, f'{operator_package}.json') + set_request_state(request_id, 'in_progress', 'Adding deprecations to from_index') + + with open(operator_deprecations_file, 'w') as output_file: + json.dump(json.loads(deprecation_schema), output_file) + + opm_validate(from_index_configs_dir) + + local_cache_path = os.path.join(temp_dir, 'cache') + generate_cache_locally( + base_dir=temp_dir, fbc_dir=from_index_configs_dir, local_cache_path=local_cache_path + ) + + log.info("Dockerfile generated from %s", from_index_configs_dir) + create_dockerfile( + fbc_dir=from_index_configs_dir, + base_dir=temp_dir, + index_db=index_db_path, + binary_image=binary_image_resolved, + dockerfile_name='index.Dockerfile', + ) diff --git a/iib/workers/tasks/utils.py b/iib/workers/tasks/utils.py index 121fd36f4..fa62e95f4 100644 --- a/iib/workers/tasks/utils.py +++ b/iib/workers/tasks/utils.py @@ -394,6 +394,36 @@ class RequestConfigFBCOperation(RequestConfig): fbc_fragment: str +class RequestConfigAddDeprecations(RequestConfig): + """ + Request config for Add Deprecations operation. + + :param str overwrite_from_index_token: the token used for overwriting the input + ``from_index`` image. This is required for + non-privileged users to use ``overwrite_from_index``. + The format of the token must be + in the format "user:password". + :param str from_index: the pull specification of the container image + containing the index that the index image build + will be based from. + :param str operator_package: the package for which deprecation information needs to be added + :param str deprecation_schema: the deprecation schema to add in index image + """ + + _attrs: List[str] = RequestConfig._attrs + [ + "overwrite_from_index_token", + "from_index", + "operator_package", + "deprecation_schema", + ] + __slots__ = _attrs + if TYPE_CHECKING: + overwrite_from_index_token: str + from_index: str + operator_package: str + deprecation_schema: str + + def get_bundles_from_deprecation_list(bundles: List[str], deprecation_list: List[str]) -> List[str]: """ Get a list of to-be-deprecated bundles based on the data from the deprecation list. diff --git a/tests/conftest.py b/tests/conftest.py index 8d116ea9b..f69506cc3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -239,6 +239,48 @@ def minimal_request_recursive_related_bundles(db): return request +@pytest.fixture() +def minimal_request_add_deprecations(db): + """ + Create and return an instance of the RequestAddDeprecations class. + + The request instance will have the minimal set of required attributes set, + and it'll be committed to the database. + + :param flask_sqlalchemy.SQLAlchemy db: the connection to the database + :return: the newly created request object + :rtype: RequestAddDeprecations + """ + binary_image = models.Image(pull_specification='quay.io/add_deprecations/binary-image:latest') + db.session.add(binary_image) + from_index_image = models.Image( + pull_specification='quay.io/add_deprecations/index-image:latest' + ) + db.session.add(from_index_image) + batch = models.Batch() + db.session.add(batch) + operator = models.Operator(name='operator') + db.session.add(operator) + deprecation_schema_json = ( + '{"schema":"olm.deprecations","package":"test-operator",' + '"entries":[{"reference":{"name":"test-operator.v1.57.7","schema":"olm.bundle"},' + '"message":"test-operator.v1.57.7 is deprecated. Uninstall and install' + ' test-operator.v1.65.6 for support"}]}' + ) + deprecation_schema = models.DeprecationSchema(schema=deprecation_schema_json) + db.session.add(deprecation_schema) + request = models.RequestAddDeprecations( + batch=batch, + binary_image=binary_image, + from_index=from_index_image, + operator_package=operator, + deprecation_schema=deprecation_schema, + ) + db.session.add(request) + db.session.commit() + return request, deprecation_schema_json + + @pytest.fixture(scope='session', autouse=True) def patch_retry(): with mock.patch.object(tenacity.nap.time, "sleep"): diff --git a/tests/test_web/test_api_v1.py b/tests/test_web/test_api_v1.py index afeb82259..f64b9e603 100644 --- a/tests/test_web/test_api_v1.py +++ b/tests/test_web/test_api_v1.py @@ -2889,13 +2889,133 @@ def test_add_deprecations_invalid_params_format(mock_smfsc, db, auth_env, client mock_smfsc.assert_not_called() -def test_add_deprecations_success(db, auth_env, client): +@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') +@mock.patch('iib.web.api_v1.handle_add_deprecations_request') +def test_add_deprecations_success(mock_had, mock_smfsc, db, auth_env, client): data = { 'from_index': 'pull:spec', 'binary_image': 'binary:image', 'operator_package': 'my-package', 'deprecation_schema': '{"valid":"json"}', + 'build_tags': ['timestamptag'], + } + expected_response = { + 'arches': [], + 'batch': 1, + 'batch_annotations': None, + 'binary_image': 'binary:image', + 'binary_image_resolved': None, + 'build_tags': ['timestamptag'], + 'deprecation_schema_url': 'http://localhost/api/v1/builds/1/deprecation-schema', + 'from_index': 'pull:spec', + 'from_index_resolved': None, + 'id': 1, + 'index_image': None, + 'index_image_resolved': None, + 'internal_index_image_copy': None, + 'internal_index_image_copy_resolved': None, + 'operator_package': 'my-package', + 'request_type': 'add-deprecations', + 'state': 'in_progress', + 'logs': { + 'url': 'http://localhost/api/v1/builds/1/logs', + 'expiration': '2020-02-15T17:03:00Z', + }, + 'state_history': [ + { + 'state': 'in_progress', + 'state_reason': 'The request was initiated', + 'updated': '2020-02-12T17:03:00Z', + } + ], + 'state_reason': 'The request was initiated', + 'updated': '2020-02-12T17:03:00Z', + 'user': 'tbrady@DOMAIN.LOCAL', } rv = client.post(f'/api/v1/builds/add-deprecations', json=data, environ_base=auth_env) - # TODO: Change this status code to 201 once the endpoint functionality is implemented - assert rv.status_code == 501 + assert rv.status_code == 201 + rv_json = rv.json + mock_had.apply_async.assert_called_once() + mock_smfsc.assert_called_once_with(mock.ANY, new_batch_msg=True) + rv_json['state_history'][0]['updated'] = '2020-02-12T17:03:00Z' + rv_json['updated'] = '2020-02-12T17:03:00Z' + rv_json['logs']['expiration'] = '2020-02-15T17:03:00Z' + assert rv.status_code == 201 + assert expected_response == rv_json + + +@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') +@mock.patch('iib.web.api_v1.handle_add_deprecations_request') +def test_add_deprecations_overwrite_token_redacted(mock_had, mock_smfsc, auth_env, client): + token = 'username:password' + data = { + 'binary_image': 'binary:image', + 'build_tags': ['build-tag-1'], + 'deprecation_schema': '{"valid":"json"}', + 'from_index': 'pull:spec', + 'operator_package': 'my-package', + 'overwrite_from_index': True, + 'overwrite_from_index_token': token, + } + + rv = client.post('/api/v1/builds/add-deprecations', json=data, environ_base=auth_env) + rv_json = rv.json + assert rv.status_code == 201 + mock_had.apply_async.assert_called_once() + # Third to last element in args is the overwrite_from_index parameter + assert mock_had.apply_async.call_args[1]['args'][-3] is True + # second to last element in args is the overwrite_from_index_token parameter + assert mock_had.apply_async.call_args[1]['args'][-2] == token + assert 'overwrite_from_index_token' not in rv_json + assert token not in json.dumps(rv_json) + assert token not in mock_had.apply_async.call_args[1]['argsrepr'] + assert '*****' in mock_had.apply_async.call_args[1]['argsrepr'] + + +@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') +def test_add_deprecations_overwrite_not_allowed_without_token(mock_smfsc, db, auth_env, client): + data = { + 'from_index': 'pull:spec', + 'binary_image': 'binary:image', + 'operator_package': 'my-package', + 'deprecation_schema': '{"valid":"json"}', + 'build_tags': ['timestamptag'], + 'overwrite_from_index': True, + } + rv = client.post( + f'/api/v1/builds/add-deprecations', json=data, environ_base={'REMOTE_USER': 'tom_hanks'} + ) + assert rv.status_code == 403 + error_msg = 'You must set "overwrite_from_index_token" to use "overwrite_from_index"' + assert error_msg == rv.json['error'] + mock_smfsc.assert_not_called() + + +def test_get_deprecation_schema_valid_request(client, minimal_request_add_deprecations): + request, dep_schema = minimal_request_add_deprecations + request_id = request.id + rv = client.get(f'/api/v1/builds/{request_id}/deprecation-schema') + expected = {'status': 200, 'mimetype': 'application/json', 'json': ['foobar']} + assert rv.status_code == expected['status'] + assert rv.mimetype == expected['mimetype'] + if 'data' in expected: + assert rv.data.decode('utf-8') == expected['data'] + if 'json' in expected: + assert rv.json == json.loads(dep_schema) + + +def test_get_deprecation_schema_invalid_request( + client, + db, + minimal_request_regenerate_bundle, +): + error_msg = ( + 'The request 1 is of type regenerate-bundle. ' + 'This endpoint is only valid for requests of type add-deprecations.' + ) + minimal_request_regenerate_bundle.add_state('complete', 'The request is complete') + db.session.commit() + request_id = minimal_request_regenerate_bundle.id + rv = client.get(f'/api/v1/builds/{request_id}/deprecation-schema') + assert rv.status_code == 400 + assert rv.json['error'] == error_msg diff --git a/tests/test_workers/test_tasks/test_build_add_deprecations.py b/tests/test_workers/test_tasks/test_build_add_deprecations.py new file mode 100644 index 000000000..1eaa251b6 --- /dev/null +++ b/tests/test_workers/test_tasks/test_build_add_deprecations.py @@ -0,0 +1,335 @@ +from unittest import mock + +import json +import os +import pytest + + +from iib.workers.tasks import build_add_deprecations +from iib.workers.tasks.utils import RequestConfigAddDeprecations, IIBError +from iib.workers.config import get_worker_config + + +@mock.patch('iib.workers.tasks.build_add_deprecations._update_index_image_pull_spec') +@mock.patch('iib.workers.tasks.build_add_deprecations._create_and_push_manifest_list') +@mock.patch('iib.workers.tasks.build_add_deprecations._push_image') +@mock.patch('iib.workers.tasks.build_add_deprecations._build_image') +@mock.patch('iib.workers.tasks.build_add_deprecations._add_label_to_index') +@mock.patch('iib.workers.tasks.build_add_deprecations.add_deprecations_to_index') +@mock.patch('iib.workers.tasks.opm_operations.verify_operators_exists') +@mock.patch('iib.workers.tasks.build_add_deprecations.verify_operators_exists') +@mock.patch('iib.workers.tasks.build_add_deprecations.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_add_deprecations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_add_deprecations.set_request_state') +@mock.patch('iib.workers.tasks.opm_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_add_deprecations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_add_deprecations._cleanup') +def test_handle_add_deprecations_request( + mock_cleanup, + mock_prfb, + mock_sov, + mock_srs, + mock_uiibs, + mock_temp_dir, + mock_voe, + mock_ovoe, + mock_adti, + mock_alti, + mock_bi, + mock_pi, + mock_cpml, + mock_uiips, + tmpdir, +): + arches = {'amd64', 's390x'} + request_id = 11 + from_index = 'from-index:latest' + from_index_resolved = 'from-index@sha256:bcdefg' + binary_image = 'binary-image:latest' + binary_image_config = {'prod': {'v4.5': 'some_image'}} + binary_image_resolved = 'binary-image@sha256:abcdef' + operator_package = 'deprecation-operator' + deprecation_schema = '{"schema":"olm.deprecations","message":"deprecation-msg"}' + + mock_prfb.return_value = { + 'arches': arches, + 'binary_image': binary_image, + 'binary_image_resolved': binary_image_resolved, + 'from_index_resolved': from_index_resolved, + 'ocp_version': 'v4.6', + 'distribution_scope': "prod", + 'operator_package': 'deprecation-operator', + 'deprecation_schema': '{"schema":"olm.deprecations","message":"deprecation-msg"}', + } + + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + mock_voe.return_value = set([operator_package]), "index/db/path" + + build_add_deprecations.handle_add_deprecations_request( + request_id=request_id, + operator_package=operator_package, + deprecation_schema=deprecation_schema, + from_index=from_index, + binary_image=binary_image, + binary_image_config=binary_image_config, + ) + mock_prfb.assert_called_once_with( + request_id, + RequestConfigAddDeprecations( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=None, + binary_image_config=binary_image_config, + distribution_scope='prod', + operator_package=operator_package, + deprecation_schema=deprecation_schema, + ), + ) + mock_sov.assert_called_once_with(from_index_resolved) + mock_cpml.assert_called_once_with(request_id, {'s390x', 'amd64'}, None) + mock_voe.assert_called_once_with(from_index_resolved, tmpdir, [operator_package], None) + mock_voe.return_value = set(operator_package), "index/db/path" + mock_adti.assert_called_once_with( + request_id, + tmpdir, + from_index_resolved, + operator_package, + deprecation_schema, + binary_image_resolved, + "index/db/path", + ) + assert mock_srs.call_count == 3 + assert mock_alti.call_count == 2 + assert mock_bi.call_count == 2 + assert mock_pi.call_count == 2 + assert mock_srs.call_args[0][1] == 'complete' + + +@mock.patch('iib.workers.tasks.build_add_deprecations._push_image') +@mock.patch('iib.workers.tasks.build_add_deprecations._build_image') +@mock.patch('iib.workers.tasks.build_add_deprecations.add_deprecations_to_index') +@mock.patch('iib.workers.tasks.opm_operations.verify_operators_exists') +@mock.patch('iib.workers.tasks.build_add_deprecations.verify_operators_exists') +@mock.patch('iib.workers.tasks.build_add_deprecations.tempfile.TemporaryDirectory') +@mock.patch('iib.workers.tasks.build_add_deprecations._update_index_image_build_state') +@mock.patch('iib.workers.tasks.build_add_deprecations.set_request_state') +@mock.patch('iib.workers.tasks.opm_operations.Opm.set_opm_version') +@mock.patch('iib.workers.tasks.build_add_deprecations.prepare_request_for_build') +@mock.patch('iib.workers.tasks.build_add_deprecations._cleanup') +def test_add_deprecation_operator_not_exist( + mock_cleanup, + mock_prfb, + mock_sov, + mock_srs, + mock_uiibs, + mock_temp_dir, + mock_voe, + mock_ovoe, + mock_adti, + mock_bi, + mock_pi, + tmpdir, +): + arches = {'amd64', 's390x'} + request_id = 11 + from_index = 'from-index:latest' + from_index_resolved = 'from-index@sha256:bcdefg' + binary_image = 'binary-image:latest' + binary_image_config = {'prod': {'v4.5': 'some_image'}} + binary_image_resolved = 'binary-image@sha256:abcdef' + operator_package = 'deprecation-operator' + deprecation_schema = '{"schema":"olm.deprecations","message":"deprecation-msg"}' + + mock_prfb.return_value = { + 'arches': arches, + 'binary_image': binary_image, + 'binary_image_resolved': binary_image_resolved, + 'from_index_resolved': from_index_resolved, + 'ocp_version': 'v4.6', + 'distribution_scope': "prod", + 'operator_package': 'deprecation-operator', + 'deprecation_schema': '{"schema":"olm.deprecations","message":"deprecation-msg"}', + } + + mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) + mock_voe.return_value = None, None + + with pytest.raises( + IIBError, + match=f"Cannot add deprecations for {operator_package}," + " It is either not present in index or opted in fbc", + ): + build_add_deprecations.handle_add_deprecations_request( + request_id=request_id, + operator_package=operator_package, + deprecation_schema=deprecation_schema, + from_index=from_index, + binary_image=binary_image, + binary_image_config=binary_image_config, + ) + mock_prfb.assert_called_once_with( + request_id, + RequestConfigAddDeprecations( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=None, + binary_image_config=binary_image_config, + distribution_scope='prod', + operator_package=operator_package, + deprecation_schema=deprecation_schema, + ), + ) + mock_sov.assert_called_once_with(from_index_resolved) + mock_adti.call_count == 0 + mock_bi.call_count == 0 + mock_pi.call_count == 0 + + +@mock.patch('iib.workers.tasks.build_add_deprecations.create_dockerfile') +@mock.patch('iib.workers.tasks.build_add_deprecations.generate_cache_locally') +@mock.patch('iib.workers.tasks.build_add_deprecations.opm_validate') +@mock.patch('iib.workers.tasks.build_add_deprecations.get_catalog_dir') +@mock.patch('iib.workers.tasks.build_add_deprecations.set_request_state') +def test_add_deprecations_to_index_dep_dir_not_exist( + mock_srs, mock_gcd, mock_ov, mock_gcl, mock_cd, tmpdir +): + request_id = 11 + from_index_resolved = 'from-index@sha256:bcdefg' + binary_image_resolved = 'binary-image@sha256:abcdef' + operator_package = 'deprecation-operator' + deprecation_schema = '{"schema": "olm.deprecations", "message": "deprecation-msg"}' + index_db_path = '/tmpdir/indexdb' + + configs_dir = os.path.join(tmpdir, 'configs') + os.makedirs(configs_dir) + mock_gcd.return_value = configs_dir + build_add_deprecations.add_deprecations_to_index( + request_id, + tmpdir, + from_index_resolved, + operator_package, + deprecation_schema, + binary_image_resolved, + index_db_path, + ) + mock_ov.assert_called_once() + mock_gcl.assert_called_once_with( + base_dir=tmpdir, + fbc_dir=mock_gcd.return_value, + local_cache_path=os.path.join(tmpdir, 'cache'), + ) + mock_cd.assert_called_once() + # assert deprecations_file and dir exist + assert os.path.exists( + os.path.join( + configs_dir, get_worker_config()['operator_deprecations_dir'], operator_package + ) + ) + + # assert file has right content + operator_deprecation_file = os.path.join( + configs_dir, + get_worker_config()['operator_deprecations_dir'], + operator_package, + f'{operator_package}.json', + ) + with open(operator_deprecation_file, 'r') as output_file: + assert output_file.read() == deprecation_schema + + +@mock.patch('iib.workers.tasks.build_add_deprecations.create_dockerfile') +@mock.patch('iib.workers.tasks.build_add_deprecations.generate_cache_locally') +@mock.patch('iib.workers.tasks.build_add_deprecations.opm_validate') +@mock.patch('iib.workers.tasks.build_add_deprecations.get_catalog_dir') +@mock.patch('iib.workers.tasks.build_add_deprecations.set_request_state') +def test_add_deprecations_to_index_dep_dir_exist( + mock_srs, mock_gcd, mock_ov, mock_gcl, mock_cd, tmpdir +): + request_id = 11 + from_index_resolved = 'from-index@sha256:bcdefg' + binary_image_resolved = 'binary-image@sha256:abcdef' + operator_package = 'deprecation-operator' + deprecation_schema = '{"schema": "olm.deprecations", "message": "deprecation-msg"}' + index_db_path = '/tmpdir/indexdb' + + configs_dir = os.path.join(tmpdir, 'configs') + os.makedirs(os.path.join(configs_dir, get_worker_config()['operator_deprecations_dir'])) + mock_gcd.return_value = configs_dir + build_add_deprecations.add_deprecations_to_index( + request_id, + tmpdir, + from_index_resolved, + operator_package, + deprecation_schema, + binary_image_resolved, + index_db_path, + ) + mock_ov.assert_called_once() + mock_gcl.assert_called_once_with( + base_dir=tmpdir, + fbc_dir=mock_gcd.return_value, + local_cache_path=os.path.join(tmpdir, 'cache'), + ) + mock_cd.assert_called_once() + # assert file has right content + operator_deprecation_file = os.path.join( + configs_dir, + get_worker_config()['operator_deprecations_dir'], + operator_package, + f'{operator_package}.json', + ) + with open(operator_deprecation_file, 'r') as output_file: + assert output_file.read() == deprecation_schema + + +@mock.patch('iib.workers.tasks.build_add_deprecations.create_dockerfile') +@mock.patch('iib.workers.tasks.build_add_deprecations.generate_cache_locally') +@mock.patch('iib.workers.tasks.build_add_deprecations.opm_validate') +@mock.patch('iib.workers.tasks.build_add_deprecations.get_catalog_dir') +@mock.patch('iib.workers.tasks.build_add_deprecations.set_request_state') +def test_add_deprecations_to_index_file_exist( + mock_srs, mock_gcd, mock_ov, mock_gcl, mock_cd, tmpdir +): + request_id = 11 + from_index_resolved = 'from-index@sha256:bcdefg' + binary_image_resolved = 'binary-image@sha256:abcdef' + operator_package = 'deprecation-operator' + old_deprecation_schema = '{"schema": "olm.deprecations", "message": "old deprecation-msg"}' + new_deprecation_schema = '{"schema": "olm.deprecations", "message": "new deprecation-msg"}' + index_db_path = '/tmpdir/indexdb' + + configs_dir = os.path.join(tmpdir, 'configs') + os.makedirs( + os.path.join( + configs_dir, get_worker_config()['operator_deprecations_dir'], operator_package + ) + ) + operator_deprecation_file = os.path.join( + configs_dir, + get_worker_config()['operator_deprecations_dir'], + operator_package, + f'{operator_package}.json', + ) + with open(operator_deprecation_file, 'w') as output_file: + json.dump(json.loads(old_deprecation_schema), output_file) + mock_gcd.return_value = configs_dir + build_add_deprecations.add_deprecations_to_index( + request_id, + tmpdir, + from_index_resolved, + operator_package, + new_deprecation_schema, + binary_image_resolved, + index_db_path, + ) + mock_ov.assert_called_once() + mock_gcl.assert_called_once_with( + base_dir=tmpdir, + fbc_dir=mock_gcd.return_value, + local_cache_path=os.path.join(tmpdir, 'cache'), + ) + mock_cd.assert_called_once() + + # assert file has right content + with open(operator_deprecation_file, 'r') as output_file: + assert output_file.read() == new_deprecation_schema