From b2890e15a4797c80402d459731c0e203ae0a203d Mon Sep 17 00:00:00 2001 From: Tian Wang Date: Mon, 25 Mar 2024 22:01:59 +0000 Subject: [PATCH 1/5] Set up pre-commit hooks for checking code styles --- .github/workflows/check_code_quality.yml | 46 ++++++++++++++++++++++++ .pre-commit-config.yaml | 26 ++++++++++++++ DEVELOPMENT.md | 17 +++++++++ environment.yml | 4 +++ 4 files changed, 93 insertions(+) create mode 100644 .github/workflows/check_code_quality.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/check_code_quality.yml b/.github/workflows/check_code_quality.yml new file mode 100644 index 00000000..17a2bda1 --- /dev/null +++ b/.github/workflows/check_code_quality.yml @@ -0,0 +1,46 @@ +name: check_code_quality + +on: + push: + branches: [ main ] + paths: + - "src/**.py" + - "test/**.py" + - "template/**.py" + + pull_request: + branches: [ main ] + paths: + - "src/**.py" + - "test/**.py" + - "template/**.py" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + name: Check Code Quality + runs-on: ubuntu-latest + if: github.repository == 'aws/sagemaker-distribution' + permissions: + pull-requests: write + contents: write + steps: + - uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./environment.yml + environment-name: sagemaker-distribution + init-shell: bash + - name: Free up disk space + run: rm -rf /opt/hostedtoolcache + - name: Activate sagemaker-distribution + run: micromamba activate sagemaker-distribution + - name: Check style with black + run: black --line-length=120 --check src test template + - name: Check style with autoflake + run: autoflake --in-place --expand-star-imports --ignore-init-module-imports --remove-all-unused-imports -rc src test template + - name: Check style with isort + run: isort --profile black -c src test template diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4c3a2233 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: detect-aws-credentials +- repo: https://github.com/humitos/mirrors-autoflake.git + rev: v1.3 + hooks: + - id: autoflake + args: ['--in-place', '--expand-star-imports', '--ignore-init-module-imports', '--remove-all-unused-imports'] +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + args: [--line-length=120] +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: # imports sorting + - id: isort + name: isort (python) + args: ["--profile", "black"] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 4bf4796b..835edb67 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -24,6 +24,23 @@ Run the following to invoke those tests: pytest --local-image-id REPLACE_ME_WITH_IMAGE_ID ``` +## Code Style + +Install pre-commit to run code style checks before each commit: + +```shell +pip install pre-commit +pre-commit install +``` + +To run formatters for all existing files, use: + +```shell +pre-commit run --all-files +``` + +pre-commit checks can be disabled for a particular commit with git commit -n. + You can also pass a `--use-gpu` flag if the test machine has Nvidia GPU(s) and necessary Nvidia drivers. ### Unit tests for the project's source code diff --git a/environment.yml b/environment.yml index fbe6253f..78a32952 100644 --- a/environment.yml +++ b/environment.yml @@ -10,3 +10,7 @@ dependencies: - pytest-mock - pytest-xdist - semver + - autoflake + - black + - isort + - pre-commit From a74466c154d01201388f5232b167538275aa74c4 Mon Sep 17 00:00:00 2001 From: Tian Wang Date: Mon, 25 Mar 2024 22:02:54 +0000 Subject: [PATCH 2/5] Run code style formetters on all existing files --- .github/workflows/PR_TEMPLATE.md | 2 +- .github/workflows/monthly-minor-release.yml | 2 +- .gitignore | 2 +- CONTRIBUTING.md | 46 +- README.md | 10 +- RELEASE.md | 2 +- src/changelog_generator.py | 78 ++- src/config.py | 38 +- src/dependency_upgrader.py | 40 +- src/main.py | 222 ++++--- src/package_staleness.py | 109 ++-- src/release_notes_generator.py | 58 +- src/utils.py | 32 +- .../code_editor_machine_settings.json | 2 +- .../v1/dirs/etc/code-editor/extensions.txt | 2 +- .../dirs/etc/jupyter/jupyter_server_config.py | 8 +- .../dirs/usr/local/bin/entrypoint-code-editor | 2 +- .../dirs/usr/local/bin/start-jupyter-server | 4 +- test/conftest.py | 15 +- .../run_glue_sessions_notebook.sh | 1 - test/test_artifacts/v0/keras.test.Dockerfile | 1 - .../v0/matplotlib.test.Dockerfile | 1 - test/test_artifacts/v0/run_pandas_tests.py | 12 +- ...-headless-execution-driver.test.Dockerfile | 2 +- .../v0/scripts/run_autogluon_tests.sh | 2 +- .../v0/scripts/run_matplotlib_tests.sh | 2 - .../run_glue_sessions_notebook.sh | 1 - .../v1/jupyterlab-git.test.Dockerfile | 1 - .../v1/matplotlib.test.Dockerfile | 1 - test/test_artifacts/v1/run_pandas_tests.py | 22 +- .../scripts/run_altair_example_notebooks.sh | 1 - .../v1/scripts/run_autogluon_tests.sh | 2 +- .../v1/scripts/run_matplotlib_tests.sh | 2 - .../run_sagemaker_code_editor_tests.sh | 2 +- test/test_artifacts/v1/serve.test.Dockerfile | 2 +- test/test_dependency_upgrader.py | 40 +- test/test_dockerfile_based_harness.py | 209 +++--- test/test_main.py | 606 +++++++++--------- test/test_package_staleness.py | 106 +-- 39 files changed, 850 insertions(+), 840 deletions(-) diff --git a/.github/workflows/PR_TEMPLATE.md b/.github/workflows/PR_TEMPLATE.md index f6bcb2e4..767b3ba5 100644 --- a/.github/workflows/PR_TEMPLATE.md +++ b/.github/workflows/PR_TEMPLATE.md @@ -1,4 +1,4 @@ This pull request was created by GitHub Actions/AWS CodeBuild! Before merging, please do the following: - [ ] Review changelog/staleness report. - [ ] Review build/test results by clicking *Build Logs* in CI Report (be patient, tests take ~4hr). -- [ ] Review ECR Scan results. \ No newline at end of file +- [ ] Review ECR Scan results. diff --git a/.github/workflows/monthly-minor-release.yml b/.github/workflows/monthly-minor-release.yml index 7b91b75c..fc443d26 100644 --- a/.github/workflows/monthly-minor-release.yml +++ b/.github/workflows/monthly-minor-release.yml @@ -41,4 +41,4 @@ jobs: uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main with: release-type: "minor" - base-version: ${{ matrix.version }} \ No newline at end of file + base-version: ${{ matrix.version }} diff --git a/.gitignore b/.gitignore index 16058d2e..120ee635 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ __pycache__ .idea -.DS_Store \ No newline at end of file +.DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a028449..324d899c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,60 +41,60 @@ GitHub provides additional document on [forking a repository](https://help.githu ## For adding new Conda packages to SageMaker Distribution -SageMaker Distribution will add new Conda packages only during a minor/major version release. +SageMaker Distribution will add new Conda packages only during a minor/major version release. New packages will not be added during a patch version release. Follow these steps for sending out a pull request for adding new packages: 1. Identify the latest version of SageMaker Distribution. 2. Create the next minor/major version's build artifacts folder here: https://github.com/aws/sagemaker-distribution/tree/main/build_artifacts -3. Currently, SageMaker Distribution is using Conda forge channel as our source (for Conda - packages). +3. Currently, SageMaker Distribution is using Conda forge channel as our source (for Conda + packages). Ensure that the new package which you are trying to add is present in Conda forge channel. https://conda-forge.org/feedstock-outputs/ -4. Create {cpu/gpu}.additional_packages_env.in file in that folder containing the new packages. +4. Create {cpu/gpu}.additional_packages_env.in file in that folder containing the new packages. Specify the new package based on the following examples: i. conda-forge::new-package - + ii. conda-forge::new-package[version='>=some-version-number, [Amazon SageMaker Studio](https://docs.aws.amazon.com/sagemaker/latest/dg/studio.html) is a web-based, integrated +> [Amazon SageMaker Studio](https://docs.aws.amazon.com/sagemaker/latest/dg/studio.html) is a web-based, integrated > development environment (IDE) for machine learning that lets you build, train, debug, deploy, and monitor your > machine learning models. @@ -125,8 +125,8 @@ RUN micromamba install sagemaker-inference --freeze-installed --yes --channel co ## FIPS -As of sagemaker-distribution: v0.12+, v1.6+, and v2+, the images come with FIPS 140-2 validated openssl provider -available for use. You can enable the FIPS provider by running: +As of sagemaker-distribution: v0.12+, v1.6+, and v2+, the images come with FIPS 140-2 validated openssl provider +available for use. You can enable the FIPS provider by running: `export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf` diff --git a/RELEASE.md b/RELEASE.md index 03286729..d6f16c01 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -51,7 +51,7 @@ python src/main.py build \ Note: -- As you can see above, the `--target-ecr-repo` parameter can be supplied zero or multiple times. If not supplied, the +- As you can see above, the `--target-ecr-repo` parameter can be supplied zero or multiple times. If not supplied, the tool will just build a local image. If supplied multiple times, it'll upload the images to all those ECR repositories. - There is also a `--skip-tests` flag which, by default, is `false`. You can supply it if you'd like to skip tests locally. However, we'll make sure the tests succeed before any image is release publicly. diff --git a/src/changelog_generator.py b/src/changelog_generator.py index e6a343b2..f4ab3bb8 100644 --- a/src/changelog_generator.py +++ b/src/changelog_generator.py @@ -1,67 +1,65 @@ import os -from utils import ( - get_dir_for_version, - get_semver, - get_match_specs, -) + from semver import Version +from utils import get_dir_for_version, get_match_specs, get_semver + -def _derive_changeset(target_version_dir, source_version_dir, image_config) -> (dict[str, - list[str]], dict[str, str]): - env_in_file_name = image_config['build_args']['ENV_IN_FILENAME'] - env_out_file_name = image_config['env_out_filename'] - required_packages_from_target = get_match_specs( - target_version_dir + "/" + env_in_file_name - ).keys() +def _derive_changeset(target_version_dir, source_version_dir, image_config) -> (dict[str, list[str]], dict[str, str]): + env_in_file_name = image_config["build_args"]["ENV_IN_FILENAME"] + env_out_file_name = image_config["env_out_filename"] + required_packages_from_target = get_match_specs(target_version_dir + "/" + env_in_file_name).keys() target_match_spec_out = get_match_specs(target_version_dir + "/" + env_out_file_name) source_match_spec_out = get_match_specs(source_version_dir + "/" + env_out_file_name) # Note: required_packages_from_source is not currently used. # In the future, If we remove any packages from env.in, at that time required_packages_from_source will be needed. # We only care about the packages which are present in the target version env.in file - installed_packages_from_target = {k: str(v.get('version')).removeprefix('==') - for k, v in target_match_spec_out.items() - if k in required_packages_from_target} + installed_packages_from_target = { + k: str(v.get("version")).removeprefix("==") + for k, v in target_match_spec_out.items() + if k in required_packages_from_target + } # Note: A required package in the target version might not be a required package in the source version # But source version could still have this package pulled as a dependency of a dependency. - installed_packages_from_source = {k: str(v.get('version')).removeprefix('==') for - k, v in source_match_spec_out.items() - if k in required_packages_from_target} - upgrades = {k: [installed_packages_from_source[k], v] for k, v in installed_packages_from_target.items() - if k in installed_packages_from_source and installed_packages_from_source[k] != v} - new_packages = {k: v for k, v in installed_packages_from_target.items() - if k not in installed_packages_from_source} + installed_packages_from_source = { + k: str(v.get("version")).removeprefix("==") + for k, v in source_match_spec_out.items() + if k in required_packages_from_target + } + upgrades = { + k: [installed_packages_from_source[k], v] + for k, v in installed_packages_from_target.items() + if k in installed_packages_from_source and installed_packages_from_source[k] != v + } + new_packages = {k: v for k, v in installed_packages_from_target.items() if k not in installed_packages_from_source} # TODO: Add support for removed packages. return upgrades, new_packages def generate_change_log(target_version: Version, image_config): target_version_dir = get_dir_for_version(target_version) - source_version_txt_file_path = f'{target_version_dir}/source-version.txt' + source_version_txt_file_path = f"{target_version_dir}/source-version.txt" if not os.path.exists(source_version_txt_file_path): - print('[WARN]: Generating CHANGELOG is skipped because \'source-version.txt\' isn\'t ' - 'found.') + print("[WARN]: Generating CHANGELOG is skipped because 'source-version.txt' isn't " "found.") return - with open(source_version_txt_file_path, 'r') as f: + with open(source_version_txt_file_path, "r") as f: source_patch_version = f.readline() source_version = get_semver(source_patch_version) source_version_dir = get_dir_for_version(source_version) - image_type = image_config['image_type'] - upgrades, new_packages = _derive_changeset(target_version_dir, source_version_dir, - image_config) - with open(f'{target_version_dir}/CHANGELOG-{image_type}.md', 'w') as f: - f.write('# Change log: ' + str(target_version) + '(' + image_type + ')\n\n') + image_type = image_config["image_type"] + upgrades, new_packages = _derive_changeset(target_version_dir, source_version_dir, image_config) + with open(f"{target_version_dir}/CHANGELOG-{image_type}.md", "w") as f: + f.write("# Change log: " + str(target_version) + "(" + image_type + ")\n\n") if len(upgrades) != 0: - f.write('## Upgrades: \n\n') - f.write('Package | Previous Version | Current Version\n') - f.write('---|---|---\n') + f.write("## Upgrades: \n\n") + f.write("Package | Previous Version | Current Version\n") + f.write("---|---|---\n") for package in upgrades: - f.write(package + '|' + upgrades[package][0] + '|' - + upgrades[package][1] + '\n') + f.write(package + "|" + upgrades[package][0] + "|" + upgrades[package][1] + "\n") if len(new_packages) != 0: - f.write('\n## What\'s new: \n\n') - f.write('Package | Version \n') - f.write('---|---\n') + f.write("\n## What's new: \n\n") + f.write("Package | Version \n") + f.write("---|---\n") for package in new_packages: - f.write(package + '|' + new_packages[package] + '\n') + f.write(package + "|" + new_packages[package] + "\n") diff --git a/src/config.py b/src/config.py index 9afb899e..1b654e6d 100644 --- a/src/config.py +++ b/src/config.py @@ -1,26 +1,26 @@ _image_generator_configs = [ { - 'build_args': { - 'TAG_FOR_BASE_MICROMAMBA_IMAGE': 'jammy-cuda-11.8.0', - 'CUDA_MAJOR_MINOR_VERSION': '11.8', # Should match the previous one. - 'ENV_IN_FILENAME': 'gpu.env.in', - 'ARG_BASED_ENV_IN_FILENAME': 'gpu.arg_based_env.in', + "build_args": { + "TAG_FOR_BASE_MICROMAMBA_IMAGE": "jammy-cuda-11.8.0", + "CUDA_MAJOR_MINOR_VERSION": "11.8", # Should match the previous one. + "ENV_IN_FILENAME": "gpu.env.in", + "ARG_BASED_ENV_IN_FILENAME": "gpu.arg_based_env.in", }, - 'additional_packages_env_in_file': 'gpu.additional_packages_env.in', - 'image_tag_generator': '{image_version}-gpu', - 'env_out_filename': 'gpu.env.out', - 'pytest_flags': ['--use-gpu'], - 'image_type': 'gpu' + "additional_packages_env_in_file": "gpu.additional_packages_env.in", + "image_tag_generator": "{image_version}-gpu", + "env_out_filename": "gpu.env.out", + "pytest_flags": ["--use-gpu"], + "image_type": "gpu", }, { - 'build_args': { - 'TAG_FOR_BASE_MICROMAMBA_IMAGE': 'jammy', - 'ENV_IN_FILENAME': 'cpu.env.in', + "build_args": { + "TAG_FOR_BASE_MICROMAMBA_IMAGE": "jammy", + "ENV_IN_FILENAME": "cpu.env.in", }, - 'additional_packages_env_in_file': 'cpu.additional_packages_env.in', - 'image_tag_generator': '{image_version}-cpu', - 'env_out_filename': 'cpu.env.out', - 'pytest_flags': [], - 'image_type': 'cpu' - } + "additional_packages_env_in_file": "cpu.additional_packages_env.in", + "image_tag_generator": "{image_version}-cpu", + "env_out_filename": "cpu.env.out", + "pytest_flags": [], + "image_type": "cpu", + }, ] diff --git a/src/dependency_upgrader.py b/src/dependency_upgrader.py index 6d7703e1..e8b0c9da 100644 --- a/src/dependency_upgrader.py +++ b/src/dependency_upgrader.py @@ -1,31 +1,31 @@ from semver import Version -_MAJOR = 'major' -_MINOR = 'minor' -_PATCH = 'patch' +_MAJOR = "major" +_MINOR = "minor" +_PATCH = "patch" def _get_dependency_upper_bound_for_runtime_upgrade(dependency_name: str, lower_bound: str, runtime_upgrade_type): metadata = _dependency_metadata.get(dependency_name, None) - version_upgrade_strategy = 'semver' if metadata is None else metadata['version_upgrade_strategy'] + version_upgrade_strategy = "semver" if metadata is None else metadata["version_upgrade_strategy"] - func = _version_upgrade_metadata[version_upgrade_strategy]['func'] + func = _version_upgrade_metadata[version_upgrade_strategy]["func"] # Version strings on conda-forge follow PEP standards rather than SemVer, which support # version strings such as X.Y.Z.postN, X.Y.Z.preN. These cause errors in semver.Version.parse # so we keep the first 3 entries as version string. - if lower_bound.count('.') > 2: - lower_bound = '.'.join(lower_bound.split('.')[:3]) + if lower_bound.count(".") > 2: + lower_bound = ".".join(lower_bound.split(".")[:3]) return func(lower_bound, runtime_upgrade_type) def _get_dependency_upper_bound_for_semver(lower_bound: str, runtime_upgrade_type): lower_semver = Version.parse(lower_bound, optional_minor_and_patch=True) if runtime_upgrade_type == _MAJOR: - return '' # No upper bound. + return "" # No upper bound. elif runtime_upgrade_type == _MINOR: - return f',<{lower_semver.bump_major()}' + return f",<{lower_semver.bump_major()}" elif runtime_upgrade_type == _PATCH: - return f',<{lower_semver.bump_minor()}' + return f",<{lower_semver.bump_minor()}" else: raise Exception() @@ -33,11 +33,11 @@ def _get_dependency_upper_bound_for_semver(lower_bound: str, runtime_upgrade_typ def _get_dependency_upper_bound_for_pythonesque(lower_bound: str, runtime_upgrade_type): lower_semver = Version.parse(lower_bound, optional_minor_and_patch=True) if runtime_upgrade_type == _MAJOR: - return '' # No upper bound. + return "" # No upper bound. elif runtime_upgrade_type == _MINOR: - return f',<{lower_semver.bump_minor()}' + return f",<{lower_semver.bump_minor()}" elif runtime_upgrade_type == _PATCH: - return f',<{lower_semver.bump_minor()}' + return f",<{lower_semver.bump_minor()}" else: raise Exception() @@ -50,21 +50,13 @@ def _get_dependency_upper_bound_for_pythonesque(lower_bound: str, runtime_upgrad # happens in the given month). So, 2nd release in December 2022 was versioned as '2022.12.1'. # In Amazon SageMaker Distribution, we want to move to a new month only during a minor version upgrade and new year # during a major version upgrade. - 'semver': { - 'func': _get_dependency_upper_bound_for_semver - }, + "semver": {"func": _get_dependency_upper_bound_for_semver}, # Some dependencies follow, for lack of a better word, "python style" release cycles. For e.g., even if Python does # a minor version upgrade from 3.9 to 3.10, we will only introduce 3.10 in Amazon SageMaker Distribution as part of # a major version upgrade. In other words, for dependencies below, minor version upgrades are treated as major # version upgrades in Amazon SageMaker Distribution. - 'pythonesque': { - 'func': _get_dependency_upper_bound_for_pythonesque - } + "pythonesque": {"func": _get_dependency_upper_bound_for_pythonesque}, } -_dependency_metadata = { - 'python': { - 'version_upgrade_strategy': 'pythonesque' - } -} +_dependency_metadata = {"python": {"version_upgrade_strategy": "pythonesque"}} diff --git a/src/main.py b/src/main.py index 05ea1e58..464053af 100644 --- a/src/main.py +++ b/src/main.py @@ -4,7 +4,6 @@ import glob import os import shutil -from typing import List import boto3 import docker @@ -13,16 +12,21 @@ from docker.errors import BuildError, ContainerError from semver import Version -from dependency_upgrader import _get_dependency_upper_bound_for_runtime_upgrade, _MAJOR, _MINOR, _PATCH from changelog_generator import generate_change_log -from release_notes_generator import generate_release_notes from config import _image_generator_configs +from dependency_upgrader import ( + _MAJOR, + _MINOR, + _PATCH, + _get_dependency_upper_bound_for_runtime_upgrade, +) from package_staleness import generate_package_staleness_report +from release_notes_generator import generate_release_notes from utils import ( get_dir_for_version, - is_exists_dir_for_version, + get_match_specs, get_semver, - get_match_specs + is_exists_dir_for_version, ) _docker_client = docker.from_env() @@ -44,8 +48,9 @@ def create_and_get_semver_dir(version: Version, exist_ok: bool = False): def _delete_all_files_except_additional_packages_input_files(base_version_dir): - additional_package_env_in_files = [image_generator_config['additional_packages_env_in_file'] - for image_generator_config in _image_generator_configs] + additional_package_env_in_files = [ + image_generator_config["additional_packages_env_in_file"] for image_generator_config in _image_generator_configs + ] for filename in os.listdir(base_version_dir): if filename not in additional_package_env_in_files: file_path = os.path.join(base_version_dir, filename) @@ -55,18 +60,18 @@ def _delete_all_files_except_additional_packages_input_files(base_version_dir): elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: - print('Failed to delete %s. Reason: %s' % (file_path, e)) + print("Failed to delete %s. Reason: %s" % (file_path, e)) def _create_new_version_artifacts(args): runtime_version_upgrade_type = args.runtime_version_upgrade_type if runtime_version_upgrade_type == _PATCH: - runtime_version_upgrade_func = 'bump_patch' + runtime_version_upgrade_func = "bump_patch" elif runtime_version_upgrade_type == _MINOR: - runtime_version_upgrade_func = 'bump_minor' + runtime_version_upgrade_func = "bump_minor" elif runtime_version_upgrade_type == _MAJOR: - runtime_version_upgrade_func = 'bump_major' + runtime_version_upgrade_func = "bump_major" else: raise Exception() @@ -85,18 +90,19 @@ def _create_new_version_artifacts(args): new_version_dir = create_and_get_semver_dir(next_version, args.force) for image_generator_config in _image_generator_configs: - _create_new_version_conda_specs(base_version_dir, new_version_dir, runtime_version_upgrade_type, - image_generator_config) + _create_new_version_conda_specs( + base_version_dir, new_version_dir, runtime_version_upgrade_type, image_generator_config + ) _copy_static_files(base_version_dir, new_version_dir, str(next_version.major), runtime_version_upgrade_type) - with open(f'{new_version_dir}/source-version.txt', 'w') as f: + with open(f"{new_version_dir}/source-version.txt", "w") as f: f.write(args.base_patch_version) def _copy_static_files(base_version_dir, new_version_dir, new_version_major, runtime_version_upgrade_type): - for f in glob.glob(f'{base_version_dir}/gpu.arg_based_env.in'): + for f in glob.glob(f"{base_version_dir}/gpu.arg_based_env.in"): shutil.copy2(f, new_version_dir) - for f in glob.glob(f'{base_version_dir}/patch_*'): + for f in glob.glob(f"{base_version_dir}/patch_*"): shutil.copy2(f, new_version_dir) # For patches, get Dockerfile+dirs from base patch @@ -104,26 +110,27 @@ def _copy_static_files(base_version_dir, new_version_dir, new_version_major, run if runtime_version_upgrade_type == _PATCH: base_path = base_version_dir else: - base_path = f'template/v{new_version_major}' - for f in glob.glob(os.path.relpath(f'{base_path}/Dockerfile')): + base_path = f"template/v{new_version_major}" + for f in glob.glob(os.path.relpath(f"{base_path}/Dockerfile")): shutil.copy2(f, new_version_dir) if int(new_version_major) >= 1: # dirs directory doesn't exist for v0. It was introduced only for v1 - dirs_relative_path = os.path.relpath(f'{base_path}/dirs') + dirs_relative_path = os.path.relpath(f"{base_path}/dirs") for f in glob.glob(dirs_relative_path): - shutil.copytree(f, os.path.join(new_version_dir, 'dirs')) + shutil.copytree(f, os.path.join(new_version_dir, "dirs")) + -def _create_new_version_conda_specs(base_version_dir, new_version_dir, runtime_version_upgrade_type, - image_generator_config): - env_in_filename = image_generator_config['build_args']['ENV_IN_FILENAME'] - additional_packages_env_in_filename = image_generator_config['additional_packages_env_in_file'] - env_out_filename = image_generator_config['env_out_filename'] +def _create_new_version_conda_specs( + base_version_dir, new_version_dir, runtime_version_upgrade_type, image_generator_config +): + env_in_filename = image_generator_config["build_args"]["ENV_IN_FILENAME"] + additional_packages_env_in_filename = image_generator_config["additional_packages_env_in_file"] + env_out_filename = image_generator_config["env_out_filename"] - base_match_specs_in = get_match_specs(f'{base_version_dir}/{env_in_filename}') + base_match_specs_in = get_match_specs(f"{base_version_dir}/{env_in_filename}") - base_match_specs_out = get_match_specs(f'{base_version_dir}/{env_out_filename}') - additional_packages_match_specs_in = \ - get_match_specs(f'{new_version_dir}/{additional_packages_env_in_filename}') + base_match_specs_out = get_match_specs(f"{base_version_dir}/{env_out_filename}") + additional_packages_match_specs_in = get_match_specs(f"{new_version_dir}/{additional_packages_env_in_filename}") # Add all the match specs from the previous version. # If a package is present in both additional packages as well as the previous version, then @@ -140,17 +147,17 @@ def _create_new_version_conda_specs(base_version_dir, new_version_dir, runtime_v else: channel = match_out.get("channel").channel_name - min_version_inclusive = match_out.get('version') - assert str(min_version_inclusive).startswith('==') - min_version_inclusive = str(min_version_inclusive).removeprefix('==') + min_version_inclusive = match_out.get("version") + assert str(min_version_inclusive).startswith("==") + min_version_inclusive = str(min_version_inclusive).removeprefix("==") - max_version_str = _get_dependency_upper_bound_for_runtime_upgrade(package_name, - min_version_inclusive, - runtime_version_upgrade_type) + max_version_str = _get_dependency_upper_bound_for_runtime_upgrade( + package_name, min_version_inclusive, runtime_version_upgrade_type + ) out.append(f"{channel}::{package_name}[version='>={min_version_inclusive}{max_version_str}']") - with open(f'{new_version_dir}/{env_in_filename}', 'w') as f: + with open(f"{new_version_dir}/{env_in_filename}", "w") as f: f.write("# This file is auto-generated.\n") f.write("\n".join(out)) f.write("\n") # This new line is pretty important. See code documentation in Dockerfile for the reasoning. @@ -170,8 +177,7 @@ def create_patch_version_artifacts(args): def build_images(args): target_version = get_semver(args.target_patch_version) - image_ids, image_versions = _build_local_images(target_version, args.target_ecr_repo, - args.force, args.skip_tests) + image_ids, image_versions = _build_local_images(target_version, args.target_ecr_repo, args.force, args.skip_tests) generate_release_notes(target_version) # Upload to ECR before running tests so that only the exact image which we tested goes to public @@ -180,55 +186,56 @@ def build_images(args): _push_images_upstream(image_versions, args.region) if not args.skip_tests: - print(f'Will now run tests against: {image_ids}') + print(f"Will now run tests against: {image_ids}") _test_local_images(image_ids, args.target_patch_version) else: - print('Will skip tests.') + print("Will skip tests.") def _push_images_upstream(image_versions_to_push: list[dict[str, str]], region: str): - print(f'Will now push the images to ECR: {image_versions_to_push}') + print(f"Will now push the images to ECR: {image_versions_to_push}") for i in image_versions_to_push: - username, password = _get_ecr_credentials(region, i['repository']) - _docker_client.images.push(repository=i['repository'], tag=i['tag'], - auth_config={'username': username, 'password': password}) + username, password = _get_ecr_credentials(region, i["repository"]) + _docker_client.images.push( + repository=i["repository"], tag=i["tag"], auth_config={"username": username, "password": password} + ) - print(f'Successfully pushed these images to ECR: {image_versions_to_push}') + print(f"Successfully pushed these images to ECR: {image_versions_to_push}") def _test_local_images(image_ids_to_test: list[str], target_version: str): assert len(image_ids_to_test) == len(_image_generator_configs) exit_codes = [] image_ids = [] - for (image_id, config) in zip(image_ids_to_test, _image_generator_configs): - exit_code = pytest.main(['-n', '2', '-m', config['image_type'], '--local-image-version', - target_version, *config['pytest_flags']]) + for image_id, config in zip(image_ids_to_test, _image_generator_configs): + exit_code = pytest.main( + ["-n", "2", "-m", config["image_type"], "--local-image-version", target_version, *config["pytest_flags"]] + ) - assert exit_code == 0, f'Tests failed with exit codes: {exit_codes} against: {image_ids}' + assert exit_code == 0, f"Tests failed with exit codes: {exit_codes} against: {image_ids}" - print(f'Tests ran successfully against: {image_ids_to_test}') + print(f"Tests ran successfully against: {image_ids_to_test}") def _get_config_for_image(target_version_dir: str, image_generator_config, force_rebuild) -> dict: - if not os.path.exists(target_version_dir + "/" + image_generator_config["env_out_filename"]) \ - or force_rebuild: + if not os.path.exists(target_version_dir + "/" + image_generator_config["env_out_filename"]) or force_rebuild: return image_generator_config config_for_image = copy.deepcopy(image_generator_config) # Use the existing env.out to create the conda environment. Pass that as env.in - config_for_image['build_args']['ENV_IN_FILENAME'] = \ - image_generator_config["env_out_filename"] + config_for_image["build_args"]["ENV_IN_FILENAME"] = image_generator_config["env_out_filename"] # Remove ARG_BASED_ENV_IN_FILENAME if it exists - config_for_image['build_args'].pop('ARG_BASED_ENV_IN_FILENAME', None) + config_for_image["build_args"].pop("ARG_BASED_ENV_IN_FILENAME", None) return config_for_image # Returns a tuple of: 1/ list of actual images generated; 2/ list of tagged images. A given image can be tagged by # multiple different strings - for e.g., a CPU image can be tagged as '1.3.2-cpu', '1.3-cpu', '1-cpu' and/or # 'latest-cpu'. Therefore, (1) is strictly a subset of (2). -def _build_local_images(target_version: Version, target_ecr_repo_list: list[str], force: bool, - skip_tests=False) -> (list[str], list[dict[str, str]]): +def _build_local_images( + target_version: Version, target_ecr_repo_list: list[str], force: bool, skip_tests=False +) -> (list[str], list[dict[str, str]]): target_version_dir = get_dir_for_version(target_version) generated_image_ids = [] @@ -237,26 +244,27 @@ def _build_local_images(target_version: Version, target_ecr_repo_list: list[str] for image_generator_config in _image_generator_configs: config = _get_config_for_image(target_version_dir, image_generator_config, force) try: - image, log_gen = _docker_client.images.build(path=target_version_dir, rm=True, - pull=True, buildargs=config['build_args']) + image, log_gen = _docker_client.images.build( + path=target_version_dir, rm=True, pull=True, buildargs=config["build_args"] + ) except BuildError as e: for line in e.build_log: - if 'stream' in line: - print(line['stream'].strip()) + if "stream" in line: + print(line["stream"].strip()) # After printing the logs, raise the exception (which is the old behavior) raise - print(f'Successfully built an image with id: {image.id}') + print(f"Successfully built an image with id: {image.id}") generated_image_ids.append(image.id) try: - container_logs = _docker_client.containers.run(image=image.id, detach=False, - auto_remove=True, - command='micromamba env export --explicit') + container_logs = _docker_client.containers.run( + image=image.id, detach=False, auto_remove=True, command="micromamba env export --explicit" + ) except ContainerError as e: - print(e.container.logs().decode('utf-8')) + print(e.container.logs().decode("utf-8")) # After printing the logs, raise the exception (which is the old behavior) raise - with open(f'{target_version_dir}/{config["env_out_filename"]}', 'wb') as f: + with open(f'{target_version_dir}/{config["env_out_filename"]}', "wb") as f: f.write(container_logs) # Generate change logs. Use the original image generator config which contains the name @@ -264,17 +272,18 @@ def _build_local_images(target_version: Version, target_ecr_repo_list: list[str] generate_change_log(target_version, image_generator_config) version_tags_to_apply = _get_version_tags(target_version, config["env_out_filename"]) - image_tags_to_apply = [config['image_tag_generator'].format(image_version=i) for i in version_tags_to_apply] + image_tags_to_apply = [config["image_tag_generator"].format(image_version=i) for i in version_tags_to_apply] if target_ecr_repo_list is not None: for target_ecr_repo in target_ecr_repo_list: for t in image_tags_to_apply: image.tag(target_ecr_repo, tag=t) - generated_image_versions.append({'repository': target_ecr_repo, 'tag': t}) + generated_image_versions.append({"repository": target_ecr_repo, "tag": t}) # Tag the image for testing - image.tag('localhost/sagemaker-distribution', - config['image_tag_generator'].format(image_version=str(target_version))) + image.tag( + "localhost/sagemaker-distribution", config["image_tag_generator"].format(image_version=str(target_version)) + ) return generated_image_ids, generated_image_versions @@ -292,7 +301,7 @@ def _get_version_tags(target_version: Version, env_out_file_name: str) -> list[s # If we were to add '2.6', check if '2.6.(x+1)' is present. if not is_exists_dir_for_version(target_version.bump_patch(), env_out_file_name): - res.append(f'{target_version.major}.{target_version.minor}') + res.append(f"{target_version.major}.{target_version.minor}") else: return res @@ -304,19 +313,19 @@ def _get_version_tags(target_version: Version, env_out_file_name: str) -> list[s # If we were to add 'latest', check if '3.0.0' is present. if not is_exists_dir_for_version(target_version.bump_major(), env_out_file_name): - res.append('latest') + res.append("latest") return res def _get_ecr_credentials(region, repository: str) -> (str, str): - _ecr_client_config_name = 'ecr-public' if repository.startswith('public.ecr.aws') else 'ecr' + _ecr_client_config_name = "ecr-public" if repository.startswith("public.ecr.aws") else "ecr" _ecr_client = boto3.client(_ecr_client_config_name, region_name=region) - _authorization_data = _ecr_client.get_authorization_token()['authorizationData'] - if _ecr_client_config_name == 'ecr': + _authorization_data = _ecr_client.get_authorization_token()["authorizationData"] + if _ecr_client_config_name == "ecr": # If we are using the ecr private client, then fetch the first index from authorizationData _authorization_data = _authorization_data[0] - return base64.b64decode(_authorization_data['authorizationToken']).decode().split(':') + return base64.b64decode(_authorization_data["authorizationToken"]).decode().split(":") def get_arg_parser(): @@ -327,85 +336,68 @@ def get_arg_parser(): subparsers = parser.add_subparsers(dest="subcommand") create_major_version_parser = subparsers.add_parser( - "create-major-version-artifacts", - help="Creates a new major version of Amazon SageMaker Distribution." + "create-major-version-artifacts", help="Creates a new major version of Amazon SageMaker Distribution." ) - create_major_version_parser.set_defaults(func=create_major_version_artifacts, - runtime_version_upgrade_type=_MAJOR) + create_major_version_parser.set_defaults(func=create_major_version_artifacts, runtime_version_upgrade_type=_MAJOR) create_minor_version_parser = subparsers.add_parser( - "create-minor-version-artifacts", - help="Creates a new minor version of Amazon SageMaker Distribution." + "create-minor-version-artifacts", help="Creates a new minor version of Amazon SageMaker Distribution." ) - create_minor_version_parser.set_defaults(func=create_minor_version_artifacts, - runtime_version_upgrade_type=_MINOR) + create_minor_version_parser.set_defaults(func=create_minor_version_artifacts, runtime_version_upgrade_type=_MINOR) create_patch_version_parser = subparsers.add_parser( - "create-patch-version-artifacts", - help="Creates a new patch version of Amazon SageMaker Distribution." + "create-patch-version-artifacts", help="Creates a new patch version of Amazon SageMaker Distribution." ) - create_patch_version_parser.set_defaults(func=create_patch_version_artifacts, - runtime_version_upgrade_type=_PATCH) + create_patch_version_parser.set_defaults(func=create_patch_version_artifacts, runtime_version_upgrade_type=_PATCH) # Common arguments for p in [create_major_version_parser, create_minor_version_parser, create_patch_version_parser]: p.add_argument( "--base-patch-version", required=True, - help="Specify the base patch version from which a new version should be created." + help="Specify the base patch version from which a new version should be created.", ) p.add_argument( "--pre-release-identifier", - help="Optionally specify the pre-release identifier for this new version that should " - "be created." + help="Optionally specify the pre-release identifier for this new version that should " "be created.", ) p.add_argument( "--force", - action='store_true', - help="Overwrites any existing directory corresponding to the new version that will be generated." + action="store_true", + help="Overwrites any existing directory corresponding to the new version that will be generated.", ) - build_image_parser = subparsers.add_parser( - "build", - help="Builds a new image from the Dockerfile." - ) + build_image_parser = subparsers.add_parser("build", help="Builds a new image from the Dockerfile.") build_image_parser.add_argument( "--target-patch-version", required=True, - help="Specify the target version of Amazon SageMaker Distribution for which an image needs to be built." + help="Specify the target version of Amazon SageMaker Distribution for which an image needs to be built.", ) build_image_parser.add_argument( - "--skip-tests", - action='store_true', - help="Disable running tests against the newly generated Docker image." + "--skip-tests", action="store_true", help="Disable running tests against the newly generated Docker image." ) build_image_parser.add_argument( "--force", - action='store_true', + action="store_true", help="Builds a new docker image which will fetch the latest versions of each package in " - "the conda environment. Any existing env.out file will be overwritten." + "the conda environment. Any existing env.out file will be overwritten.", ) build_image_parser.add_argument( "--target-ecr-repo", - action='append', - help="Specify the AWS ECR repository in which this image needs to be uploaded." - ) - build_image_parser.add_argument( - "--region", - help="Specify the region of the ECR repository." + action="append", + help="Specify the AWS ECR repository in which this image needs to be uploaded.", ) + build_image_parser.add_argument("--region", help="Specify the region of the ECR repository.") build_image_parser.set_defaults(func=build_images) package_staleness_parser = subparsers.add_parser( "generate-staleness-report", - help="Generates package staleness report for each of the marquee packages in the given " - "image version." + help="Generates package staleness report for each of the marquee packages in the given " "image version.", ) package_staleness_parser.set_defaults(func=generate_package_staleness_report) package_staleness_parser.add_argument( "--target-patch-version", required=True, - help="Specify the base patch version for which the package staleness report needs to be " - "generated." + help="Specify the base patch version for which the package staleness report needs to be " "generated.", ) return parser @@ -418,5 +410,5 @@ def parse_args(parser): args.func(args) -if __name__ == '__main__': +if __name__ == "__main__": parse_args(get_arg_parser()) diff --git a/src/package_staleness.py b/src/package_staleness.py index 21d4482a..6a113860 100644 --- a/src/package_staleness.py +++ b/src/package_staleness.py @@ -1,17 +1,14 @@ -import conda.cli.python_api import json -from utils import ( - get_dir_for_version, - get_semver, - get_match_specs, -) + +import conda.cli.python_api +from conda.models.match_spec import MatchSpec + from config import _image_generator_configs from dependency_upgrader import _dependency_metadata -from conda.models.match_spec import MatchSpec +from utils import get_dir_for_version, get_match_specs, get_semver -def _get_package_versions_in_upstream(target_packages_match_spec_out, target_version) \ - -> dict[str, str]: +def _get_package_versions_in_upstream(target_packages_match_spec_out, target_version) -> dict[str, str]: package_to_version_mapping = {} is_major_version_release = target_version.minor == 0 and target_version.patch == 0 is_minor_version_release = target_version.patch == 0 and not is_major_version_release @@ -19,13 +16,13 @@ def _get_package_versions_in_upstream(target_packages_match_spec_out, target_ver # Execute a conda search api call in the linux-64 subdirectory # packages such as pytorch-gpu are present only in linux-64 sub directory match_spec_out = target_packages_match_spec_out[package] - package_version = str(match_spec_out.get("version")).removeprefix('==') + package_version = str(match_spec_out.get("version")).removeprefix("==") package_version = get_semver(package_version) channel = match_spec_out.get("channel").channel_name - subdir_filter = '[subdir=' + match_spec_out.get('subdir') + ']' - search_result = conda.cli.python_api.run_command('search', channel + '::' + package + - '>=' + str(package_version) + - subdir_filter, '--json') + subdir_filter = "[subdir=" + match_spec_out.get("subdir") + "]" + search_result = conda.cli.python_api.run_command( + "search", channel + "::" + package + ">=" + str(package_version) + subdir_filter, "--json" + ) # Load the first result as json. The API sends a json string inside an array package_metadata = json.loads(search_result[0])[package] # Response is of the structure @@ -33,58 +30,62 @@ def _get_package_versions_in_upstream(target_packages_match_spec_out, target_ver # }, ..., {'url':, 'dependencies': , 'version': # }] # We only care about the version number in the last index - package_version_in_conda = '' + package_version_in_conda = "" if is_major_version_release: - latest_package_version_in_conda = package_metadata[-1]['version'] + latest_package_version_in_conda = package_metadata[-1]["version"] elif is_minor_version_release: package_major_version_prefix = str(package_version.major) + "." - latest_package_version_in_conda = [x['version'] for x in package_metadata if - x['version'].startswith(package_major_version_prefix)][-1] + latest_package_version_in_conda = [ + x["version"] for x in package_metadata if x["version"].startswith(package_major_version_prefix) + ][-1] else: - package_minor_version_prefix = \ - ".".join([str(package_version.major), str(package_version.minor)]) + "." - latest_package_version_in_conda = [x['version'] for x in package_metadata if - x['version'].startswith(package_minor_version_prefix)][-1] + package_minor_version_prefix = ".".join([str(package_version.major), str(package_version.minor)]) + "." + latest_package_version_in_conda = [ + x["version"] for x in package_metadata if x["version"].startswith(package_minor_version_prefix) + ][-1] package_to_version_mapping[package] = latest_package_version_in_conda return package_to_version_mapping -def _generate_report(package_versions_in_upstream, target_packages_match_spec_out, image_config, - version): - print('\n# Staleness Report: ' + str(version) + '(' + image_config['image_type'] + ')\n') - print('Package | Current Version in the Distribution image | Latest Relevant Version in ' - 'Upstream') - print('---|---|---') +def _generate_report(package_versions_in_upstream, target_packages_match_spec_out, image_config, version): + print("\n# Staleness Report: " + str(version) + "(" + image_config["image_type"] + ")\n") + print("Package | Current Version in the Distribution image | Latest Relevant Version in " "Upstream") + print("---|---|---") for package in package_versions_in_upstream: - version_in_sagemaker_distribution = \ - str(target_packages_match_spec_out[package].get('version')).removeprefix('==') + version_in_sagemaker_distribution = str(target_packages_match_spec_out[package].get("version")).removeprefix( + "==" + ) if version_in_sagemaker_distribution == package_versions_in_upstream[package]: - print(package + '|' + version_in_sagemaker_distribution + '|' + - package_versions_in_upstream[package]) + print(package + "|" + version_in_sagemaker_distribution + "|" + package_versions_in_upstream[package]) else: - print('${\color{red}' + package + '}$' + '|' + version_in_sagemaker_distribution + - '|' + package_versions_in_upstream[package]) + print( + "${\color{red}" + + package + + "}$" + + "|" + + version_in_sagemaker_distribution + + "|" + + package_versions_in_upstream[package] + ) -def _get_installed_package_versions_and_conda_versions(image_config, target_version_dir, - target_version) \ - -> (dict[str, MatchSpec], dict[str, str]): - env_in_file_name = image_config['build_args']['ENV_IN_FILENAME'] - env_out_file_name = image_config['env_out_filename'] - required_packages_from_target = get_match_specs( - target_version_dir + "/" + env_in_file_name - ).keys() +def _get_installed_package_versions_and_conda_versions( + image_config, target_version_dir, target_version +) -> (dict[str, MatchSpec], dict[str, str]): + env_in_file_name = image_config["build_args"]["ENV_IN_FILENAME"] + env_out_file_name = image_config["env_out_filename"] + required_packages_from_target = get_match_specs(target_version_dir + "/" + env_in_file_name).keys() match_spec_out = get_match_specs(target_version_dir + "/" + env_out_file_name) # We only care about packages which are present in env.in # Remove Python from the dictionary, we don't want to track python version as part of our # staleness report. - target_packages_match_spec_out = {k: v for k, v in match_spec_out.items() - if k in required_packages_from_target and k - not in _dependency_metadata} - latest_package_versions_in_upstream = \ - _get_package_versions_in_upstream(target_packages_match_spec_out, - target_version) + target_packages_match_spec_out = { + k: v for k, v in match_spec_out.items() if k in required_packages_from_target and k not in _dependency_metadata + } + latest_package_versions_in_upstream = _get_package_versions_in_upstream( + target_packages_match_spec_out, target_version + ) return target_packages_match_spec_out, latest_package_versions_in_upstream @@ -92,8 +93,10 @@ def generate_package_staleness_report(args): target_version = get_semver(args.target_patch_version) target_version_dir = get_dir_for_version(target_version) for image_config in _image_generator_configs: - target_packages_match_spec_out, latest_package_versions_in_upstream = \ - _get_installed_package_versions_and_conda_versions(image_config, target_version_dir, - target_version) - _generate_report(latest_package_versions_in_upstream, target_packages_match_spec_out, - image_config, target_version) + ( + target_packages_match_spec_out, + latest_package_versions_in_upstream, + ) = _get_installed_package_versions_and_conda_versions(image_config, target_version_dir, target_version) + _generate_report( + latest_package_versions_in_upstream, target_packages_match_spec_out, image_config, target_version + ) diff --git a/src/release_notes_generator.py b/src/release_notes_generator.py index 83c45c3c..9993aa92 100644 --- a/src/release_notes_generator.py +++ b/src/release_notes_generator.py @@ -1,25 +1,25 @@ import os -from config import _image_generator_configs -from utils import ( - get_dir_for_version, - get_match_specs, -) + from semver import Version +from config import _image_generator_configs +from utils import get_dir_for_version, get_match_specs + -def _get_installed_packages(target_version_dir, image_config) -> (dict[str, str]): - env_in_file_name = image_config['build_args']['ENV_IN_FILENAME'] - env_out_file_name = image_config['env_out_filename'] +def _get_installed_packages(target_version_dir, image_config) -> dict[str, str]: + env_in_file_name = image_config["build_args"]["ENV_IN_FILENAME"] + env_out_file_name = image_config["env_out_filename"] env_in_file_path = target_version_dir + "/" + env_in_file_name if not os.path.exists(env_in_file_path): return {} - required_packages_from_target = get_match_specs( - target_version_dir + "/" + env_in_file_name - ).keys() + required_packages_from_target = get_match_specs(target_version_dir + "/" + env_in_file_name).keys() target_match_spec_out = get_match_specs(target_version_dir + "/" + env_out_file_name) # We only care about the packages which are present in the target version env.in file - return {k: str(v.get('version')).removeprefix('==') - for k, v in target_match_spec_out.items() if k in required_packages_from_target} + return { + k: str(v.get("version")).removeprefix("==") + for k, v in target_match_spec_out.items() + if k in required_packages_from_target + } def _get_package_to_image_type_mapping(image_type_package_metadata): @@ -39,8 +39,9 @@ def _get_package_to_image_type_mapping(image_type_package_metadata): def _get_image_type_package_metadata(target_version_dir): image_type_package_metadata = {} for image_generator_config in _image_generator_configs: - image_type_package_metadata[image_generator_config['image_type']] = _get_installed_packages( - target_version_dir, image_generator_config) + image_type_package_metadata[image_generator_config["image_type"]] = _get_installed_packages( + target_version_dir, image_generator_config + ) return image_type_package_metadata @@ -51,19 +52,22 @@ def generate_release_notes(target_version: Version): image_type_package_metadata = _get_image_type_package_metadata(target_version_dir) package_to_image_type_mapping = _get_package_to_image_type_mapping(image_type_package_metadata) - with open(f'{target_version_dir}/RELEASE.md', 'w') as f: - f.write('# Release notes: ' + str(target_version) + '\n\n') - f.write('Package ') - table_separator = '---' + with open(f"{target_version_dir}/RELEASE.md", "w") as f: + f.write("# Release notes: " + str(target_version) + "\n\n") + f.write("Package ") + table_separator = "---" for image_type in image_type_package_metadata.keys(): - f.write('| ' + image_type) - table_separator += '|---' - f.write('\n') - f.write(table_separator + '\n') + f.write("| " + image_type) + table_separator += "|---" + f.write("\n") + f.write(table_separator + "\n") for package in package_to_image_type_mapping.keys(): f.write(package) for image_type in image_type_package_metadata.keys(): - version = package_to_image_type_mapping[package][image_type] if image_type in \ - package_to_image_type_mapping[package] else ' ' - f.write('|' + version) - f.write('\n') + version = ( + package_to_image_type_mapping[package][image_type] + if image_type in package_to_image_type_mapping[package] + else " " + ) + f.write("|" + version) + f.write("\n") diff --git a/src/utils.py b/src/utils.py index eec0776d..daf1e17e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,33 +1,36 @@ import os -from semver import Version -from conda.models.match_spec import MatchSpec + from conda.env.specs import RequirementsSpec +from conda.models.match_spec import MatchSpec +from semver import Version def get_dir_for_version(version: Version) -> str: - version_prerelease_suffix = f'/v{version.major}.{version.minor}.{version.patch}-' \ - f'{version.prerelease}' if version.prerelease else '' - return os.path.relpath(f'build_artifacts/v{version.major}/v{version.major}.{version.minor}/' - f'v{version.major}.{version.minor}.{version.patch}' - f'{version_prerelease_suffix}') + version_prerelease_suffix = ( + f"/v{version.major}.{version.minor}.{version.patch}-" f"{version.prerelease}" if version.prerelease else "" + ) + return os.path.relpath( + f"build_artifacts/v{version.major}/v{version.major}.{version.minor}/" + f"v{version.major}.{version.minor}.{version.patch}" + f"{version_prerelease_suffix}" + ) -def is_exists_dir_for_version(version: Version, file_name_to_verify_existence='Dockerfile') -> bool: +def is_exists_dir_for_version(version: Version, file_name_to_verify_existence="Dockerfile") -> bool: dir_path = get_dir_for_version(version) # Also validate whether this directory is not generated due to any pre-release builds/ # additional packages. # This can be validated by checking whether {cpu/gpu}.env.{in/out}/Dockerfile exists in the # directory. - return os.path.exists(dir_path) and os.path.exists(dir_path + "/" + - file_name_to_verify_existence) + return os.path.exists(dir_path) and os.path.exists(dir_path + "/" + file_name_to_verify_existence) def get_semver(version_str) -> Version: # Version strings on conda-forge follow PEP standards rather than SemVer, which support # version strings such as X.Y.Z.postN, X.Y.Z.preN. These cause errors in semver.Version.parse # so we keep the first 3 entries as version string. - if version_str.count('.') > 2: - version_str = '.'.join(version_str.split('.')[:3]) + if version_str.count(".") > 2: + version_str = ".".join(version_str.split(".")[:3]) version = Version.parse(version_str) if version.build is not None: raise Exception() @@ -44,7 +47,6 @@ def get_match_specs(file_path) -> dict[str, MatchSpec]: requirement_spec = read_env_file(file_path) assert len(requirement_spec.environment.dependencies) == 1 - assert 'conda' in requirement_spec.environment.dependencies + assert "conda" in requirement_spec.environment.dependencies - return {MatchSpec(i).get('name'): MatchSpec(i) for i in - requirement_spec.environment.dependencies['conda']} + return {MatchSpec(i).get("name"): MatchSpec(i) for i in requirement_spec.environment.dependencies["conda"]} diff --git a/template/v1/dirs/etc/code-editor/code_editor_machine_settings.json b/template/v1/dirs/etc/code-editor/code_editor_machine_settings.json index a391da92..44fb8ef7 100644 --- a/template/v1/dirs/etc/code-editor/code_editor_machine_settings.json +++ b/template/v1/dirs/etc/code-editor/code_editor_machine_settings.json @@ -1,4 +1,4 @@ { "python.terminal.activateEnvironment": false, "python.defaultInterpreterPath": "/opt/conda/bin/python" -} \ No newline at end of file +} diff --git a/template/v1/dirs/etc/code-editor/extensions.txt b/template/v1/dirs/etc/code-editor/extensions.txt index ee18444b..29d683eb 100644 --- a/template/v1/dirs/etc/code-editor/extensions.txt +++ b/template/v1/dirs/etc/code-editor/extensions.txt @@ -1,3 +1,3 @@ https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -https://open-vsx.org/api/amazonwebservices/aws-toolkit-vscode/1.99.0/file/amazonwebservices.aws-toolkit-vscode-1.99.0.vsix \ No newline at end of file +https://open-vsx.org/api/amazonwebservices/aws-toolkit-vscode/1.99.0/file/amazonwebservices.aws-toolkit-vscode-1.99.0.vsix diff --git a/template/v1/dirs/etc/jupyter/jupyter_server_config.py b/template/v1/dirs/etc/jupyter/jupyter_server_config.py index e05997f2..8e45762f 100644 --- a/template/v1/dirs/etc/jupyter/jupyter_server_config.py +++ b/template/v1/dirs/etc/jupyter/jupyter_server_config.py @@ -1,10 +1,10 @@ # Default Jupyter server config -# Note: those config can be overridden by user-level configs. +# Note: those config can be overridden by user-level configs. -c.ServerApp.terminado_settings = { 'shell_command': ['/bin/bash'] } -c.ServerApp.tornado_settings = { 'compress_response': True } +c.ServerApp.terminado_settings = {"shell_command": ["/bin/bash"]} +c.ServerApp.tornado_settings = {"compress_response": True} -# Do not delete files to trash. Instead, permanently delete files. +# Do not delete files to trash. Instead, permanently delete files. c.FileContentsManager.delete_to_trash = False # Allow deleting non-empty directory via file browser diff --git a/template/v1/dirs/usr/local/bin/entrypoint-code-editor b/template/v1/dirs/usr/local/bin/entrypoint-code-editor index 677c9803..bf55a371 100755 --- a/template/v1/dirs/usr/local/bin/entrypoint-code-editor +++ b/template/v1/dirs/usr/local/bin/entrypoint-code-editor @@ -13,4 +13,4 @@ micromamba activate base export SAGEMAKER_APP_TYPE_LOWERCASE=$(echo $SAGEMAKER_APP_TYPE | tr '[:upper:]' '[:lower:]') mkdir -p $STUDIO_LOGGING_DIR/$SAGEMAKER_APP_TYPE_LOWERCASE/supervisord -exec supervisord -c /etc/supervisor/conf.d/supervisord-code-editor.conf -n \ No newline at end of file +exec supervisord -c /etc/supervisor/conf.d/supervisord-code-editor.conf -n diff --git a/template/v1/dirs/usr/local/bin/start-jupyter-server b/template/v1/dirs/usr/local/bin/start-jupyter-server index 377a86d4..626c3f8e 100755 --- a/template/v1/dirs/usr/local/bin/start-jupyter-server +++ b/template/v1/dirs/usr/local/bin/start-jupyter-server @@ -9,8 +9,8 @@ micromamba activate base # Start Jupyter server in rtc mode for shared spaces if [ -n "$SAGEMAKER_APP_TYPE_LOWERCASE" ] && [ "$SAGEMAKER_SPACE_TYPE_LOWERCASE" == "shared" ]; then # SAGEMAKER_APP_TYPE is set, indicating the server is running within a SageMaker - # app. Configure the base url to be `//default`. - # SAGEMAKER_SPACE_TYPE_LOWERCASE flag is used to determine if the server should start + # app. Configure the base url to be `//default`. + # SAGEMAKER_SPACE_TYPE_LOWERCASE flag is used to determine if the server should start # in real-time-collaboration mode for a given space. jupyter lab --ip 0.0.0.0 --port 8888 \ --ServerApp.base_url="/$SAGEMAKER_APP_TYPE_LOWERCASE/default" \ diff --git a/test/conftest.py b/test/conftest.py index 36c3fa75..df98aa2f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,15 +3,14 @@ def pytest_addoption(parser): parser.addoption( - "--local-image-version", help="Version of a locally available SageMaker Distribution " - "Docker image against which the tests should run. Note: " - "We expect the local docker image to have " - "'localhost/sagemaker-distribution' as the repository and " - "'-{cpu/gpu}' as the tag" - ) - parser.addoption( - "--use-gpu", action='store_true', help="Boolean on whether to use GPUs or not" + "--local-image-version", + help="Version of a locally available SageMaker Distribution " + "Docker image against which the tests should run. Note: " + "We expect the local docker image to have " + "'localhost/sagemaker-distribution' as the repository and " + "'-{cpu/gpu}' as the tag", ) + parser.addoption("--use-gpu", action="store_true", help="Boolean on whether to use GPUs or not") @pytest.fixture diff --git a/test/test_artifacts/v0/aws-glue-sessions/run_glue_sessions_notebook.sh b/test/test_artifacts/v0/aws-glue-sessions/run_glue_sessions_notebook.sh index fbf5faf2..1aa73e37 100644 --- a/test/test_artifacts/v0/aws-glue-sessions/run_glue_sessions_notebook.sh +++ b/test/test_artifacts/v0/aws-glue-sessions/run_glue_sessions_notebook.sh @@ -8,4 +8,3 @@ nb='script' for kernel in ${kernels[@]}; do papermill 'glue_notebook.ipynb' 'nb_output.ipynb' -k $kernel done - diff --git a/test/test_artifacts/v0/keras.test.Dockerfile b/test/test_artifacts/v0/keras.test.Dockerfile index 1bf1713b..74690920 100644 --- a/test/test_artifacts/v0/keras.test.Dockerfile +++ b/test/test_artifacts/v0/keras.test.Dockerfile @@ -17,4 +17,3 @@ COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_keras_tests.sh . RUN chmod +x run_keras_tests.sh # Run tests in run_keras_tests.sh CMD ["./run_keras_tests.sh"] - diff --git a/test/test_artifacts/v0/matplotlib.test.Dockerfile b/test/test_artifacts/v0/matplotlib.test.Dockerfile index 7c987704..4d290016 100644 --- a/test/test_artifacts/v0/matplotlib.test.Dockerfile +++ b/test/test_artifacts/v0/matplotlib.test.Dockerfile @@ -16,4 +16,3 @@ COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_matplotlib_tests.sh . RUN chmod +x run_matplotlib_tests.sh # Run tests in run_matplotlib_tests.sh CMD ["./run_matplotlib_tests.sh"] - diff --git a/test/test_artifacts/v0/run_pandas_tests.py b/test/test_artifacts/v0/run_pandas_tests.py index 4cc8086c..08953750 100644 --- a/test/test_artifacts/v0/run_pandas_tests.py +++ b/test/test_artifacts/v0/run_pandas_tests.py @@ -1,4 +1,8 @@ -import pandas, sys, os, site +import os +import site +import sys + +import pandas # We change the working directory here because there is at least one test (`test_html_template_extends_options`) which # expects the directory to be 'pandas'. Ideally, we would have changed directories through a `WORKDIR` in Dockerfile @@ -13,8 +17,8 @@ # expectation is just ">=3.6.1". Our image contains v3.7.1, so it meets the latter requirement but not the former. This # particular test, however, only works with the former requirement. (We verified that the test succeeds if we manually # drop the version to v3.6.x) So, we skip it. -tests_succeeded = pandas.test([ - '-m', '(not slow and not network and not db)', - '-k', '(not test_network and not s3 and not test_plain_axes)']) +tests_succeeded = pandas.test( + ["-m", "(not slow and not network and not db)", "-k", "(not test_network and not s3 and not test_plain_axes)"] +) sys.exit(not tests_succeeded) diff --git a/test/test_artifacts/v0/sagemaker-headless-execution-driver.test.Dockerfile b/test/test_artifacts/v0/sagemaker-headless-execution-driver.test.Dockerfile index e91d8914..094b1066 100644 --- a/test/test_artifacts/v0/sagemaker-headless-execution-driver.test.Dockerfile +++ b/test/test_artifacts/v0/sagemaker-headless-execution-driver.test.Dockerfile @@ -5,4 +5,4 @@ ARG MAMBA_DOCKERFILE_ACTIVATE=1 # Execute execution_driver module to for sagemaker-headless-execution-driver installation -CMD ["python", "-c", "import sagemaker_headless_execution_driver.headless_execution as execution_driver"] \ No newline at end of file +CMD ["python", "-c", "import sagemaker_headless_execution_driver.headless_execution as execution_driver"] diff --git a/test/test_artifacts/v0/scripts/run_autogluon_tests.sh b/test/test_artifacts/v0/scripts/run_autogluon_tests.sh index 036d1432..09d70554 100644 --- a/test/test_artifacts/v0/scripts/run_autogluon_tests.sh +++ b/test/test_artifacts/v0/scripts/run_autogluon_tests.sh @@ -12,6 +12,6 @@ python -c "import torch; exit(0) if torch.cuda.is_available() else exit(1)" ret=$? if [ $ret -eq 0 ] -then +then jupyter nbconvert --execute --to python docs/tutorials/multimodal/multimodal_prediction/multimodal-quick-start.ipynb fi diff --git a/test/test_artifacts/v0/scripts/run_matplotlib_tests.sh b/test/test_artifacts/v0/scripts/run_matplotlib_tests.sh index 13aa3fb3..848e7421 100644 --- a/test/test_artifacts/v0/scripts/run_matplotlib_tests.sh +++ b/test/test_artifacts/v0/scripts/run_matplotlib_tests.sh @@ -3,5 +3,3 @@ for file in *.py; do python "$file" || exit $? done - - diff --git a/test/test_artifacts/v1/aws-glue-sessions/run_glue_sessions_notebook.sh b/test/test_artifacts/v1/aws-glue-sessions/run_glue_sessions_notebook.sh index fbf5faf2..1aa73e37 100644 --- a/test/test_artifacts/v1/aws-glue-sessions/run_glue_sessions_notebook.sh +++ b/test/test_artifacts/v1/aws-glue-sessions/run_glue_sessions_notebook.sh @@ -8,4 +8,3 @@ nb='script' for kernel in ${kernels[@]}; do papermill 'glue_notebook.ipynb' 'nb_output.ipynb' -k $kernel done - diff --git a/test/test_artifacts/v1/jupyterlab-git.test.Dockerfile b/test/test_artifacts/v1/jupyterlab-git.test.Dockerfile index cb04575f..7d5cbd96 100644 --- a/test/test_artifacts/v1/jupyterlab-git.test.Dockerfile +++ b/test/test_artifacts/v1/jupyterlab-git.test.Dockerfile @@ -4,4 +4,3 @@ FROM $SAGEMAKER_DISTRIBUTION_IMAGE ARG MAMBA_DOCKERFILE_ACTIVATE=1 CMD ["python", "-c", "import jupyterlab_git"] - diff --git a/test/test_artifacts/v1/matplotlib.test.Dockerfile b/test/test_artifacts/v1/matplotlib.test.Dockerfile index 7c987704..4d290016 100644 --- a/test/test_artifacts/v1/matplotlib.test.Dockerfile +++ b/test/test_artifacts/v1/matplotlib.test.Dockerfile @@ -16,4 +16,3 @@ COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_matplotlib_tests.sh . RUN chmod +x run_matplotlib_tests.sh # Run tests in run_matplotlib_tests.sh CMD ["./run_matplotlib_tests.sh"] - diff --git a/test/test_artifacts/v1/run_pandas_tests.py b/test/test_artifacts/v1/run_pandas_tests.py index 00127f2d..2995581f 100644 --- a/test/test_artifacts/v1/run_pandas_tests.py +++ b/test/test_artifacts/v1/run_pandas_tests.py @@ -1,4 +1,8 @@ -import pandas, sys, os, site +import os +import site +import sys + +import pandas # We change the working directory here because there is at least one test (`test_html_template_extends_options`) which # expects the directory to be 'pandas'. Ideally, we would have changed directories through a `WORKDIR` in Dockerfile @@ -14,10 +18,16 @@ # particular test, however, only works with the former requirement. (We verified that the test succeeds if we manually # drop the version to v3.6.x) So, we skip it. # Also skipping specific TestFrameFlexArithmetic test; failing due to known issue https://github.com/pandas-dev/pandas/issues/54546 -tests_succeeded = pandas.test([ - '-m', '(not slow and not network and not db)', - '-k', '(not test_network and not s3 and not test_plain_axes)', - '--no-strict-data-files', - '--ignore', 'pandas/tests/frame/test_arithmetic.py::TestFrameFlexArithmetic::test_floordiv_axis0_numexpr_path']) +tests_succeeded = pandas.test( + [ + "-m", + "(not slow and not network and not db)", + "-k", + "(not test_network and not s3 and not test_plain_axes)", + "--no-strict-data-files", + "--ignore", + "pandas/tests/frame/test_arithmetic.py::TestFrameFlexArithmetic::test_floordiv_axis0_numexpr_path", + ] +) sys.exit(not tests_succeeded) diff --git a/test/test_artifacts/v1/scripts/run_altair_example_notebooks.sh b/test/test_artifacts/v1/scripts/run_altair_example_notebooks.sh index 47762d0f..1bb4f370 100644 --- a/test/test_artifacts/v1/scripts/run_altair_example_notebooks.sh +++ b/test/test_artifacts/v1/scripts/run_altair_example_notebooks.sh @@ -15,4 +15,3 @@ example_notebooks=('02-Tutorial.ipynb' for nb in ${example_notebooks[@]}; do papermill $nb 'nb_output.ipynb' done - diff --git a/test/test_artifacts/v1/scripts/run_autogluon_tests.sh b/test/test_artifacts/v1/scripts/run_autogluon_tests.sh index ea68f6a4..7a136c25 100644 --- a/test/test_artifacts/v1/scripts/run_autogluon_tests.sh +++ b/test/test_artifacts/v1/scripts/run_autogluon_tests.sh @@ -12,6 +12,6 @@ python -c "import torch; exit(0) if torch.cuda.is_available() else exit(1)" ret=$? if [ $ret -eq 0 ] -then +then jupyter nbconvert --execute --to python docs/tutorials/multimodal/multimodal_prediction/multimodal-quick-start.ipynb fi diff --git a/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh b/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh index 13aa3fb3..848e7421 100644 --- a/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh +++ b/test/test_artifacts/v1/scripts/run_matplotlib_tests.sh @@ -3,5 +3,3 @@ for file in *.py; do python "$file" || exit $? done - - diff --git a/test/test_artifacts/v1/scripts/run_sagemaker_code_editor_tests.sh b/test/test_artifacts/v1/scripts/run_sagemaker_code_editor_tests.sh index b6db3bd6..0b7dda58 100644 --- a/test/test_artifacts/v1/scripts/run_sagemaker_code_editor_tests.sh +++ b/test/test_artifacts/v1/scripts/run_sagemaker_code_editor_tests.sh @@ -67,4 +67,4 @@ if [ ! -d "$ARTIFACTS_DIR" ]; then else echo "Error: Directory $ARTIFACTS_DIR still exists." exit 1 -fi \ No newline at end of file +fi diff --git a/test/test_artifacts/v1/serve.test.Dockerfile b/test/test_artifacts/v1/serve.test.Dockerfile index cab72baf..19dd8d5d 100644 --- a/test/test_artifacts/v1/serve.test.Dockerfile +++ b/test/test_artifacts/v1/serve.test.Dockerfile @@ -3,4 +3,4 @@ FROM $SAGEMAKER_DISTRIBUTION_IMAGE ARG MAMBA_DOCKERFILE_ACTIVATE=1 -CMD ["python", "-c", "import fastapi, uvicorn, langchain"] \ No newline at end of file +CMD ["python", "-c", "import fastapi, uvicorn, langchain"] diff --git a/test/test_dependency_upgrader.py b/test/test_dependency_upgrader.py index b3e2a62b..caebf730 100644 --- a/test/test_dependency_upgrader.py +++ b/test/test_dependency_upgrader.py @@ -4,50 +4,50 @@ pytestmark = pytest.mark.unit from dependency_upgrader import ( - _get_dependency_upper_bound_for_runtime_upgrade, - _get_dependency_upper_bound_for_semver, - _get_dependency_upper_bound_for_pythonesque, _MAJOR, _MINOR, - _PATCH + _PATCH, + _get_dependency_upper_bound_for_pythonesque, + _get_dependency_upper_bound_for_runtime_upgrade, + _get_dependency_upper_bound_for_semver, ) def test_get_dependency_upper_bound_for_runtime_upgrade(): # case 1: Runtime upgrade type is MINOR, dependency name is python => use pythonesque - assert _get_dependency_upper_bound_for_runtime_upgrade('python', '1.2.5', _MINOR) == ',<1.3.0' + assert _get_dependency_upper_bound_for_runtime_upgrade("python", "1.2.5", _MINOR) == ",<1.3.0" # case 1: Runtime upgrade type is MINOR, dependency name is node => use semver - assert _get_dependency_upper_bound_for_runtime_upgrade('node', '1.2.5', _MINOR) == ',<2.0.0' + assert _get_dependency_upper_bound_for_runtime_upgrade("node", "1.2.5", _MINOR) == ",<2.0.0" def test_get_dependency_upper_bound_for_semver(): # case 1: Runtime upgrade type is MAJOR. - assert _get_dependency_upper_bound_for_semver('1.2.5', _MAJOR) == '' + assert _get_dependency_upper_bound_for_semver("1.2.5", _MAJOR) == "" # case 2: Runtime upgrade type is MINOR. - assert _get_dependency_upper_bound_for_semver('1.2.5', _MINOR) == ',<2.0.0' + assert _get_dependency_upper_bound_for_semver("1.2.5", _MINOR) == ",<2.0.0" # case 3: Runtime upgrade type is PATCH. - assert _get_dependency_upper_bound_for_semver('1.2.5', _PATCH) == ',<1.3.0' + assert _get_dependency_upper_bound_for_semver("1.2.5", _PATCH) == ",<1.3.0" # case 4: Throw exception for any other runtime_upgrade_type with pytest.raises(Exception): - _get_dependency_upper_bound_for_semver('1.2.5', 'prerelease') + _get_dependency_upper_bound_for_semver("1.2.5", "prerelease") def test_get_dependency_upper_bound_for_pythonesque(): # case 1: Runtime upgrade type is MAJOR. - assert _get_dependency_upper_bound_for_pythonesque('1.2.5', _MAJOR) == '' + assert _get_dependency_upper_bound_for_pythonesque("1.2.5", _MAJOR) == "" # case 2: Runtime upgrade type is MINOR. - assert _get_dependency_upper_bound_for_pythonesque('1.2.5', _MINOR) == ',<1.3.0' + assert _get_dependency_upper_bound_for_pythonesque("1.2.5", _MINOR) == ",<1.3.0" # case 3: Runtime upgrade type is PATCH. For pythonesque, we will bump minor version. - assert _get_dependency_upper_bound_for_pythonesque('1.2.5', _PATCH) == ',<1.3.0' + assert _get_dependency_upper_bound_for_pythonesque("1.2.5", _PATCH) == ",<1.3.0" # case 4: Throw exception for any other runtime_upgrade_type with pytest.raises(Exception): - _get_dependency_upper_bound_for_pythonesque('1.2.5', 'prerelease') + _get_dependency_upper_bound_for_pythonesque("1.2.5", "prerelease") def test_get_dependency_upper_bound_for_post_pre_release(): - assert _get_dependency_upper_bound_for_runtime_upgrade('python', '1.2.5.post1', _PATCH) == ',<1.3.0' - assert _get_dependency_upper_bound_for_runtime_upgrade('node', '1.2.5.post1', _PATCH) == ',<1.3.0' - assert _get_dependency_upper_bound_for_runtime_upgrade('python', '1.2.5.post1', _MINOR) == ',<1.3.0' - assert _get_dependency_upper_bound_for_runtime_upgrade('node', '1.2.5.post1', _MINOR) == ',<2.0.0' - assert _get_dependency_upper_bound_for_runtime_upgrade('python', '1.2.5.post1', _MAJOR) == '' - assert _get_dependency_upper_bound_for_runtime_upgrade('node', '1.2.5.post1', _MAJOR) == '' + assert _get_dependency_upper_bound_for_runtime_upgrade("python", "1.2.5.post1", _PATCH) == ",<1.3.0" + assert _get_dependency_upper_bound_for_runtime_upgrade("node", "1.2.5.post1", _PATCH) == ",<1.3.0" + assert _get_dependency_upper_bound_for_runtime_upgrade("python", "1.2.5.post1", _MINOR) == ",<1.3.0" + assert _get_dependency_upper_bound_for_runtime_upgrade("node", "1.2.5.post1", _MINOR) == ",<2.0.0" + assert _get_dependency_upper_bound_for_runtime_upgrade("python", "1.2.5.post1", _MAJOR) == "" + assert _get_dependency_upper_bound_for_runtime_upgrade("node", "1.2.5.post1", _MAJOR) == "" diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index 8026ff2b..5e1c4b88 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -1,84 +1,91 @@ -import subprocess import os +import subprocess +from typing import List + import docker import pytest -from config import _image_generator_configs from docker.errors import BuildError from semver import Version -from typing import List -from utils import ( - get_dir_for_version, - get_semver, - get_match_specs, -) + +from config import _image_generator_configs +from utils import get_dir_for_version, get_match_specs, get_semver _docker_client = docker.from_env() @pytest.mark.cpu -@pytest.mark.parametrize("dockerfile_path, required_packages", [ - ("keras.test.Dockerfile", ['keras']), - ("autogluon.test.Dockerfile", ['autogluon']), - ("matplotlib.test.Dockerfile", ['matplotlib']), - ("sagemaker-headless-execution-driver.test.Dockerfile", ['sagemaker-headless-execution-driver']), - ("scipy.test.Dockerfile", ['scipy']), - ("numpy.test.Dockerfile", ['numpy']), - ("boto3.test.Dockerfile", ['boto3']), - ("pandas.test.Dockerfile", ['pandas']), - ("sm-python-sdk.test.Dockerfile", ['sagemaker-python-sdk']), - ("pytorch.examples.Dockerfile", ['pytorch']), - ("tensorflow.examples.Dockerfile", ['tensorflow']), - ("jupyter-ai.test.Dockerfile", ['jupyter-ai']), - ("jupyter-dash.test.Dockerfile", ['jupyter-dash']), - ("jupyterlab-lsp.test.Dockerfile", ['jupyterlab-lsp']), - ("jupyter-lsp-server.test.Dockerfile", ['jupyter-lsp-server']), - ("sagemaker-code-editor.test.Dockerfile", ['sagemaker-code-editor']), - ("notebook.test.Dockerfile", ['notebook']), - ("glue-sessions.test.Dockerfile", ['aws-glue-sessions']), - ("altair.test.Dockerfile", ['altair']), - ("sagemaker-studio-analytics-extension.test.Dockerfile", ['sagemaker-studio-analytics-extension']), - ("amazon-codewhisperer-jupyterlab-ext.test.Dockerfile", ['amazon-codewhisperer-jupyterlab-ext']), - ("jupyterlab-git.test.Dockerfile", ['jupyterlab-git']), - ("amazon-sagemaker-sql-magic.test.Dockerfile", ['amazon-sagemaker-sql-magic']), - ("amazon_sagemaker_sql_editor.test.Dockerfile", ['amazon_sagemaker_sql_editor']), - ("serve.test.Dockerfile", ['serve-langchain'])]) -def test_dockerfiles_for_cpu(dockerfile_path: str, required_packages: List[str], - local_image_version: str, use_gpu: bool): - _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, 'cpu') +@pytest.mark.parametrize( + "dockerfile_path, required_packages", + [ + ("keras.test.Dockerfile", ["keras"]), + ("autogluon.test.Dockerfile", ["autogluon"]), + ("matplotlib.test.Dockerfile", ["matplotlib"]), + ("sagemaker-headless-execution-driver.test.Dockerfile", ["sagemaker-headless-execution-driver"]), + ("scipy.test.Dockerfile", ["scipy"]), + ("numpy.test.Dockerfile", ["numpy"]), + ("boto3.test.Dockerfile", ["boto3"]), + ("pandas.test.Dockerfile", ["pandas"]), + ("sm-python-sdk.test.Dockerfile", ["sagemaker-python-sdk"]), + ("pytorch.examples.Dockerfile", ["pytorch"]), + ("tensorflow.examples.Dockerfile", ["tensorflow"]), + ("jupyter-ai.test.Dockerfile", ["jupyter-ai"]), + ("jupyter-dash.test.Dockerfile", ["jupyter-dash"]), + ("jupyterlab-lsp.test.Dockerfile", ["jupyterlab-lsp"]), + ("jupyter-lsp-server.test.Dockerfile", ["jupyter-lsp-server"]), + ("sagemaker-code-editor.test.Dockerfile", ["sagemaker-code-editor"]), + ("notebook.test.Dockerfile", ["notebook"]), + ("glue-sessions.test.Dockerfile", ["aws-glue-sessions"]), + ("altair.test.Dockerfile", ["altair"]), + ("sagemaker-studio-analytics-extension.test.Dockerfile", ["sagemaker-studio-analytics-extension"]), + ("amazon-codewhisperer-jupyterlab-ext.test.Dockerfile", ["amazon-codewhisperer-jupyterlab-ext"]), + ("jupyterlab-git.test.Dockerfile", ["jupyterlab-git"]), + ("amazon-sagemaker-sql-magic.test.Dockerfile", ["amazon-sagemaker-sql-magic"]), + ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), + ("serve.test.Dockerfile", ["serve-langchain"]), + ], +) +def test_dockerfiles_for_cpu( + dockerfile_path: str, required_packages: List[str], local_image_version: str, use_gpu: bool +): + _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, "cpu") @pytest.mark.gpu -@pytest.mark.parametrize("dockerfile_path, required_packages", [ - ("keras.test.Dockerfile", ['keras']), - ("autogluon.test.Dockerfile", ['autogluon']), - ("matplotlib.test.Dockerfile", ['matplotlib']), - ("sagemaker-headless-execution-driver.test.Dockerfile", ['sagemaker-headless-execution-driver']), - ("scipy.test.Dockerfile", ['scipy']), - ("numpy.test.Dockerfile", ['numpy']), - ("boto3.test.Dockerfile", ['boto3']), - ("pandas.test.Dockerfile", ['pandas']), - ("sm-python-sdk.test.Dockerfile", ['sagemaker-python-sdk']), - ("pytorch.examples.Dockerfile", ['pytorch']), - ("tensorflow.examples.Dockerfile", ['tensorflow']), - ("glue-sessions.test.Dockerfile", ['aws-glue-sessions']), - ("jupyter-ai.test.Dockerfile", ['jupyter-ai']), - ("jupyter-dash.test.Dockerfile", ['jupyter-dash']), - ("jupyterlab-lsp.test.Dockerfile", ['jupyterlab-lsp']), - ("jupyter-lsp-server.test.Dockerfile", ['jupyter-lsp-server']), - ("sagemaker-code-editor.test.Dockerfile", ['sagemaker-code-editor']), - ("notebook.test.Dockerfile", ['notebook']), - ("glue-sessions.test.Dockerfile", ['aws-glue-sessions']), - ("altair.test.Dockerfile", ['altair']), - ("sagemaker-studio-analytics-extension.test.Dockerfile", ['sagemaker-studio-analytics-extension']), - ("amazon-codewhisperer-jupyterlab-ext.test.Dockerfile", ['amazon-codewhisperer-jupyterlab-ext']), - ("jupyterlab-git.test.Dockerfile", ['jupyterlab-git']), - ("amazon-sagemaker-sql-magic.test.Dockerfile", ['amazon-sagemaker-sql-magic']), - ("amazon_sagemaker_sql_editor.test.Dockerfile", ['amazon_sagemaker_sql_editor']), - ("serve.test.Dockerfile", ['serve-langchain'])]) -def test_dockerfiles_for_gpu(dockerfile_path: str, required_packages: List[str], - local_image_version: str, use_gpu: bool): - _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, 'gpu') - +@pytest.mark.parametrize( + "dockerfile_path, required_packages", + [ + ("keras.test.Dockerfile", ["keras"]), + ("autogluon.test.Dockerfile", ["autogluon"]), + ("matplotlib.test.Dockerfile", ["matplotlib"]), + ("sagemaker-headless-execution-driver.test.Dockerfile", ["sagemaker-headless-execution-driver"]), + ("scipy.test.Dockerfile", ["scipy"]), + ("numpy.test.Dockerfile", ["numpy"]), + ("boto3.test.Dockerfile", ["boto3"]), + ("pandas.test.Dockerfile", ["pandas"]), + ("sm-python-sdk.test.Dockerfile", ["sagemaker-python-sdk"]), + ("pytorch.examples.Dockerfile", ["pytorch"]), + ("tensorflow.examples.Dockerfile", ["tensorflow"]), + ("glue-sessions.test.Dockerfile", ["aws-glue-sessions"]), + ("jupyter-ai.test.Dockerfile", ["jupyter-ai"]), + ("jupyter-dash.test.Dockerfile", ["jupyter-dash"]), + ("jupyterlab-lsp.test.Dockerfile", ["jupyterlab-lsp"]), + ("jupyter-lsp-server.test.Dockerfile", ["jupyter-lsp-server"]), + ("sagemaker-code-editor.test.Dockerfile", ["sagemaker-code-editor"]), + ("notebook.test.Dockerfile", ["notebook"]), + ("glue-sessions.test.Dockerfile", ["aws-glue-sessions"]), + ("altair.test.Dockerfile", ["altair"]), + ("sagemaker-studio-analytics-extension.test.Dockerfile", ["sagemaker-studio-analytics-extension"]), + ("amazon-codewhisperer-jupyterlab-ext.test.Dockerfile", ["amazon-codewhisperer-jupyterlab-ext"]), + ("jupyterlab-git.test.Dockerfile", ["jupyterlab-git"]), + ("amazon-sagemaker-sql-magic.test.Dockerfile", ["amazon-sagemaker-sql-magic"]), + ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), + ("serve.test.Dockerfile", ["serve-langchain"]), + ], +) +def test_dockerfiles_for_gpu( + dockerfile_path: str, required_packages: List[str], local_image_version: str, use_gpu: bool +): + _validate_docker_images(dockerfile_path, required_packages, local_image_version, use_gpu, "gpu") # The following is a simple function to check whether the local machine has at least 1 GPU and some Nvidia driver @@ -86,80 +93,84 @@ def test_dockerfiles_for_gpu(dockerfile_path: str, required_packages: List[str], def _is_nvidia_drivers_available() -> bool: exitcode, output = subprocess.getstatusoutput("nvidia-smi --query-gpu=driver_version --format=csv,noheader --id=0") if exitcode == 0: - print(f'Found Nvidia driver version: {output}') + print(f"Found Nvidia driver version: {output}") else: - print(f'No Nvidia drivers found on the machine. Error output: {output}') + print(f"No Nvidia drivers found on the machine. Error output: {output}") return exitcode == 0 def _check_docker_file_existence(dockerfile_name: str, test_artifacts_path: str): - if not os.path.exists(f'{test_artifacts_path}/{dockerfile_name}'): - pytest.skip(f'Skipping test because {dockerfile_name} does not exist.') + if not os.path.exists(f"{test_artifacts_path}/{dockerfile_name}"): + pytest.skip(f"Skipping test because {dockerfile_name} does not exist.") -def _check_required_package_constraints(target_version: Version, required_packages: List[str], - image_type: str): +def _check_required_package_constraints(target_version: Version, required_packages: List[str], image_type: str): target_version_dir = get_dir_for_version(target_version) if not os.path.exists(target_version_dir): - pytest.skip(f'Skipping test because {target_version_dir} does not exist.') + pytest.skip(f"Skipping test because {target_version_dir} does not exist.") # fetch the env.out file for this image_type - env_out_file_name = next(config['env_out_filename'] for config in _image_generator_configs if - config['image_type'] == image_type) - env_out_path = f'{target_version_dir}/{env_out_file_name}' + env_out_file_name = next( + config["env_out_filename"] for config in _image_generator_configs if config["image_type"] == image_type + ) + env_out_path = f"{target_version_dir}/{env_out_file_name}" if not os.path.exists(env_out_path): - pytest.skip(f'Skipping test because {env_out_path} does not exist.') + pytest.skip(f"Skipping test because {env_out_path} does not exist.") target_match_spec_out = get_match_specs(env_out_path) for required_package in required_packages: if required_package not in target_match_spec_out: - pytest.skip(f'Skipping test because {required_package} is not present in {env_out_file_name}') + pytest.skip(f"Skipping test because {required_package} is not present in {env_out_file_name}") -def _validate_docker_images(dockerfile_path: str, required_packages: List[str], - local_image_version: str, use_gpu: bool, image_type: str): +def _validate_docker_images( + dockerfile_path: str, required_packages: List[str], local_image_version: str, use_gpu: bool, image_type: str +): target_version = get_semver(local_image_version) - test_artifacts_path = f'test/test_artifacts/v{str(target_version.major)}' + test_artifacts_path = f"test/test_artifacts/v{str(target_version.major)}" _check_docker_file_existence(dockerfile_path, test_artifacts_path) _check_required_package_constraints(target_version, required_packages, image_type) - image_tag_generator_from_config = next(config['image_tag_generator'] for config in - _image_generator_configs if config['image_type'] == - image_type) + image_tag_generator_from_config = next( + config["image_tag_generator"] for config in _image_generator_configs if config["image_type"] == image_type + ) docker_image_tag = image_tag_generator_from_config.format(image_version=local_image_version) - docker_image_identifier = f'localhost/sagemaker-distribution:{docker_image_tag}' - print(f'Will start running test for: {dockerfile_path} against: {docker_image_identifier}') + docker_image_identifier = f"localhost/sagemaker-distribution:{docker_image_tag}" + print(f"Will start running test for: {dockerfile_path} against: {docker_image_identifier}") try: - image, _ = _docker_client.images.build(path=test_artifacts_path, - dockerfile=dockerfile_path, shmsize='512000000', - tag=dockerfile_path.lower().replace('.', '-'), - rm=True, buildargs={'SAGEMAKER_DISTRIBUTION_IMAGE': docker_image_identifier}) + image, _ = _docker_client.images.build( + path=test_artifacts_path, + dockerfile=dockerfile_path, + shmsize="512000000", + tag=dockerfile_path.lower().replace(".", "-"), + rm=True, + buildargs={"SAGEMAKER_DISTRIBUTION_IMAGE": docker_image_identifier}, + ) except BuildError as e: for line in e.build_log: - if 'stream' in line: - print(line['stream'].strip()) + if "stream" in line: + print(line["stream"].strip()) # After printing the logs raise the exception (which is the old behavior) raise - print(f'Built a test image: {image.id}, will now execute its default CMD.') + print(f"Built a test image: {image.id}, will now execute its default CMD.") # Execute the new image once. Mark the current test successful/failed based on container's exit code. (We assume # that the image would have supplied the right entrypoint. device_requests = [] if use_gpu and _is_nvidia_drivers_available(): # Pass all available GPUs, if available. - device_requests.append(docker.types.DeviceRequest(count=-1, capabilities=[['gpu']])) + device_requests.append(docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])) # We assume that the image above would have supplied the right entrypoint, so we just run it as is. If the container # didn't execute successfully, the Docker client below will throw an error and fail the test. # A consequence of this design decision is that any test assertions should go inside the container's entry-point. - container = _docker_client.containers.run(image=image.id, detach=True, stderr=True, - device_requests=device_requests) + container = _docker_client.containers.run(image=image.id, detach=True, stderr=True, device_requests=device_requests) # Wait till container completes execution result = container.wait() exit_code = result["StatusCode"] if exit_code != 0: # Print STD out only during test failure - print(container.logs().decode('utf-8')) + print(container.logs().decode("utf-8")) # Remove the container. container.remove(force=True) _docker_client.images.remove(image=image.id, force=True) diff --git a/test/test_main.py b/test/test_main.py index 23451010..e1ed2752 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,33 +1,35 @@ from __future__ import absolute_import import base64 -import boto3 + import pytest pytestmark = pytest.mark.unit +import os +from unittest.mock import MagicMock, Mock, patch + +from changelog_generator import _derive_changeset +from config import _image_generator_configs from main import ( - create_and_get_semver_dir, + _get_config_for_image, _get_version_tags, + _push_images_upstream, + build_images, + create_and_get_semver_dir, create_major_version_artifacts, create_minor_version_artifacts, create_patch_version_artifacts, - build_images, - _push_images_upstream, - _get_config_for_image ) -from config import _image_generator_configs -from changelog_generator import _derive_changeset -from release_notes_generator import _get_image_type_package_metadata, \ - _get_package_to_image_type_mapping +from release_notes_generator import ( + _get_image_type_package_metadata, + _get_package_to_image_type_mapping, +) from utils import get_semver -import os -from unittest.mock import patch, Mock, MagicMock class CreateVersionArgs: - def __init__(self, runtime_version_upgrade_type, base_patch_version, - pre_release_identifier=None, force=False): + def __init__(self, runtime_version_upgrade_type, base_patch_version, pre_release_identifier=None, force=False): self.base_patch_version = base_patch_version self.runtime_version_upgrade_type = runtime_version_upgrade_type self.pre_release_identifier = pre_release_identifier @@ -42,101 +44,110 @@ def __init__(self, target_patch_version, target_ecr_repo=None, force=False): self.force = force -def _create_docker_cpu_env_in_file(file_path, required_package='conda-forge::ipykernel'): - with open(file_path, 'w') as env_in_file: - env_in_file.write(f'{required_package}\n') +def _create_docker_cpu_env_in_file(file_path, required_package="conda-forge::ipykernel"): + with open(file_path, "w") as env_in_file: + env_in_file.write(f"{required_package}\n") -def _create_docker_cpu_env_out_file(file_path, - package_metadata='https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.3-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67'): - with open(file_path, 'w') as env_out_file: - env_out_file.write(f'''# This file may be used to create an environment using: +def _create_docker_cpu_env_out_file( + file_path, + package_metadata="https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.3-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67", +): + with open(file_path, "w") as env_out_file: + env_out_file.write( + f"""# This file may be used to create an environment using: # $ conda create --name --file # platform: linux-64 @EXPLICIT -{package_metadata}\n''') +{package_metadata}\n""" + ) def _create_docker_gpu_env_in_file(file_path): - with open(file_path, 'w') as env_in_file: - env_in_file.write('conda-forge::numpy\n') + with open(file_path, "w") as env_in_file: + env_in_file.write("conda-forge::numpy\n") -def _create_additional_packages_gpu_env_in_file(file_path, package_metadata='conda-forge::sagemaker-python-sdk'): - with open(file_path, 'w') as env_in_file: - env_in_file.write(package_metadata + '\n') +def _create_additional_packages_gpu_env_in_file(file_path, package_metadata="conda-forge::sagemaker-python-sdk"): + with open(file_path, "w") as env_in_file: + env_in_file.write(package_metadata + "\n") def _create_docker_gpu_env_out_file(file_path): - with open(file_path, 'w') as env_out_file: - env_out_file.write('''# This file may be used to create an environment using: + with open(file_path, "w") as env_out_file: + env_out_file.write( + """# This file may be used to create an environment using: # $ conda create --name --file # platform: linux-64 @EXPLICIT -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.2-py38h10c12cc_0.conda#05592c85b9f6931dc2df1e80c0d56294\n''') +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.2-py38h10c12cc_0.conda#05592c85b9f6931dc2df1e80c0d56294\n""" + ) def _create_template_docker_file(file_path): - with open(file_path, 'w') as docker_file: - docker_file.write('''ARG TAG_FOR_BASE_MICROMAMBA_IMAGE - FROM mambaorg / micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE\ntemplate_dockerfile\n''') + with open(file_path, "w") as docker_file: + docker_file.write( + """ARG TAG_FOR_BASE_MICROMAMBA_IMAGE + FROM mambaorg / micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE\ntemplate_dockerfile\n""" + ) def _create_prev_docker_file(file_path): - with open(file_path, 'w') as docker_file: - docker_file.write('''ARG TAG_FOR_BASE_MICROMAMBA_IMAGE - FROM mambaorg / micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE\nprevious_dockerfile\n''') + with open(file_path, "w") as docker_file: + docker_file.write( + """ARG TAG_FOR_BASE_MICROMAMBA_IMAGE + FROM mambaorg / micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE\nprevious_dockerfile\n""" + ) -def _create_new_version_artifacts_helper(mocker, tmp_path, version, target_version): +def _create_new_version_artifacts_helper(mocker, tmp_path, version, target_version): def mock_get_dir_for_version(base_version): - pre_release_suffix = '/v' + str(base_version) if base_version.prerelease else '' - version_string = f'v{base_version.major}.{base_version.minor}.{base_version.patch}' + \ - pre_release_suffix + pre_release_suffix = "/v" + str(base_version) if base_version.prerelease else "" + version_string = f"v{base_version.major}.{base_version.minor}.{base_version.patch}" + pre_release_suffix # get_dir_for_version returns a str and not PosixPath return str(tmp_path) + "/" + version_string - mocker.patch('main.get_dir_for_version', side_effect=mock_get_dir_for_version) + mocker.patch("main.get_dir_for_version", side_effect=mock_get_dir_for_version) input_version = get_semver(version) # Create directory for base version input_version_dir = create_and_get_semver_dir(input_version) # Create env.in and env.out for base version - _create_docker_cpu_env_in_file(input_version_dir + '/cpu.env.in') - _create_docker_gpu_env_in_file(input_version_dir + '/gpu.env.in') - _create_docker_cpu_env_out_file(input_version_dir + '/cpu.env.out') - _create_docker_gpu_env_out_file(input_version_dir + '/gpu.env.out') - _create_prev_docker_file(input_version_dir + '/Dockerfile') - os.makedirs(tmp_path / 'template') + _create_docker_cpu_env_in_file(input_version_dir + "/cpu.env.in") + _create_docker_gpu_env_in_file(input_version_dir + "/gpu.env.in") + _create_docker_cpu_env_out_file(input_version_dir + "/cpu.env.out") + _create_docker_gpu_env_out_file(input_version_dir + "/gpu.env.out") + _create_prev_docker_file(input_version_dir + "/Dockerfile") + os.makedirs(tmp_path / "template") next_version = get_semver(target_version) - next_major_version = 'v' + str(next_version.major) - os.makedirs(tmp_path / 'template' / next_major_version) + next_major_version = "v" + str(next_version.major) + os.makedirs(tmp_path / "template" / next_major_version) if next_version.major == 1: # Create dirs directory under template - os.makedirs(tmp_path / 'template' / next_major_version / 'dirs') - _create_template_docker_file(tmp_path / 'template' / next_major_version / 'Dockerfile') + os.makedirs(tmp_path / "template" / next_major_version / "dirs") + _create_template_docker_file(tmp_path / "template" / next_major_version / "Dockerfile") -def _create_additional_packages_env_in_file_helper(mocker, tmp_path, version, - include_additional_package=False, - use_existing_package_as_additional_package=False): +def _create_additional_packages_env_in_file_helper( + mocker, tmp_path, version, include_additional_package=False, use_existing_package_as_additional_package=False +): if include_additional_package: + def mock_get_dir_for_version(base_version): - pre_release_suffix = '/v' + str(base_version) if base_version.prerelease else '' - version_string = f'v{base_version.major}.{base_version.minor}.{base_version.patch}' + \ - pre_release_suffix + pre_release_suffix = "/v" + str(base_version) if base_version.prerelease else "" + version_string = f"v{base_version.major}.{base_version.minor}.{base_version.patch}" + pre_release_suffix # get_dir_for_version returns a str and not PosixPath return str(tmp_path) + "/" + version_string - mocker.patch('main.get_dir_for_version', side_effect=mock_get_dir_for_version) + mocker.patch("main.get_dir_for_version", side_effect=mock_get_dir_for_version) input_version = get_semver(version) # Create directory for new version input_version_dir = create_and_get_semver_dir(input_version) - additional_env_in_file_path = input_version_dir + '/gpu.additional_packages_env.in' + additional_env_in_file_path = input_version_dir + "/gpu.additional_packages_env.in" if use_existing_package_as_additional_package: # Using an older version of numpy. - _create_additional_packages_gpu_env_in_file(additional_env_in_file_path, - 'conda-forge::numpy[version=\'>=1.0.4,' - '<1.1.0\']') + _create_additional_packages_gpu_env_in_file( + additional_env_in_file_path, "conda-forge::numpy[version='>=1.0.4," "<1.1.0']" + ) else: _create_additional_packages_gpu_env_in_file(additional_env_in_file_path) @@ -144,54 +155,53 @@ def mock_get_dir_for_version(base_version): def test_get_semver_version(): # Test invalid version string. with pytest.raises(Exception): - get_semver('1.124.5d') + get_semver("1.124.5d") # Test version string with build with pytest.raises(Exception): - get_semver('1.124.5+25') + get_semver("1.124.5+25") # Test version string with build and prerelease with pytest.raises(Exception): - get_semver('1.124.5-prerelease+25') + get_semver("1.124.5-prerelease+25") # Test valid version string. - assert get_semver('1.124.5') is not None - assert get_semver('1.124.5-beta') is not None + assert get_semver("1.124.5") is not None + assert get_semver("1.124.5-beta") is not None def test_new_version_artifacts_for_an_input_prerelease_version(): - input_version = '1.23.0-beta' - args = CreateVersionArgs('patch', input_version) + input_version = "1.23.0-beta" + args = CreateVersionArgs("patch", input_version) with pytest.raises(Exception): create_patch_version_artifacts(args) - args = CreateVersionArgs('minor', input_version) + args = CreateVersionArgs("minor", input_version) with pytest.raises(Exception): create_minor_version_artifacts(args) - args = CreateVersionArgs('major', input_version) + args = CreateVersionArgs("major", input_version) with pytest.raises(Exception): create_major_version_artifacts(args) -@patch('os.path.exists') -@patch('os.path.isdir') -@patch('shutil.rmtree') -@patch('os.makedirs') -@patch('os.listdir') -def test_create_and_get_semver_dir(mock_list_dir, mock_make_dirs, mock_rmtree, - mock_path_is_dir, mock_path_exists): +@patch("os.path.exists") +@patch("os.path.isdir") +@patch("shutil.rmtree") +@patch("os.makedirs") +@patch("os.listdir") +def test_create_and_get_semver_dir(mock_list_dir, mock_make_dirs, mock_rmtree, mock_path_is_dir, mock_path_exists): # case 1: Directory exists and exist_ok is False => Throws Exception mock_path_exists.return_value = True with pytest.raises(Exception): - create_and_get_semver_dir(get_semver('1.124.5')) + create_and_get_semver_dir(get_semver("1.124.5")) # Case 2: Instead of a directory in the path, a file exists. mock_path_is_dir.return_value = False with pytest.raises(Exception): - create_and_get_semver_dir(get_semver('1.124.5'), True) + create_and_get_semver_dir(get_semver("1.124.5"), True) # Happy case mock_path_is_dir.return_value = True mock_list_dir.return_value = [] - assert create_and_get_semver_dir(get_semver('1.124.5'), True) is not None + assert create_and_get_semver_dir(get_semver("1.124.5"), True) is not None def test_create_new_version_artifacts_for_invalid_upgrade_type(): - input = CreateVersionArgs('test_upgrade', '1.2.3') + input = CreateVersionArgs("test_upgrade", "1.2.3") with pytest.raises(Exception): create_major_version_artifacts(input) with pytest.raises(Exception): @@ -200,60 +210,62 @@ def test_create_new_version_artifacts_for_invalid_upgrade_type(): create_patch_version_artifacts(input) -def _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, - pre_release_identifier=None, - include_additional_package=False, - use_existing_package_as_additional_package=False): - input_version = '0.2.5' - next_version = get_semver('0.2.6') - new_version_dir = tmp_path / ('v' + str(next_version)) - base_version_dir = tmp_path / ('v' + input_version) +def _create_and_assert_patch_version_upgrade( + rel_path, + mocker, + tmp_path, + pre_release_identifier=None, + include_additional_package=False, + use_existing_package_as_additional_package=False, +): + input_version = "0.2.5" + next_version = get_semver("0.2.6") + new_version_dir = tmp_path / ("v" + str(next_version)) + base_version_dir = tmp_path / ("v" + input_version) if pre_release_identifier: - new_version_dir = new_version_dir / ('v' + str(next_version) + '-' + pre_release_identifier) - next_major_version_dir_name = 'v' + str(next_version.major) + new_version_dir = new_version_dir / ("v" + str(next_version) + "-" + pre_release_identifier) + next_major_version_dir_name = "v" + str(next_version.major) if next_version.major == 0: - rel_path.side_effect = [str(base_version_dir / 'Dockerfile')] + rel_path.side_effect = [str(base_version_dir / "Dockerfile")] else: - rel_path.side_effect = [str(base_version_dir, 'Dockerfile'), str(base_version_dir / 'dirs')] + rel_path.side_effect = [str(base_version_dir, "Dockerfile"), str(base_version_dir / "dirs")] _create_new_version_artifacts_helper(mocker, tmp_path, input_version, str(next_version)) - _create_additional_packages_env_in_file_helper(mocker, tmp_path, str(next_version), - include_additional_package, - use_existing_package_as_additional_package) + _create_additional_packages_env_in_file_helper( + mocker, tmp_path, str(next_version), include_additional_package, use_existing_package_as_additional_package + ) if include_additional_package: - args = CreateVersionArgs('patch', input_version, - pre_release_identifier=pre_release_identifier, force=True) + args = CreateVersionArgs("patch", input_version, pre_release_identifier=pre_release_identifier, force=True) else: - args = CreateVersionArgs('patch', input_version, - pre_release_identifier=pre_release_identifier) + args = CreateVersionArgs("patch", input_version, pre_release_identifier=pre_release_identifier) create_patch_version_artifacts(args) # Assert new version directory is created assert os.path.exists(new_version_dir) # Check cpu.env.in and gpu.env.in exists in the new directory new_version_dir_files = os.listdir(new_version_dir) - assert 'cpu.env.in' in new_version_dir_files - assert 'gpu.env.in' in new_version_dir_files - assert 'Dockerfile' in new_version_dir_files - with open(new_version_dir / 'Dockerfile', 'r') as f: - assert 'previous_dockerfile' in f.read() + assert "cpu.env.in" in new_version_dir_files + assert "gpu.env.in" in new_version_dir_files + assert "Dockerfile" in new_version_dir_files + with open(new_version_dir / "Dockerfile", "r") as f: + assert "previous_dockerfile" in f.read() if next_version.major >= 1: - assert 'dirs' in new_version_dir_files + assert "dirs" in new_version_dir_files if include_additional_package: - assert 'gpu.additional_packages_env.in' in new_version_dir_files - with open(new_version_dir / 'cpu.env.in', 'r') as f: + assert "gpu.additional_packages_env.in" in new_version_dir_files + with open(new_version_dir / "cpu.env.in", "r") as f: contents = f.read() # version of ipykernel in cpu.env.out is 6.21.3 # so we expect the version string to be >=6.21.3,<6.22.0 - expected_version_string = '>=6.21.3,<6.22.0' + expected_version_string = ">=6.21.3,<6.22.0" assert contents.find(expected_version_string) != -1 - with open(new_version_dir / 'gpu.env.in', 'r') as f: + with open(new_version_dir / "gpu.env.in", "r") as f: contents = f.read() # version of numpy in gpu.env.out is 1.24.2 # so we expect the version string to be >=1.24.2,<1.25.0 - expected_version_string = '>=1.24.2,<1.25.0' + expected_version_string = ">=1.24.2,<1.25.0" assert contents.find(expected_version_string) != -1 if include_additional_package and not use_existing_package_as_additional_package: - assert contents.find('sagemaker-python-sdk') != -1 + assert contents.find("sagemaker-python-sdk") != -1 @patch("os.path.relpath") @@ -262,83 +274,79 @@ def test_create_new_version_artifacts_for_patch_version_upgrade(rel_path, mocker @patch("os.path.relpath") -def test_create_new_patch_version_upgrade_with_existing_package_as_additional_packages(rel_path, - mocker, - tmp_path): - _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True, - use_existing_package_as_additional_package=True) +def test_create_new_patch_version_upgrade_with_existing_package_as_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_patch_version_upgrade( + rel_path, mocker, tmp_path, include_additional_package=True, use_existing_package_as_additional_package=True + ) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_patch_version_upgrade_with_additional_packages( - rel_path, mocker, tmp_path): - _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True) +def test_create_new_version_artifacts_for_patch_version_upgrade_with_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, include_additional_package=True) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_patch_version_upgrade_with_prerelease(rel_path, mocker, - tmp_path): - _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, 'beta') - - -def _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, - pre_release_identifier=None, - include_additional_package=False, - use_existing_package_as_additional_package=False - ): - input_version = '1.2.5' - next_version = get_semver('1.3.0') - new_version_dir = tmp_path / ('v' + str(next_version)) +def test_create_new_version_artifacts_for_patch_version_upgrade_with_prerelease(rel_path, mocker, tmp_path): + _create_and_assert_patch_version_upgrade(rel_path, mocker, tmp_path, "beta") + + +def _create_and_assert_minor_version_upgrade( + rel_path, + mocker, + tmp_path, + pre_release_identifier=None, + include_additional_package=False, + use_existing_package_as_additional_package=False, +): + input_version = "1.2.5" + next_version = get_semver("1.3.0") + new_version_dir = tmp_path / ("v" + str(next_version)) if pre_release_identifier: - new_version_dir = new_version_dir / ('v' + str(next_version) + '-' + pre_release_identifier) - next_major_version_dir_name = 'v' + str(next_version.major) + new_version_dir = new_version_dir / ("v" + str(next_version) + "-" + pre_release_identifier) + next_major_version_dir_name = "v" + str(next_version.major) if next_version.major == 0: - rel_path.side_effect = [str(tmp_path / 'template' / next_major_version_dir_name / - 'Dockerfile')] + rel_path.side_effect = [str(tmp_path / "template" / next_major_version_dir_name / "Dockerfile")] else: - rel_path.side_effect = [str(tmp_path / 'template' / next_major_version_dir_name / - 'Dockerfile'), str(tmp_path / 'template' / - next_major_version_dir_name / 'dirs')] - _create_new_version_artifacts_helper(mocker, tmp_path, input_version, '1.3.0') - _create_additional_packages_env_in_file_helper(mocker, tmp_path, '1.3.0', - include_additional_package, - use_existing_package_as_additional_package) + rel_path.side_effect = [ + str(tmp_path / "template" / next_major_version_dir_name / "Dockerfile"), + str(tmp_path / "template" / next_major_version_dir_name / "dirs"), + ] + _create_new_version_artifacts_helper(mocker, tmp_path, input_version, "1.3.0") + _create_additional_packages_env_in_file_helper( + mocker, tmp_path, "1.3.0", include_additional_package, use_existing_package_as_additional_package + ) if include_additional_package: - args = CreateVersionArgs('minor', input_version, - pre_release_identifier=pre_release_identifier, force=True) + args = CreateVersionArgs("minor", input_version, pre_release_identifier=pre_release_identifier, force=True) else: - args = CreateVersionArgs('minor', input_version, - pre_release_identifier=pre_release_identifier) + args = CreateVersionArgs("minor", input_version, pre_release_identifier=pre_release_identifier) create_minor_version_artifacts(args) # Assert new version directory is created assert os.path.exists(new_version_dir) # Check cpu.env.in and gpu.env.in exists in the new directory new_version_dir_files = os.listdir(new_version_dir) - assert 'cpu.env.in' in new_version_dir_files - assert 'gpu.env.in' in new_version_dir_files - assert 'Dockerfile' in new_version_dir_files - with open(new_version_dir / 'Dockerfile', 'r') as f: - assert 'template_dockerfile' in f.read() + assert "cpu.env.in" in new_version_dir_files + assert "gpu.env.in" in new_version_dir_files + assert "Dockerfile" in new_version_dir_files + with open(new_version_dir / "Dockerfile", "r") as f: + assert "template_dockerfile" in f.read() if next_version.major >= 1: - assert 'dirs' in new_version_dir_files + assert "dirs" in new_version_dir_files if include_additional_package: - assert 'gpu.additional_packages_env.in' in new_version_dir_files - with open(new_version_dir / 'cpu.env.in', 'r') as f: + assert "gpu.additional_packages_env.in" in new_version_dir_files + with open(new_version_dir / "cpu.env.in", "r") as f: contents = f.read() # version of ipykernel in cpu.env.out is 6.21.3 # so we expect the version string to be >=6.21.3,<7.0.0 - expected_version_string = '>=6.21.3,<7.0.0' + expected_version_string = ">=6.21.3,<7.0.0" assert contents.find(expected_version_string) != -1 - with open(new_version_dir / 'gpu.env.in', 'r') as f: + with open(new_version_dir / "gpu.env.in", "r") as f: contents = f.read() # version of numpy in gpu.env.out is 1.24.2 # so we expect the version string to be >=1.24.2,<2.0.0 - expected_version_string = '>=1.24.2,<2.0.0' + expected_version_string = ">=1.24.2,<2.0.0" assert contents.find(expected_version_string) != -1 if include_additional_package and not use_existing_package_as_additional_package: - assert contents.find('sagemaker-python-sdk') != -1 + assert contents.find("sagemaker-python-sdk") != -1 @patch("os.path.relpath") @@ -347,84 +355,79 @@ def test_create_new_version_artifacts_for_minor_version_upgrade(rel_path, mocker @patch("os.path.relpath") -def test_create_new_minor_version_upgrade_with_existing_package_as_additional_packages(rel_path, - mocker, - tmp_path): - _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True, - use_existing_package_as_additional_package=True) +def test_create_new_minor_version_upgrade_with_existing_package_as_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_minor_version_upgrade( + rel_path, mocker, tmp_path, include_additional_package=True, use_existing_package_as_additional_package=True + ) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_minor_version_upgrade_with_additional_packages( - rel_path, mocker, tmp_path): - _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True) - - +def test_create_new_version_artifacts_for_minor_version_upgrade_with_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, include_additional_package=True) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_minor_version_upgrade_with_prerelease(rel_path, mocker, - tmp_path): - _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, 'beta') - - -def _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, - pre_release_identifier=None, - include_additional_package=False, - use_existing_package_as_additional_package=False): - input_version = '0.2.5' - next_version = get_semver('1.0.0') - new_version_dir = tmp_path / ('v' + str(next_version)) +def test_create_new_version_artifacts_for_minor_version_upgrade_with_prerelease(rel_path, mocker, tmp_path): + _create_and_assert_minor_version_upgrade(rel_path, mocker, tmp_path, "beta") + + +def _create_and_assert_major_version_upgrade( + rel_path, + mocker, + tmp_path, + pre_release_identifier=None, + include_additional_package=False, + use_existing_package_as_additional_package=False, +): + input_version = "0.2.5" + next_version = get_semver("1.0.0") + new_version_dir = tmp_path / ("v" + str(next_version)) if pre_release_identifier: - new_version_dir = new_version_dir / ('v' + str(next_version) + '-' + pre_release_identifier) - next_major_version_dir_name = 'v' + str(next_version.major) + new_version_dir = new_version_dir / ("v" + str(next_version) + "-" + pre_release_identifier) + next_major_version_dir_name = "v" + str(next_version.major) if next_version.major == 0: - rel_path.side_effect = [str(tmp_path / 'template' / next_major_version_dir_name / - 'Dockerfile')] + rel_path.side_effect = [str(tmp_path / "template" / next_major_version_dir_name / "Dockerfile")] else: - rel_path.side_effect = [str(tmp_path / 'template' / next_major_version_dir_name / - 'Dockerfile'), str(tmp_path / 'template' / - next_major_version_dir_name / 'dirs')] + rel_path.side_effect = [ + str(tmp_path / "template" / next_major_version_dir_name / "Dockerfile"), + str(tmp_path / "template" / next_major_version_dir_name / "dirs"), + ] _create_new_version_artifacts_helper(mocker, tmp_path, input_version, str(next_version)) - _create_additional_packages_env_in_file_helper(mocker, tmp_path, str(next_version), - include_additional_package, - use_existing_package_as_additional_package) + _create_additional_packages_env_in_file_helper( + mocker, tmp_path, str(next_version), include_additional_package, use_existing_package_as_additional_package + ) if include_additional_package: - args = CreateVersionArgs('major', input_version, - pre_release_identifier=pre_release_identifier, force=True) + args = CreateVersionArgs("major", input_version, pre_release_identifier=pre_release_identifier, force=True) else: - args = CreateVersionArgs('major', input_version, - pre_release_identifier=pre_release_identifier) + args = CreateVersionArgs("major", input_version, pre_release_identifier=pre_release_identifier) create_major_version_artifacts(args) # Assert new version directory is created assert os.path.exists(new_version_dir) # Check cpu.env.in and gpu.env.in exists in the new directory new_version_dir_files = os.listdir(new_version_dir) - assert 'cpu.env.in' in new_version_dir_files - assert 'gpu.env.in' in new_version_dir_files - assert 'Dockerfile' in new_version_dir_files - with open(new_version_dir / 'Dockerfile', 'r') as f: - assert 'template_dockerfile' in f.read() + assert "cpu.env.in" in new_version_dir_files + assert "gpu.env.in" in new_version_dir_files + assert "Dockerfile" in new_version_dir_files + with open(new_version_dir / "Dockerfile", "r") as f: + assert "template_dockerfile" in f.read() if next_version.major >= 1: - assert 'dirs' in new_version_dir_files + assert "dirs" in new_version_dir_files if include_additional_package: - assert 'gpu.additional_packages_env.in' in new_version_dir_files - with open(new_version_dir / 'cpu.env.in', 'r') as f: + assert "gpu.additional_packages_env.in" in new_version_dir_files + with open(new_version_dir / "cpu.env.in", "r") as f: contents = f.read() # version of ipykernel in cpu.env.out is 6.21.3 # so we expect the version string to be >=6.21.3, - expected_version_string = '>=6.21.3\'' + expected_version_string = ">=6.21.3'" assert contents.find(expected_version_string) != -1 - with open(new_version_dir / 'gpu.env.in', 'r') as f: + with open(new_version_dir / "gpu.env.in", "r") as f: contents = f.read() # version of numpy in gpu.env.out is 1.24.2 # so we expect the version string to be >=1.24.2, - expected_version_string = '>=1.24.2\'' + expected_version_string = ">=1.24.2'" assert contents.find(expected_version_string) != -1 if include_additional_package and not use_existing_package_as_additional_package: - assert contents.find('sagemaker-python-sdk') != -1 + assert contents.find("sagemaker-python-sdk") != -1 @patch("os.path.relpath") @@ -433,192 +436,187 @@ def test_create_new_version_artifacts_for_major_version_upgrade(rel_path, mocker @patch("os.path.relpath") -def test_create_new_major_version_upgrade_with_existing_package_as_additional_packages(rel_path, - mocker, - tmp_path): - _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True, - use_existing_package_as_additional_package=True) +def test_create_new_major_version_upgrade_with_existing_package_as_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_major_version_upgrade( + rel_path, mocker, tmp_path, include_additional_package=True, use_existing_package_as_additional_package=True + ) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_major_version_upgrade_with_additional_packages( - rel_path, mocker, tmp_path): - _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, - include_additional_package=True) - +def test_create_new_version_artifacts_for_major_version_upgrade_with_additional_packages(rel_path, mocker, tmp_path): + _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, include_additional_package=True) @patch("os.path.relpath") -def test_create_new_version_artifacts_for_major_version_upgrade_with_prerelease(rel_path, mocker, - tmp_path): - _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, 'beta') +def test_create_new_version_artifacts_for_major_version_upgrade_with_prerelease(rel_path, mocker, tmp_path): + _create_and_assert_major_version_upgrade(rel_path, mocker, tmp_path, "beta") def test_build_images(mocker, tmp_path): - mock_docker_from_env = MagicMock(name='_docker_client') - mocker.patch('main._docker_client', new=mock_docker_from_env) - version = '1.124.5' + mock_docker_from_env = MagicMock(name="_docker_client") + mocker.patch("main._docker_client", new=mock_docker_from_env) + version = "1.124.5" args = BuildImageArgs(version) def mock_get_dir_for_version(base_version): - version_string = f'v{base_version.major}.{base_version.minor}.{base_version.patch}' + version_string = f"v{base_version.major}.{base_version.minor}.{base_version.patch}" # get_dir_for_version returns a str and not a PosixPath return str(tmp_path) + "/" + version_string - mocker.patch('main.get_dir_for_version', side_effect=mock_get_dir_for_version) + mocker.patch("main.get_dir_for_version", side_effect=mock_get_dir_for_version) input_version = get_semver(version) # Create directory for base version input_version_dir = create_and_get_semver_dir(input_version) # Create env.in for base version - _create_docker_cpu_env_in_file(input_version_dir + '/cpu.env.in') - _create_docker_cpu_env_in_file(input_version_dir + '/gpu.env.in') - _create_prev_docker_file(input_version_dir + '/Dockerfile') + _create_docker_cpu_env_in_file(input_version_dir + "/cpu.env.in") + _create_docker_cpu_env_in_file(input_version_dir + "/gpu.env.in") + _create_prev_docker_file(input_version_dir + "/Dockerfile") # Assert env.out doesn't exist - assert os.path.exists(input_version_dir + '/cpu.env.out') is False - assert os.path.exists(input_version_dir + '/gpu.env.out') is False + assert os.path.exists(input_version_dir + "/cpu.env.out") is False + assert os.path.exists(input_version_dir + "/gpu.env.out") is False mock_image_1 = Mock() - mock_image_1.id.return_value = 'img1' + mock_image_1.id.return_value = "img1" mock_image_2 = Mock() - mock_image_2.id.return_value = 'img2' - mock_docker_from_env.images.build.side_effect = [(mock_image_1, 'logs1'), (mock_image_2, 'logs2')] - mock_docker_from_env.containers.run.side_effect = ['container_logs1'.encode('utf-8'), - 'container_logs2'.encode('utf-8')] + mock_image_2.id.return_value = "img2" + mock_docker_from_env.images.build.side_effect = [(mock_image_1, "logs1"), (mock_image_2, "logs2")] + mock_docker_from_env.containers.run.side_effect = [ + "container_logs1".encode("utf-8"), + "container_logs2".encode("utf-8"), + ] # Invoke build images build_images(args) # Assert env.out exists - assert os.path.exists(input_version_dir + '/cpu.env.out') - assert os.path.exists(input_version_dir + '/gpu.env.out') + assert os.path.exists(input_version_dir + "/cpu.env.out") + assert os.path.exists(input_version_dir + "/gpu.env.out") # Validate the contents of env.out actual_output = set() - with open(input_version_dir + '/cpu.env.out', 'r') as f: + with open(input_version_dir + "/cpu.env.out", "r") as f: actual_output.add(f.read()) - with open(input_version_dir + '/gpu.env.out', 'r') as f: + with open(input_version_dir + "/gpu.env.out", "r") as f: actual_output.add(f.read()) - expected_output = {'container_logs1', 'container_logs2'} + expected_output = {"container_logs1", "container_logs2"} assert actual_output == expected_output -@patch('os.path.exists') +@patch("os.path.exists") def test_get_version_tags(mock_path_exists): - version = get_semver('1.124.5') - file_name = 'cpu.env.out' + version = get_semver("1.124.5") + file_name = "cpu.env.out" # case 1: The given version is the latest for patch, minor and major mock_path_exists.side_effect = [False, False, False] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124', '1', 'latest'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124", "1", "latest"] # case 2: The given version is the latest for patch, minor but not major # case 2.1 The major version is a prerelease version mock_path_exists.side_effect = [False, False, True, False] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124', '1', 'latest'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124", "1", "latest"] # case 2.2 The major version is not a prerelease version mock_path_exists.side_effect = [False, False, True, True] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124', '1'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124", "1"] # case 3: The given version is the latest for patch and major but not for minor # case 3.1 The minor version is a prerelease version (we need to mock path.exists for major # version twice - one for the actual directory, one for the docker file) mock_path_exists.side_effect = [False, True, False, True, True] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124', '1'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124", "1"] # case 3.2 The minor version is not a prerelease version mock_path_exists.side_effect = [False, True, True] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124"] # case 4: The given version is not the latest for patch, minor, major # case 4.1 The patch version is a prerelease version (we need to mock path.exists for minor # and major twice - one for the actual directory, one for the docker file) mock_path_exists.side_effect = [True, False, True, True, True, True] - assert _get_version_tags(version, file_name) == ['1.124.5', '1.124'] + assert _get_version_tags(version, file_name) == ["1.124.5", "1.124"] # case 4.2 The patch version is not a prerelease version mock_path_exists.side_effect = [True, True] - assert _get_version_tags(version, file_name) == ['1.124.5'] + assert _get_version_tags(version, file_name) == ["1.124.5"] # case 5: The given version includes a prerelease identifier - assert _get_version_tags(get_semver('1.124.5-beta'), file_name) == ['1.124.5-beta'] + assert _get_version_tags(get_semver("1.124.5-beta"), file_name) == ["1.124.5-beta"] def _test_push_images_upstream(mocker, repository): boto3_client = MagicMock() - expected_client_name = 'ecr-public' if repository.startswith('public.ecr.aws') else 'ecr' - boto3_mocker = mocker.patch('boto3.client', return_value=boto3_client) - mock_docker_from_env = MagicMock(name='_docker_client') - mocker.patch('main._docker_client', new=mock_docker_from_env) - authorization_token_string = 'username:password' - encoded_authorization_token = base64.b64encode(authorization_token_string.encode('ascii')) - authorization_data = { - 'authorizationToken': encoded_authorization_token - } - if expected_client_name == 'ecr': + expected_client_name = "ecr-public" if repository.startswith("public.ecr.aws") else "ecr" + boto3_mocker = mocker.patch("boto3.client", return_value=boto3_client) + mock_docker_from_env = MagicMock(name="_docker_client") + mocker.patch("main._docker_client", new=mock_docker_from_env) + authorization_token_string = "username:password" + encoded_authorization_token = base64.b64encode(authorization_token_string.encode("ascii")) + authorization_data = {"authorizationToken": encoded_authorization_token} + if expected_client_name == "ecr": # Private ECR client returns a list of authorizationData. authorization_data = [authorization_data] - boto3_client.get_authorization_token.return_value = { - 'authorizationData': authorization_data - } + boto3_client.get_authorization_token.return_value = {"authorizationData": authorization_data} mock_docker_from_env.images.push.side_effect = None - _push_images_upstream([{'repository': repository, 'tag': '0.1'}], 'us-west-2') + _push_images_upstream([{"repository": repository, "tag": "0.1"}], "us-west-2") assert boto3_mocker.call_args[0][0] == expected_client_name def test_push_images_upstream_for_private_ecr_repository(mocker): - repository = 'aws_account_id.dkr.ecr.us-west-2.amazonaws.com/my-repository' + repository = "aws_account_id.dkr.ecr.us-west-2.amazonaws.com/my-repository" _test_push_images_upstream(mocker, repository) def test_push_images_upstream_for_public_ecr_repository(mocker): - repository = 'public.ecr.aws/registry_alias/my-repository' + repository = "public.ecr.aws/registry_alias/my-repository" _test_push_images_upstream(mocker, repository) -@patch('os.path.exists') +@patch("os.path.exists") def test_get_build_config_for_image(mock_path_exists, tmp_path): - - input_version_dir = str(tmp_path) + '/v2.0.0' + input_version_dir = str(tmp_path) + "/v2.0.0" image_generator_config = _image_generator_configs[0] # Case 1: Mock os.path.exists to return False mock_path_exists.return_value = False - assert image_generator_config == _get_config_for_image(input_version_dir, - image_generator_config, False) + assert image_generator_config == _get_config_for_image(input_version_dir, image_generator_config, False) # Case 2: Mock os.path.exists to return True but force is True mock_path_exists.return_value = True - assert image_generator_config == _get_config_for_image(input_version_dir, - image_generator_config, True) + assert image_generator_config == _get_config_for_image(input_version_dir, image_generator_config, True) # Case 3: Mock os.path.exists to return True and force is False mock_path_exists.return_value = True response = _get_config_for_image(input_version_dir, image_generator_config, False) - assert response['build_args']['ENV_IN_FILENAME'] == image_generator_config["env_out_filename"] - assert 'ARG_BASED_ENV_IN_FILENAME' not in response['build_args'] + assert response["build_args"]["ENV_IN_FILENAME"] == image_generator_config["env_out_filename"] + assert "ARG_BASED_ENV_IN_FILENAME" not in response["build_args"] def test_derive_changeset(tmp_path): - target_version_dir = str(tmp_path / 'v1.0.6') - source_version_dir = str(tmp_path / 'v1.0.5') + target_version_dir = str(tmp_path / "v1.0.6") + source_version_dir = str(tmp_path / "v1.0.5") os.makedirs(target_version_dir) os.makedirs(source_version_dir) # Create env.in of the source version _create_docker_cpu_env_in_file(source_version_dir + "/cpu.env.in") # Create env.out of the source version - _create_docker_cpu_env_out_file(source_version_dir + "/cpu.env.out", - package_metadata='https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.3-pyh210e3f_0.conda#8c1f6bf32a6ca81232c4853d4165ca67') + _create_docker_cpu_env_out_file( + source_version_dir + "/cpu.env.out", + package_metadata="https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.3-pyh210e3f_0.conda#8c1f6bf32a6ca81232c4853d4165ca67", + ) # Create env.in of the target version, which has additional dependency on boto3 - target_env_in_packages = 'conda-forge::ipykernel\nconda-forge::boto3' + target_env_in_packages = "conda-forge::ipykernel\nconda-forge::boto3" _create_docker_cpu_env_in_file(target_version_dir + "/cpu.env.in", required_package=target_env_in_packages) - target_env_out_packages = 'https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.6-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67\n' \ - 'https://conda.anaconda.org/conda-forge/linux-64/boto3-1.2-cuda112py38hd_0.conda#8c1f6bf32a6ca81232c4853d4165ca67' + target_env_out_packages = ( + "https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.6-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67\n" + "https://conda.anaconda.org/conda-forge/linux-64/boto3-1.2-cuda112py38hd_0.conda#8c1f6bf32a6ca81232c4853d4165ca67" + ) _create_docker_cpu_env_out_file(target_version_dir + "/cpu.env.out", package_metadata=target_env_out_packages) - expected_upgrades = {'ipykernel': ['6.21.3', '6.21.6']} - expected_new_packages = {'boto3': '1.2'} - actual_upgrades, actual_new_packages = _derive_changeset(target_version_dir, - source_version_dir, - _image_generator_configs[1]) + expected_upgrades = {"ipykernel": ["6.21.3", "6.21.6"]} + expected_new_packages = {"boto3": "1.2"} + actual_upgrades, actual_new_packages = _derive_changeset( + target_version_dir, source_version_dir, _image_generator_configs[1] + ) assert expected_upgrades == actual_upgrades assert expected_new_packages == actual_new_packages def test_generate_release_notes(tmp_path): - target_version_dir = str(tmp_path / 'v1.0.6') + target_version_dir = str(tmp_path / "v1.0.6") os.makedirs(target_version_dir) # Create env.in of the target version, which has additional dependency on boto3 - target_env_in_packages = 'conda-forge::ipykernel\nconda-forge::boto3' + target_env_in_packages = "conda-forge::ipykernel\nconda-forge::boto3" _create_docker_cpu_env_in_file(target_version_dir + "/cpu.env.in", required_package=target_env_in_packages) - target_env_out_packages = 'https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.6-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67\n' \ - 'https://conda.anaconda.org/conda-forge/linux-64/boto3-1.23.4' \ - '-cuda112py38hd_0.conda#8c1f6bf32a6ca81232c4853d4165ca67' + target_env_out_packages = ( + "https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.6-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67\n" + "https://conda.anaconda.org/conda-forge/linux-64/boto3-1.23.4" + "-cuda112py38hd_0.conda#8c1f6bf32a6ca81232c4853d4165ca67" + ) _create_docker_cpu_env_out_file(target_version_dir + "/cpu.env.out", package_metadata=target_env_out_packages) # GPU contains only numpy _create_docker_gpu_env_in_file(target_version_dir + "/gpu.env.in") @@ -626,11 +624,11 @@ def test_generate_release_notes(tmp_path): # Verify _get_image_type_package_metadata image_type_package_metadata = _get_image_type_package_metadata(target_version_dir) assert len(image_type_package_metadata) == 2 - assert image_type_package_metadata['gpu'] == {'numpy': '1.24.2'} - assert image_type_package_metadata['cpu'] == {'ipykernel': '6.21.6', 'boto3': '1.23.4'} + assert image_type_package_metadata["gpu"] == {"numpy": "1.24.2"} + assert image_type_package_metadata["cpu"] == {"ipykernel": "6.21.6", "boto3": "1.23.4"} # Verify _get_package_to_image_type_mapping package_to_image_type_mapping = _get_package_to_image_type_mapping(image_type_package_metadata) assert len(package_to_image_type_mapping) == 3 - assert package_to_image_type_mapping['numpy'] == {'gpu': '1.24.2'} - assert package_to_image_type_mapping['ipykernel'] == {'cpu': '6.21.6'} - assert package_to_image_type_mapping['boto3'] == {'cpu': '1.23.4'} + assert package_to_image_type_mapping["numpy"] == {"gpu": "1.24.2"} + assert package_to_image_type_mapping["ipykernel"] == {"cpu": "6.21.6"} + assert package_to_image_type_mapping["boto3"] == {"cpu": "1.23.4"} diff --git a/test/test_package_staleness.py b/test/test_package_staleness.py index a1a07b19..b26283a5 100644 --- a/test/test_package_staleness.py +++ b/test/test_package_staleness.py @@ -1,87 +1,95 @@ from __future__ import absolute_import -from conda.cli.python_api import run_command import pytest pytestmark = pytest.mark.unit -from utils import get_semver, get_match_specs -from package_staleness import _get_installed_package_versions_and_conda_versions +from unittest.mock import patch + from config import _image_generator_configs -from unittest.mock import patch, Mock, MagicMock +from package_staleness import _get_installed_package_versions_and_conda_versions +from utils import get_match_specs, get_semver def _create_env_in_docker_file(file_path): - with open(file_path, 'w') as env_in_file: - env_in_file.write(f'''# This file is auto-generated. + with open(file_path, "w") as env_in_file: + env_in_file.write( + f"""# This file is auto-generated. conda-forge::ipykernel -conda-forge::numpy[version=\'>=1.0.17,<2.0.0\']''') +conda-forge::numpy[version=\'>=1.0.17,<2.0.0\']""" + ) def _create_env_out_docker_file(file_path): - with open(file_path, 'w') as env_out_file: - env_out_file.write(f'''# This file may be used to create an environment using: + with open(file_path, "w") as env_out_file: + env_out_file.write( + f"""# This file may be used to create an environment using: # $ conda create --name --file # platform: linux-64 https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.21.3-pyh210e3f2_0.conda#8c1f6bf32a6ca81232c4853d4165ca67 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.2-py38h10c12cc_0.conda#05592c85b9f6931dc2df1e80c0d56294\n''') +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.2-py38h10c12cc_0.conda#05592c85b9f6931dc2df1e80c0d56294\n""" + ) def test_get_match_specs(tmp_path): - env_out_file_path = tmp_path / 'env.out' + env_out_file_path = tmp_path / "env.out" _create_env_out_docker_file(env_out_file_path) match_spec_out = get_match_specs(env_out_file_path) - ipykernel_match_spec = match_spec_out['ipykernel'] - assert str(ipykernel_match_spec.get('version')).removeprefix('==') == '6.21.3' - numpy_match_spec = match_spec_out['numpy'] - assert str(numpy_match_spec.get('version')).removeprefix('==') == '1.24.2' - assert ipykernel_match_spec.get('subdir') == 'noarch' - assert numpy_match_spec.get('subdir') == 'linux-64' + ipykernel_match_spec = match_spec_out["ipykernel"] + assert str(ipykernel_match_spec.get("version")).removeprefix("==") == "6.21.3" + numpy_match_spec = match_spec_out["numpy"] + assert str(numpy_match_spec.get("version")).removeprefix("==") == "1.24.2" + assert ipykernel_match_spec.get("subdir") == "noarch" + assert numpy_match_spec.get("subdir") == "linux-64" assert len(match_spec_out) == 2 # Test bad file path - env_out_file_path = tmp_path / 'bad.env.out' + env_out_file_path = tmp_path / "bad.env.out" match_spec_out = get_match_specs(env_out_file_path) assert len(match_spec_out) == 0 @patch("conda.cli.python_api.run_command") def test_get_installed_package_versions_and_conda_versions(mock_run_command, tmp_path): - mock_run_command.return_value = ('{"ipykernel":[{"version": "6.21.3"}], "numpy":[{"version": ' - '"1.24.3"},{"version":"1.26.0"},{"version":"2.1.0"}]}', '', 0) - env_in_file_path = tmp_path / 'cpu.env.in' + mock_run_command.return_value = ( + '{"ipykernel":[{"version": "6.21.3"}], "numpy":[{"version": ' + '"1.24.3"},{"version":"1.26.0"},{"version":"2.1.0"}]}', + "", + 0, + ) + env_in_file_path = tmp_path / "cpu.env.in" _create_env_in_docker_file(env_in_file_path) - env_out_file_path = tmp_path / 'cpu.env.out' + env_out_file_path = tmp_path / "cpu.env.out" _create_env_out_docker_file(env_out_file_path) # Validate results for patch version release # _image_generator_configs[1] is for CPU - match_spec_out, latest_package_versions_in_conda_forge = \ - _get_installed_package_versions_and_conda_versions(_image_generator_configs[1], - str(tmp_path), get_semver('0.4.2')) - ipykernel_match_spec = match_spec_out['ipykernel'] - assert str(ipykernel_match_spec.get('version')).removeprefix('==') == '6.21.3' - numpy_match_spec = match_spec_out['numpy'] - assert str(numpy_match_spec.get('version')).removeprefix('==') == '1.24.2' - assert latest_package_versions_in_conda_forge['ipykernel'] == '6.21.3' - assert latest_package_versions_in_conda_forge['numpy'] == '1.24.3' + match_spec_out, latest_package_versions_in_conda_forge = _get_installed_package_versions_and_conda_versions( + _image_generator_configs[1], str(tmp_path), get_semver("0.4.2") + ) + ipykernel_match_spec = match_spec_out["ipykernel"] + assert str(ipykernel_match_spec.get("version")).removeprefix("==") == "6.21.3" + numpy_match_spec = match_spec_out["numpy"] + assert str(numpy_match_spec.get("version")).removeprefix("==") == "1.24.2" + assert latest_package_versions_in_conda_forge["ipykernel"] == "6.21.3" + assert latest_package_versions_in_conda_forge["numpy"] == "1.24.3" # Validate results for minor version release - match_spec_out, latest_package_versions_in_conda_forge = \ - _get_installed_package_versions_and_conda_versions(_image_generator_configs[1], - str(tmp_path), get_semver('0.5.0')) - ipykernel_match_spec = match_spec_out['ipykernel'] - assert str(ipykernel_match_spec.get('version')).removeprefix('==') == '6.21.3' - numpy_match_spec = match_spec_out['numpy'] - assert str(numpy_match_spec.get('version')).removeprefix('==') == '1.24.2' - assert latest_package_versions_in_conda_forge['ipykernel'] == '6.21.3' + match_spec_out, latest_package_versions_in_conda_forge = _get_installed_package_versions_and_conda_versions( + _image_generator_configs[1], str(tmp_path), get_semver("0.5.0") + ) + ipykernel_match_spec = match_spec_out["ipykernel"] + assert str(ipykernel_match_spec.get("version")).removeprefix("==") == "6.21.3" + numpy_match_spec = match_spec_out["numpy"] + assert str(numpy_match_spec.get("version")).removeprefix("==") == "1.24.2" + assert latest_package_versions_in_conda_forge["ipykernel"] == "6.21.3" # Only for numpy there is a new minor version available. - assert latest_package_versions_in_conda_forge['numpy'] == '1.26.0' + assert latest_package_versions_in_conda_forge["numpy"] == "1.26.0" # Validate results for major version release - match_spec_out, latest_package_versions_in_conda_forge = \ - _get_installed_package_versions_and_conda_versions(_image_generator_configs[1], - str(tmp_path), get_semver('1.0.0')) - ipykernel_match_spec = match_spec_out['ipykernel'] - assert str(ipykernel_match_spec.get('version')).removeprefix('==') == '6.21.3' - numpy_match_spec = match_spec_out['numpy'] - assert str(numpy_match_spec.get('version')).removeprefix('==') == '1.24.2' - assert latest_package_versions_in_conda_forge['ipykernel'] == '6.21.3' + match_spec_out, latest_package_versions_in_conda_forge = _get_installed_package_versions_and_conda_versions( + _image_generator_configs[1], str(tmp_path), get_semver("1.0.0") + ) + ipykernel_match_spec = match_spec_out["ipykernel"] + assert str(ipykernel_match_spec.get("version")).removeprefix("==") == "6.21.3" + numpy_match_spec = match_spec_out["numpy"] + assert str(numpy_match_spec.get("version")).removeprefix("==") == "1.24.2" + assert latest_package_versions_in_conda_forge["ipykernel"] == "6.21.3" # Only for numpy there is a new major version available. - assert latest_package_versions_in_conda_forge['numpy'] == '2.1.0' + assert latest_package_versions_in_conda_forge["numpy"] == "2.1.0" From 05d06c738917eb506ca3da16df2d4df6f1643f8b Mon Sep 17 00:00:00 2001 From: Tian Wang Date: Mon, 25 Mar 2024 22:04:43 +0000 Subject: [PATCH 3/5] Add .git-blame-ignore-revs to preserve git blame information --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..c519eda5 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Migrate code style to Black +a74466c154d01201388f5232b167538275aa74c4 From 0690182a696df97cf38c173441768a178fd5f20f Mon Sep 17 00:00:00 2001 From: Tian Wang Date: Mon, 25 Mar 2024 22:21:41 +0000 Subject: [PATCH 4/5] Fix check code quality workflow --- .github/workflows/check_code_quality.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/check_code_quality.yml b/.github/workflows/check_code_quality.yml index 17a2bda1..6239f821 100644 --- a/.github/workflows/check_code_quality.yml +++ b/.github/workflows/check_code_quality.yml @@ -19,6 +19,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +defaults: + run: + shell: bash -l {0} + jobs: build: name: Check Code Quality From e575ba50a9ed53a822ad34ed0ec1df70436fd259 Mon Sep 17 00:00:00 2001 From: Tian Wang <133085652+aws-tianquaw@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:42:02 -0700 Subject: [PATCH 5/5] Remove unnecessary pip install in DEVELOPMENT.md --- DEVELOPMENT.md | 1 - 1 file changed, 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 835edb67..2b9d61df 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -29,7 +29,6 @@ pytest --local-image-id REPLACE_ME_WITH_IMAGE_ID Install pre-commit to run code style checks before each commit: ```shell -pip install pre-commit pre-commit install ```