From 4edc7aa8846c8de160ad5ba0d9c0b77614cb3ff6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 28 May 2024 14:02:16 +0100 Subject: [PATCH 01/39] Skip running cleanup on pull requests --- .github/workflows/test_local_integration.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml index 81810abfe..c5d3596b6 100644 --- a/.github/workflows/test_local_integration.yaml +++ b/.github/workflows/test_local_integration.yaml @@ -192,7 +192,9 @@ jobs: ### CLEANUP AFTER TESTS - name: Cleanup nebari deployment - if: always() + # Since this is not critical for most pull requests and takes more than half of the time + # in the CI, it makes sense to only run on merge to main to speed up feedback cycle + if: github.event_name != 'pull_request' working-directory: local-deployment run: | nebari destroy --config nebari-config.yaml --disable-prompt From 3e0f9c4fb243dc3dda99e8d70ee45a69fa0d6a78 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 31 May 2024 12:56:23 +0100 Subject: [PATCH 02/39] update triggers for local integration tests --- .github/workflows/test_local_integration.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml index c5d3596b6..6d605787a 100644 --- a/.github/workflows/test_local_integration.yaml +++ b/.github/workflows/test_local_integration.yaml @@ -33,6 +33,7 @@ on: pr_number: required: true type: string + workflow_dispatch: # When the cancel-in-progress: true option is specified, any concurrent jobs or workflows using the same # concurrency group will cancel both the pending and currently running jobs or workflows. This allows only @@ -193,8 +194,9 @@ jobs: ### CLEANUP AFTER TESTS - name: Cleanup nebari deployment # Since this is not critical for most pull requests and takes more than half of the time - # in the CI, it makes sense to only run on merge to main to speed up feedback cycle - if: github.event_name != 'pull_request' + # in the CI, it makes sense to only run on merge to main or workflow_dispatch to speed + # up feedback cycle + if: github.ref_name == 'develop' || github.event_name == 'workflow_dispatch' working-directory: local-deployment run: | nebari destroy --config nebari-config.yaml --disable-prompt From 55c9e8c95668a4415e6ea4150fd23dbe69fa989a Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Fri, 14 Jun 2024 16:48:40 +0530 Subject: [PATCH 03/39] #1792 Add suggested docstring on the top of `upgrade.py` --- src/_nebari/upgrade.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 4ae683b47..3fc490b2c 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1,3 +1,9 @@ +"""This file contains the upgrade logic for Nebari. + +This file contains the upgrade logic for Nebari. Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. +When a user runs `nebari upgrade -c nebari-config.yaml`, then the do_upgrade function will then run through all required upgrade steps to bring the config file up to date with the current version of Nebari. +""" + import json import logging import re From e6af7b5379a0e6de8fe68a346383fdda2d3ebec1 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Fri, 14 Jun 2024 16:50:28 +0530 Subject: [PATCH 04/39] #1792 Add docstring to `do_upgrade` --- src/_nebari/upgrade.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 3fc490b2c..9c7d4471f 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1,6 +1,6 @@ -"""This file contains the upgrade logic for Nebari. - -This file contains the upgrade logic for Nebari. Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. +""" +This file contains the upgrade logic for Nebari. +Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. When a user runs `nebari upgrade -c nebari-config.yaml`, then the do_upgrade function will then run through all required upgrade steps to bring the config file up to date with the current version of Nebari. """ @@ -45,6 +45,20 @@ def do_upgrade(config_filename, attempt_fixes=False): + """ + Perform an upgrade of the Nebari configuration file. + + This function loads the YAML configuration file, checks for deprecated keys, + validates the current version, and if necessary, upgrades the configuration + to the latest version of Nebari. + + Parameters: + config_filename (str): The path to the configuration file. + attempt_fixes (bool): Whether to attempt automatic fixes for validation errors. + + Returns: + None + """ config = load_yaml(config_filename) if config.get("qhub_version"): rich.print( From 3a10d1b5cdf862150943c1067133b95bb502223b Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sat, 15 Jun 2024 13:51:20 +0530 Subject: [PATCH 05/39] #1792 Add docstring to `UpgradeStep` class --- src/_nebari/upgrade.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 9c7d4471f..58495544d 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -107,10 +107,23 @@ def do_upgrade(config_filename, attempt_fixes=False): class UpgradeStep(ABC): + """ + Abstract base class representing an upgrade step. + + Attributes: + _steps (ClassVar[Dict[str, Any]]): Class variable holding registered upgrade steps. + version (ClassVar[str]): The version of the upgrade step. + """ _steps: ClassVar[Dict[str, Any]] = {} version: ClassVar[str] = "" def __init_subclass__(cls): + """ + Initializes a subclass of UpgradeStep. + + This method validates the version string and registers the subclass + in the _steps dictionary. + """ try: parsed_version = Version(cls.version) except ValueError as exc: @@ -132,6 +145,15 @@ def clear_steps_registry(cls): @classmethod def has_step(cls, version): + """ + Checks if there is an upgrade step for a given version. + + Parameters: + version (str): The version to check. + + Returns: + bool: True if the step exists, False otherwise. + """ return version in cls._steps @classmethod @@ -141,6 +163,16 @@ def upgrade( """ Runs through all required upgrade steps (i.e. relevant subclasses of UpgradeStep). Calls UpgradeStep.upgrade_step for each. + + Parameters: + config (dict): The current configuration dictionary. + start_version (str): The starting version of the configuration. + finish_version (str): The target version for the configuration. + config_filename (str): The path to the configuration file. + attempt_fixes (bool): Whether to attempt automatic fixes for validation errors. + + Returns: + dict: The updated configuration dictionary. """ starting_ver = rounded_ver_parse(start_version or "0.0.0") finish_ver = rounded_ver_parse(finish_version) @@ -176,9 +208,21 @@ def upgrade( return config def get_version(self): + """ + Returns the version of the upgrade step. + + Returns: + str: The version of the upgrade step. + """ return self.version def requires_nebari_version_field(self): + """ + Checks if the nebari_version field is required for this upgrade step. + + Returns: + bool: True if the nebari_version field is required, False otherwise. + """ return rounded_ver_parse(self.version) > rounded_ver_parse("0.3.13") def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): @@ -194,6 +238,14 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): It should normally be left as-is for all upgrades. Use _version_specific_upgrade below for any actions that are only required for the particular upgrade you are creating. + + Parameters: + config (dict): The current configuration dictionary. + start_version (str): The starting version of the configuration. + config_filename (str): The path to the configuration file. + + Returns: + dict: The updated configuration dictionary. """ finish_version = self.get_version() __rounded_finish_version__ = str(rounded_ver_parse(finish_version)) @@ -216,6 +268,7 @@ def contains_image_and_tag(s: str) -> bool: return bool(re.match(pattern, s)) def replace_image_tag_legacy(image, start_version, new_version): + """Replace legacy image tags with the new version.""" start_version_regex = start_version.replace(".", "\\.") if not start_version: start_version_regex = "0\\.[0-3]\\.[0-9]{1,2}" @@ -230,6 +283,7 @@ def replace_image_tag_legacy(image, start_version, new_version): return None def replace_image_tag(s: str, new_version: str, config_path: str) -> str: + """Replace the image tag with the new version.""" legacy_replacement = replace_image_tag_legacy(s, start_version, new_version) if legacy_replacement: return legacy_replacement @@ -250,6 +304,7 @@ def replace_image_tag(s: str, new_version: str, config_path: str) -> str: return s def set_nested_item(config: dict, config_path: list, value: str): + """Set a nested item in the configuration dictionary.""" config_path = config_path.split(".") for k in config_path[:-1]: try: @@ -264,6 +319,7 @@ def set_nested_item(config: dict, config_path: list, value: str): config[config_path[-1]] = value def update_image_tag(config, config_path, current_image, new_version): + """Update the image tag in the configuration.""" new_image = replace_image_tag(current_image, new_version, config_path) if new_image != current_image: set_nested_item(config, config_path, new_image) @@ -308,7 +364,17 @@ def _version_specific_upgrade( self, config, start_version, config_filename, *args, **kwargs ): """ + Perform version-specific upgrade tasks. + Override this method in subclasses if you need to do anything specific to your version. + + Parameters: + config (dict): The current configuration dictionary. + start_version (str): The starting version of the configuration. + config_filename (str): The path to the configuration file. + + Returns: + dict: The updated configuration dictionary. """ return config From 52b127160dc56c4166a7841390d81a1abb94268a Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sat, 15 Jun 2024 14:59:40 +0530 Subject: [PATCH 06/39] #1792 Add docstring to the rest of the `UpgradeStep` classes --- src/_nebari/upgrade.py | 109 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 58495544d..ea9140d5b 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -380,6 +380,11 @@ def _version_specific_upgrade( class Upgrade_0_3_12(UpgradeStep): + """ + Upgrade step for Nebari version 0.3.12 + + This class handles the specific upgrade tasks required for transitioning to version 0.3.12 + """ version = "0.3.12" def _version_specific_upgrade( @@ -400,13 +405,19 @@ def _version_specific_upgrade( class Upgrade_0_4_0(UpgradeStep): + """ + Upgrade step for Nebari version 0.4.0 + + This class handles the specific upgrade tasks required for transitioning to version 0.4.0 + """ version = "0.4.0" def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): """ - Upgrade to Keycloak. + This version of Nebari introduces Keycloak for authentication, removes deprecated fields, + and generates a default password for the Keycloak root user. """ security = config.get("security", {}) users = security.get("users", {}) @@ -532,6 +543,11 @@ def _version_specific_upgrade( class Upgrade_0_4_1(UpgradeStep): + """ + Upgrade step for Nebari version 0.4.1 + + This class handles the specific upgrade tasks required for transitioning to version 0.4.1 + """ version = "0.4.1" def _version_specific_upgrade( @@ -558,6 +574,11 @@ def _version_specific_upgrade( class Upgrade_2023_4_2(UpgradeStep): + """ + Upgrade step for Nebari version 2023.4.2 + + This class handles the specific upgrade tasks required for transitioning to version 2023.4.2 + """ version = "2023.4.2" def _version_specific_upgrade( @@ -592,6 +613,11 @@ def _version_specific_upgrade( class Upgrade_2023_7_1(UpgradeStep): + """ + Upgrade step for Nebari version 2023.7.1 + + This class handles the specific upgrade tasks required for transitioning to version 2023.7.1 + """ version = "2023.7.1" def _version_specific_upgrade( @@ -610,6 +636,11 @@ def _version_specific_upgrade( class Upgrade_2023_7_2(UpgradeStep): + """ + Upgrade step for Nebari version 2023.7.2 + + This class handles the specific upgrade tasks required for transitioning to version 2023.7.2 + """ version = "2023.7.2" def _version_specific_upgrade( @@ -633,6 +664,17 @@ def _version_specific_upgrade( class Upgrade_2023_10_1(UpgradeStep): + """ + Upgrade step for Nebari version 2023.10.1 + + This class handles the specific upgrade tasks required for transitioning to version 2023.10.1 + + Note: + Upgrading to 2023.10.1 is considered high-risk because it includes a major refactor + to introduce the extension mechanism system. This version introduces significant + changes, including the support for third-party plugins, upgrades JupyterHub to version 3.1, + and deprecates certain components such as CDS Dashboards, ClearML, Prefect, and kbatch. + """ version = "2023.10.1" # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) # This released has been tested against 1.26 @@ -740,6 +782,14 @@ def _version_specific_upgrade( class Upgrade_2023_11_1(UpgradeStep): + """ + Upgrade step for Nebari version 2023.11.1 + + This class handles the specific upgrade tasks required for transitioning to version 2023.11.1 + + Note: + - ClearML, Prefect, and kbatch are no longer supported in this version. + """ version = "2023.11.1" def _version_specific_upgrade( @@ -758,6 +808,14 @@ def _version_specific_upgrade( class Upgrade_2023_12_1(UpgradeStep): + """ + Upgrade step for Nebari version 2023.12.1 + + This class handles the specific upgrade tasks required for transitioning to version 2023.12.1 + + Note: + - This is the last version that supports the jupyterlab-videochat extension. + """ version = "2023.12.1" def _version_specific_upgrade( @@ -777,6 +835,14 @@ def _version_specific_upgrade( class Upgrade_2024_1_1(UpgradeStep): + """ + Upgrade step for Nebari version 2024.1.1 + + This class handles the specific upgrade tasks required for transitioning to version 2024.1.1 + + Note: + - jupyterlab-videochat, retrolab, jupyter-tensorboard, jupyterlab-conda-store, and jupyter-nvdashboard are no longer supported. + """ version = "2024.1.1" def _version_specific_upgrade( @@ -797,6 +863,11 @@ def _version_specific_upgrade( class Upgrade_2024_3_1(UpgradeStep): + """ + Upgrade step for Nebari version 2024.3.1 + + This class handles the specific upgrade tasks required for transitioning to version 2024.3.1 + """ version = "2024.3.1" def _version_specific_upgrade( @@ -808,6 +879,11 @@ def _version_specific_upgrade( class Upgrade_2024_3_2(UpgradeStep): + """ + Upgrade step for Nebari version 2024.3.2 + + This class handles the specific upgrade tasks required for transitioning to version 2024.3.2 + """ version = "2024.3.2" def _version_specific_upgrade( @@ -819,6 +895,11 @@ def _version_specific_upgrade( class Upgrade_2024_3_3(UpgradeStep): + """ + Upgrade step for Nebari version 2024.3.3 + + This class handles the specific upgrade tasks required for transitioning to version 2024.3.3 + """ version = "2024.3.3" def _version_specific_upgrade( @@ -830,6 +911,14 @@ def _version_specific_upgrade( class Upgrade_2024_4_1(UpgradeStep): + """ + Upgrade step for Nebari version 2024.4.1 + + This class handles the specific upgrade tasks required for transitioning to version 2024.4.1 + + Note: + - Adds default configuration for node groups if not already defined. + """ version = "2024.4.1" def _version_specific_upgrade( @@ -861,6 +950,11 @@ def _version_specific_upgrade( class Upgrade_2024_5_1(UpgradeStep): + """ + Upgrade step for Nebari version 2024.5.1 + + This class handles the specific upgrade tasks required for transitioning to version 2024.5.1 + """ version = "2024.5.1" def _version_specific_upgrade( @@ -872,6 +966,13 @@ def _version_specific_upgrade( class Upgrade_2024_6_1(UpgradeStep): + """ + Upgrade step for version 2024.6.1 + + This upgrade includes: + - Manual updates for kube-prometheus-stack CRDs if monitoring is enabled. + - Prompts to upgrade GCP node groups to more cost-efficient instances. + """ version = "2024.6.1" def _version_specific_upgrade( @@ -1003,4 +1104,10 @@ def _version_specific_upgrade( # Always have a way to upgrade to the latest full version number, even if no customizations # Don't let dev/prerelease versions cloud things class UpgradeLatest(UpgradeStep): + """ + Upgrade step for the latest available version. + + This class ensures there is always an upgrade path to the latest version, + even if no specific upgrade steps are defined for the current version. + """ version = __rounded_version__ From 88dad92bf7c57d4fc87bf0b5ca4d853ba8439944 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 12:53:09 +0000 Subject: [PATCH 07/39] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/upgrade.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index ea9140d5b..e1cb91a61 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1,6 +1,6 @@ """ This file contains the upgrade logic for Nebari. -Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. +Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. When a user runs `nebari upgrade -c nebari-config.yaml`, then the do_upgrade function will then run through all required upgrade steps to bring the config file up to date with the current version of Nebari. """ @@ -114,6 +114,7 @@ class UpgradeStep(ABC): _steps (ClassVar[Dict[str, Any]]): Class variable holding registered upgrade steps. version (ClassVar[str]): The version of the upgrade step. """ + _steps: ClassVar[Dict[str, Any]] = {} version: ClassVar[str] = "" @@ -163,7 +164,7 @@ def upgrade( """ Runs through all required upgrade steps (i.e. relevant subclasses of UpgradeStep). Calls UpgradeStep.upgrade_step for each. - + Parameters: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. @@ -238,7 +239,7 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): It should normally be left as-is for all upgrades. Use _version_specific_upgrade below for any actions that are only required for the particular upgrade you are creating. - + Parameters: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. @@ -385,6 +386,7 @@ class Upgrade_0_3_12(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 0.3.12 """ + version = "0.3.12" def _version_specific_upgrade( @@ -410,6 +412,7 @@ class Upgrade_0_4_0(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 0.4.0 """ + version = "0.4.0" def _version_specific_upgrade( @@ -548,6 +551,7 @@ class Upgrade_0_4_1(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 0.4.1 """ + version = "0.4.1" def _version_specific_upgrade( @@ -579,6 +583,7 @@ class Upgrade_2023_4_2(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2023.4.2 """ + version = "2023.4.2" def _version_specific_upgrade( @@ -618,6 +623,7 @@ class Upgrade_2023_7_1(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2023.7.1 """ + version = "2023.7.1" def _version_specific_upgrade( @@ -641,6 +647,7 @@ class Upgrade_2023_7_2(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2023.7.2 """ + version = "2023.7.2" def _version_specific_upgrade( @@ -675,6 +682,7 @@ class Upgrade_2023_10_1(UpgradeStep): changes, including the support for third-party plugins, upgrades JupyterHub to version 3.1, and deprecates certain components such as CDS Dashboards, ClearML, Prefect, and kbatch. """ + version = "2023.10.1" # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) # This released has been tested against 1.26 @@ -790,6 +798,7 @@ class Upgrade_2023_11_1(UpgradeStep): Note: - ClearML, Prefect, and kbatch are no longer supported in this version. """ + version = "2023.11.1" def _version_specific_upgrade( @@ -816,6 +825,7 @@ class Upgrade_2023_12_1(UpgradeStep): Note: - This is the last version that supports the jupyterlab-videochat extension. """ + version = "2023.12.1" def _version_specific_upgrade( @@ -843,6 +853,7 @@ class Upgrade_2024_1_1(UpgradeStep): Note: - jupyterlab-videochat, retrolab, jupyter-tensorboard, jupyterlab-conda-store, and jupyter-nvdashboard are no longer supported. """ + version = "2024.1.1" def _version_specific_upgrade( @@ -868,6 +879,7 @@ class Upgrade_2024_3_1(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2024.3.1 """ + version = "2024.3.1" def _version_specific_upgrade( @@ -884,6 +896,7 @@ class Upgrade_2024_3_2(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2024.3.2 """ + version = "2024.3.2" def _version_specific_upgrade( @@ -900,6 +913,7 @@ class Upgrade_2024_3_3(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2024.3.3 """ + version = "2024.3.3" def _version_specific_upgrade( @@ -919,6 +933,7 @@ class Upgrade_2024_4_1(UpgradeStep): Note: - Adds default configuration for node groups if not already defined. """ + version = "2024.4.1" def _version_specific_upgrade( @@ -955,6 +970,7 @@ class Upgrade_2024_5_1(UpgradeStep): This class handles the specific upgrade tasks required for transitioning to version 2024.5.1 """ + version = "2024.5.1" def _version_specific_upgrade( @@ -973,6 +989,7 @@ class Upgrade_2024_6_1(UpgradeStep): - Manual updates for kube-prometheus-stack CRDs if monitoring is enabled. - Prompts to upgrade GCP node groups to more cost-efficient instances. """ + version = "2024.6.1" def _version_specific_upgrade( @@ -1110,4 +1127,5 @@ class UpgradeLatest(UpgradeStep): This class ensures there is always an upgrade path to the latest version, even if no specific upgrade steps are defined for the current version. """ + version = __rounded_version__ From cf0f7541f0d99af852e94c518d2ef7849331b852 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:19:34 -0500 Subject: [PATCH 08/39] Explicit config (#2294) Co-authored-by: Fangchen Li Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Amit Kumar Co-authored-by: Vinicius D. Cerutti <51954708+viniciusdc@users.noreply.github.com> --- src/_nebari/config.py | 3 ++- src/_nebari/subcommands/init.py | 19 ++++++++++++++++- src/nebari/plugins.py | 2 +- src/nebari/schema.py | 4 +++- tests/tests_unit/test_cli_init.py | 34 +++++++++++++++++++------------ 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/_nebari/config.py b/src/_nebari/config.py index 7c27274f3..9d7dec4bd 100644 --- a/src/_nebari/config.py +++ b/src/_nebari/config.py @@ -103,7 +103,8 @@ def write_configuration( """Write the nebari configuration file to disk""" with config_filename.open(mode) as f: if isinstance(config, pydantic.BaseModel): - yaml.dump(config.model_dump(), f) + config_dict = config.model_dump() + yaml.dump(config_dict, f) else: config = dump_nested_model(config) yaml.dump(config, f) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 9040f3d20..8c3de6d5b 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -106,6 +106,7 @@ class InitInputs(schema.Base): ssl_cert_email: Optional[schema.email_pydantic] = None disable_prompt: bool = False output: pathlib.Path = pathlib.Path("nebari-config.yaml") + explicit: int = 0 def enum_to_list(enum_cls): @@ -152,7 +153,7 @@ def handle_init(inputs: InitInputs, config_schema: BaseModel): try: write_configuration( inputs.output, - config, + config if not inputs.explicit else config_schema(**config), mode="x", ) except FileExistsError: @@ -565,6 +566,13 @@ def init( "-o", help="Output file path for the rendered config file.", ), + explicit: int = typer.Option( + 0, + "--explicit", + "-e", + count=True, + help="Write explicit nebari config file (advanced users only).", + ), ): """ Create and initialize your [purple]nebari-config.yaml[/purple] file. @@ -604,6 +612,7 @@ def init( inputs.ssl_cert_email = ssl_cert_email inputs.disable_prompt = disable_prompt inputs.output = output + inputs.explicit = explicit from nebari.plugins import nebari_plugin_manager @@ -894,6 +903,14 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): ) inputs.kubernetes_version = kubernetes_version + # EXPLICIT CONFIG + inputs.explicit = questionary.confirm( + "Would you like the nebari config to show all available options? (recommended for advanced users only)", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask() + from nebari.plugins import nebari_plugin_manager config_schema = nebari_plugin_manager.config_schema diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index c5148e9e1..a523c0324 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -128,7 +128,7 @@ def config_schema(self): classes = [schema.Main] + [ _.input_schema for _ in self.ordered_stages if _.input_schema is not None ] - return type("ConfigSchema", tuple(classes), {}) + return type("ConfigSchema", tuple(classes[::-1]), {}) nebari_plugin_manager = NebariPluginManager() diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 70b9589e6..2cc1c1ea3 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -25,7 +25,9 @@ class Base(pydantic.BaseModel): model_config = ConfigDict( - extra="forbid", validate_assignment=True, populate_by_name=True + extra="forbid", + validate_assignment=True, + populate_by_name=True, ) diff --git a/tests/tests_unit/test_cli_init.py b/tests/tests_unit/test_cli_init.py index 0cd0fe03d..9afab5ddc 100644 --- a/tests/tests_unit/test_cli_init.py +++ b/tests/tests_unit/test_cli_init.py @@ -51,6 +51,8 @@ (["--ssl-cert-email"], 2, ["requires an argument"]), (["--output"], 2, ["requires an argument"]), (["-o"], 2, ["requires an argument"]), + (["--explicit"], 2, ["Missing option"]), + (["-e"], 2, ["Missing option"]), ], ) def test_cli_init_stdout(args: List[str], exit_code: int, content: List[str]): @@ -90,20 +92,22 @@ def generate_test_data_test_cli_init_happy_path(): ) in get_kubernetes_versions(provider) + [ "latest" ]: - test_data.append( - ( - provider, - region, - project_name, - domain_name, - namespace, - auth_provider, - ci_provider, - terraform_state, - email, - kubernetes_version, + for explicit in [True, False]: + test_data.append( + ( + provider, + region, + project_name, + domain_name, + namespace, + auth_provider, + ci_provider, + terraform_state, + email, + kubernetes_version, + explicit, + ) ) - ) keys = [ "provider", @@ -116,6 +120,7 @@ def generate_test_data_test_cli_init_happy_path(): "terraform_state", "email", "kubernetes_version", + "explicit", ] return {"keys": keys, "test_data": test_data} @@ -131,6 +136,7 @@ def test_cli_init_happy_path( terraform_state: str, email: str, kubernetes_version: str, + explicit: bool, ): app = create_cli() args = [ @@ -159,6 +165,8 @@ def test_cli_init_happy_path( "--region", region, ] + if explicit: + args += ["--explicit"] expected_yaml = f""" provider: {provider} From 71c2079bbe6a1782e2e5b1e17a487445cc4c5cdb Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:24:43 -0500 Subject: [PATCH 09/39] fix general node scaling bug for azure (#2517) --- .../template/azure/modules/kubernetes/main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf index cd3948830..1e3d78356 100644 --- a/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf @@ -31,13 +31,12 @@ resource "azurerm_kubernetes_cluster" "main" { default_node_pool { vnet_subnet_id = var.vnet_subnet_id name = var.node_groups[0].name - node_count = 1 vm_size = var.node_groups[0].instance_type enable_auto_scaling = "true" - min_count = 1 - max_count = 1 + min_count = var.node_groups[0].min_size + max_count = var.node_groups[0].max_size max_pods = var.max_pods - # node_labels = var.node_labels + orchestrator_version = var.kubernetes_version node_labels = { "azure-node-pool" = var.node_groups[0].name From a7b039eb5216798360585eaec09a63ca7114a63d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:38:09 -0700 Subject: [PATCH 10/39] set's min TLS version for azure storage account to TLS 1.2 --- .../template/azure/modules/terraform-state/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf index a13f613cf..544aa578c 100644 --- a/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf +++ b/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf @@ -12,6 +12,7 @@ resource "azurerm_storage_account" "terraform-state-storage-account" { account_tier = "Standard" account_replication_type = "GRS" tags = var.tags + min_tls_version = "TLS1_2" identity { type = "SystemAssigned" From 52ea69e511a1ed309f03931edf857dd0914483dc Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 19 Jun 2024 12:37:18 +0530 Subject: [PATCH 11/39] - Specify as `Args` instead of `Parameters` - Use `@override` for `_version_specific_upgrade_steps` - Remove upgrade docstrings unless a note --- src/_nebari/upgrade.py | 112 +++++++++-------------------------------- 1 file changed, 25 insertions(+), 87 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index e1cb91a61..0ada8ba58 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1,6 +1,6 @@ """ This file contains the upgrade logic for Nebari. -Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. +Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. When a user runs `nebari upgrade -c nebari-config.yaml`, then the do_upgrade function will then run through all required upgrade steps to bring the config file up to date with the current version of Nebari. """ @@ -12,7 +12,7 @@ import textwrap from abc import ABC from pathlib import Path -from typing import Any, ClassVar, Dict +from typing import Any, ClassVar, Dict, override import rich from packaging.version import Version @@ -52,7 +52,7 @@ def do_upgrade(config_filename, attempt_fixes=False): validates the current version, and if necessary, upgrades the configuration to the latest version of Nebari. - Parameters: + Args: config_filename (str): The path to the configuration file. attempt_fixes (bool): Whether to attempt automatic fixes for validation errors. @@ -114,7 +114,6 @@ class UpgradeStep(ABC): _steps (ClassVar[Dict[str, Any]]): Class variable holding registered upgrade steps. version (ClassVar[str]): The version of the upgrade step. """ - _steps: ClassVar[Dict[str, Any]] = {} version: ClassVar[str] = "" @@ -149,7 +148,7 @@ def has_step(cls, version): """ Checks if there is an upgrade step for a given version. - Parameters: + Args: version (str): The version to check. Returns: @@ -164,8 +163,8 @@ def upgrade( """ Runs through all required upgrade steps (i.e. relevant subclasses of UpgradeStep). Calls UpgradeStep.upgrade_step for each. - - Parameters: + + Args: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. finish_version (str): The target version for the configuration. @@ -210,8 +209,6 @@ def upgrade( def get_version(self): """ - Returns the version of the upgrade step. - Returns: str: The version of the upgrade step. """ @@ -239,8 +236,8 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): It should normally be left as-is for all upgrades. Use _version_specific_upgrade below for any actions that are only required for the particular upgrade you are creating. - - Parameters: + + Args: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. config_filename (str): The path to the configuration file. @@ -369,7 +366,7 @@ def _version_specific_upgrade( Override this method in subclasses if you need to do anything specific to your version. - Parameters: + Args: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. config_filename (str): The path to the configuration file. @@ -381,14 +378,9 @@ def _version_specific_upgrade( class Upgrade_0_3_12(UpgradeStep): - """ - Upgrade step for Nebari version 0.3.12 - - This class handles the specific upgrade tasks required for transitioning to version 0.3.12 - """ - version = "0.3.12" + @override def _version_specific_upgrade( self, config, start_version, config_filename, *args, **kwargs ): @@ -407,14 +399,9 @@ def _version_specific_upgrade( class Upgrade_0_4_0(UpgradeStep): - """ - Upgrade step for Nebari version 0.4.0 - - This class handles the specific upgrade tasks required for transitioning to version 0.4.0 - """ - version = "0.4.0" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -546,14 +533,9 @@ def _version_specific_upgrade( class Upgrade_0_4_1(UpgradeStep): - """ - Upgrade step for Nebari version 0.4.1 - - This class handles the specific upgrade tasks required for transitioning to version 0.4.1 - """ - version = "0.4.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -578,14 +560,9 @@ def _version_specific_upgrade( class Upgrade_2023_4_2(UpgradeStep): - """ - Upgrade step for Nebari version 2023.4.2 - - This class handles the specific upgrade tasks required for transitioning to version 2023.4.2 - """ - version = "2023.4.2" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -618,14 +595,9 @@ def _version_specific_upgrade( class Upgrade_2023_7_1(UpgradeStep): - """ - Upgrade step for Nebari version 2023.7.1 - - This class handles the specific upgrade tasks required for transitioning to version 2023.7.1 - """ - version = "2023.7.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -642,14 +614,9 @@ def _version_specific_upgrade( class Upgrade_2023_7_2(UpgradeStep): - """ - Upgrade step for Nebari version 2023.7.2 - - This class handles the specific upgrade tasks required for transitioning to version 2023.7.2 - """ - version = "2023.7.2" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -674,20 +641,18 @@ class Upgrade_2023_10_1(UpgradeStep): """ Upgrade step for Nebari version 2023.10.1 - This class handles the specific upgrade tasks required for transitioning to version 2023.10.1 - Note: Upgrading to 2023.10.1 is considered high-risk because it includes a major refactor to introduce the extension mechanism system. This version introduces significant changes, including the support for third-party plugins, upgrades JupyterHub to version 3.1, and deprecates certain components such as CDS Dashboards, ClearML, Prefect, and kbatch. """ - version = "2023.10.1" # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) # This released has been tested against 1.26 min_k8s_version = 1.26 + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -793,14 +758,12 @@ class Upgrade_2023_11_1(UpgradeStep): """ Upgrade step for Nebari version 2023.11.1 - This class handles the specific upgrade tasks required for transitioning to version 2023.11.1 - Note: - ClearML, Prefect, and kbatch are no longer supported in this version. """ - version = "2023.11.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -820,14 +783,12 @@ class Upgrade_2023_12_1(UpgradeStep): """ Upgrade step for Nebari version 2023.12.1 - This class handles the specific upgrade tasks required for transitioning to version 2023.12.1 - Note: - This is the last version that supports the jupyterlab-videochat extension. """ - version = "2023.12.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -848,14 +809,12 @@ class Upgrade_2024_1_1(UpgradeStep): """ Upgrade step for Nebari version 2024.1.1 - This class handles the specific upgrade tasks required for transitioning to version 2024.1.1 - Note: - jupyterlab-videochat, retrolab, jupyter-tensorboard, jupyterlab-conda-store, and jupyter-nvdashboard are no longer supported. """ - version = "2024.1.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -874,14 +833,9 @@ def _version_specific_upgrade( class Upgrade_2024_3_1(UpgradeStep): - """ - Upgrade step for Nebari version 2024.3.1 - - This class handles the specific upgrade tasks required for transitioning to version 2024.3.1 - """ - version = "2024.3.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -891,14 +845,9 @@ def _version_specific_upgrade( class Upgrade_2024_3_2(UpgradeStep): - """ - Upgrade step for Nebari version 2024.3.2 - - This class handles the specific upgrade tasks required for transitioning to version 2024.3.2 - """ - version = "2024.3.2" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -908,14 +857,9 @@ def _version_specific_upgrade( class Upgrade_2024_3_3(UpgradeStep): - """ - Upgrade step for Nebari version 2024.3.3 - - This class handles the specific upgrade tasks required for transitioning to version 2024.3.3 - """ - version = "2024.3.3" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -928,14 +872,12 @@ class Upgrade_2024_4_1(UpgradeStep): """ Upgrade step for Nebari version 2024.4.1 - This class handles the specific upgrade tasks required for transitioning to version 2024.4.1 - Note: - Adds default configuration for node groups if not already defined. """ - version = "2024.4.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -965,14 +907,9 @@ def _version_specific_upgrade( class Upgrade_2024_5_1(UpgradeStep): - """ - Upgrade step for Nebari version 2024.5.1 - - This class handles the specific upgrade tasks required for transitioning to version 2024.5.1 - """ - version = "2024.5.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): @@ -992,6 +929,7 @@ class Upgrade_2024_6_1(UpgradeStep): version = "2024.6.1" + @override def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): From 5158fe00bd43c21b26cc8a1656d40f7e2227432c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 07:07:38 +0000 Subject: [PATCH 12/39] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/upgrade.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 0ada8ba58..9c79bd46d 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1,6 +1,6 @@ """ This file contains the upgrade logic for Nebari. -Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. +Each release of Nebari requires an upgrade step class (which is a child class of UpgradeStep) to be created. When a user runs `nebari upgrade -c nebari-config.yaml`, then the do_upgrade function will then run through all required upgrade steps to bring the config file up to date with the current version of Nebari. """ @@ -114,6 +114,7 @@ class UpgradeStep(ABC): _steps (ClassVar[Dict[str, Any]]): Class variable holding registered upgrade steps. version (ClassVar[str]): The version of the upgrade step. """ + _steps: ClassVar[Dict[str, Any]] = {} version: ClassVar[str] = "" @@ -163,7 +164,7 @@ def upgrade( """ Runs through all required upgrade steps (i.e. relevant subclasses of UpgradeStep). Calls UpgradeStep.upgrade_step for each. - + Args: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. @@ -236,7 +237,7 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): It should normally be left as-is for all upgrades. Use _version_specific_upgrade below for any actions that are only required for the particular upgrade you are creating. - + Args: config (dict): The current configuration dictionary. start_version (str): The starting version of the configuration. @@ -647,6 +648,7 @@ class Upgrade_2023_10_1(UpgradeStep): changes, including the support for third-party plugins, upgrades JupyterHub to version 3.1, and deprecates certain components such as CDS Dashboards, ClearML, Prefect, and kbatch. """ + version = "2023.10.1" # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) # This released has been tested against 1.26 @@ -761,6 +763,7 @@ class Upgrade_2023_11_1(UpgradeStep): Note: - ClearML, Prefect, and kbatch are no longer supported in this version. """ + version = "2023.11.1" @override @@ -786,6 +789,7 @@ class Upgrade_2023_12_1(UpgradeStep): Note: - This is the last version that supports the jupyterlab-videochat extension. """ + version = "2023.12.1" @override @@ -812,6 +816,7 @@ class Upgrade_2024_1_1(UpgradeStep): Note: - jupyterlab-videochat, retrolab, jupyter-tensorboard, jupyterlab-conda-store, and jupyter-nvdashboard are no longer supported. """ + version = "2024.1.1" @override @@ -875,6 +880,7 @@ class Upgrade_2024_4_1(UpgradeStep): Note: - Adds default configuration for node groups if not already defined. """ + version = "2024.4.1" @override From 8f834431526654cccc52a1ac596d72d1aba9f063 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 19 Jun 2024 13:13:22 +0530 Subject: [PATCH 13/39] #1792 Add docstring to some additional methods --- src/_nebari/upgrade.py | 63 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 9c79bd46d..d154635dd 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -262,12 +262,30 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs): config["nebari_version"] = self.version def contains_image_and_tag(s: str) -> bool: - # match on `quay.io/nebari/nebari-<...>:YYYY.MM.XX`` + """ + Check if the string matches the Nebari image pattern. + + Args: + s (str): The string to check. + + Returns: + bool: True if the string matches the pattern, False otherwise. + """ pattern = r"^quay\.io\/nebari\/nebari-(jupyterhub|jupyterlab|dask-worker)(-gpu)?:\d{4}\.\d+\.\d+$" return bool(re.match(pattern, s)) - def replace_image_tag_legacy(image, start_version, new_version): - """Replace legacy image tags with the new version.""" + def replace_image_tag_legacy(image: str, start_version: str, new_version: str) -> str: + """ + Replace legacy image tags with the new version. + + Args: + image (str): The current image string. + start_version (str): The starting version of the image. + new_version (str): The new version to replace with. + + Returns: + str: The updated image string with the new version, or None if no match. + """ start_version_regex = start_version.replace(".", "\\.") if not start_version: start_version_regex = "0\\.[0-3]\\.[0-9]{1,2}" @@ -282,7 +300,17 @@ def replace_image_tag_legacy(image, start_version, new_version): return None def replace_image_tag(s: str, new_version: str, config_path: str) -> str: - """Replace the image tag with the new version.""" + """ + Replace the image tag with the new version. + + Args: + s (str): The current image string. + new_version (str): The new version to replace with. + config_path (str): The path to the configuration file. + + Returns: + str: The updated image string with the new version, or the original string if no changes. + """ legacy_replacement = replace_image_tag_legacy(s, start_version, new_version) if legacy_replacement: return legacy_replacement @@ -303,7 +331,17 @@ def replace_image_tag(s: str, new_version: str, config_path: str) -> str: return s def set_nested_item(config: dict, config_path: list, value: str): - """Set a nested item in the configuration dictionary.""" + """ + Set a nested item in the configuration dictionary. + + Args: + config (dict): The configuration dictionary. + config_path (list): The path to the item to set. + value (str): The value to set. + + Returns: + None + """ config_path = config_path.split(".") for k in config_path[:-1]: try: @@ -317,8 +355,19 @@ def set_nested_item(config: dict, config_path: list, value: str): pass config[config_path[-1]] = value - def update_image_tag(config, config_path, current_image, new_version): - """Update the image tag in the configuration.""" + def update_image_tag(config: dict, config_path: str, current_image: str, new_version: str) -> dict: + """ + Update the image tag in the configuration. + + Args: + config (dict): The configuration dictionary. + config_path (str): The path to the item to update. + current_image (str): The current image string. + new_version (str): The new version to replace with. + + Returns: + dict: The updated configuration dictionary. + """ new_image = replace_image_tag(current_image, new_version, config_path) if new_image != current_image: set_nested_item(config, config_path, new_image) From ff859256fa5a8affcb079220efb3b8ec52c9deca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 07:43:41 +0000 Subject: [PATCH 14/39] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/upgrade.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index d154635dd..5f34da74e 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -274,7 +274,9 @@ def contains_image_and_tag(s: str) -> bool: pattern = r"^quay\.io\/nebari\/nebari-(jupyterhub|jupyterlab|dask-worker)(-gpu)?:\d{4}\.\d+\.\d+$" return bool(re.match(pattern, s)) - def replace_image_tag_legacy(image: str, start_version: str, new_version: str) -> str: + def replace_image_tag_legacy( + image: str, start_version: str, new_version: str + ) -> str: """ Replace legacy image tags with the new version. @@ -355,7 +357,9 @@ def set_nested_item(config: dict, config_path: list, value: str): pass config[config_path[-1]] = value - def update_image_tag(config: dict, config_path: str, current_image: str, new_version: str) -> dict: + def update_image_tag( + config: dict, config_path: str, current_image: str, new_version: str + ) -> dict: """ Update the image tag in the configuration. From 941d7eb8c5f3fe3e7051edb07d2e104217064365 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 19 Jun 2024 20:21:34 +0530 Subject: [PATCH 15/39] #1792 Use `@override` from `typing-extensions` --- pyproject.toml | 1 + src/_nebari/upgrade.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 91b0fe4ed..e4e20a304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ dependencies = [ "ruamel.yaml==0.18.6", "typer==0.9.0", "packaging==23.2", + "typing-extensions==4.12.0", ] [project.optional-dependencies] diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 5f34da74e..f0c3e7b79 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -12,12 +12,13 @@ import textwrap from abc import ABC from pathlib import Path -from typing import Any, ClassVar, Dict, override +from typing import Any, ClassVar, Dict import rich from packaging.version import Version from pydantic import ValidationError from rich.prompt import Prompt +from typing_extensions import override from _nebari.config import backup_configuration from _nebari.stages.infrastructure import ( From aaf28edc65e687fab03d6766cfc976d9c8c46933 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 19 Jun 2024 20:41:03 +0530 Subject: [PATCH 16/39] #1792 Pin `typing-extensions==4.11.0` as 4.12 not available on conda --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e4e20a304..844c57e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ dependencies = [ "ruamel.yaml==0.18.6", "typer==0.9.0", "packaging==23.2", - "typing-extensions==4.12.0", + "typing-extensions==4.11.0", ] [project.optional-dependencies] From 3e94a6a07007da345e9941b5df0a25b28b184a0b Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:11:57 -0700 Subject: [PATCH 17/39] restrict public access and add buckett encryption using cmk --- .../template/aws/modules/s3/main.tf | 15 ++++++++++++ .../aws/modules/terraform-state/main.tf | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf index 6f349fbf1..e348f399c 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf @@ -1,3 +1,8 @@ +resource "aws_kms_key" "main" { + description = "KMS key for ${var.name}" + enable_key_rotation = true +} + resource "aws_s3_bucket" "main" { bucket = var.name acl = var.public ? "public-read" : "private" @@ -6,6 +11,16 @@ resource "aws_s3_bucket" "main" { enabled = true } + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.main.arn + sse_algorithm = "aws:kms" + } + } + + } + tags = merge({ Name = var.name Description = "S3 bucket for ${var.name}" diff --git a/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf index 2b0561dd7..dccf229cc 100644 --- a/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf +++ b/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf @@ -1,3 +1,7 @@ +resource "aws_kms_key" "tf-state-key" { + enable_key_rotation = true +} + resource "aws_s3_bucket" "terraform-state" { bucket = "${var.name}-terraform-state" @@ -16,6 +20,25 @@ resource "aws_s3_bucket" "terraform-state" { } } +resource "aws_s3_bucket_server_side_encryption_configuration" "terraform-state" { + bucket = aws_s3_bucket.terraform-state.id + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.tf-state-key.arn + sse_algorithm = "aws:kms" + } + } +} + +resource "aws_s3_bucket_public_access_block" "terraform-state" { + bucket = aws_s3_bucket.terraform-state.id + ignore_public_acls = true + block_public_acls = true + block_public_policy = true + restrict_public_buckets = true +} + resource "aws_dynamodb_table" "terraform-state-lock" { name = "${var.name}-terraform-state-lock" From 4f4dc41e4d877daa07059b78d1f7a18f6575d037 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:14:10 -0700 Subject: [PATCH 18/39] add cmk and block public access --- .../template/aws/modules/s3/main.tf | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf index e348f399c..54167d3ce 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf @@ -11,18 +11,27 @@ resource "aws_s3_bucket" "main" { enabled = true } - server_side_encryption_configuration { - rule { - apply_server_side_encryption_by_default { - kms_master_key_id = aws_kms_key.main.arn - sse_algorithm = "aws:kms" - } - } - - } - tags = merge({ Name = var.name Description = "S3 bucket for ${var.name}" }, var.tags) } + +resource "aws_s3_bucket_server_side_encryption_configuration" "main" { + bucket = aws_s3_bucket.main.id + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.main.arn + sse_algorithm = "aws:kms" + } + } +} + +resource "aws_s3_bucket_public_access_block" "main" { + bucket = aws_s3_bucket.main.id + ignore_public_acls = true + block_public_acls = true + block_public_policy = true + restrict_public_buckets = true +} From cbf57cd8ec1265e851d6a4c567a2572a6eed0acb Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:46:14 -0700 Subject: [PATCH 19/39] add overwrite to AWS coredns addon --- .../infrastructure/template/aws/modules/kubernetes/main.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf index 43e553850..521096cae 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf @@ -89,8 +89,11 @@ resource "aws_eks_addon" "aws-ebs-csi-driver" { } resource "aws_eks_addon" "coredns" { - addon_name = "coredns" - cluster_name = aws_eks_cluster.main.name + addon_name = "coredns" + cluster_name = aws_eks_cluster.main.name + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + configuration_values = jsonencode({ nodeSelector = { From 3a36f9b5d5930a2d03735297b640a81ef2577b05 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Mon, 24 Jun 2024 18:37:08 -0500 Subject: [PATCH 20/39] Replace data source ids. --- .../dashboards/Main/conda_store.json | 40 +++++++++---------- .../monitoring/dashboards/Main/traefik.json | 14 +++---- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json index 724b130bc..b7655c18d 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json @@ -112,7 +112,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_queued", @@ -123,7 +123,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_building", @@ -136,7 +136,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_completed", @@ -149,7 +149,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_failed", @@ -165,7 +165,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -216,7 +216,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_environments", @@ -231,7 +231,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -282,7 +282,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_queued", @@ -297,7 +297,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -348,7 +348,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_building", @@ -363,7 +363,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -414,7 +414,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_completed", @@ -429,7 +429,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -480,7 +480,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_build_failed", @@ -508,7 +508,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -559,7 +559,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_disk_usage / conda_store_disk_total", @@ -574,7 +574,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -621,7 +621,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_disk_total / (2.0^30)", @@ -637,7 +637,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -688,7 +688,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "conda_store_disk_usage / (2^30)", diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json index 503b48af7..188491c73 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json @@ -31,7 +31,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "gridPos": { "h": 1, @@ -108,7 +108,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "exemplar": true, "expr": "count(kube_pod_status_ready{namespace=\"$namespace\",condition=\"true\",pod=~\"nebari-traefik-ingress-.*\", job=\"kube-state-metrics\"})", @@ -211,7 +211,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "gridPos": { "h": 1, @@ -506,7 +506,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "gridPos": { "h": 1, @@ -806,7 +806,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "gridPos": { "h": 1, @@ -917,7 +917,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "gridPos": { "h": 1, @@ -1168,7 +1168,7 @@ }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "prometheus" }, "definition": "label_values(kube_pod_container_info{pod=~\".*traefik.*\"}, namespace)", "hide": 0, From 6e4076e449b00e93483205bedeb8e0d52d9a89b7 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 25 Jun 2024 20:09:57 -0500 Subject: [PATCH 21/39] Add option to run CRDs updates and DaemonSet deletion on user's behalf. --- src/_nebari/upgrade.py | 128 ++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 32 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index f0c3e7b79..41ca8ebd8 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -14,6 +14,9 @@ from pathlib import Path from typing import Any, ClassVar, Dict +import kubernetes.client +import kubernetes.config +import requests import rich from packaging.version import Version from pydantic import ValidationError @@ -35,6 +38,7 @@ from nebari.schema import ProviderEnum, is_version_accepted logger = logging.getLogger(__name__) +kubernetes.config.load_kube_config() NEBARI_WORKFLOW_CONTROLLER_DOCS = ( "https://www.nebari.dev/docs/how-tos/using-argo/#jupyterflow-override-beta" @@ -995,52 +999,112 @@ def _version_specific_upgrade( ): # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled if config.get("monitoring", {}).get("enabled", True): - rich.print( - "\n ⚠️ Warning ⚠️" - "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]" - "\n-> [red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]" - ) + + crd_urls = [ + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml", + "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml", + ] + daemonset_name = "prometheus-node-exporter" + namespace = config.get("namespace", "default") # We're upgrading from version 30.1.0 to 58.4.0. This is a major upgrade and requires manual intervention. # See https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/README.md#upgrading-chart # for more information on why the following commands are necessary. - commands = textwrap.dedent( - f""" - [cyan bold] - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml - kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml - kubectl delete daemonset -l app=prometheus-node-exporter --namespace {config['namespace']} - [/cyan bold] - """ + commands = "[cyan bold]" + for url in crd_urls: + commands += f"kubectl apply --server-side --force-conflicts -f {url}\n" + commands += f"kubectl delete daemonset -l app={daemonset_name} --namespace {namespace}\n" + commands += "[/cyan bold]" + + rich.print( + "\n ⚠️ Warning ⚠️" + "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]" + f"\n-> [red bold]Before upgrading, kube-prometheus-stack CRDs need to be updated and the {daemonset_name} daemonset needs to be deleted.[/red bold]" + ) + run_commands = Prompt.ask( + "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually.", + choices=["y", "N"], + default="N", ) # By default, rich wraps lines by splitting them into multiple lines. This is # far from ideal, as users copy-pasting the commands will get errors when running them. # To avoid this, we use a rich console with a larger width to print the entire commands # and let the terminal wrap them if needed. - Prompt.ask("Hit enter to show the commands") console = rich.console.Console(width=220) - console.print(commands) + if run_commands == "y": + current_kube_context = kubernetes.config.list_kube_config_contexts()[1] + cluster_name = current_kube_context["context"]["cluster"] + rich.print( + f"The following commands will be run for the [cyan bold]{cluster_name}[/cyan bold] cluster" + ) + Prompt.ask("Hit enter to show the commands") + console.print(commands) + + Prompt.ask("Hit enter to continue") + # We need to add a special constructor to the yaml loader to handle a specific + # tag as otherwise the kubernetes API will fail when updating the CRD. + yaml.constructor.add_constructor( + "tag:yaml.org,2002:value", lambda loader, node: node.value + ) + for url in crd_urls: + response = requests.get(url) + response.raise_for_status() + crd = yaml.load(response.text) + crd_name = crd["metadata"]["name"] + api_instance = kubernetes.client.ApiextensionsV1Api() + try: + api_response = api_instance.read_custom_resource_definition( + name=crd_name + ) + except kubernetes.client.exceptions.ApiException: + api_response = api_instance.create_custom_resource_definition( + body=crd + ) + else: + api_response = api_instance.patch_custom_resource_definition( + name=crd["metadata"]["name"], body=crd + ) - Prompt.ask("Hit enter to continue") - continue_ = Prompt.ask( - "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?", - choices=["y", "N"], - default="N", - ) - if not continue_ == "y": + api_instance = kubernetes.client.AppsV1Api() + api_response = api_instance.list_namespaced_daemon_set( + namespace=namespace, label_selector=f"app={daemonset_name}" + ) + if api_response.items: + api_instance.delete_namespaced_daemon_set( + name=api_response.items[0].metadata.name, + namespace=namespace, + ) + + rich.print( + f"The kube-prometheus-stack CRDs have been updated and the {daemonset_name} daemonset has been deleted." + ) + else: rich.print( - f"[red bold]You must back up your custom dashboards (if necessary), delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]" + "[red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]" ) - exit() + Prompt.ask("Hit enter to show the commands") + console.print(commands) + + Prompt.ask("Hit enter to continue") + continue_ = Prompt.ask( + f"Have you backed up your custom dashboards (if necessary), deleted the {daemonset_name} daemonset and updated the kube-prometheus-stack CRDs?", + choices=["y", "N"], + default="N", + ) + if not continue_ == "y": + rich.print( + f"[red bold]You must back up your custom dashboards (if necessary), delete the {daemonset_name} daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]" + ) + exit() # Prompt users to upgrade to the new default node groups for GCP if (provider := config.get("provider", "")) == ProviderEnum.gcp.value: From f19dfb23744ddcffb57d475d49ff88f2ad40ca11 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 25 Jun 2024 20:33:47 -0500 Subject: [PATCH 22/39] Move kube config loading inside upgrade function. --- src/_nebari/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 41ca8ebd8..3a1f63f05 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -38,7 +38,6 @@ from nebari.schema import ProviderEnum, is_version_accepted logger = logging.getLogger(__name__) -kubernetes.config.load_kube_config() NEBARI_WORKFLOW_CONTROLLER_DOCS = ( "https://www.nebari.dev/docs/how-tos/using-argo/#jupyterflow-override-beta" @@ -1041,6 +1040,7 @@ def _version_specific_upgrade( # and let the terminal wrap them if needed. console = rich.console.Console(width=220) if run_commands == "y": + kubernetes.config.load_kube_config() current_kube_context = kubernetes.config.list_kube_config_contexts()[1] cluster_name = current_kube_context["context"]["cluster"] rich.print( From c38517267986e15ee3589bf553d293a35ac0d2eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:30:48 +0100 Subject: [PATCH 23/39] Implement support for jupyterlab-gallery config (#2501) --- .../stages/kubernetes_services/__init__.py | 21 +++++++++++++++++++ .../template/jupyterhub.tf | 19 +++++++++++++++++ .../services/jupyterhub/configmaps.tf | 13 ++++++++++-- .../services/jupyterhub/variables.tf | 17 +++++++++++++++ ...min.happy.jupyterlab.gallery_settings.yaml | 10 +++++++++ 5 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 tests/tests_unit/cli_validate/min.happy.jupyterlab.gallery_settings.yaml diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index fae8955de..edfe865a7 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -235,8 +235,25 @@ class IdleCuller(schema.Base): server_shutdown_no_activity_timeout: int = 15 +class JupyterLabGalleryExhibit(schema.Base): + git: str + title: str + homepage: Optional[str] = None + description: Optional[str] = None + icon: Optional[str] = None + account: Optional[str] = None + token: Optional[str] = None + + +class JupyterLabGallerySettings(schema.Base): + title: str = "Examples" + destination: str = "examples" + exhibits: List[JupyterLabGalleryExhibit] = [] + + class JupyterLab(schema.Base): default_settings: Dict[str, Any] = {} + gallery_settings: JupyterLabGallerySettings = JupyterLabGallerySettings() idle_culler: IdleCuller = IdleCuller() initial_repositories: List[Dict[str, str]] = [] preferred_dir: Optional[str] = None @@ -368,6 +385,9 @@ class JupyterhubInputVars(schema.Base): jupyterlab_default_settings: Dict[str, Any] = Field( alias="jupyterlab-default-settings" ) + jupyterlab_gallery_settings: JupyterLabGallerySettings = Field( + alias="jupyterlab-gallery-settings" + ) initial_repositories: str = Field(alias="initial-repositories") jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage") @@ -534,6 +554,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): jhub_apps_enabled=self.config.jhub_apps.enabled, initial_repositories=str(self.config.jupyterlab.initial_repositories), jupyterlab_default_settings=self.config.jupyterlab.default_settings, + jupyterlab_gallery_settings=self.config.jupyterlab.gallery_settings, jupyterlab_preferred_dir=self.config.jupyterlab.preferred_dir, ) diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf index 4f8bebb9e..425c60bed 100644 --- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf +++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf @@ -54,6 +54,23 @@ variable "jupyterlab-default-settings" { type = map(any) } +variable "jupyterlab-gallery-settings" { + description = "Server-side settings for jupyterlab-gallery extension" + type = object({ + title = optional(string) + destination = optional(string) + exhibits = list(object({ + git = string + title = string + homepage = optional(string) + description = optional(string) + icon = optional(string) + account = optional(string) + token = optional(string) + })) + }) +} + variable "jupyterhub-hub-extraEnv" { description = "Extracted overrides to merge with jupyterhub.hub.extraEnv" type = string @@ -149,6 +166,8 @@ module "jupyterhub" { jupyterlab-default-settings = var.jupyterlab-default-settings + jupyterlab-gallery-settings = var.jupyterlab-gallery-settings + jupyterlab-pioneer-enabled = var.jupyterlab-pioneer-enabled jupyterlab-pioneer-log-format = var.jupyterlab-pioneer-log-format diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf index 4b8f9145b..bfee219e9 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf @@ -47,6 +47,13 @@ resource "local_file" "jupyter_jupyterlab_pioneer_config_py" { } } +resource "local_sensitive_file" "jupyter_gallery_config_json" { + content = jsonencode({ + "GalleryManager" = var.jupyterlab-gallery-settings + }) + filename = "${path.module}/files/jupyter/jupyter_gallery_config.json" +} + resource "local_file" "overrides_json" { content = jsonencode(local.jupyterlab-overrides-json-object) @@ -70,7 +77,8 @@ resource "kubernetes_config_map" "etc-ipython" { locals { etc-jupyter-config-data = merge( { - "jupyter_server_config.py" = local_file.jupyter_server_config_py.content, + "jupyter_server_config.py" = local_file.jupyter_server_config_py.content, + "jupyter_gallery_config.json" = local_sensitive_file.jupyter_gallery_config_json.content, }, var.jupyterlab-pioneer-enabled ? { # quotes are must here, as terraform would otherwise think py is a property of @@ -89,7 +97,8 @@ locals { resource "kubernetes_config_map" "etc-jupyter" { depends_on = [ local_file.jupyter_server_config_py, - local_file.jupyter_jupyterlab_pioneer_config_py + local_file.jupyter_jupyterlab_pioneer_config_py, + local_sensitive_file.jupyter_gallery_config_json ] metadata { diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf index 577dedc8e..af6f06654 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf @@ -163,6 +163,23 @@ variable "jupyterlab-default-settings" { type = map(any) } +variable "jupyterlab-gallery-settings" { + description = "Server-side settings for jupyterlab-gallery extension" + type = object({ + title = optional(string) + destination = optional(string) + exhibits = list(object({ + git = string + title = string + homepage = optional(string) + description = optional(string) + icon = optional(string) + account = optional(string) + token = optional(string) + })) + }) +} + variable "jupyterlab-pioneer-enabled" { description = "Enable JupyterLab Pioneer for telemetry" type = bool diff --git a/tests/tests_unit/cli_validate/min.happy.jupyterlab.gallery_settings.yaml b/tests/tests_unit/cli_validate/min.happy.jupyterlab.gallery_settings.yaml new file mode 100644 index 000000000..02df930fd --- /dev/null +++ b/tests/tests_unit/cli_validate/min.happy.jupyterlab.gallery_settings.yaml @@ -0,0 +1,10 @@ +project_name: test +jupyterlab: + gallery_settings: + title: Example repositories + destination: examples + exhibits: + - title: Nebari + git: https://github.com/nebari-dev/nebari.git + homepage: https://github.com/nebari-dev/nebari + description: 🪴 Nebari - your open source data science platform From b1aefa2981f44a8307722418f0cedfdb13c68921 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 26 Jun 2024 08:37:26 -0500 Subject: [PATCH 24/39] Handle config exception. --- src/_nebari/upgrade.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 3a1f63f05..6639f2a65 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1040,7 +1040,13 @@ def _version_specific_upgrade( # and let the terminal wrap them if needed. console = rich.console.Console(width=220) if run_commands == "y": - kubernetes.config.load_kube_config() + try: + kubernetes.config.load_kube_config() + except kubernetes.config.config_exception.ConfigException: + rich.print( + "[red bold]No default kube configuration file was found. Make sure to have one pointing to your Nebari cluster before upgrading.[/red bold]" + ) + exit() current_kube_context = kubernetes.config.list_kube_config_contexts()[1] cluster_name = current_kube_context["context"]["cluster"] rich.print( From 37615faa6c2b005b86c8065b1aaf55c458b04619 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 26 Jun 2024 09:54:58 -0500 Subject: [PATCH 25/39] Add link to docs on how to generate the kube config. --- src/_nebari/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 6639f2a65..7e45dfa80 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -1044,7 +1044,7 @@ def _version_specific_upgrade( kubernetes.config.load_kube_config() except kubernetes.config.config_exception.ConfigException: rich.print( - "[red bold]No default kube configuration file was found. Make sure to have one pointing to your Nebari cluster before upgrading.[/red bold]" + "[red bold]No default kube configuration file was found. Make sure to [link=https://www.nebari.dev/docs/how-tos/debug-nebari#generating-the-kubeconfig]have one pointing to your Nebari cluster[/link] before upgrading.[/red bold]" ) exit() current_kube_context = kubernetes.config.list_kube_config_contexts()[1] From 0d8350f590a269e3414e9fe1a1c1aca83e398c22 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 26 Jun 2024 09:55:14 -0500 Subject: [PATCH 26/39] Mock different prompts. --- tests/tests_unit/test_upgrade.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index 4871a1fe0..a19095726 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from rich.prompt import Prompt from _nebari.upgrade import do_upgrade from _nebari.version import __version__, rounded_ver_parse @@ -48,8 +49,29 @@ def test_upgrade_4_0( qhub_users_import_json, monkeypatch, ): - # Return "y" when asked if you've deleted the Argo CRDs - monkeypatch.setattr("builtins.input", lambda: "y") + + def mock_input(prompt, **kwargs): + # Mock different upgrade steps prompt answers + if ( + prompt + == "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] " + ): + return "y" + elif ( + prompt + == "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually." + ): + return "N" + elif ( + prompt + == "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?" + ): + return "y" + # All other prompts will be answered with "y" + else: + return "y" + + monkeypatch.setattr(Prompt, "ask", mock_input) old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str From 6733b578fdf51ae8a6233e485a3faa2284c2357e Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 26 Jun 2024 15:48:03 -0500 Subject: [PATCH 27/39] Update release notes and constants. --- RELEASE.md | 39 +++++++++++++++++++++++++++++++++++++++ src/_nebari/constants.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index e1bac3acc..a6cd9deac 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,45 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> --- +### Release 2024.6.1 - June 26, 2024 + +> NOTE: This release includes an upgrade to the `kube-prometheus-stack` Helm chart, resulting in a newer version of Grafana. When upgrading your Nebari cluster, you will be prompted to have Nebari update some CRDs and delete a DaemonSet on your behalf. If you prefer, you can also run the commands yourself, which will be shown to you. If you have any custom dashboards, you'll also need to back them up by [exporting them as JSON](https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json), so you can [import them](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard) after upgrading. + +### What's Changed +* Fetch JupyterHub roles from Keycloak by @krassowski in https://github.com/nebari-dev/nebari/pull/2447 +* Update selector for Start server button to use button tag by @krassowski in https://github.com/nebari-dev/nebari/pull/2464 +* Reduce GCP Fixed Costs by 50% by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2453 +* Restore JupyterHub updates from PR-2427 by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2465 +* Workload identity by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2460 +* Fix test using a non-specific selector by @krassowski in https://github.com/nebari-dev/nebari/pull/2475 +* add verify=false since we use self signed cert in tests by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2481 +* fix forward auth when using custom cert by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2479 +* Upgrade to JupyterHub 5.0.0b2 by @krassowski in https://github.com/nebari-dev/nebari/pull/2468 +* upgrade instructions for PR 2453 by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2466 +* Use Helm Chart for JupyterHub 5.0.0 final by @krassowski in https://github.com/nebari-dev/nebari/pull/2484 +* Parse and insert keycloak roles scopes into JupyterHub by @aktech in https://github.com/nebari-dev/nebari/pull/2471 +* Add CITATION file by @pavithraes in https://github.com/nebari-dev/nebari/pull/2455 +* CI: add azure integration by @fangchenli in https://github.com/nebari-dev/nebari/pull/2061 +* Create trivy.yml by @dcmcand in https://github.com/nebari-dev/nebari/pull/2458 +* don't run azure deployment on PRs, only on schedule and manual trigger by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2498 +* add cloud provider deployment status badges to README.md by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2407 +* Upgrade kube-prometheus-stack helm chart by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2472 +* upgrade note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2502 +* Remove VSCode from jhub_apps default services by @jbouder in https://github.com/nebari-dev/nebari/pull/2503 +* Explicit config by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2294 +* fix general node scaling bug for azure by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2517 +* Skip running cleanup on pull requests by @aktech in https://github.com/nebari-dev/nebari/pull/2488 +* 1792 Add docstrings to `upgrade.py` by @arjxn-py in https://github.com/nebari-dev/nebari/pull/2512 +* set's min TLS version for azure storage account to TLS 1.2 by @dcmcand in https://github.com/nebari-dev/nebari/pull/2522 +* Fix conda-store and Traefik Grafana Dashboards by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2540 +* Implement support for jupyterlab-gallery config by @krassowski in https://github.com/nebari-dev/nebari/pull/2501 +* Add option to run CRDs updates and DaemonSet deletion on user's behalf. by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2544 + +### New Contributors +* @arjxn-py made their first contribution in https://github.com/nebari-dev/nebari/pull/2512 + +**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.5.1...2024.6.1 + ### Release 2024.5.1 - May 13, 2024 ## What's Changed diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index d0e3f3744..8194d2032 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -1,4 +1,4 @@ -CURRENT_RELEASE = "2024.5.1" +CURRENT_RELEASE = "2024.6.1" # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes # implemented in August 2023. From 85849a3d29b6aa489dc362c430455d55e7cd2337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Thu, 27 Jun 2024 23:11:14 +0100 Subject: [PATCH 28/39] Enable authentication by default in jupyter-server (#2288) Co-authored-by: Marcelo Villa --- .../jupyterhub/files/jupyter/jupyter_server_config.py.tpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl index d5e089dfa..f8206a3ec 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl @@ -4,6 +4,13 @@ # Extra config available at: # https://zero-to-jupyterhub.readthedocs.io/en/1.x/jupyterhub/customizing/user-management.html#culling-user-pods + +# Refuse to serve content from handlers missing authentication guards, unless +# the handler is explicitly allow-listed with `@allow_unauthenticated`; this +# prevents accidental exposure of information by extensions installed in the +# single-user server when their handlers are missing authentication guards. +c.ServerApp.allow_unauthenticated_access = False + # Enable Show Hidden Files menu option in View menu c.ContentsManager.allow_hidden = True c.FileContentsManager.allow_hidden = True From 3beafc029f704a54e5ba08637e984031b117e847 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:34:41 -0500 Subject: [PATCH 29/39] remove dns sleep (#2550) --- src/_nebari/stages/kubernetes_ingress/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 628d38383..ea5f8fa33 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -93,17 +93,18 @@ def _attempt_dns_lookup( attempt = 0 while not _attempt_dns_lookup(domain_name, ip): - sleeptime = 60 * (2**attempt) - if not disable_prompt: + if disable_prompt: + sleeptime = 60 * (2**attempt) + print(f"Will attempt to poll DNS again in {sleeptime} seconds...") + time.sleep(sleeptime) + else: input( f"After attempting to poll the DNS, the record for domain={domain_name} appears not to exist, " f"has recently been updated, or has yet to fully propagate. This non-deterministic behavior is likely due to " - f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again in {sleeptime} seconds " - f"[Press Enter].\n\n...otherwise kill the process and run the deployment again later..." + f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again [Press Enter].\n\n" + f"...otherwise kill the process and run the deployment again later..." ) - print(f"Will attempt to poll DNS again in {sleeptime} seconds...") - time.sleep(sleeptime) attempt += 1 if attempt == 5: print( From ebcd68369746ee48548179958e2a04121ae6c86a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 01:45:42 +0000 Subject: [PATCH 30/39] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.7 → v0.5.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.7...v0.5.0) - [github.com/antonbabenko/pre-commit-terraform: v1.90.0 → v1.92.0](https://github.com/antonbabenko/pre-commit-terraform/compare/v1.90.0...v1.92.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e53c3da0f..96795fb23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: args: ["--line-length=88", "--exclude=/src/_nebari/template/"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.5.0 hooks: - id: ruff args: ["--fix"] @@ -73,7 +73,7 @@ repos: # terraform - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.90.0 + rev: v1.92.0 hooks: - id: terraform_fmt args: From c42a1040960e27f43dc348a5310fe3d198e4e790 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 2 Jul 2024 15:34:08 +0100 Subject: [PATCH 31/39] Conda-store permissions v2 + load roles from keycloak (#2531) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../stages/kubernetes_services/__init__.py | 6 + .../conda-store/config/conda_store_config.py | 293 +++++++++++++++++- .../kubernetes/services/conda-store/server.tf | 36 ++- tests/common/conda_store_utils.py | 41 +++ tests/tests_deployment/conftest.py | 1 + .../test_conda_store_roles_loaded.py | 66 ++++ 6 files changed, 426 insertions(+), 17 deletions(-) create mode 100644 tests/common/conda_store_utils.py create mode 100644 tests/tests_deployment/test_conda_store_roles_loaded.py diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index edfe865a7..6cbf03dbb 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -488,6 +488,12 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "*/*": ["viewer"], }, }, + "conda-store-service-account": { + "primary_namespace": "", + "role_bindings": { + "*/*": ["admin"], + }, + }, } # Compound any logout URLs from extensions so they are are logged out in succession diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index 6ed6232ba..ad9b79843 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -1,6 +1,12 @@ +import dataclasses import json import logging +import re import tempfile +import typing +import urllib +import urllib.parse +import urllib.request from pathlib import Path import requests @@ -17,7 +23,6 @@ def conda_store_config(path="/var/lib/conda-store/config.json"): config = conda_store_config() - # ================================== # conda-store settings # ================================== @@ -49,11 +54,15 @@ def conda_store_config(path="/var/lib/conda-store/config.json"): "conda-forge", "https://repo.anaconda.com/pkgs/main", ] +c.RBACAuthorizationBackend.role_mappings_version = 2 # ================================== # server settings # ================================== c.CondaStoreServer.log_level = logging.INFO +c.CondaStoreServer.log_format = ( + "%(asctime)s %(levelname)9s %(name)s:%(lineno)4s: %(message)s" +) c.CondaStoreServer.enable_ui = True c.CondaStoreServer.enable_api = True c.CondaStoreServer.enable_registry = True @@ -64,7 +73,6 @@ def conda_store_config(path="/var/lib/conda-store/config.json"): # This MUST start with `/` c.CondaStoreServer.url_prefix = "/conda-store" - # ================================== # auth settings # ================================== @@ -83,10 +91,280 @@ def conda_store_config(path="/var/lib/conda-store/config.json"): c.GenericOAuthAuthentication.user_data_key = "preferred_username" c.GenericOAuthAuthentication.tls_verify = False +CONDA_STORE_ROLE_PERMISSIONS_ORDER = ["viewer", "developer", "admin"] + + +@dataclasses.dataclass +class CondaStoreNamespaceRole: + namespace: str + role: str + + +@dataclasses.dataclass +class KeyCloakCondaStoreRoleScopes: + scopes: str + log: logging.Logger + + def _validate_role(self, role): + valid = role in CONDA_STORE_ROLE_PERMISSIONS_ORDER + self.log.info(f"role: {role} is {'valid' if valid else 'invalid'}") + return valid + + def parse_role_and_namespace( + self, text + ) -> typing.Optional[CondaStoreNamespaceRole]: + # The regex pattern + pattern = r"^(\w+)!namespace=([^!]+)$" + + # Perform the regex search + match = re.search(pattern, text) + + # Extract the permission and namespace if there is a match + if match and self._validate_role(match.group(1)): + return CondaStoreNamespaceRole( + namespace=match.group(2), role=match.group(1) + ) + else: + return None + + def parse_scope(self) -> typing.List[CondaStoreNamespaceRole]: + """Parsed scopes from keycloak role's attribute and returns a list of role/namespace + if scopes' syntax is valid otherwise return [] + + Example: + Given scopes as "viewer!namespace=scipy,admin!namespace=pycon", the function will + return [{"role": "viewer", "namespace": "scipy"}, {"role": "admin", "namespace": "pycon"}] + """ + if not self.scopes: + self.log.info(f"No scope found: {self.scopes}, skipping role") + return [] + scope_list = self.scopes.split(",") + parsed_scopes = [] + self.log.info(f"Scopes to parse: {scope_list}") + for scope_text in scope_list: + parsed_scope = self.parse_role_and_namespace(scope_text) + parsed_scopes.append(parsed_scope) + if not parsed_scope: + self.log.info(f"Unable to parse: {scope_text}, skipping keycloak role") + return [] + return parsed_scopes + class KeyCloakAuthentication(GenericOAuthAuthentication): + conda_store_api_url = f"https://{config['external-url']}/conda-store/api/v1" + access_token_url = config["token_url_internal"] + realm_api_url = config["realm_api_url_internal"] + service_account_token = config["service-tokens-mapping"][ + "conda-store-service-account" + ] + + def _get_conda_store_client_id(self, token: str) -> str: + # Get the clients list to find the "id" of "conda-store" client. + self.log.info("Getting conda store client id") + clients_data = self._fetch_api(endpoint="clients/", token=token) + conda_store_clients = [ + client for client in clients_data if client["clientId"] == "conda_store" + ] + self.log.info(f"conda store clients: {conda_store_clients}") + assert len(conda_store_clients) == 1 + conda_store_client_id = conda_store_clients[0]["id"] + return conda_store_client_id + + async def _delete_conda_store_roles(self, request, namespace: str, username: str): + self.log.info( + f"Delete all conda-store roles on namespace: {namespace} for user: {username}" + ) + conda_store = await get_conda_store(request) + with conda_store.session_factory() as db: + api.delete_namespace_role(db, namespace, other=username) + db.commit() + + async def _create_conda_store_role( + self, request, namespace_role: CondaStoreNamespaceRole, username: str + ): + self.log.info( + f"Creating conda-store roles on namespace: {namespace_role.namespace} for user: {username}" + ) + conda_store = await get_conda_store(request) + with conda_store.session_factory() as db: + api.create_namespace_role( + db, namespace_role.namespace, username, namespace_role.role + ) + db.commit() + + def _get_keycloak_token(self) -> str: + body = urllib.parse.urlencode( + { + "client_id": self.client_id, + "client_secret": self.client_secret, + "grant_type": "client_credentials", + } + ) + self.log.info(f"Getting token from access token url: {self.access_token_url}") + req = urllib.request.Request(self.access_token_url, data=body.encode()) + response = urllib.request.urlopen(req) + data = json.loads(response.read()) + return data["access_token"] # type: ignore[no-any-return] + + def _fetch_api(self, endpoint: str, token: str): + request_url = f"{self.realm_api_url}/{endpoint}" + req = urllib.request.Request( + request_url, + method="GET", + headers={"Authorization": f"Bearer {token}"}, + ) + self.log.info(f"Making request to: {request_url}") + with urllib.request.urlopen(req) as response: + data = json.loads(response.read()) + return data + + async def _remove_current_bindings(self, request, username): + """Remove current roles for the user to make sure only the roles defined in + keycloak are applied: + - to avoid inconsistency in user roles + - single source of truth + - roles that are added in keycloak and then later removed from keycloak are actually removed from conda-store. + """ + entity_bindings = self._get_current_entity_bindings(username) + self.log.info("Remove current role bindings for the user") + for entity, role in entity_bindings.items(): + if entity not in {"default/*", "filesystem/*"}: + namespace = entity.split("/")[0] + self.log.info( + f"Removing current role {role} on namespace {namespace} " + f"for user {username}" + ) + await self._delete_conda_store_roles(request, namespace, username) + + async def _apply_roles_from_keycloak(self, request, user_data): + token = self._get_keycloak_token() + conda_store_client_id = self._get_conda_store_client_id(token) + conda_store_client_roles = self._get_conda_store_client_roles_for_user( + user_data["sub"], conda_store_client_id, token + ) + await self._remove_current_bindings(request, user_data["preferred_username"]) + await self._apply_conda_store_roles_from_keycloak( + request, conda_store_client_roles, user_data["preferred_username"] + ) + + def _filter_duplicate_namespace_roles_with_max_permissions( + self, namespace_roles: typing.List[CondaStoreNamespaceRole] + ): + """Filter duplicate roles in keycloak such that to apply only the one with the highest + permissions. + + Example: + role 1: namespace: foo, role: viewer + role 2: namespace: foo, role: admin + We need to apply only the role 2 as that one has higher permissions. + """ + self.log.info("Filtering duplicate roles for same namespace") + namespace_role_mapping: typing.Dict[str:CondaStoreNamespaceRole] = {} + for namespace_role in namespace_roles: + namespace = namespace_role.namespace + new_role = namespace_role.role + + existing_role: CondaStoreNamespaceRole = namespace_role_mapping.get( + namespace + ) + if not existing_role: + # Add if not already added + namespace_role_mapping[namespace] = namespace_role + else: + # Only add if the permissions of this role is higher than existing + new_role_priority = CONDA_STORE_ROLE_PERMISSIONS_ORDER.index(new_role) + existing_role_priority = CONDA_STORE_ROLE_PERMISSIONS_ORDER.index( + existing_role.role + ) + if new_role_priority > existing_role_priority: + namespace_role_mapping[namespace] = new_role + return list(namespace_role_mapping.values()) + + def _get_permissions_from_keycloak_role( + self, keycloak_role + ) -> typing.List[CondaStoreNamespaceRole]: + self.log.info(f"Getting permissions from keycloak role: {keycloak_role}") + role_attributes = keycloak_role["attributes"] + # scopes returns a list with a value say ["viewer!namespace=pycon,developer!namespace=scipy"] + scopes = role_attributes.get("scopes", [""])[0] + k_cstore_scopes = KeyCloakCondaStoreRoleScopes(scopes=scopes, log=self.log) + return k_cstore_scopes.parse_scope() + + async def _apply_conda_store_roles_from_keycloak( + self, request, conda_store_client_roles, username + ): + self.log.info( + f"Apply conda store roles from keycloak roles: {conda_store_client_roles}, user: {username}" + ) + role_permissions: typing.List[CondaStoreNamespaceRole] = [] + for conda_store_client_role in conda_store_client_roles: + role_permissions += self._get_permissions_from_keycloak_role( + conda_store_client_role + ) + + self.log.info("Filtering duplicate namespace role for max permissions") + filtered_namespace_role: typing.List[CondaStoreNamespaceRole] = ( + self._filter_duplicate_namespace_roles_with_max_permissions( + role_permissions + ) + ) + self.log.info(f"Final role permissions to apply: {filtered_namespace_role}") + for namespace_role in filtered_namespace_role: + if namespace_role.namespace.lower() == username.lower(): + self.log.info("Role for given user's namespace, skipping") + continue + try: + await self._delete_conda_store_roles( + request, namespace_role.namespace, username + ) + await self._create_conda_store_role(request, namespace_role, username) + except ValueError as e: + self.log.error( + f"Failed to add permissions for namespace: {namespace_role.namespace} to user: {username}" + ) + self.log.exception(e) + + def _get_keycloak_conda_store_roles_with_attributes( + self, roles: dict, client_id: str, token: str + ): + """This fetches all roles by id to fetch their attributes.""" + roles_rich = [] + for role in roles: + # If this takes too much time, which isn't the case right now, we can + # also do multi-threaded requests + role_rich = self._fetch_api( + endpoint=f"roles-by-id/{role['id']}?client={client_id}", token=token + ) + roles_rich.append(role_rich) + return roles_rich + + def _get_conda_store_client_roles_for_user( + self, user_id, conda_store_client_id, token + ): + """Get roles for the client named 'conda-store' for the given user_id.""" + self.log.info( + f"Get conda store client roles for user: {user_id}, conda_store_client_id: {conda_store_client_id}" + ) + user_roles = self._fetch_api( + endpoint=f"users/{user_id}/role-mappings/clients/{conda_store_client_id}/composite", + token=token, + ) + client_roles_rich = self._get_keycloak_conda_store_roles_with_attributes( + user_roles, client_id=conda_store_client_id, token=token + ) + self.log.info(f"conda store client roles: {client_roles_rich}") + return client_roles_rich + + def _get_current_entity_bindings(self, username): + entity = schema.AuthenticationToken( + primary_namespace=username, role_bindings={} + ) + self.log.info(f"entity: {entity}") + entity_bindings = self.authorization.get_entity_bindings(entity) + self.log.info(f"current entity_bindings: {entity_bindings}") + return entity_bindings + async def authenticate(self, request): - # 1. using the callback_url code and state in request oauth_access_token = self._get_oauth_token(request) if oauth_access_token is None: return None # authentication failed @@ -98,9 +376,14 @@ async def authenticate(self, request): ) response.raise_for_status() user_data = response.json() - username = user_data["preferred_username"] + try: + await self._apply_roles_from_keycloak(request, user_data=user_data) + except Exception as e: + self.log.error("Adding roles from keycloak failed") + self.log.exception(e) + # superadmin gets access to everything if "conda_store_superadmin" in user_data.get("roles", []): return schema.AuthenticationToken( @@ -119,7 +402,9 @@ async def authenticate(self, request): if role in role_mappings } default_namespace = config["default-namespace"] + self.log.info(f"default_namespace: {default_namespace}") namespaces = {username, "global", default_namespace} + self.log.info(f"namespaces: {namespaces}") role_bindings = { f"{username}/*": {"admin"}, f"{default_namespace}/*": {"viewer"}, diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf index ab9edd87e..8a29bc2d4 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf @@ -13,22 +13,28 @@ resource "kubernetes_secret" "conda-store-secret" { data = { "config.json" = jsonencode({ - external-url = var.external-url - minio-username = module.minio.root_username - minio-password = module.minio.root_password - minio-service = module.minio.service - redis-password = module.redis.root_password - redis-service = module.redis.service - postgres-username = module.postgresql.root_username - postgres-password = module.postgresql.root_password - postgres-service = module.postgresql.service - openid-config = module.conda-store-openid-client.config - extra-settings = var.extra-settings - extra-config = var.extra-config - default-namespace = var.default-namespace-name + external-url = var.external-url + minio-username = module.minio.root_username + minio-password = module.minio.root_password + minio-service = module.minio.service + redis-password = module.redis.root_password + redis-service = module.redis.service + postgres-username = module.postgresql.root_username + postgres-password = module.postgresql.root_password + postgres-service = module.postgresql.service + openid-config = module.conda-store-openid-client.config + extra-settings = var.extra-settings + extra-config = var.extra-config + default-namespace = var.default-namespace-name + token_url_internal = "http://keycloak-http.${var.namespace}.svc/auth/realms/${var.realm_id}/protocol/openid-connect/token" + realm_api_url_internal = "http://keycloak-http.${var.namespace}.svc/auth/admin/realms/${var.realm_id}" service-tokens = { for service, value in var.services : base64encode(random_password.conda_store_service_token[service].result) => value } + # So that the mapping can be used in conda-store config itself + service-tokens-mapping = { + for service, _ in var.services : service => base64encode(random_password.conda_store_service_token[service].result) + } extra-settings = var.extra-settings extra-config = var.extra-config }) @@ -63,6 +69,10 @@ module "conda-store-openid-client" { callback-url-paths = [ "https://${var.external-url}/conda-store/oauth_callback" ] + service-accounts-enabled = true + service-account-roles = [ + "view-realm", "view-users", "view-clients" + ] } diff --git a/tests/common/conda_store_utils.py b/tests/common/conda_store_utils.py new file mode 100644 index 000000000..c150b4f6f --- /dev/null +++ b/tests/common/conda_store_utils.py @@ -0,0 +1,41 @@ +import re + +import requests + +from tests.tests_deployment import constants + + +def get_conda_store_session(): + """Log into conda-store using the test account and get session""" + session = requests.Session() + r = session.get( + f"https://{constants.NEBARI_HOSTNAME}/conda-store/login/?next=", verify=False + ) + auth_url = re.search('action="([^"]+)"', r.content.decode("utf8")).group(1) + response = session.post( + auth_url.replace("&", "&"), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={ + "username": constants.KEYCLOAK_USERNAME, + "password": constants.KEYCLOAK_PASSWORD, + "credentialId": "", + }, + verify=False, + ) + assert response.status_code == 200 + return session + + +def get_conda_store_user_permissions(): + """Log into conda-store using the test account and get session and using the token in + session call conda-store API to get permissions. + """ + session = get_conda_store_session() + token = session.cookies.get("conda-store-auth") + response = requests.get( + f"https://{constants.NEBARI_HOSTNAME}/conda-store/api/v1/permission/", + headers={"Authorization": f"Bearer {token}"}, + verify=False, + ) + assert response.status_code == 200 + return response.json() diff --git a/tests/tests_deployment/conftest.py b/tests/tests_deployment/conftest.py index fa7130282..7464f6bb4 100644 --- a/tests/tests_deployment/conftest.py +++ b/tests/tests_deployment/conftest.py @@ -9,3 +9,4 @@ def cleanup_keycloak_roles(): yield # teardown delete_client_keycloak_test_roles(client_name="jupyterhub") + delete_client_keycloak_test_roles(client_name="conda_store") diff --git a/tests/tests_deployment/test_conda_store_roles_loaded.py b/tests/tests_deployment/test_conda_store_roles_loaded.py new file mode 100644 index 000000000..732b0b015 --- /dev/null +++ b/tests/tests_deployment/test_conda_store_roles_loaded.py @@ -0,0 +1,66 @@ +import pytest + +from tests.common.conda_store_utils import get_conda_store_user_permissions +from tests.tests_deployment import constants +from tests.tests_deployment.keycloak_utils import ( + assign_keycloak_client_role_to_user, + create_keycloak_role, +) + + +@pytest.mark.parametrize( + "scopes,changed_scopes", + ( + [ + "admin!namespace=analyst,developer!namespace=nebari-git", + {"nebari-git/*": ["developer"], "analyst/*": ["admin"]}, + ], + [ + "admin!namespace=analyst,developer!namespace=invalid-namespace", + {"analyst/*": ["admin"]}, + ], + [ + # duplicate namespace role, chose highest permissions + "admin!namespace=analyst,developer!namespace=analyst", + {"analyst/*": ["admin"]}, + ], + ["invalid-role!namespace=analyst", {}], + ), +) +@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") +@pytest.mark.filterwarnings( + "ignore:.*auto_refresh_token is deprecated:DeprecationWarning" +) +@pytest.mark.filterwarnings("ignore::ResourceWarning") +def test_conda_store_roles_loaded_from_keycloak( + scopes: str, changed_scopes: dict, cleanup_keycloak_roles +): + + # Verify permissions/roles are different from what we're about to set + # So that this test is actually testing the change + permissions = get_conda_store_user_permissions() + entity_roles = permissions["data"]["entity_roles"] + for namespace, role in changed_scopes.items(): + assert entity_roles[namespace] != role + + role = create_keycloak_role( + client_name="conda_store", + # Note: we're clearing this role after every test case, and we're clearing + # it by name, so it must start with test- to be deleted afterwards + role_name="test-custom-role", + scopes=scopes, + component="conda-store", + ) + assert role + # assign created role to the user + assign_keycloak_client_role_to_user( + constants.KEYCLOAK_USERNAME, client_name="conda_store", role=role + ) + permissions = get_conda_store_user_permissions() + updated_entity_roles = permissions["data"]["entity_roles"] + + # Verify permissions/roles are set to expectation + assert updated_entity_roles == { + **entity_roles, + **changed_scopes, + } From 4f8fc54189df896fdb2c74b0fb15516b34c660b0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 12 Jul 2024 14:28:29 +0100 Subject: [PATCH 32/39] Add a default roles at initialisation (#2546) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Krassowski <5832902+krassowski@users.noreply.github.com> --- .../jupyterhub/files/jupyterhub/04-auth.py | 4 +- .../kubernetes/services/jupyterhub/main.tf | 26 +++++++++++++ .../services/keycloak-client/main.tf | 39 ++++++++++++++++++- .../services/keycloak-client/variables.tf | 11 ++++++ tests/tests_deployment/keycloak_utils.py | 8 ++++ tests/tests_deployment/test_jupyterhub_api.py | 21 ++++++++++ 6 files changed, 106 insertions(+), 3 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py index bc6fb6a72..cbd20a441 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py @@ -179,11 +179,11 @@ def validate_scopes(self, role_scopes): return [] async def _get_roles_with_attributes(self, roles: dict, client_id: str, token: str): - """This fetches all roles by id to fetch there attributes.""" + """This fetches all roles by id to fetch their attributes.""" roles_rich = [] for role in roles: # If this takes too much time, which isn't the case right now, we can - # also do multi-threaded requests + # also do multithreaded requests role_rich = await self._fetch_api( endpoint=f"roles-by-id/{role['id']}?client={client_id}", token=token ) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf index fe7716cf8..8c310c5ed 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf @@ -279,6 +279,32 @@ module "jupyterhub-openid-client" { "developer" = ["jupyterhub_developer", "dask_gateway_developer"] "analyst" = ["jupyterhub_developer"] } + client_roles = [ + { + "name" : "allow-app-sharing-role", + "description" : "Allow app sharing for apps created via JupyterHub App Launcher (jhub-apps)", + "groups" : [], + "attributes" : { + # grants permissions to share server + # grants permissions to read other user's names + # grants permissions to read other groups' names + # The later two are required for sharing with a group or user + "scopes" : "shares,read:users:name,read:groups:name" + "component" : "jupyterhub" + } + }, + { + "name" : "allow-read-access-to-services-role", + "description" : "Allow read access to services, such that they are visible on the home page e.g. conda-store", + # Adding it to analyst group such that it's applied to every user. + "groups" : ["analyst"], + "attributes" : { + # grants permissions to read services + "scopes" : "read:services", + "component" : "jupyterhub" + } + }, + ] callback-url-paths = [ "https://${var.external-url}/hub/oauth_callback", var.jupyterhub-logout-redirect-url diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf index 7a2c3e648..e23aeb13c 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf @@ -99,7 +99,6 @@ resource "keycloak_role" "main" { description = each.key } - data "keycloak_group" "main" { for_each = var.role_mapping @@ -117,3 +116,41 @@ resource "keycloak_group_roles" "group_roles" { exhaustive = false } + +resource "keycloak_role" "default_client_roles" { + for_each = { for role in var.client_roles : role.name => role } + realm_id = var.realm_id + client_id = keycloak_openid_client.main.id + name = each.value.name + description = each.value.description + attributes = each.value.attributes +} + +locals { + group_role_mapping = flatten([ + for role_object in var.client_roles : [ + for group_name in role_object.groups : { + group : group_name + role_name : role_object.name + } + ] + ]) + + client_roles_groups = toset([ + for index, value in local.group_role_mapping : value.group + ]) +} + +data "keycloak_group" "client_role_groups" { + for_each = local.client_roles_groups + realm_id = var.realm_id + name = each.value +} + +resource "keycloak_group_roles" "assign_roles" { + for_each = { for idx, value in local.group_role_mapping : idx => value } + realm_id = var.realm_id + group_id = data.keycloak_group.client_role_groups[each.value.group].id + role_ids = [keycloak_role.default_client_roles[each.value.role_name].id] + exhaustive = false +} diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf index b4e709c6a..7626cc2b9 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf @@ -46,3 +46,14 @@ variable "jupyterlab_profiles_mapper" { type = bool default = false } + +variable "client_roles" { + description = "Create roles for the client and assign it to groups" + default = [] + type = list(object({ + name = string + description = string + groups = optional(list(string)) + attributes = map(any) + })) +} diff --git a/tests/tests_deployment/keycloak_utils.py b/tests/tests_deployment/keycloak_utils.py index 6e6f6c21e..b11c64b93 100644 --- a/tests/tests_deployment/keycloak_utils.py +++ b/tests/tests_deployment/keycloak_utils.py @@ -81,6 +81,14 @@ def create_keycloak_role(client_name: str, role_name: str, scopes: str, componen ) +def get_keycloak_client_roles(client_name): + keycloak_admin = get_keycloak_admin() + client_details = get_keycloak_client_details_by_name( + client_name=client_name, keycloak_admin=keycloak_admin + ) + return keycloak_admin.get_client_roles(client_id=client_details["id"]) + + def delete_client_keycloak_test_roles(client_name): keycloak_admin = get_keycloak_admin() client_details = get_keycloak_client_details_by_name( diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py index 5e1a54562..4144fd4fe 100644 --- a/tests/tests_deployment/test_jupyterhub_api.py +++ b/tests/tests_deployment/test_jupyterhub_api.py @@ -4,6 +4,7 @@ from tests.tests_deployment.keycloak_utils import ( assign_keycloak_client_role_to_user, create_keycloak_role, + get_keycloak_client_roles, ) from tests.tests_deployment.utils import create_jupyterhub_token, get_jupyterhub_session @@ -30,9 +31,29 @@ def test_jupyterhub_loads_roles_from_keycloak(): "grafana_developer", "manage-account-links", "view-profile", + # default roles + "allow-read-access-to-services-role", } +@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") +def test_default_user_role_scopes(): + token_response = create_jupyterhub_token(note="get-default-scopes") + token_scopes = set(token_response.json()["scopes"]) + assert "read:services" in token_scopes + + +@pytest.mark.filterwarnings( + "ignore:.*auto_refresh_token is deprecated:DeprecationWarning" +) +@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") +def test_check_default_roles_added_in_keycloak(): + client_roles = get_keycloak_client_roles(client_name="jupyterhub") + role_names = [role["name"] for role in client_roles] + assert "allow-app-sharing-role" in role_names + assert "allow-read-access-to-services-role" in role_names + + @pytest.mark.parametrize( "component,scopes,expected_scopes_difference", ( From 8a2bfbbebd8adf5d89674fc0b625004dd0149e89 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 23 Jul 2024 06:50:19 -0500 Subject: [PATCH 33/39] Upgrade conda store to latest version --- src/_nebari/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 8194d2032..98b95937a 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -15,7 +15,7 @@ DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.3.1" +DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.6.1" LATEST_SUPPORTED_PYTHON_VERSION = "3.10" From cdd7cf9d37d90cb20314936862abbdfd3eb29036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:49:01 +0100 Subject: [PATCH 34/39] Hide gallery section if no exhibits are configured (#2549) --- src/_nebari/stages/kubernetes_services/__init__.py | 1 + .../stages/kubernetes_services/template/jupyterhub.tf | 5 +++-- .../modules/kubernetes/services/jupyterhub/variables.tf | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 6cbf03dbb..75234b2ce 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -249,6 +249,7 @@ class JupyterLabGallerySettings(schema.Base): title: str = "Examples" destination: str = "examples" exhibits: List[JupyterLabGalleryExhibit] = [] + hide_gallery_without_exhibits: bool = True class JupyterLab(schema.Base): diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf index 425c60bed..e95a55573 100644 --- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf +++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf @@ -57,8 +57,9 @@ variable "jupyterlab-default-settings" { variable "jupyterlab-gallery-settings" { description = "Server-side settings for jupyterlab-gallery extension" type = object({ - title = optional(string) - destination = optional(string) + title = optional(string) + destination = optional(string) + hide_gallery_without_exhibits = optional(bool) exhibits = list(object({ git = string title = string diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf index af6f06654..f43a25b90 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf @@ -166,8 +166,9 @@ variable "jupyterlab-default-settings" { variable "jupyterlab-gallery-settings" { description = "Server-side settings for jupyterlab-gallery extension" type = object({ - title = optional(string) - destination = optional(string) + title = optional(string) + destination = optional(string) + hide_gallery_without_exhibits = optional(bool) exhibits = list(object({ git = string title = string From 77ddd0173eea45c292312773ac5be64879906110 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:17:14 -0500 Subject: [PATCH 35/39] add note about ~/.bash_profile (#2575) --- .../modules/kubernetes/services/jupyterhub/files/skel/.bashrc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc index a42797222..58a612cc4 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc @@ -4,6 +4,10 @@ # ~/.bashrc: executed by bash(1) for non-login shells. # see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) # for examples +# +# Regular Nebari users do not have write permissions to modify the ~/.bashrc file. +# Users can add their own customizations to the ~/.bash_profile file. +# More details can be found in the docs https://www.nebari.dev/docs/faq#can-i-modify-the-bashrc-file-on-nebari. # If not running interactively, don't do anything case $- in From 02124139b8010d2894acc42f329db4b09e5c07bb Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Tue, 23 Jul 2024 23:35:09 +0530 Subject: [PATCH 36/39] #2566 Upgrade Jupyterhub ssh image --- .../modules/kubernetes/services/jupyterhub-ssh/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf index 430eb1488..3c389b598 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf @@ -30,7 +30,7 @@ variable "jupyterhub-ssh-image" { }) default = { name = "quay.io/jupyterhub-ssh/ssh" - tag = "0.0.1-0.dev.git.136.ha610981" + tag = "0.0.1-0.dev.git.149.he5107a4" } } From 5f65e784bd325b6be4c1d83900946715d9462eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:20:20 +0100 Subject: [PATCH 37/39] Expose jupyterlab-gallery branch and depth options (#2556) --- src/_nebari/stages/kubernetes_services/__init__.py | 2 ++ src/_nebari/stages/kubernetes_services/template/jupyterhub.tf | 2 ++ .../modules/kubernetes/services/jupyterhub/variables.tf | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 75234b2ce..206a48325 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -243,6 +243,8 @@ class JupyterLabGalleryExhibit(schema.Base): icon: Optional[str] = None account: Optional[str] = None token: Optional[str] = None + branch: Optional[str] = None + depth: Optional[int] = None class JupyterLabGallerySettings(schema.Base): diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf index e95a55573..29450ddfa 100644 --- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf +++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf @@ -68,6 +68,8 @@ variable "jupyterlab-gallery-settings" { icon = optional(string) account = optional(string) token = optional(string) + branch = optional(string) + depth = optional(number) })) }) } diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf index f43a25b90..21767723c 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf @@ -177,6 +177,8 @@ variable "jupyterlab-gallery-settings" { icon = optional(string) account = optional(string) token = optional(string) + branch = optional(string) + depth = optional(number) })) }) } From d64cbdbae8250d77fa2a0065988ed62cf8ece24c Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 24 Jul 2024 08:49:04 -0500 Subject: [PATCH 38/39] Use is comparison to be ruff compliant --- tests/tests_unit/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_unit/test_init.py b/tests/tests_unit/test_init.py index 8d880162d..7f8192727 100644 --- a/tests/tests_unit/test_init.py +++ b/tests/tests_unit/test_init.py @@ -15,7 +15,7 @@ ], ) def test_render_config(mock_all_cloud_methods, k8s_version, cloud_provider, expected): - if type(expected) == type and issubclass(expected, Exception): + if type(expected) is type and issubclass(expected, Exception): with pytest.raises(expected): config = render_config( project_name="test", From 200099631c2f25e9d0eaa0a23b62df4cf6597651 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:48:38 -0500 Subject: [PATCH 39/39] stop copying unneccesary files into user home directory (#2578) --- .../jupyterhub/files/jupyterhub/03-profiles.py | 10 ++++++++-- tests/tests_deployment/test_jupyterhub_ssh.py | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 8ccac70ca..22193e79d 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -48,10 +48,16 @@ def base_profile_home_mounts(username): } MKDIR_OWN_DIRECTORY = ( - "mkdir -p /mnt/{path} && chmod 777 /mnt/{path} && cp -r /etc/skel/. /mnt/{path}" + "mkdir -p /mnt/{path} && chmod 777 /mnt/{path} && " + # Copy skel files/folders not starting with '..' to user home directory. + # Filtering out ..* removes some unneeded folders (k8s configmap mount implementation details). + "find /etc/skel/. -maxdepth 1 -not -name '.' -not -name '..*' -exec " + "cp -rL {escaped_brackets} /mnt/{path} \;" ) command = MKDIR_OWN_DIRECTORY.format( - path=pvc_home_mount_path.format(username=username) + # have to escape the brackets since this string will be formatted later by KubeSpawner + escaped_brackets="{{}}", + path=pvc_home_mount_path.format(username=username), ) init_containers = [ { diff --git a/tests/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py index fd6b0799d..1e2822643 100644 --- a/tests/tests_deployment/test_jupyterhub_ssh.py +++ b/tests/tests_deployment/test_jupyterhub_ssh.py @@ -125,6 +125,9 @@ def test_contains_jupyterhub_ssh(paramiko_object): ("cat ~/.bashrc", "Managed by Nebari"), ("cat ~/.profile", "Managed by Nebari"), ("cat ~/.bash_logout", "Managed by Nebari"), + # ensure we don't copy over extra files from /etc/skel in init container + ("ls -la ~/..202*", "No such file or directory"), + ("ls -la ~/..data", "No such file or directory"), ] for command, output in commands_contain: