Merge branch 'nebari-dev:develop' into eks-private-cluster

.github/workflows/test_local_integration.yaml

@@ -33,6 +33,7 @@ on:
       pr_number:
         required: true
         type: string
+  workflow_dispatch:
 
 # When the cancel-in-progress: true option is specified, any concurrent jobs or workflows using the same
 # concurrency group will cancel both the pending and currently running jobs or workflows. This allows only
@@ -192,7 +193,10 @@ jobs:
 
       ### CLEANUP AFTER TESTS
       - name: Cleanup nebari deployment
-        if: always()
+        # Since this is not critical for most pull requests and takes more than half of the time
+        # in the CI, it makes sense to only run on merge to main or workflow_dispatch to speed
+        # up feedback cycle
+        if: github.ref_name == 'develop' || github.event_name == 'workflow_dispatch'
         working-directory: local-deployment
         run: |
           nebari destroy --config nebari-config.yaml --disable-prompt

.pre-commit-config.yaml

@@ -57,7 +57,7 @@ repos:
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.7
+    rev: v0.5.0
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -73,7 +73,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.90.0
+    rev: v1.92.0
     hooks:
       - id: terraform_fmt
         args:

RELEASE.md

@@ -9,6 +9,45 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+### Release 2024.6.1 - June 26, 2024
+
+> NOTE: This release includes an upgrade to the `kube-prometheus-stack` Helm chart, resulting in a newer version of Grafana. When upgrading your Nebari cluster, you will be prompted to have Nebari update some CRDs and delete a DaemonSet on your behalf. If you prefer, you can also run the commands yourself, which will be shown to you. If you have any custom dashboards, you'll also need to back them up by [exporting them as JSON](https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json), so you can [import them](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard) after upgrading.
+
+### What's Changed
+* Fetch JupyterHub roles from Keycloak by @krassowski in https://github.com/nebari-dev/nebari/pull/2447
+* Update selector for Start server button to use button tag by @krassowski in https://github.com/nebari-dev/nebari/pull/2464
+* Reduce GCP Fixed Costs by 50% by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2453
+* Restore JupyterHub updates from PR-2427 by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2465
+* Workload identity by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2460
+* Fix test using a non-specific selector by @krassowski in https://github.com/nebari-dev/nebari/pull/2475
+* add verify=false since we use self signed cert in tests by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2481
+* fix forward auth when using custom cert by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2479
+* Upgrade to JupyterHub 5.0.0b2 by @krassowski in https://github.com/nebari-dev/nebari/pull/2468
+* upgrade instructions for PR 2453 by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2466
+* Use Helm Chart for JupyterHub 5.0.0 final by @krassowski in https://github.com/nebari-dev/nebari/pull/2484
+* Parse and insert keycloak roles scopes into JupyterHub by @aktech in https://github.com/nebari-dev/nebari/pull/2471
+* Add CITATION file by @pavithraes in https://github.com/nebari-dev/nebari/pull/2455
+* CI: add azure integration by @fangchenli in https://github.com/nebari-dev/nebari/pull/2061
+* Create trivy.yml by @dcmcand in https://github.com/nebari-dev/nebari/pull/2458
+* don't run azure deployment on PRs, only on schedule and manual trigger by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2498
+* add cloud provider deployment status badges to README.md by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2407
+* Upgrade kube-prometheus-stack helm chart by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2472
+* upgrade note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2502
+* Remove VSCode from jhub_apps default services by @jbouder in https://github.com/nebari-dev/nebari/pull/2503
+* Explicit config by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2294
+* fix general node scaling bug for azure by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2517
+* Skip running cleanup on pull requests by @aktech in https://github.com/nebari-dev/nebari/pull/2488
+* 1792 Add docstrings to `upgrade.py` by @arjxn-py in https://github.com/nebari-dev/nebari/pull/2512
+* set's min TLS version for azure storage account to TLS 1.2 by @dcmcand in https://github.com/nebari-dev/nebari/pull/2522
+* Fix conda-store and Traefik Grafana Dashboards by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2540
+* Implement support for jupyterlab-gallery config by @krassowski in https://github.com/nebari-dev/nebari/pull/2501
+* Add option to run CRDs updates and DaemonSet deletion on user's behalf. by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2544
+
+### New Contributors
+* @arjxn-py made their first contribution in https://github.com/nebari-dev/nebari/pull/2512
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.5.1...2024.6.1
+
 ### Release 2024.5.1 - May 13, 2024
 
 ## What's Changed

pyproject.toml

@@ -72,6 +72,7 @@ dependencies = [
     "ruamel.yaml==0.18.6",
     "typer==0.9.0",
     "packaging==23.2",
+    "typing-extensions==4.11.0",
 ]
 
 [project.optional-dependencies]

src/_nebari/config.py

@@ -103,7 +103,8 @@ def write_configuration(
     """Write the nebari configuration file to disk"""
     with config_filename.open(mode) as f:
         if isinstance(config, pydantic.BaseModel):
-            yaml.dump(config.model_dump(), f)
+            config_dict = config.model_dump()
+            yaml.dump(config_dict, f)
         else:
             config = dump_nested_model(config)
             yaml.dump(config, f)

src/_nebari/constants.py

@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2024.5.1"
+CURRENT_RELEASE = "2024.6.1"
 
 # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
 # implemented in August 2023.
@@ -15,7 +15,7 @@
 DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE
 DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE
 
-DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.3.1"
+DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.6.1"
 
 LATEST_SUPPORTED_PYTHON_VERSION = "3.10"
 

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -90,8 +90,11 @@ resource "aws_eks_addon" "aws-ebs-csi-driver" {
 }
 
 resource "aws_eks_addon" "coredns" {
-  addon_name   = "coredns"
-  cluster_name = aws_eks_cluster.main.name
+  addon_name                  = "coredns"
+  cluster_name                = aws_eks_cluster.main.name
+  resolve_conflicts_on_create = "OVERWRITE"
+  resolve_conflicts_on_update = "OVERWRITE"
+
 
   configuration_values = jsonencode({
     nodeSelector = {

src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf

@@ -1,3 +1,8 @@
+resource "aws_kms_key" "main" {
+  description         = "KMS key for ${var.name}"
+  enable_key_rotation = true
+}
+
 resource "aws_s3_bucket" "main" {
   bucket = var.name
   acl    = var.public ? "public-read" : "private"
@@ -11,3 +16,22 @@ resource "aws_s3_bucket" "main" {
     Description = "S3 bucket for ${var.name}"
   }, var.tags)
 }
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "main" {
+  bucket = aws_s3_bucket.main.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      kms_master_key_id = aws_kms_key.main.arn
+      sse_algorithm     = "aws:kms"
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "main" {
+  bucket                  = aws_s3_bucket.main.id
+  ignore_public_acls      = true
+  block_public_acls       = true
+  block_public_policy     = true
+  restrict_public_buckets = true
+}

src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf

@@ -31,13 +31,12 @@ resource "azurerm_kubernetes_cluster" "main" {
   default_node_pool {
     vnet_subnet_id      = var.vnet_subnet_id
     name                = var.node_groups[0].name
-    node_count          = 1
     vm_size             = var.node_groups[0].instance_type
     enable_auto_scaling = "true"
-    min_count           = 1
-    max_count           = 1
+    min_count           = var.node_groups[0].min_size
+    max_count           = var.node_groups[0].max_size
     max_pods            = var.max_pods
-    # node_labels          = var.node_labels
+
     orchestrator_version = var.kubernetes_version
     node_labels = {
       "azure-node-pool" = var.node_groups[0].name

src/_nebari/stages/kubernetes_ingress/__init__.py

@@ -93,17 +93,18 @@ def _attempt_dns_lookup(
 
     attempt = 0
     while not _attempt_dns_lookup(domain_name, ip):
-        sleeptime = 60 * (2**attempt)
-        if not disable_prompt:
+        if disable_prompt:
+            sleeptime = 60 * (2**attempt)
+            print(f"Will attempt to poll DNS again in {sleeptime} seconds...")
+            time.sleep(sleeptime)
+        else:
             input(
                 f"After attempting to poll the DNS, the record for domain={domain_name} appears not to exist, "
                 f"has recently been updated, or has yet to fully propagate. This non-deterministic behavior is likely due to "
-                f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again in {sleeptime} seconds "
-                f"[Press Enter].\n\n...otherwise kill the process and run the deployment again later..."
+                f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again [Press Enter].\n\n"
+                f"...otherwise kill the process and run the deployment again later..."
             )
 
-        print(f"Will attempt to poll DNS again in {sleeptime} seconds...")
-        time.sleep(sleeptime)
         attempt += 1
         if attempt == 5:
             print(

src/_nebari/stages/kubernetes_services/__init__.py

@@ -235,8 +235,28 @@ class IdleCuller(schema.Base):
     server_shutdown_no_activity_timeout: int = 15
 
 
+class JupyterLabGalleryExhibit(schema.Base):
+    git: str
+    title: str
+    homepage: Optional[str] = None
+    description: Optional[str] = None
+    icon: Optional[str] = None
+    account: Optional[str] = None
+    token: Optional[str] = None
+    branch: Optional[str] = None
+    depth: Optional[int] = None
+
+
+class JupyterLabGallerySettings(schema.Base):
+    title: str = "Examples"
+    destination: str = "examples"
+    exhibits: List[JupyterLabGalleryExhibit] = []
+    hide_gallery_without_exhibits: bool = True
+
+
 class JupyterLab(schema.Base):
     default_settings: Dict[str, Any] = {}
+    gallery_settings: JupyterLabGallerySettings = JupyterLabGallerySettings()
     idle_culler: IdleCuller = IdleCuller()
     initial_repositories: List[Dict[str, str]] = []
     preferred_dir: Optional[str] = None
@@ -368,6 +388,9 @@ class JupyterhubInputVars(schema.Base):
     jupyterlab_default_settings: Dict[str, Any] = Field(
         alias="jupyterlab-default-settings"
     )
+    jupyterlab_gallery_settings: JupyterLabGallerySettings = Field(
+        alias="jupyterlab-gallery-settings"
+    )
     initial_repositories: str = Field(alias="initial-repositories")
     jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides")
     jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage")
@@ -468,6 +491,12 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                     "*/*": ["viewer"],
                 },
             },
+            "conda-store-service-account": {
+                "primary_namespace": "",
+                "role_bindings": {
+                    "*/*": ["admin"],
+                },
+            },
         }
 
         # Compound any logout URLs from extensions so they are are logged out in succession
@@ -534,6 +563,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             jhub_apps_enabled=self.config.jhub_apps.enabled,
             initial_repositories=str(self.config.jupyterlab.initial_repositories),
             jupyterlab_default_settings=self.config.jupyterlab.default_settings,
+            jupyterlab_gallery_settings=self.config.jupyterlab.gallery_settings,
             jupyterlab_preferred_dir=self.config.jupyterlab.preferred_dir,
         )
 

src/_nebari/stages/kubernetes_services/template/jupyterhub.tf

@@ -54,6 +54,26 @@ variable "jupyterlab-default-settings" {
   type        = map(any)
 }
 
+variable "jupyterlab-gallery-settings" {
+  description = "Server-side settings for jupyterlab-gallery extension"
+  type = object({
+    title                         = optional(string)
+    destination                   = optional(string)
+    hide_gallery_without_exhibits = optional(bool)
+    exhibits = list(object({
+      git         = string
+      title       = string
+      homepage    = optional(string)
+      description = optional(string)
+      icon        = optional(string)
+      account     = optional(string)
+      token       = optional(string)
+      branch      = optional(string)
+      depth       = optional(number)
+    }))
+  })
+}
+
 variable "jupyterhub-hub-extraEnv" {
   description = "Extracted overrides to merge with jupyterhub.hub.extraEnv"
   type        = string
@@ -149,6 +169,8 @@ module "jupyterhub" {
 
   jupyterlab-default-settings = var.jupyterlab-default-settings
 
+  jupyterlab-gallery-settings = var.jupyterlab-gallery-settings
+
   jupyterlab-pioneer-enabled    = var.jupyterlab-pioneer-enabled
   jupyterlab-pioneer-log-format = var.jupyterlab-pioneer-log-format