feat: Support 3.12 python

.github/workflows/continuous-integration.yaml

@@ -21,13 +21,13 @@ on:
           - pr
 
       linux_python_versions:
-        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on linux"
+        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11, 3.12 are supported) to launch on linux"
         required: false
         type: string
         default: "3.8"
 
       macos_python_versions:
-        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on macos (intel)"
+        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11, 3.12 are supported) to launch on macos (intel)"
         required: false
         type: string
         default: "3.8"
@@ -165,6 +165,7 @@ jobs:
       needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }}
       needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }}
       needs-311-linux-runner: ${{ steps.set-matrix.outputs.needs-311-linux-runner }}
+      needs-312-linux-runner: ${{ steps.set-matrix.outputs.needs-312-linux-runner }}
       instance-type: ${{ steps.set-matrix.outputs.instance-type }}
       linux-python-versions: ${{ steps.set-matrix.outputs.linux-python-versions }}
       macos-python-versions: ${{ steps.set-matrix.outputs.macos-python-versions }}
@@ -213,10 +214,10 @@ jobs:
             LINUX_PYTHON_VERSIONS="3.8"
             MACOS_PYTHON_VERSIONS=""
           elif [[ "${BUILD_TYPE}" == "weekly" ]]; then
-            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11"
+            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11 3.12"
             MACOS_PYTHON_VERSIONS="3.9"
           elif [[ "${BUILD_TYPE}" == "release" ]] || [[ "${BUILD_TYPE}" == "published_release" ]]; then
-            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11"
+            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11 3.12"
             MACOS_PYTHON_VERSIONS=""
           elif [[ "${BUILD_TYPE}" == "push_to_main" ]]; then
             LINUX_PYTHON_VERSIONS="3.8"
@@ -266,6 +267,8 @@ jobs:
             jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0')
           NEEDS_LINUX_311_RUNNER=$(echo "${LINUX_MATRIX}" | \
             jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.11")) | length > 0')
+          NEEDS_LINUX_312_RUNNER=$(echo "${LINUX_MATRIX}" | \
+            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.12")) | length > 0')
 
           echo "Needs Linux 3.8 runner:"
           echo "${NEEDS_LINUX_38_RUNNER}"
@@ -279,10 +282,14 @@ jobs:
           echo "Needs Linux 3.11 runner:"
           echo "${NEEDS_LINUX_311_RUNNER}"
 
+          echo "Needs Linux 3.12 runner:"
+          echo "${NEEDS_LINUX_312_RUNNER}"
+
           echo "needs-38-linux-runner=${NEEDS_LINUX_38_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-39-linux-runner=${NEEDS_LINUX_39_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-310-linux-runner=${NEEDS_LINUX_310_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-311-linux-runner=${NEEDS_LINUX_311_RUNNER}" >> $GITHUB_OUTPUT
+          echo "needs-312-linux-runner=${NEEDS_LINUX_312_RUNNER}" >> $GITHUB_OUTPUT
           echo "instance-type=${INSTANCE_TYPE}" >> $GITHUB_OUTPUT
 
   start-runner-linux:
@@ -299,6 +306,8 @@ jobs:
       ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }}
       label-311: ${{ steps.start-ec2-runner-311.outputs.label }}
       ec2-instance-id-311: ${{ steps.start-ec2-runner-311.outputs.ec2-instance-id || '' }}
+      label-312: ${{ steps.start-ec2-runner-312.outputs.label }}
+      ec2-instance-id-312: ${{ steps.start-ec2-runner-312.outputs.ec2-instance-id || '' }}
       matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }}
     steps:
       - name: Checkout Code
@@ -407,6 +416,29 @@ jobs:
               {"Key": "Team", "Value": "CML"}
             ]
 
+      - name: Start EC2 runner python 3.12
+        id: start-ec2-runner-312
+        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-312-linux-runner) }}
+        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
+        with:
+          mode: start
+          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
+          ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
+          ec2-instance-type: ${{ needs.matrix-preparation.outputs.instance-type }}
+          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
+          security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
+          aws-resource-tags: >
+            [
+              {"Key": "Name", "Value": "cml-ci-ec2-github-runner-py312"},
+              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
+              {"Key": "Python version", "Value": "3.12"},
+              {"Key": "Actor", "Value": "${{ github.actor }}"},
+              {"Key": "Action", "Value": "${{ github.action }}"},
+              {"Key": "GitHash", "Value": "${{ github.sha }}"},
+              {"Key": "RefName", "Value": "${{ github.ref_name }}"},
+              {"Key": "RunId", "Value": "${{ github.run_id }}"},
+              {"Key": "Team", "Value": "CML"}
+            ]
 
       - name: Update Linux runs_on Matrix
         id: update-linux-matrix
@@ -421,6 +453,8 @@ jobs:
           '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )')
           MATRIX=$(echo "${MATRIX}" | jq -rc \
           '(. | map(select(.os_kind=="linux" and .python_version=="3.11") |= . + {"runs_on": "${{ steps.start-ec2-runner-311.outputs.label }}"}) )')
+          MATRIX=$(echo "${MATRIX}" | jq -rc \
+          '(. | map(select(.os_kind=="linux" and .python_version=="3.12") |= . + {"runs_on": "${{ steps.start-ec2-runner-312.outputs.label }}"}) )')
 
           echo "Updated matrix:"
           echo "${MATRIX}"
@@ -1058,6 +1092,15 @@ jobs:
           ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-311 }}
           mode: stop
 
+      - name: Stop EC2 runner python 3.12
+        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
+        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-312 }}
+        with:
+          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
+          label: ${{ needs.start-runner-linux.outputs.label-312 }}
+          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-312 }}
+          mode: stop
+
   build-macos-intel:
     name: Python ${{ matrix.python_version }} (macOS, intel)
     needs: [matrix-preparation]

Makefile

@@ -1,9 +1,16 @@
 SHELL:=$(shell /usr/bin/env which bash)
 
+DEV_DOCKER_PYTHON?=py38
+ifeq ($(DEV_DOCKER_PYTHON),py312)
+UBUNTU_BASE=24.10
+else 
+UBUNTU_BASE=20.04
+endif 
+
 DEV_DOCKER_IMG:=concrete-ml-dev
 DEV_DOCKERFILE:=docker/Dockerfile.dev
-DEV_CONTAINER_VENV_VOLUME:=concrete-ml-venv
-DEV_CONTAINER_CACHE_VOLUME:=concrete-ml-cache
+DEV_CONTAINER_VENV_VOLUME:=concrete-ml-venv-$(DEV_DOCKER_PYTHON)
+DEV_CONTAINER_CACHE_VOLUME:=concrete-ml-cache-$(DEV_DOCKER_PYTHON)
 DOCKER_VENV_PATH:="$${HOME}"/dev_venv/
 SRC_DIR:=src
 TEST?=tests
@@ -342,12 +349,12 @@ mypy_ci:
 
 .PHONY: docker_build # Build dev docker
 docker_build:
-	BUILD_ARGS=; \
+	BUILD_ARGS="--build-arg UBUNTU_BASE=$(UBUNTU_BASE) ";\
 	if [[ $$(uname) == "Linux" ]]; then \
-		BUILD_ARGS="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
+		BUILD_ARGS+="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
 	fi; \
 	DOCKER_BUILDKIT=1 docker build $${BUILD_ARGS:+$$BUILD_ARGS} \
-	--pull -t $(DEV_DOCKER_IMG) -f $(DEV_DOCKERFILE) .
+	--pull -t $(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) -f $(DEV_DOCKERFILE) .
 
 .PHONY: docker_rebuild # Rebuild docker
 docker_rebuild: docker_clean_volumes
@@ -356,7 +363,7 @@ docker_rebuild: docker_clean_volumes
 		BUILD_ARGS="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
 	fi; \
 	DOCKER_BUILDKIT=1 docker build $${BUILD_ARGS:+$$BUILD_ARGS} \
-	--pull --no-cache -t $(DEV_DOCKER_IMG) -f $(DEV_DOCKERFILE) .
+	--pull --no-cache -t $(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) -f $(DEV_DOCKERFILE) .
 
 .PHONY: docker_start # Launch docker
 docker_start:
@@ -371,7 +378,7 @@ docker_start:
 	--volume /"$$(pwd)":/src \
 	--volume $(DEV_CONTAINER_VENV_VOLUME):/home/dev_user/dev_venv \
 	--volume $(DEV_CONTAINER_CACHE_VOLUME):/home/dev_user/.cache \
-	$(DEV_DOCKER_IMG) || rm -f "$${EV_FILE}"
+	$(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) || rm -f "$${EV_FILE}"
 
 .PHONY: docker_build_and_start # Docker build and start
 docker_build_and_start: docker_build docker_start
@@ -642,7 +649,7 @@ docker_publish_measurements: docker_rebuild
 	docker run --volume /"$$(pwd)":/src \
 	--volume $(DEV_CONTAINER_VENV_VOLUME):/home/dev_user/dev_venv \
 	--volume $(DEV_CONTAINER_CACHE_VOLUME):/home/dev_user/.cache \
-	$(DEV_DOCKER_IMG) \
+	$(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) \
 	/bin/bash ./script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh \
 	"$${LAUNCH_COMMAND}"
 

README.md

@@ -83,7 +83,7 @@ Depending on your OS, Concrete ML may be installed with Docker or with pip:
 |            macOS 11+ (Intel)            |         Yes         |       Yes        |
 | macOS 11+ (Apple Silicon: M1, M2, etc.) |     Coming soon     |       Yes        |
 
-Note: Concrete ML only supports Python `3.8`, `3.9`, `3.10` and `3.11`.
+Note: Concrete ML only supports Python `3.8`, `3.9`, `3.10`, `3.11` and `3.12`.
 Concrete ML can be installed on Kaggle ([see this question on the community for more details](https://community.zama.ai/t/how-do-we-use-concrete-ml-on-kaggle/332)) and on Google Colab.
 
 #### Docker

benchmarks/common.py

@@ -757,6 +757,8 @@ def benchmark_name_generator(
 
     elif model_name in DEEP_LEARNING_NAMES:
         config_str = f"_{config['n_bits']}"
+    else:
+        config_str = "UNKNOWN"
 
     # We remove underscores to make sure to not have any conflict when splitting
     return model_name.replace("_", "-") + config_str + joiner + dataset_name.replace("_", "-")

benchmarks/glm.py

@@ -8,6 +8,7 @@
 import numpy as np
 import pandas
 import py_progress_tracker as progress
+import sklearn
 from common import (
     BENCHMARK_CONFIGURATION,
     GLMS_STRING_TO_CLASS,
@@ -29,6 +30,10 @@
     StandardScaler,
 )
 
+sklearn_sparse_arg = (
+    {"sparse": False} if "1.1." in sklearn.__version__ else {"sparse_output": False}
+)
+
 
 # pylint: disable=redefined-outer-name
 def get_data() -> Tuple[pandas.DataFrame, str]:
@@ -102,7 +107,7 @@ def get_preprocessor() -> ColumnTransformer:
             ("log_scaled_numeric", log_scale_transformer, ["Density"]),
             (
                 "onehot_categorical",
-                OneHotEncoder(sparse=False),
+                OneHotEncoder(**sklearn_sparse_arg),
                 ["VehBrand", "VehPower", "VehGas", "Region", "Area"],
             ),
         ],

deps_licenses/licenses_linux_user.txt

@@ -7,22 +7,22 @@ certifi, 2024.8.30, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.4.0, MIT License
 coloredlogs, 15.0.1, MIT License
 concrete-ml-extensions, 0.1.2, BSD-3-Clause-Clear
-concrete-python, 2.8.1, BSD-3-Clause
+concrete-python, 2.9.0rc1, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.9, BSD License
 filelock, 3.16.1, The Unlicense (Unlicense)
-flatbuffers, 2.0.7, Apache Software License
-fsspec, 2024.9.0, BSD License
-huggingface-hub, 0.25.2, Apache Software License
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.10.0, BSD License
+huggingface-hub, 0.26.2, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.11, MIT License
 idna, 3.10, BSD License
 importlib_resources, 6.4.5, Apache Software License
 joblib, 1.4.2, BSD License
-jsonpickle, 3.3.0, BSD License
+jsonpickle, 4.0.0, BSD License
 mpmath, 1.3.0, BSD License
 networkx, 3.1, BSD License
-numpy, 1.24.2, BSD License
+numpy, 1.26.4, BSD License
 nvidia-cublas-cu12, 12.1.3.1, Other/Proprietary License
 nvidia-cuda-cupti-cu12, 12.1.105, Other/Proprietary License
 nvidia-cuda-nvrtc-cu12, 12.1.105, Other/Proprietary License
@@ -35,20 +35,20 @@ nvidia-cusparse-cu12, 12.1.0.106, Other/Proprietary License
 nvidia-nccl-cu12, 2.20.5, Other/Proprietary License
 nvidia-nvjitlink-cu12, 12.6.77, Other/Proprietary License
 nvidia-nvtx-cu12, 12.1.105, Other/Proprietary License
-onnx, 1.16.1, Apache License v2.0
+onnx, 1.17.0, Apache License v2.0
 onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.12.0, Apache Software License
 onnxoptimizer, 0.3.13, Apache License v2.0
 onnxruntime, 1.18.0, MIT License
-packaging, 24.1, Apache Software License; BSD License
+packaging, 24.2, Apache Software License; BSD License
 pandas, 2.0.3, BSD License
-protobuf, 4.25.5, 3-Clause BSD License
+protobuf, 5.28.3, 3-Clause BSD License
 psutil, 5.9.8, BSD License
 python-dateutil, 2.9.0.post0, Apache Software License; BSD License
 pytz, 2024.2, MIT License
 requests, 2.32.3, Apache Software License
-scikit-learn, 1.1.3, BSD License
-scipy, 1.10.1, BSD License
+scikit-learn, 1.4.0, BSD License
+scipy, 1.13.1, BSD License
 six, 1.16.0, MIT License
 skl2onnx, 1.17.0, Apache Software License
 skops, 0.5.0, MIT
@@ -57,8 +57,7 @@ sympy, 1.13.3, BSD License
 tabulate, 0.8.10, MIT License
 threadpoolctl, 3.5.0, BSD License
 torch, 2.3.1, BSD License
-tqdm, 4.66.5, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-triton, 2.3.1, MIT License
+tqdm, 4.67.0, MIT License; Mozilla Public License 2.0 (MPL 2.0)
 typing_extensions, 4.12.2, Python Software Foundation License
 tzdata, 2024.2, Apache Software License
 unfoldNd, 0.2.2, MIT License

deps_licenses/licenses_linux_user.txt.md5

@@ -1 +1 @@
-8ea8aec4f5aac03565c2dcb9f3f8a1da
+585ea4f42b69f489934e29adb6efb4ac

docker/Dockerfile.dev

@@ -1,4 +1,5 @@
-FROM ubuntu:20.04
+ARG UBUNTU_BASE=20.04
+FROM ubuntu:${UBUNTU_BASE}
 
 ENV TZ=Europe/Paris
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@@ -7,10 +8,6 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 # original archive showed performance issues and is farther away
 RUN sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
 
-COPY ./script/make_utils/setup_os_deps.sh ./setup_os_deps.sh
-COPY ./script/make_utils/detect_docker.sh ./detect_docker.sh
-RUN ./setup_os_deps.sh --linux-install-python && rm ./setup_os_deps.sh && rm ./detect_docker.sh
-
 ENV SRC_DIR=/src
 
 # Default to Ubuntu default uid for first user
@@ -19,25 +16,38 @@ ARG BUILD_UID=1000
 
 # Get sudo and vim for our future user
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y sudo vim && \
+    apt-get install --no-install-recommends -y sudo vim adduser && \
     rm -rf /var/lib/apt/lists/*
 
+# For ubuntu>23 the ubuntu user will exist and use uid/gid=1000
+# we need to remove this user if it exists
+RUN userdel -f -r ubuntu; exit 0
+
 # From https://dev.to/emmanuelnk/using-sudo-without-password-prompt-as-non-root-docker-user-52bg
 # Create dev_user and add it to relevant groups
 # Create /src and make the dev user own it
 # Ensure sudo group users are not asked for a password when using
 # sudo command by ammending sudoers file
-RUN groupadd -g "${BUILD_GID}" dev_user && \
+
+RUN groupadd -f -g "${BUILD_GID}" dev_user && \
     adduser --disabled-password \
     --uid "${BUILD_UID}" --gid "${BUILD_GID}" --shell /bin/bash --gecos "" dev_user && \
     usermod -aG sudo dev_user && \
     mkdir -p "${SRC_DIR}" && \
     chown dev_user "${SRC_DIR}" && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
+RUN groupadd docker && usermod -aG docker dev_user
+
 # Now switch to the newly created user
 USER dev_user
 
+RUN cd /home/dev_user
+
+COPY ./script/make_utils/setup_os_deps.sh ./setup_os_deps.sh
+COPY ./script/make_utils/detect_docker.sh ./detect_docker.sh
+RUN ./setup_os_deps.sh --linux-install-python && sudo rm ./setup_os_deps.sh && sudo rm ./detect_docker.sh
+
 RUN echo "source ~/dev_venv/bin/activate" >> ~/.bashrc && \
     echo "if [[ \"\$?\" != \"0\" ]]; then" >> ~/.bashrc && \
     echo "    python3 -m venv ~/dev_venv" >> ~/.bashrc && \

docker/release_resources/sanity_check.py

@@ -94,7 +94,7 @@ def ml_check(args, keyring_dir_as_str):
     print(f"Prediction sklearn       : {y_pred[:nb_samples]}")
 
     print(
-        f"{numpy.sum(y_pred_simulated==y_pred[:nb_samples])}/{nb_samples} "
+        f"{numpy.sum(y_pred_simulated == y_pred[:nb_samples])}/{nb_samples} "
         "predictions are similar between the FHE simulated model and the clear sklearn model."
     )
 

docs/advanced_examples/DecisionTreeClassifier.ipynb

@@ -319,7 +319,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -332,7 +332,7 @@
    ],
    "source": [
     "print(\n",
-    "    f\"{numpy.sum(y_pred_fhe==y_pred)}/\"\n",
+    "    f\"{numpy.sum(y_pred_fhe == y_pred)}/\"\n",
     "    \"10 predictions are similar between the FHE model and the clear sklearn model.\"\n",
     ")"
    ]