Bump python to 3.12

commit-id:acf01ecc
mosaicml · Oct 19, 2024 · 0eb1029 · 0eb1029
1 parent f0952b4
commit 0eb1029
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 35 deletions.
diff --git a/.github/workflows/daily.yaml b/.github/workflows/daily.yaml
@@ -55,8 +55,18 @@ jobs:
           markers: daily and (remote or not remote) and not gpu and not doctest
           pytest_command: coverage run -m pytest
           composer_package_name: composer
+        - name: daily-cpu-3.12-2.5
+          container: mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
+          markers: daily and (remote or not remote) and not gpu and not doctest
+          pytest_command: coverage run -m pytest
+          composer_package_name: mosaicml
+        - name: daily-cpu-3.12-2.5-composer
+          container: mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
+          markers: daily and (remote or not remote) and not gpu and not doctest
+          pytest_command: coverage run -m pytest
+          composer_package_name: composer
         - name: daily-cpu-doctest
-          container: mosaicml/pytorch:2.4.1_cpu-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
           markers: daily and (remote or not remote) and not gpu and doctest
           pytest_command: coverage run -m pytest tests/test_docs.py
           composer_package_name: mosaicml
@@ -109,6 +119,12 @@ jobs:
           pytest_command: "coverage run -m pytest"
           composer_package_name: "mosaicml"
           gpu_num: 1
+        - name: "gpu-3.12-2.5-1-gpu"
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
+          markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
+          pytest_command: "coverage run -m pytest"
+          composer_package_name: "mosaicml"
+          gpu_num: 1
         - name: "gpu-3.11-2.3-2-gpu"
           container: mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04
           markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
@@ -121,6 +137,12 @@ jobs:
           pytest_command: "coverage run -m pytest"
           composer_package_name: "mosaicml"
           gpu_num: 2
+        - name: "gpu-3.12-2.5-2-gpu"
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
+          markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
+          pytest_command: "coverage run -m pytest"
+          composer_package_name: "mosaicml"
+          gpu_num: 2
         - name: "gpu-3.11-2.3-4-gpu"
           container: mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04
           markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
@@ -133,6 +155,12 @@ jobs:
           pytest_command: "coverage run -m pytest"
           composer_package_name: "mosaicml"
           gpu_num: 4
+        - name: "gpu-3.12-2.5-4-gpu"
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
+          markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
+          pytest_command: "coverage run -m pytest"
+          composer_package_name: "mosaicml"
+          gpu_num: 4
     steps:
     - name: Checkout code
       uses: actions/checkout@v3

diff --git a/.github/workflows/pr-cpu.yaml b/.github/workflows/pr-cpu.yaml
@@ -25,11 +25,11 @@ jobs:
           markers: not daily and not remote and not gpu and not doctest
           pytest_command: coverage run -m pytest
         - name: cpu-3.11-2.5
-          container: mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
           markers: not daily and not remote and not gpu and not doctest
           pytest_command: coverage run -m pytest
         - name: cpu-doctest
-          container: mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
           markers: not daily and not remote and not gpu and doctest
           pytest_command: coverage run -m pytest tests/test_docs.py
     steps:

diff --git a/.github/workflows/pr-gpu.yaml b/.github/workflows/pr-gpu.yaml
@@ -16,7 +16,7 @@ jobs:
       matrix:
         include:
         - name: gpu-3.11-2.5-1
-          container: mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
           markers: not daily and not remote and gpu and (doctest or not doctest)
           pytest_command: coverage run -m pytest
           composer_package_name: mosaicml
@@ -45,7 +45,7 @@ jobs:
       matrix:
         include:
         - name: gpu-3.11-2.5-2
-          container: mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
           markers: not daily and not remote and gpu and (doctest or not doctest)
           pytest_command: coverage run -m pytest
           composer_package_name: mosaicml
@@ -75,7 +75,7 @@ jobs:
       matrix:
         include:
         - name: gpu-3.11-2.5-4
-          container: mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
           markers: not daily and not remote and gpu and (doctest or not doctest)
           pytest_command: coverage run -m pytest
           composer_package_name: mosaicml

diff --git a/docker/README.md b/docker/README.md
@@ -30,9 +30,9 @@ To install composer, once inside the image, run `pip install mosaicml`.
 <!-- BEGIN_PYTORCH_BUILD_MATRIX -->
 | Linux Distro   | Flavor   | PyTorch Version   | CUDA Version        | Python Version   | Docker Tags                                                                                                                                                                                                          |
 |----------------|----------|-------------------|---------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (Infiniband) | 3.11             | `mosaicml/pytorch:latest`, `mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04`                 |
-| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (EFA)        | 3.11             | `mosaicml/pytorch:latest-aws`, `mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws` |
-| Ubuntu 20.04   | Base     | 2.5.0             | cpu                 | 3.11             | `mosaicml/pytorch:latest_cpu`, `mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04`             |
+| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (Infiniband) | 3.12             | `mosaicml/pytorch:latest`, `mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04`                 |
+| Ubuntu 20.04   | Base     | 2.5.0             | 12.4.1 (EFA)        | 3.12             | `mosaicml/pytorch:latest-aws`, `mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04-aws` |
+| Ubuntu 20.04   | Base     | 2.5.0             | cpu                 | 3.12             | `mosaicml/pytorch:latest_cpu`, `mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04`             |
 | Ubuntu 20.04   | Base     | 2.4.1             | 12.4.1 (Infiniband) | 3.11             | `mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04`                                                                                        |
 | Ubuntu 20.04   | Base     | 2.4.1             | 12.4.1 (EFA)        | 3.11             | `mosaicml/pytorch:2.4.1_cu124-python3.11-ubuntu20.04-aws`                                                                                |
 | Ubuntu 20.04   | Base     | 2.4.1             | cpu                 | 3.11             | `mosaicml/pytorch:2.4.1_cpu-python3.11-ubuntu20.04`                                                                                            |

diff --git a/docker/build_matrix.yaml b/docker/build_matrix.yaml
@@ -5,13 +5,13 @@
   IMAGE_NAME: torch-2-5-0-cu124
   MOFED_VERSION: latest-23.10
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.12'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.5.0
   TAGS:
-  - mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
-  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.12-ubuntu20.04
   - mosaicml/pytorch:latest
   - ghcr.io/databricks-mosaic/pytorch:latest
   TARGET: pytorch_stage
@@ -22,13 +22,13 @@
   IMAGE_NAME: torch-2-5-0-cu124-aws
   MOFED_VERSION: ''
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.12'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.5.0
   TAGS:
-  - mosaicml/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws
-  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.11-ubuntu20.04-aws
+  - mosaicml/pytorch:2.5.0_cu124-python3.12-ubuntu20.04-aws
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cu124-python3.12-ubuntu20.04-aws
   - mosaicml/pytorch:latest-aws
   - ghcr.io/databricks-mosaic/pytorch:latest-aws
   TARGET: pytorch_stage
@@ -39,13 +39,13 @@
   IMAGE_NAME: torch-2-5-0-cpu
   MOFED_VERSION: ''
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.12'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.5.0
   TAGS:
-  - mosaicml/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
-  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cpu-python3.11-ubuntu20.04
+  - mosaicml/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
+  - ghcr.io/databricks-mosaic/pytorch:2.5.0_cpu-python3.12-ubuntu20.04
   - mosaicml/pytorch:latest_cpu
   - ghcr.io/databricks-mosaic/pytorch:latest_cpu
   TARGET: pytorch_stage
@@ -173,7 +173,7 @@
   IMAGE_NAME: composer-0-25-0
   MOFED_VERSION: latest-23.10
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.12'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.5.0
@@ -191,7 +191,7 @@
   IMAGE_NAME: composer-0-25-0-cpu
   MOFED_VERSION: latest-23.10
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.12'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.5.0

diff --git a/docker/generate_build_matrix.py b/docker/generate_build_matrix.py
@@ -19,7 +19,7 @@
 import tabulate
 import yaml
 
-PRODUCTION_PYTHON_VERSION = '3.11'
+PRODUCTION_PYTHON_VERSION = '3.12'
 PRODUCTION_PYTORCH_VERSION = '2.5.0'
 
 
@@ -87,19 +87,6 @@ def _get_cuda_override(cuda_version: str):
         )
 
         return cuda_121_override_string
-    elif cuda_version == '11.8.0':
-        cuda_118_override_string = (
-            'cuda>=11.8 brand=tesla,driver>=470,driver<471 '
-            'brand=tesla,driver>=515,driver<516 brand=unknown,driver>=470,driver<471 '
-            'brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=470,driver<471 '
-            'brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=470,driver<471 '
-            'brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=470,driver<471 '
-            'brand=geforce,driver>=515,driver<516 brand=quadro,driver>=470,driver<471 '
-            'brand=quadro,driver>=515,driver<516 brand=titan,driver>=470,driver<471 '
-            'brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=470,driver<471 '
-            'brand=titanrtx,driver>=515,driver<516'
-        )
-        return cuda_118_override_string
     return ''
 
 
@@ -180,7 +167,7 @@ def _write_table(table_tag: str, table_contents: str):
 
 
 def _main():
-    python_pytorch_versions = [('3.11', '2.5.0'), ('3.11', '2.4.1'), ('3.11', '2.3.1')]
+    python_pytorch_versions = [('3.12', '2.5.0'), ('3.11', '2.4.1'), ('3.11', '2.3.1')]
     cuda_options = [True, False]
     stages = ['pytorch_stage']
     interconnects = ['mellanox', 'EFA']  # mellanox is default, EFA needed for AWS