diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e4f855..5dab8a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,24 @@ version: 2 jobs: build: docker: - - image: cimg/python:3.11 + - image: cimg/python:3.12 steps: - checkout - setup_remote_docker + - run: + name: Check that .ds_python_version and Dockerfile are consistent + command: | + python -u << HERE + import re + v1 = re.search(r"(\d+\.\d+.\d+)", open(".ds_python_version").read()).group(1) + v2 = re.search(r"ARG DS_PYTHON_IMG_VERSION\=(\d+\.\d+.\d+)", open("Dockerfile").read()).group(1) + if v1 == v2: + print(f".ds_python_version and Dockerfile match for the ds-py image version: {v1}") + else: + raise ValueError( + f".ds_python_version and Dockerfile don't match for the ds-py image version: {v1} and {v2}" + ) + HERE - run: name: "Get civisanalytics/datascience-python image version" command: echo 'export DS_PYTHON_IMG_VERSION="$(head -n 1 ./.ds_python_version)"' >> "$BASH_ENV" diff --git a/.ds_python_version b/.ds_python_version index 66ce77b..1502020 100644 --- a/.ds_python_version +++ b/.ds_python_version @@ -1 +1 @@ -7.0.0 +7.3.0 diff --git a/.github/ISSUE_TEMPLATE/general.md b/.github/ISSUE_TEMPLATE/general.md new file mode 100644 index 0000000..abbd5f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.md @@ -0,0 +1,11 @@ +--- +name: General +about: Ask a question, report a potential issue, etc. +title: '' +labels: '' +assignees: '' + +--- + +**Note:** Civis employees should _not_ use the GitHub Issues feature at this public codebase +to file a ticket, and should instead use the internal ticketing system. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..98a01f3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ + + +--- + +- [ ] (For Civis employees only) Reference to a relevant ticket in the pull request title +- [ ] Changelog entry added to `CHANGELOG.md` at the repo's root level +- [ ] Description of change in the pull request description +- [ ] If applicable, unit tests have been added and/or updated +- [ ] The CircleCI builds have all passed \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 91f6bb4..a66cca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +## [3.1.0] - 2024-06-18 +### Changed +- Updated base datascience-python image version to v7.3.0 (#54) +- Updated core Python dependency versions (#54): + * civis-jupyter-notebook 2.1.1 -> 2.2.0 + * matplotlib 3.7.2 -> 3.9.0 + * seaborn 0.12.2 -> 0.13.2 + ## [3.0.0] - 2023-08-22 ### Changed - update base datascience-python version to v7.0.0 (#50) diff --git a/Dockerfile b/Dockerfile index c7dd170..a1c3cce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Unfortunately Dockerhub isn't as flexible as CircleCi or AWS Codebuild, so we have to hardcode this value here. # So if you in the future need to update this value, make sure you also edit the value in .ds_python_version. # These values should be kept in sync. -ARG DS_PYTHON_IMG_VERSION=7.0.0 +ARG DS_PYTHON_IMG_VERSION=7.3.0 FROM civisanalytics/datascience-python:${DS_PYTHON_IMG_VERSION} @@ -33,8 +33,7 @@ RUN chmod +x /tini COPY requirements-full.txt . -RUN pip install -r requirements-full.txt && \ - pip cache purge && \ +RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \ rm requirements-full.txt && \ civis-jupyter-notebooks-install diff --git a/README.md b/README.md index 258bf37..19c623a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Civis Jupyter Notebook Docker Image for Python 3 -[![CircleCI](https://circleci.com/gh/civisanalytics/civis-jupyter-python3/tree/master.svg?style=svg)](https://circleci.com/gh/civisanalytics/civis-jupyter-python3/tree/master) +[![CircleCI](https://circleci.com/gh/civisanalytics/civis-jupyter-python3/tree/master.svg?style=shield)](https://circleci.com/gh/civisanalytics/civis-jupyter-python3/tree/master) ## Installation @@ -24,7 +24,7 @@ The version number has been pulled out into a dedicated file to centralize consu To update the version simply change the version number in `.ds_python_version` -## Testing Integration with the Civis Platform +## Testing Integration with Civis Platform If you would like to test the image locally follow the steps below: @@ -46,10 +46,29 @@ and describe any changes in the [change log](CHANGELOG.md). ### For Maintainers +#### Updating Dependencies + +Updating the `civis-jupyter-python3` Docker image entails the following: + +* Update the base `datascience-python` image to the latest one; +* Update `requirements-core.txt`, the Python dependencies specific to this `civis-jupyter-python3` image. +* Update `requirements-full.txt` so that it has all transitive dependencies pinned + while respecting those already pinned at both `datascience-python` and `requirements-core.txt`. + +To execute these updates, follow these steps: + +* Update the version number of the `datascience-python` image in both `Dockerfile` and `.ds_python_version`. +* Update the Python dependencies in `requirements-core.txt` as necessary. +* Locally, have Docker Desktop running in prep for the next step. +* Run `sh generate-requirements-full.sh` to update `requirements-full.txt`. +* To verify the new `civis-jupyter-python3` image would successfully build with your changes, locally run `sh build_docker_file.sh`. + +#### Making a New Releases + This repo has autobuild enabled. Any PR that is merged to master will be built as the `latest` tag on Dockerhub. Once you are ready to create a new version, go to the "releases" tab of the repository and click -"Draft a new release". Github will prompt you to create a new tag, release title, and release +"Draft a new release". GitHub will prompt you to create a new tag, release title, and release description. The tag should use semantic versioning in the form "vX.X.X"; "major.minor.micro". The title of the release should be the same as the tag. Include a change log in the release description. Once the release is tagged, DockerHub will automatically build three identical containers, with labels @@ -59,4 +78,4 @@ Once the release is tagged, DockerHub will automatically build three identical c BSD-3 -See [LICENSE.md](LICENSE.md) for details. +See [LICENSE.txt](LICENSE.txt) for details. diff --git a/generate-requirements-full.sh b/generate-requirements-full.sh index 1714bd0..fb6ee13 100755 --- a/generate-requirements-full.sh +++ b/generate-requirements-full.sh @@ -7,4 +7,4 @@ docker run \ "pip freeze >> /tmp/requirements-aggregated-core.txt && \ cat /src/requirements-core.txt >> /tmp/requirements-aggregated-core.txt && \ pip install pip-tools && \ - pip-compile --output-file=/src/requirements-full.txt --pip-args='--prefer-binary' /tmp/requirements-aggregated-core.txt" + pip-compile --output-file=/src/requirements-full.txt --pip-args='--prefer-binary' --upgrade /tmp/requirements-aggregated-core.txt" diff --git a/requirements-core.txt b/requirements-core.txt index 3f02e20..bc1dc7b 100755 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -1,3 +1,3 @@ -matplotlib==3.7.2 -seaborn==0.12.2 -civis-jupyter-notebook==2.1.1 +civis-jupyter-notebook==2.2.0 +matplotlib==3.9.0 +seaborn==0.13.2 diff --git a/requirements-full.txt b/requirements-full.txt index ac19b68..4ed7bed 100755 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -1,82 +1,79 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=/src/requirements-full.txt --pip-args='--prefer-binary' /tmp/requirements-aggregated-core.txt # -anyio==4.0.0 +anyio==4.4.0 # via jupyter-server argon2-cffi==23.1.0 # via # jupyter-server - # nbclassic # notebook argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.2.3 +arrow==1.3.0 # via isoduration -asttokens==2.4.0 +asttokens==2.4.1 # via stack-data -attrs==23.1.0 +attrs==23.2.0 # via # -r /tmp/requirements-aggregated-core.txt # jsonschema # referencing -awscli==1.29.5 +awscli==1.33.9 # via -r /tmp/requirements-aggregated-core.txt -backcall==0.2.0 - # via ipython -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert -bleach==6.0.0 +bleach==6.1.0 # via nbconvert -boto3==1.28.5 +boto3==1.34.127 # via -r /tmp/requirements-aggregated-core.txt -botocore==1.31.5 +botocore==1.34.127 # via # -r /tmp/requirements-aggregated-core.txt # awscli # boto3 # s3transfer -certifi==2023.5.7 +certifi==2024.6.2 # via # -r /tmp/requirements-aggregated-core.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via argon2-cffi-bindings -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via # -r /tmp/requirements-aggregated-core.txt # requests -civis==1.16.1 +civis==2.3.0 # via # -r /tmp/requirements-aggregated-core.txt # civis-jupyter-extensions # civis-jupyter-notebook -civis-jupyter-extensions==1.1.0 +civis-jupyter-extensions==1.2.0 # via civis-jupyter-notebook -civis-jupyter-notebook==2.1.1 +civis-jupyter-notebook==2.2.0 # via -r /tmp/requirements-aggregated-core.txt -click==8.1.6 +click==8.1.7 # via # -r /tmp/requirements-aggregated-core.txt # civis # civis-jupyter-notebook -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via # -r /tmp/requirements-aggregated-core.txt # civis -colorama==0.4.4 +colorama==0.4.6 # via # -r /tmp/requirements-aggregated-core.txt # awscli -comm==0.1.4 +comm==0.2.2 # via ipykernel -contourpy==1.1.0 +contourpy==1.2.1 # via matplotlib -cycler==0.11.0 +cycler==0.12.1 # via matplotlib -debugpy==1.6.7.post1 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -88,29 +85,29 @@ docutils==0.16 # awscli entrypoints==0.4 # via jupyter-client -executing==1.2.0 +executing==2.0.1 # via stack-data -fastjsonschema==2.18.0 +fastjsonschema==2.20.0 # via nbformat -fonttools==4.42.1 +fonttools==4.53.0 # via matplotlib fqdn==1.5.1 # via jsonschema -gitdb==4.0.10 +gitdb==4.0.11 # via gitpython -gitpython==3.1.34 +gitpython==3.1.43 # via civis-jupyter-notebook -idna==3.4 +idna==3.7 # via # -r /tmp/requirements-aggregated-core.txt # anyio # jsonschema # requests -ipykernel==6.25.2 +ipykernel==6.29.4 # via # nbclassic # notebook -ipython==8.15.0 +ipython==8.25.0 # via # civis-jupyter-extensions # ipykernel @@ -120,12 +117,11 @@ ipython-genutils==0.2.0 # notebook isoduration==20.11.0 # via jsonschema -jedi==0.19.0 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jinja2==3.1.4 # via # jupyter-server - # nbclassic # nbconvert # notebook jmespath==1.0.1 @@ -133,24 +129,24 @@ jmespath==1.0.1 # -r /tmp/requirements-aggregated-core.txt # boto3 # botocore -joblib==1.2.0 +joblib==1.4.2 # via # -r /tmp/requirements-aggregated-core.txt # civis # scikit-learn -jsonpointer==2.4 +jsonpointer==3.0.0 # via jsonschema -jsonref==0.2 +jsonref==1.1.0 # via # -r /tmp/requirements-aggregated-core.txt # civis -jsonschema[format-nongpl]==4.18.4 +jsonschema[format-nongpl]==4.22.0 # via # -r /tmp/requirements-aggregated-core.txt # civis # jupyter-events # nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.12.1 # via # -r /tmp/requirements-aggregated-core.txt # jsonschema @@ -158,73 +154,67 @@ jupyter-client==7.4.9 # via # ipykernel # jupyter-server - # nbclassic # nbclient # notebook -jupyter-core==5.3.1 +jupyter-core==5.7.2 # via # civis-jupyter-notebook # ipykernel # jupyter-client # jupyter-server - # nbclassic # nbclient # nbconvert # nbformat # notebook -jupyter-events==0.7.0 +jupyter-events==0.10.0 # via jupyter-server -jupyter-server==2.7.3 - # via - # nbclassic - # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server==2.14.1 + # via notebook-shim +jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert kiwisolver==1.4.5 # via matplotlib -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # nbconvert -matplotlib==3.7.2 +matplotlib==3.9.0 # via # -r /tmp/requirements-aggregated-core.txt # seaborn -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via # ipykernel # ipython -mistune==3.0.1 +mistune==3.0.2 # via nbconvert -nbclassic==1.0.0 +nbclassic==1.1.0 # via notebook -nbclient==0.8.0 +nbclient==0.10.0 # via nbconvert -nbconvert==7.8.0 +nbconvert==7.16.4 # via # jupyter-server - # nbclassic # notebook -nbformat==5.9.2 +nbformat==5.10.4 # via # jupyter-server - # nbclassic # nbclient # nbconvert # notebook -nest-asyncio==1.5.7 +nest-asyncio==1.6.0 # via # ipykernel # jupyter-client # nbclassic # notebook -notebook==6.5.5 +notebook==6.5.7 # via civis-jupyter-notebook -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via nbclassic -numpy==1.25.1 +numpy==2.0.0 # via # -r /tmp/requirements-aggregated-core.txt # contourpy @@ -233,39 +223,36 @@ numpy==1.25.1 # scikit-learn # scipy # seaborn -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server -packaging==23.1 +packaging==24.1 # via # ipykernel # jupyter-server # matplotlib # nbconvert -pandas==2.0.3 +pandas==2.2.2 # via # -r /tmp/requirements-aggregated-core.txt # civis-jupyter-extensions # seaborn -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert -parso==0.8.3 +parso==0.8.4 # via jedi -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -pickleshare==0.7.5 - # via ipython -pillow==10.0.0 +pillow==10.3.0 # via matplotlib -platformdirs==3.10.0 +platformdirs==4.2.2 # via jupyter-core -prometheus-client==0.17.1 +prometheus-client==0.20.0 # via # jupyter-server - # nbclassic # notebook -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.47 # via ipython -psutil==5.9.5 +psutil==5.9.8 # via ipykernel ptyprocess==0.7.0 # via @@ -273,19 +260,19 @@ ptyprocess==0.7.0 # terminado pure-eval==0.2.2 # via stack-data -pyasn1==0.5.0 +pyasn1==0.6.0 # via # -r /tmp/requirements-aggregated-core.txt # rsa -pycparser==2.21 +pycparser==2.22 # via cffi -pygments==2.16.1 +pygments==2.18.0 # via # ipython # nbconvert -pyparsing==3.0.9 +pyparsing==3.1.2 # via matplotlib -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r /tmp/requirements-aggregated-core.txt # arrow @@ -295,7 +282,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3 +pytz==2024.1 # via # -r /tmp/requirements-aggregated-core.txt # pandas @@ -305,20 +292,19 @@ pyyaml==6.0.1 # awscli # civis # jupyter-events -pyzmq==24.0.1 +pyzmq==26.0.3 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic # notebook -referencing==0.30.0 +referencing==0.35.1 # via # -r /tmp/requirements-aggregated-core.txt # jsonschema # jsonschema-specifications # jupyter-events -requests==2.31.0 +requests==2.32.3 # via # -r /tmp/requirements-aggregated-core.txt # civis @@ -331,7 +317,7 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rpds-py==0.9.2 +rpds-py==0.18.1 # via # -r /tmp/requirements-aggregated-core.txt # jsonschema @@ -340,23 +326,22 @@ rsa==4.7.2 # via # -r /tmp/requirements-aggregated-core.txt # awscli -s3transfer==0.6.1 +s3transfer==0.10.1 # via # -r /tmp/requirements-aggregated-core.txt # awscli # boto3 -scikit-learn==1.3.0 +scikit-learn==1.5.0 # via -r /tmp/requirements-aggregated-core.txt -scipy==1.11.1 +scipy==1.13.1 # via # -r /tmp/requirements-aggregated-core.txt # scikit-learn -seaborn==0.12.2 +seaborn==0.13.2 # via -r /tmp/requirements-aggregated-core.txt -send2trash==1.8.2 +send2trash==1.8.3 # via # jupyter-server - # nbclassic # notebook six==1.16.0 # via @@ -365,40 +350,38 @@ six==1.16.0 # bleach # python-dateutil # rfc3339-validator -smmap==5.0.0 +smmap==5.0.1 # via gitdb -sniffio==1.3.0 +sniffio==1.3.1 # via anyio soupsieve==2.5 # via beautifulsoup4 -stack-data==0.6.2 +stack-data==0.6.3 # via ipython -tenacity==8.2.2 +tenacity==8.4.1 # via # -r /tmp/requirements-aggregated-core.txt # civis -terminado==0.17.1 +terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic # notebook -threadpoolctl==3.2.0 +threadpoolctl==3.5.0 # via # -r /tmp/requirements-aggregated-core.txt # scikit-learn -tinycss2==1.2.1 +tinycss2==1.3.0 # via nbconvert -tornado==6.3.3 +tornado==6.4.1 # via # civis-jupyter-notebook # ipykernel # jupyter-client # jupyter-server - # nbclassic # notebook # terminado -traitlets==5.9.0 +traitlets==5.14.3 # via # comm # ipykernel @@ -408,29 +391,35 @@ traitlets==5.9.0 # jupyter-events # jupyter-server # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat # notebook -tzdata==2023.3 +types-python-dateutil==2.9.0.20240316 + # via arrow +tzdata==2024.1 # via # -r /tmp/requirements-aggregated-core.txt # pandas uri-template==1.3.0 # via jsonschema -urllib3==1.26.16 +urllib3==2.2.2 # via # -r /tmp/requirements-aggregated-core.txt # botocore # requests -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 +webcolors==24.6.0 # via jsonschema webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.2 +websocket-client==1.8.0 # via jupyter-server +wheel==0.43.0 + # via -r /tmp/requirements-aggregated-core.txt + +# The following packages are considered to be unsafe in a requirements file: +# setuptools