Skip to content

Commit

Permalink
Merge branch 'master' into hidden-1091-continued
Browse files Browse the repository at this point in the history
  • Loading branch information
ethho committed Sep 12, 2024
2 parents 1edecda + b9d8488 commit 5948a41
Show file tree
Hide file tree
Showing 94 changed files with 15,137 additions and 253 deletions.
11 changes: 6 additions & 5 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# Note: You can use any Debian/Ubuntu based image you want.
FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye

ARG PY_VER
ARG DISTRO
FROM mcr.microsoft.com/devcontainers/python:${PY_VER}-${DISTRO}
RUN \
apt update && \
apt-get install bash-completion graphviz default-mysql-client -y && \
pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \
pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint jupyterlab && \
pip uninstall datajoint -y

USER root
ENV DJ_HOST fakeservices.datajoint.io
ENV DJ_USER root
ENV DJ_PASS simple
ENV DJ_PASS password
46 changes: 31 additions & 15 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,34 +1,50 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
{
"name": "Development",
"dockerComposeFile": "docker-compose.yaml",
"name": "Existing Docker Compose (Extend)",
// Update the 'dockerComposeFile' list if you have more compose files or use different names.
// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
"dockerComposeFile": [
"../LNX-docker-compose.yml",
"docker-compose.yml"
],
// The 'service' property is the name of the service for the container that VS Code should
// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
"service": "app",
// The optional 'workspaceFolder' property is the path VS Code should open by default when
// connected. This is typically a file mount in .devcontainer/docker-compose.yml
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
// Use this environment variable if you need to bind mount your local source code into a new container.
"remoteEnv": {
"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
},
// https://containers.dev/features
"features": {
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
"ghcr.io/devcontainers/features/git:1": {},
"ghcr.io/eitsupi/devcontainer-features/jq-likes:1": {},
"ghcr.io/guiyomh/features/vim:0": {}
},
"onCreateCommand": "pip install -e .",
"postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait",
// Features to add to the dev container. More info: https://containers.dev/features.
// "features": {},
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [
80,
443,
3306,
8080,
9000
],
// Uncomment the next line if you want start specific services in your Docker Compose config.
// "runServices": [],
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
"shutdownAction": "stopCompose",
"onCreateCommand": "python3 -m pip install -e .",
"features": {
"ghcr.io/devcontainers/features/git:1": {},
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
"ghcr.io/devcontainers/features/github-cli:1": {},
},
// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": [
"ms-python.python"
]
}
},
"remoteEnv": {
"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
}
// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "devcontainer"
}
10 changes: 0 additions & 10 deletions .devcontainer/docker-compose.yaml

This file was deleted.

32 changes: 32 additions & 0 deletions .devcontainer/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
version: '2.4'
services:
# Update this to the name of the service you want to work with in your docker-compose.yml file
app:
# Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
# folder. Note that the path of the Dockerfile and context is relative to the *primary*
# docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
# array). The sample below assumes your primary file is in the root of your project.
container_name: datajoint-python-devcontainer
image: datajoint/datajoint-python-devcontainer:${PY_VER:-3.11}-${DISTRO:-buster}
build:
context: .
dockerfile: .devcontainer/Dockerfile
args:
- PY_VER=${PY_VER:-3.11}
- DISTRO=${DISTRO:-buster}

volumes:
# Update this to wherever you want VS Code to mount the folder of your project
- ..:/workspaces:cached

# Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
# cap_add:
# - SYS_PTRACE
# security_opt:
# - seccomp:unconfined

user: root

# Overrides default command so things don't shut down after the process ends.
command: /bin/sh -c "while sleep 1000; do :; done"

54 changes: 36 additions & 18 deletions .github/workflows/development.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
DOCKER_CLIENT_TIMEOUT: "120"
COMPOSE_HTTP_TIMEOUT: "120"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Validate version and release notes
run: |
DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
Expand All @@ -42,7 +42,7 @@ jobs:
- name: Build pip artifacts
run: |
export HOST_UID=$(id -u)
docker-compose -f docker-compose-build.yaml up --exit-code-from app --build
docker compose -f docker-compose-build.yaml up --exit-code-from app --build
echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
- if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
name: Add pip artifacts
Expand All @@ -58,14 +58,16 @@ jobs:
py_ver: ["3.9"]
mysql_ver: ["8.0", "5.7"]
include:
- py_ver: "3.11"
mysql_ver: "8.0"
- py_ver: "3.10"
mysql_ver: "8.0"
- py_ver: "3.8"
mysql_ver: "5.7"
- py_ver: "3.7"
mysql_ver: "5.7"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{matrix.py_ver}}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{matrix.py_ver}}
- name: Install dependencies
Expand All @@ -77,30 +79,46 @@ jobs:
- name: Run primary tests
env:
PY_VER: ${{matrix.py_ver}}
DJ_PASS: simple
DJ_PASS: password
MYSQL_VER: ${{matrix.mysql_ver}}
DISTRO: alpine
MINIO_VER: RELEASE.2021-09-03T03-56-13Z
DOCKER_CLIENT_TIMEOUT: "120"
COMPOSE_HTTP_TIMEOUT: "120"
run: |
export HOST_UID=$(id -u)
docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
lint:
runs-on: ubuntu-latest
strategy:
matrix:
py_ver: ["3.11"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{matrix.py_ver}}
uses: actions/setup-python@v5
with:
python-version: ${{matrix.py_ver}}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 black==24.2.0
- name: Run syntax tests
run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
- name: Run style tests
run: |
flake8 --ignore=E203,E722,W503 datajoint \
--count --max-complexity=62 --max-line-length=127 --statistics
black datajoint --check -v
black tests --check -v
black tests_old --check -v
--count --max-complexity=62 --max-line-length=127 --statistics \
--per-file-ignores='datajoint/diagram.py:C901'
black --required-version '24.2.0' --check -v datajoint tests tests_old
codespell:
name: Check for spelling errors
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Codespell
uses: codespell-project/actions-codespell@v2
publish-docs:
Expand All @@ -113,7 +131,7 @@ jobs:
DOCKER_CLIENT_TIMEOUT: "120"
COMPOSE_HTTP_TIMEOUT: "120"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Deploy docs
run: |
export MODE=BUILD
Expand Down Expand Up @@ -145,9 +163,9 @@ jobs:
outputs:
release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{matrix.py_ver}}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{matrix.py_ver}}
- name: Determine package version
Expand All @@ -172,7 +190,7 @@ jobs:
prerelease: false
draft: false
- name: Fetch pip artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4.1.7
with:
name: pip-datajoint-${{env.DJ_VERSION}}
path: dist
Expand Down Expand Up @@ -201,7 +219,7 @@ jobs:
- name: Publish pip release
run: |
export HOST_UID=$(id -u)
docker-compose -f docker-compose-build.yaml run \
docker compose -f docker-compose-build.yaml run \
-e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
sh -c "pip install twine && python -m twine upload dist/*"
- name: Login to DockerHub
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
DOCKER_CLIENT_TIMEOUT: "120"
COMPOSE_HTTP_TIMEOUT: "120"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Deploy docs
run: |
export MODE=BUILD
Expand Down
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,6 @@ docs/site

!.vscode/settings.json
!.vscode/launch.json
!.devcontainer/devcontainer.json
!.devcontainer/devcontainer.json
!.devcontainer/docker-compose.yml

12 changes: 11 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
## Release notes

### Upcoming
### 0.14.3 -- TBD
- Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177)

### 0.14.2 -- Aug 19, 2024
- Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142)
- Added - Codespell GitHub Actions workflow
- Added - GitHub Actions workflow to manually release docs
- Changed - Update `datajoint/nginx` to `v0.2.6`
- Changed - Migrate docs from `https://docs.datajoint.org/python` to `https://datajoint.com/docs/core/datajoint-python`
- Fixed - [DevContainer](https://containers.dev/) configuration - PR [#1115](https://github.com/datajoint/datajoint-python/pull/1115)
- Fixed - Updated set_password to work on MySQL 8 - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106)
- Added - Missing tests for set_password - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106)
- Changed - Returning success count after the .populate() call - PR [#1050](https://github.com/datajoint/datajoint-python/pull/1050)
- Fixed - `Autopopulate.populate` excludes `reserved` jobs in addition to `ignore` and `error` jobs
- Fixed - Issue [#1159](https://github.com/datajoint/datajoint-python/pull/1159) (cascading delete) - PR [#1160](https://github.com/datajoint/datajoint-python/pull/1160)
- Changed - Minimum Python version for Datajoint-Python is now 3.8 PR #1163
- Fixed - `docker compose` commands in CI [#1164](https://github.com/datajoint/datajoint-python/pull/1164)
- Changed - Default delete behavior now includes masters of part tables - PR [#1158](https://github.com/datajoint/datajoint-python/pull/1158)

### 0.14.1 -- Jun 02, 2023
- Fixed - Fix altering a part table that uses the "master" keyword - PR [#991](https://github.com/datajoint/datajoint-python/pull/991)
Expand Down
17 changes: 8 additions & 9 deletions LNX-docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ x-net:
services:
db:
<<: *net
image: datajoint/mysql:${MYSQL_VER}
image: datajoint/mysql:${MYSQL_VER:-8.0}
environment:
- MYSQL_ROOT_PASSWORD=${DJ_PASS}
- MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
command: mysqld --default-authentication-plugin=mysql_native_password
# ports:
# - "3306:3306"
# volumes:
Expand All @@ -21,7 +22,7 @@ services:
interval: 15s
minio:
<<: *net
image: minio/minio:${MINIO_VER}
image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
environment:
- MINIO_ACCESS_KEY=datajoint
- MINIO_SECRET_KEY=datajoint
Expand All @@ -44,7 +45,7 @@ services:
interval: 15s
fakeservices.datajoint.io:
<<: *net
image: datajoint/nginx:v0.2.6
image: datajoint/nginx:latest
environment:
- ADD_db_TYPE=DATABASE
- ADD_db_ENDPOINT=db:3306
Expand All @@ -58,7 +59,7 @@ services:
# - "3306:3306"
app:
<<: *net
image: datajoint/djtest:py${PY_VER}-${DISTRO}
image: datajoint/djtest:py${PY_VER:-3.8}-${DISTRO:-alpine}
depends_on:
db:
condition: service_healthy
Expand All @@ -69,7 +70,7 @@ services:
environment:
- DJ_HOST=fakeservices.datajoint.io
- DJ_USER=root
- DJ_PASS
- DJ_PASS=password
- DJ_TEST_HOST=fakeservices.datajoint.io
- DJ_TEST_USER=datajoint
- DJ_TEST_PASSWORD=datajoint
Expand All @@ -86,14 +87,12 @@ services:
- -c
- |
set -e
pip install --user nose nose-cov
pip install -e .
pip list --format=freeze | grep datajoint
pytest -sv --cov-report term-missing --cov=datajoint tests
nosetests -vsw tests_old --with-coverage --cover-package=datajoint
# ports:
# - "8888:8888"
user: ${HOST_UID}:anaconda
user: ${HOST_UID:-1000}:anaconda
volumes:
- .:/src
- /tmp/.X11-unix:/tmp/.X11-unix:rw
Expand Down
5 changes: 3 additions & 2 deletions datajoint/autopopulate.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""This module defines class dj.AutoPopulate"""

import logging
import datetime
import traceback
Expand Down Expand Up @@ -207,12 +208,12 @@ def handler(signum, frame):

keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)

# exclude "error" or "ignore" jobs
# exclude "error", "ignore" or "reserved" jobs
if reserve_jobs:
exclude_key_hashes = (
jobs
& {"table_name": self.target.table_name}
& 'status in ("error", "ignore")'
& 'status in ("error", "ignore", "reserved")'
).fetch("key_hash")
keys = [key for key in keys if key_hash(key) not in exclude_key_hashes]

Expand Down
Loading

0 comments on commit 5948a41

Please sign in to comment.