Add package publishing workflows for Rust and Python

.github/workflows/release_pypi.yaml

@@ -4,24 +4,98 @@ on:
   release:
     types:
       - created
+
 jobs:
-  release-job:
-    name: Build and publish on PyPi
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-11]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.12.3
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* cp312-*
+          CIBW_ARCHS_LINUX: x86_64 i686 aarch64
+          CIBW_ARCHS_WINDOWS: AMD64 x86
+          CIBW_ARCHS_MACOS: x86_64 arm64
+          CIBW_BEFORE_BUILD: pip install setuptools-rust
+          CIBW_ENVIRONMENT: PATH="$HOME/.cargo/bin:$PATH"
+          CIBW_TEST_COMMAND: python -c "import outlines_core; print(outlines_core.__version__)"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+          name: wheels
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel setuptools-rust
+      - name: Build sdist
+        run: python setup.py sdist
+      - uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.tar.gz
+          name: sdist
+
+  release:
+    name: Release to PyPI
+    needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.10"
-    - name: Build SDist and Wheel
-      run: ./.github/scripts/build_sdist_and_wheel.sh
-    - name: Check that the package version matches the Release name
-      run: |
-        grep -Rq "^Version: ${GITHUB_REF:10}$" outlines_core.egg-info/PKG-INFO
-    - name: Publish to PyPi
-      uses: pypa/[email protected]
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools
+      - name: Generate egg-info
+        run: python setup.py egg_info
+      - name: Check that the package version matches the Release name
+        run: |
+          grep -Rq "^Version: ${GITHUB_REF:10}$" *.egg-info/PKG-INFO
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+      - uses: actions/download-artifact@v3
+        with:
+          name: sdist
+          path: dist
+      - name: Publish to PyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_SECRET }}

.github/workflows/release_rust.yaml

@@ -0,0 +1,42 @@
+name: Release Rust Package
+
+on:
+  release:
+    types:
+      - created
+
+jobs:
+  release-rust:
+    name: Build and publish on crates.io
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+
+    - name: Cache Cargo dependencies
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Check Cargo.toml version matches Release tag
+      run: |
+        CARGO_VERSION=$(grep '^version =' Cargo.toml | sed 's/.*"\(.*\)".*/\1/')
+        if [ "${GITHUB_REF#refs/tags/v}" != "$CARGO_VERSION" ]; then
+          echo "Version mismatch: Cargo.toml ($CARGO_VERSION) doesn't match Release tag (${GITHUB_REF#refs/tags/v})"
+          exit 1
+        fi
+
+    - name: Publish to crates.io
+      env:
+        CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+      run: cargo publish

.github/workflows/tests.yml

@@ -102,3 +102,31 @@ jobs:
       - uses: actions/checkout@v3
       - name: Build SDist and Wheel
         run: ./.github/scripts/build_sdist_and_wheel.sh
+
+  cargo-test:
+      name: Run Cargo tests
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - name: Run cargo test
+        run: cargo test
+
+  cargo-audit:
+    name: Run Cargo audit
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Install cargo-audit
+      run: cargo install cargo-audit
+    - name: Run cargo audit
+      run: cargo audit

Cargo.toml

@@ -2,6 +2,9 @@
 name = "outlines-core-rs"
 version = "0.1.0"
 edition = "2021"
+description = "Structured Generation"
+license-file = "LICENSE"
+repository = "https://github.com/outlines-dev/outlines-core"
 
 [lib]
 name = "outlines_core_rs"

RELEASE.md

@@ -0,0 +1,11 @@
+# How to release
+
+## Python
+The python package uses the setuptools-scm plugin to automatically determine the version from git tags. When a release is created, it checks the tag and the CI automatically builds and publishes the package to PyPI.
+
+No internvention should be required.
+
+## Rust
+The rust crate is similarly pushed through a Github Action that triggers on a release. But the version is determined by the Cargo.toml file, which has to be updated manually. Generally, the version should be the same as the python package but this isn't a strict requirement.
+
+Currently we fail the rust release if the version in Cargo.toml doesn't match the tag. If that happens, just manually update the Cargo.toml version to match the tag. Push a new commit and rerun the job.

pyproject.toml

@@ -68,7 +68,7 @@ packages = ["outlines_core"]
 package-dir = {"" = "python"}
 
 [tool.setuptools.package-data]
-"outlines" = ["py.typed"]
+"outlines_core" = ["py.typed", "**/*.pyi"]
 
 [tool.setuptools_scm]
 write_to = "python/outlines_core/_version.py"
@@ -105,8 +105,6 @@ module = [
     "datasets.*",
     "setuptools.*",
     "setuptools_rust.*",
-    # TODO: Add type info for the Rust extension
-    "outlines_core.fsm.outlines_core_rs.*",
 ]
 ignore_missing_imports = true
 

python/outlines_core/fsm/outlines_core_rs.pyi

@@ -0,0 +1,66 @@
+from typing import Dict, List, Optional, Set, Tuple
+
+class FSMInfo:
+    initial: int
+    finals: Set[int]
+    transitions: Dict[Tuple[int, int], int]
+    alphabet_anything_value: int
+    alphabet_symbol_mapping: Dict[str, int]
+
+    def __init__(
+        self,
+        initial: int,
+        finals: Set[int],
+        transitions: Dict[Tuple[int, int], int],
+        alphabet_anything_value: int,
+        alphabet_symbol_mapping: Dict[str, int],
+    ) -> None: ...
+
+def build_regex_from_schema(
+    json: str, whitespace_pattern: Optional[str] = None
+) -> str: ...
+def to_regex(json: Dict, whitespace_pattern: Optional[str] = None) -> str: ...
+def _walk_fsm(
+    fsm_transitions: Dict[Tuple[int, int], int],
+    fsm_initial: int,
+    fsm_finals: Set[int],
+    token_transition_keys: List[int],
+    start_state: int,
+    full_match: bool,
+) -> List[int]: ...
+def state_scan_tokens(
+    fsm_transitions: Dict[Tuple[int, int], int],
+    fsm_initial: int,
+    fsm_finals: Set[int],
+    vocabulary: List[Tuple[str, List[int]]],
+    vocabulary_transition_keys: List[List[int]],
+    start_state: int,
+) -> Set[Tuple[int, int]]: ...
+def get_token_transition_keys(
+    alphabet_symbol_mapping: Dict[str, int],
+    alphabet_anything_value: int,
+    token_str: str,
+) -> List[int]: ...
+def get_vocabulary_transition_keys(
+    alphabet_symbol_mapping: Dict[str, int],
+    alphabet_anything_value: int,
+    vocabulary: List[Tuple[str, List[int]]],
+    frozen_tokens: Set[str],
+) -> List[List[int]]: ...
+def create_fsm_index_end_to_end(
+    fsm_info: FSMInfo,
+    vocabulary: List[Tuple[str, List[int]]],
+    frozen_tokens: frozenset[str],
+) -> Dict[int, Dict[int, int]]: ...
+
+BOOLEAN: str
+DATE: str
+DATE_TIME: str
+INTEGER: str
+NULL: str
+NUMBER: str
+STRING: str
+STRING_INNER: str
+TIME: str
+UUID: str
+WHITESPACE: str