big reshuffle but this should work

.gitignore

@@ -8,3 +8,4 @@ docs/build
 .venv
 benchmarks/results
 **/target/
+**/dist/

benchmarks/bench_numba_compile.py

@@ -2,7 +2,6 @@
 
 import interegular
 import numba
-
 from outlines_core.fsm import regex
 
 from .common import setup_tokenizer

benchmarks/common.py

@@ -1,7 +1,6 @@
-from transformers import AutoTokenizer
-
 from outlines_core.fsm.guide import RegexGuide
 from outlines_core.models.transformers import TransformerTokenizer
+from transformers import AutoTokenizer
 
 
 def setup_tokenizer():

bindings/python/Cargo.toml

@@ -1,13 +1,17 @@
 [package]
-name = "outlines-core-python"
+name = "python-bindings"
 version = "0.1.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[lib]
-name = "outlines_core"
-crate-type = ["cdylib"]
-
 [dependencies]
 pyo3 = "0.22.0"
-outlines-core = { path = "../../outlines-core" }
+# outlines-core = { path = "../../outlines-core" }
+
+[profile.release-lto]
+inherits = "release"
+lto = true
+
+[lib]
+name = "_lib"
+crate-type = ["cdylib"]
+path = "rust/lib.rs"

bindings/python/Manifest.in

@@ -0,0 +1,6 @@
+graft python
+graft rust
+graft tests
+include Cargo.toml
+global-exclude */__pycache__/*
+global-exclude *.pyc

bindings/python/pyproject.toml

@@ -1,15 +1,134 @@
 [build-system]
-requires = ["maturin>=1.7,<2.0"]
-build-backend = "maturin"
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "setuptools-rust"]
+build-backend = "setuptools.build_meta"
 
 [project]
 name = "outlines_core"
+authors= [{name = "Outlines Developers"}]
+description = "Structured Text Generation in Rust"
 requires-python = ">=3.8"
+license = {text = "Apache-2.0"}
+keywords=[
+    "machine learning",
+    "deep learning",
+    "language models",
+    "structured generation",
+]
 classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Science/Research",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+   "interegular",
+   "numpy<2.0.0",
+   "cloudpickle",
+   "diskcache",
+   "pydantic>=2.0",
+   "numba",
+   "referencing",
+   "jsonschema",
+   "tqdm",
+   "datasets",
+   "typing_extensions",
 ]
 dynamic = ["version"]
-[tool.maturin]
-features = ["pyo3/extension-module"]
+
+[project.optional-dependencies]
+test = [
+    "pre-commit",
+    "pytest",
+    "pytest-benchmark",
+    "pytest-cov",
+    "pytest-mock",
+    "coverage[toml]>=5.1",
+    "diff-cover",
+    "accelerate",
+    "beartype<0.16.0",
+    "huggingface_hub",
+    "torch",
+    "transformers",
+    "pillow",
+]
+
+[project.urls]
+homepage = "https://github.com/outlines-dev/outlines-core"
+documentation = "https://outlines-dev.github.io/outlines-core/"
+repository = "https://github.com/outlines-dev/outlines-core/"
+
+[project.readme]
+file="README.md"
+content-type = "text/markdown"
+
+[tool.setuptools]
+packages = ["outlines_core"]
+package-dir = {"" = "src"}
+
+[tool.setuptools.package-data]
+"outlines" = ["py.typed"]
+
+[tool.setuptools_scm]
+write_to = "src/outlines_core/_version.py"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+filterwarnings = [
+    "error",
+    "ignore::numba.core.errors.NumbaPendingDeprecationWarning",
+    "ignore::pydantic.warnings.PydanticDeprecatedSince20",
+    "ignore::FutureWarning:transformers.*",
+    "ignore::FutureWarning:huggingface_hub.*",
+    "ignore::UserWarning",
+    "ignore::DeprecationWarning:pyairports.*",
+]
+
+[tool.mypy]
+exclude=["examples"]
+enable_incomplete_feature = ["Unpack"]
+
+[[tool.mypy.overrides]]
+module = [
+    "jsonschema.*",
+    "numpy.*",
+    "cloudpickle.*",
+    "diskcache.*",
+    "pydantic.*",
+    "pytest",
+    "referencing.*",
+    "torch.*",
+    "transformers.*",
+    "huggingface_hub",
+    "interegular.*",
+    "datasets.*",
+    "numba.*",
+]
+ignore_missing_imports = true
+
+[tool.coverage.run]
+omit = [
+    "src/outlines_core/_version.py",
+    "tests/*",
+]
+branch = true
+
+[tool.coverage.report]
+omit = [
+    "tests/*",
+]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "\\.\\.\\.",
+]
+show_missing = true
+
+[tool.diff_cover]
+compare_branch = "origin/main"
+diff_range_notation = ".."
+
+[[tool.setuptools-rust.ext-modules]]
+target = "outlines_core._lib"  # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package.

bindings/python/rust/lib.rs

@@ -0,0 +1,71 @@
+// use std::collections::{BTreeMap, BTreeSet};
+
+// use ::outlines_core as core_lib;
+// use pyo3::types::{PyAnyMethods, PyDict, PyModule, PyModuleMethods, PySet};
+use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyResult};
+
+
+#[pymodule]
+mod _lib {
+    use pyo3::prelude::*;
+
+    /// Formats the sum of two numbers as string.
+    #[pyfunction]
+    fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
+        Ok((a + b).to_string())
+    }
+}
+
+// #[pymodule]
+// fn outlines_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
+//     m.add_function(wrap_pyfunction!(add, m)?)?;
+//     Ok(())
+// }
+
+// #[pyfunction]
+// Create an FSM index end-to-end.
+//
+// Args:
+//     fsm_transitions (Dict[Tuple[int, int], int]): FSM transitions mapping.
+//     alphabet_symbol_mapping (Dict[str, int]): Alphabet symbol mapping.
+//     alphabet_anything_value (int): Value representing 'anything' in the alphabet.
+//     fsm_initial (int): Initial state of the FSM.
+//     fsm_finals (Set[int]): Set of final states in the FSM.
+//     vocabulary (Dict[str, List[int]]): Vocabulary mapping.
+//
+// Returns:
+//     Dict[int, Set[Tuple[int, int]]]: The created FSM index.
+//
+// Raises:
+//     ValueError: If the input types are incorrect or conversion fails.
+// fn create_fsm_index_end_to_end(
+//     fsm_transitions: Bound<PyDict>,
+//     alphabet_symbol_mapping: Bound<PyDict>,
+//     alphabet_anything_value: i32,
+//     fsm_initial: i32,
+//     fsm_finals: Bound<PySet>,
+//     vocabulary: Bound<PyDict>,
+// ) -> PyResult<BTreeMap<i32, BTreeSet<(i32, i32)>>> {
+//     let fsm_transitions_map = fsm_transitions.extract::<BTreeMap<(i32, i32), i32>>()?;
+//     let alphabet_symbol_mapping_map = alphabet_symbol_mapping.extract::<BTreeMap<char, i32>>()?;
+//     let fsm_finals_set = fsm_finals.extract::<BTreeSet<i32>>()?;
+//     let vocabulary_map = vocabulary.extract::<BTreeMap<String, Vec<i32>>>()?;
+
+//     let res = core_lib::create_fsm_index_end_to_end_rust(
+//         &fsm_transitions_map,
+//         &alphabet_symbol_mapping_map,
+//         alphabet_anything_value,
+//         fsm_initial,
+//         &fsm_finals_set,
+//         &vocabulary_map,
+//     );
+
+//     Ok(res)
+// }
+
+// Outlines is a Generative Model Programming Framework.
+// #[pymodule]
+// fn outlines_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
+//     m.add_function(wrap_pyfunction!(create_fsm_index_end_to_end, m)?)?;
+//     Ok(())
+// }

setup.cfg → bindings/python/setup.cfg

@@ -6,3 +6,6 @@ per-file-ignores =
     **/__init__.py:F401,F403
 exclude =
     normalai/_version.py
+
+[bdist_wheel]
+py_limited_api=cp38