Port necessary interegular functionality

.cargo/config.toml

@@ -0,0 +1,3 @@
+# enables `cargo build -F python-bindings -v` on arm macs
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]

python/outlines_core/fsm/guide.py

@@ -12,14 +12,15 @@
     Union,
 )
 
-import interegular
 import torch
 from outlines_core.fsm.regex import (
     create_fsm_index_tokenizer,
     make_byte_level_fsm,
     make_deterministic_fsm,
 )
 
+import interegular
+
 if TYPE_CHECKING:
     from outlines_core.models.tokenizer import Tokenizer
 

python/outlines_core/fsm/outlines_core_rs.pyi

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 class FSMInfo:
     initial: int
@@ -52,6 +52,8 @@ def create_fsm_index_end_to_end(
     vocabulary: Vocabulary,
     frozen_tokens: frozenset[str],
 ) -> Dict[int, Dict[int, int]]: ...
+def parse_pattern(pattern: str) -> Any: ...
+def parse_pattern_to_fsm(pattern: str) -> Any: ...
 
 BOOLEAN: str
 DATE: str

python/outlines_core/fsm/regex.py

@@ -23,13 +23,15 @@
     anything_else,
 )
 
-from .outlines_core_rs import (  # noqa: F401
+from .outlines_core_rs import (  # noqa: F401; TODO: likely temporary; just to ensure that the fsm creation works
     FSMInfo,
     Vocabulary,
     _walk_fsm,
     create_fsm_index_end_to_end,
     get_token_transition_keys,
     get_vocabulary_transition_keys,
+    parse_pattern,
+    parse_pattern_to_fsm,
     state_scan_tokens,
 )