*** replace tokens?

gbenson · May 23, 2024 · 3460b24 · 3460b24
1 parent 5a3e0f1
commit 3460b24
Showing 1 changed file with 23 additions and 2 deletions.
diff --git a/src/dom_tokenizers/pre_tokenizers/dom_snapshot.py b/src/dom_tokenizers/pre_tokenizers/dom_snapshot.py
@@ -22,8 +22,29 @@ class DOMSnapshotPreTokenizer(PreTokenizer):
     """Pre-tokenizer that consumes JSON-serialized DOM snapshots
     and emits tokenized representations of the snapshotted DOMs.
     """
-    elem_token = "[TAG]"       # beginning of element name
-    attr_token = "[ATTR]"      # beginning of attribute
+    open_tag_token = "<"  # an element name is next
+    close_tag_token = "/"  # an element name is next
+    attr_name_token = "_"  # an attribute name is next
+    attr_value_token = "="  # an attribute value is next
+    tag_done_token = ">"  # end of open/close element
+
+    elem_token = open_tag_token  # XXX
+    attr_token = attr_name_token  # XXX
+
+    # or, prefixed tokens, like:
+    tokens = [
+        "<html>",
+        "<a",
+        "_href=",
+        ">",
+        # text,
+        "[base64]",
+        "</a>",
+        "</html>",
+        ]
+
+    start_comment_token = "!"
+
     comm_token = "[COMMENT]"   # beginning of comment
     base64_token = "[BASE64]"  # beginning of some base64
     long_token = "[LONG]"      # elided long token