Implement comments

WIP: CLI interface
WIP: CI pipeline

.github/workflows/test.yaml

@@ -33,3 +33,21 @@ jobs:
         PYTHONPATH: "${{ github.workspace }}/python/src"
       run: |
         pytest
+
+    - name: Check build
+      working-directory: python/
+      env:
+        PYTHONPATH: "${{ github.workspace }}/python/src"
+      run: |
+        pip install twine
+        python setup.py sdist bdist_wheel
+        twine check dist/*
+
+    - name: Install and run
+      working-directory: python/
+      env:
+        PYTHONPATH: "${{ github.workspace }}/python/src"
+      run: |
+
+        pip install -e .
+        metaprompt

README.md

@@ -57,7 +57,7 @@ This is an early work-in-progress. Follow [me on twitter](https://x.com/klntsky)
   - [x] `[$ meta-prompt]`
   - [x] `[:use module :param1=value1]`
   - [ ] `[:model model-id ...]` for dynamic model selection
-  - [ ] `[# comments]`
+  - [x] `[# comments]`
   - [ ] `[:status some-status]` - to show during prompt evaluation
   - [ ] `[:call ffi-function :param1=foo :param2=bar]`
 - [ ] Implement an evaluator

python/README.md

@@ -0,0 +1,3 @@
+# MetaPrompt
+
+Python implementation of MetaPrompt - visit https://metaprompt-lang.org/

python/requirements.txt

@@ -1,63 +1,35 @@
-aiohappyeyeballs==2.4.3
-aiohttp==3.10.10
-aiosignal==1.3.1
 annotated-types==0.7.0
 antlr4-python3-runtime==4.13.2
 antlr4-tools==0.2.1
 anyio==4.6.2.post1
-attrs==24.2.0
 black==24.10.0
-build==1.2.2.post1
 certifi==2024.8.30
-charset-normalizer==3.4.0
 click==8.1.7
-dataclasses-json==0.6.7
 distro==1.9.0
-frozenlist==1.5.0
-greenlet==3.1.1
+flake8==7.1.1
 h11==0.14.0
 httpcore==1.0.6
 httpx==0.27.2
-httpx-sse==0.4.0
 idna==3.10
 iniconfig==2.0.0
 install-jdk==1.1.0
 invoke==2.2.0
-jiter==0.7.0
-jsonpatch==1.33
-jsonpointer==3.0.0
-langchain==0.3.7
-langchain-community==0.3.5
-langchain-core==0.3.15
-langchain-text-splitters==0.3.2
-langsmith==0.1.142
-marshmallow==3.23.1
-multidict==6.1.0
+jiter==0.7.1
+mccabe==0.7.0
+metaprompt==0.0.1
 mypy-extensions==1.0.0
-numpy==1.26.4
 openai==1.54.3
-orjson==3.10.11
 packaging==24.2
 pathspec==0.12.1
-pip-tools==7.4.1
 platformdirs==4.3.6
 pluggy==1.5.0
-propcache==0.2.0
+pycodestyle==2.12.1
 pydantic==2.9.2
-pydantic-settings==2.6.1
 pydantic_core==2.23.4
-pyproject_hooks==1.2.0
+pyflakes==3.2.0
 pytest==8.3.3
 pytest-asyncio==0.24.0
 python-dotenv==1.0.1
-PyYAML==6.0.2
-requests==2.32.3
-requests-toolbelt==1.0.0
 sniffio==1.3.1
-SQLAlchemy==2.0.35
-tenacity==9.0.0
 tqdm==4.67.0
-typing-inspect==0.9.0
 typing_extensions==4.12.2
-urllib3==2.2.3
-yarl==1.17.1

python/setup.cfg

@@ -1,2 +1,2 @@
 [metadata]
-description-file = README.md
+description_file = README.md

python/setup.py

@@ -6,5 +6,25 @@
     description="A template engine for LLM prompts",
     packages=find_packages("src"),
     package_dir={"": "src"},
-    install_requires=[],
+    install_requires=[
+        "antlr4-python3-runtime>=4.13.2",
+        "openai>=1.54.3",
+        "python-dotenv>=1.0.1",
+        "setuptools>=68.1.2",
+    ],
+    extras_require={
+        "dev": [
+            "invoke>=2.2.0",
+            "pytest>=8.3.3",
+            "pytest-asyncio>=0.24.0",
+            "flake8>=3.8.4",
+            "black>=24.10.0",
+            "antlr4-tools==0.2.1",
+        ],
+    },
+    entry_points={
+        'console_scripts': [
+            'metaprompt=main:main',
+        ],
+    },
 )

python/src/loader.py

@@ -1,9 +1,11 @@
-def discover_variables(ast):
+def _discover_variables(ast):
     if isinstance(ast, list):
         for node in ast:
-            yield from discover_variables(node)
+            yield from _discover_variables(node)
     elif isinstance(ast, dict):
-        if ast["type"] == "var":
+        if ast["type"] == "comment":
+            return
+        elif ast["type"] == "var":
             yield {
                 'type': 'var',
                 'name': ast['name']
@@ -14,12 +16,12 @@ def discover_variables(ast):
                 "name": ast["name"]
             }
         for key in ast:
-            yield from discover_variables(ast[key])
+            yield from _discover_variables(ast[key])
 
 def extract_variables(ast):
     variables = set()
     assigned = set()
-    for item in discover_variables(ast):
+    for item in _discover_variables(ast):
         match item:
             case { 'name': name, 'type': "var" }:
                 if name not in assigned:

python/src/main.py

@@ -0,0 +1,51 @@
+import argparse
+import os
+
+class ParseSetAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        var_dict = getattr(namespace, self.dest, {})
+        if var_dict is None:
+            var_dict = {}
+        # Split on the first '=' to handle cases where values may contain '='
+        name, value = values.split("=", 1)
+        var_dict[name] = value
+        setattr(namespace, self.dest, var_dict)
+
+def parse_arguments():
+    """
+    Parse command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="MetaPrompt is a template engine for LLM prompts that supports writing prompts with prompts. Visit https://metaprompt-lang.org/ for more info.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    # Adding positional arguments
+    parser.add_argument(
+        "INPUT_FILES",
+        nargs="*",
+        help="A list of metaprompt files."
+    )
+
+    parser.add_argument(
+        "--set",
+        action=ParseSetAction,
+        dest="variables",
+        metavar="VAR_NAME=value",
+        help="define a variable in the form VAR_NAME=value.",
+    )
+
+    # Parsing arguments
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    print(args)
+    for file_path in args.INPUT_FILES:
+        if os.path.isfile(file_path):
+            with open(file_path, 'utf-8') as file:
+                content = file.read()
+                ast = parse_metaprompt(content)
+
+if __name__ == "__main__":
+    main()