From 48e43c10f45daf0fedc9640e265e664ddb909ade Mon Sep 17 00:00:00 2001 From: benzguo Date: Wed, 1 May 2024 10:58:58 -0400 Subject: [PATCH 1/5] update readme --- README.md | 4 ++-- examples/notebooks/Makefile | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1101bcc..9299807 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,6 @@ print(summary_out.text) To run the above example as a notebook, navigate to the `examples/notebooks` directory and run: ```sh -make ensure # install dependencies -make basic # run the notebook +make ensure # install dependencies +marimo edit basic.py # run the notebook ``` diff --git a/examples/notebooks/Makefile b/examples/notebooks/Makefile index f908535..bb36a7d 100644 --- a/examples/notebooks/Makefile +++ b/examples/notebooks/Makefile @@ -8,7 +8,6 @@ help: @echo "Substrate notebooks" @echo "" @echo " ensure Install dependencies" - @echo " basic Basic notebook with the example from the README" @echo " update Update substrate-python to the latest version" poetry.lock: pyproject.toml @@ -20,7 +19,3 @@ ensure: poetry.lock .PHONY: update update: poetry cache clear pypi --all && poetry update substrate - -.PHONY: basic -basic: - poetry run marimo edit basic.py From 632090336080e615eb1a22f9cac9cfa4d6711358 Mon Sep 17 00:00:00 2001 From: benzguo Date: Wed, 1 May 2024 12:47:36 -0400 Subject: [PATCH 2/5] GenerateText -> RunCode --- README.md | 4 +- examples/notebooks/code-tool.py | 119 ++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 examples/notebooks/code-tool.py diff --git a/README.md b/README.md index 9299807..31bac5a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,6 @@ print(summary_out.text) To run the above example as a notebook, navigate to the `examples/notebooks` directory and run: ```sh -make ensure # install dependencies -marimo edit basic.py # run the notebook +make ensure # install dependencies +poetry run marimo edit basic.py # run the notebook ``` diff --git a/examples/notebooks/code-tool.py b/examples/notebooks/code-tool.py new file mode 100644 index 0000000..2e7c104 --- /dev/null +++ b/examples/notebooks/code-tool.py @@ -0,0 +1,119 @@ +import marimo + +__generated_with = "0.3.12" +app = marimo.App(width="medium") + + +@app.cell +def __(): + import os + import json + import base64 + import marimo as mo + from substrate import Substrate, GenerateJSON, RunCode, sb + + api_key = os.environ.get("SUBSTRATE_API_KEY") + api_key = api_key or "YOUR_API_KEY" + mo.md(f"`{api_key}`") + return ( + GenerateJSON, + RunCode, + Substrate, + api_key, + base64, + json, + mo, + os, + sb, + ) + + +@app.cell +def __(Substrate, api_key): + substrate = Substrate( + api_key=api_key, + ) + return substrate, + + +@app.cell +def __(mo): + question = mo.ui.text( + placeholder="Question", + value="What is the 88th fibonacci number?", + full_width=True, + ).form() + question + return question, + + +@app.cell +def __(GenerateJSON, RunCode, question, sb): + prompt = f""" + {question.value} + + Think step by step and run Python code to check your work or solve problems before returning an answer. Print the output. The Python runtime does not have network or filesystem access, but does include the entire standard library. Read input from stdin and write output to stdout. Remember to print the output. + """ + print(prompt) + json_schema = { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question.", + }, + "python_code": { + "type": "string", + "description": "Python code to check your work or solve the problem.", + }, + "should_run_code": { + "type": "boolean", + "description": "Set to true if you need to run `python_code` before giving your final answer.", + }, + }, + "required": ["answer", "python_code", "should_run_code"], + } + gj = GenerateJSON( + { + "prompt": prompt, + "json_schema": json_schema, + "node": "Mixtral8x7BInstruct", + # "node": "Mistral7BInstruct", + } + ) + code = RunCode( + { + "code": sb.jq(gj.future.json_object, ".python_code"), + } + ) + return code, gj, json_schema, prompt + + +@app.cell +def __(code, gj, substrate): + res = substrate.run(gj, code) + return res, + + +@app.cell +def __(code, gj, json, res): + print(json.dumps(res.json, indent=2)) + gj_out = res.get(gj) + code_out = res.get(code) + return code_out, gj_out + + +@app.cell +def __(gj_out, json): + json.dumps(gj_out.json_object, indent=2) + return + + +@app.cell +def __(code_out, json): + json.dumps(code_out.dict(), indent=2) + return + + +if __name__ == "__main__": + app.run() From 4eeb2462f3e0183548d04e3c6e97e605a04f3601 Mon Sep 17 00:00:00 2001 From: benzguo Date: Wed, 1 May 2024 14:56:11 -0400 Subject: [PATCH 3/5] gen image chain --- examples/notebooks/code-tool.py | 119 -------------------------- examples/notebooks/gen-image-chain.py | 100 ++++++++++++++++++++++ examples/notebooks/run-code-chain.py | 113 ++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 119 deletions(-) delete mode 100644 examples/notebooks/code-tool.py create mode 100644 examples/notebooks/gen-image-chain.py create mode 100644 examples/notebooks/run-code-chain.py diff --git a/examples/notebooks/code-tool.py b/examples/notebooks/code-tool.py deleted file mode 100644 index 2e7c104..0000000 --- a/examples/notebooks/code-tool.py +++ /dev/null @@ -1,119 +0,0 @@ -import marimo - -__generated_with = "0.3.12" -app = marimo.App(width="medium") - - -@app.cell -def __(): - import os - import json - import base64 - import marimo as mo - from substrate import Substrate, GenerateJSON, RunCode, sb - - api_key = os.environ.get("SUBSTRATE_API_KEY") - api_key = api_key or "YOUR_API_KEY" - mo.md(f"`{api_key}`") - return ( - GenerateJSON, - RunCode, - Substrate, - api_key, - base64, - json, - mo, - os, - sb, - ) - - -@app.cell -def __(Substrate, api_key): - substrate = Substrate( - api_key=api_key, - ) - return substrate, - - -@app.cell -def __(mo): - question = mo.ui.text( - placeholder="Question", - value="What is the 88th fibonacci number?", - full_width=True, - ).form() - question - return question, - - -@app.cell -def __(GenerateJSON, RunCode, question, sb): - prompt = f""" - {question.value} - - Think step by step and run Python code to check your work or solve problems before returning an answer. Print the output. The Python runtime does not have network or filesystem access, but does include the entire standard library. Read input from stdin and write output to stdout. Remember to print the output. - """ - print(prompt) - json_schema = { - "type": "object", - "properties": { - "answer": { - "type": "string", - "description": "The answer to the question.", - }, - "python_code": { - "type": "string", - "description": "Python code to check your work or solve the problem.", - }, - "should_run_code": { - "type": "boolean", - "description": "Set to true if you need to run `python_code` before giving your final answer.", - }, - }, - "required": ["answer", "python_code", "should_run_code"], - } - gj = GenerateJSON( - { - "prompt": prompt, - "json_schema": json_schema, - "node": "Mixtral8x7BInstruct", - # "node": "Mistral7BInstruct", - } - ) - code = RunCode( - { - "code": sb.jq(gj.future.json_object, ".python_code"), - } - ) - return code, gj, json_schema, prompt - - -@app.cell -def __(code, gj, substrate): - res = substrate.run(gj, code) - return res, - - -@app.cell -def __(code, gj, json, res): - print(json.dumps(res.json, indent=2)) - gj_out = res.get(gj) - code_out = res.get(code) - return code_out, gj_out - - -@app.cell -def __(gj_out, json): - json.dumps(gj_out.json_object, indent=2) - return - - -@app.cell -def __(code_out, json): - json.dumps(code_out.dict(), indent=2) - return - - -if __name__ == "__main__": - app.run() diff --git a/examples/notebooks/gen-image-chain.py b/examples/notebooks/gen-image-chain.py new file mode 100644 index 0000000..e0a3693 --- /dev/null +++ b/examples/notebooks/gen-image-chain.py @@ -0,0 +1,100 @@ +import marimo + +__generated_with = "0.3.12" +app = marimo.App(width="medium") + + +@app.cell +def __(): + import os + import json + import base64 + import marimo as mo + import substrate as sb + + api_key = os.environ.get("SUBSTRATE_API_KEY") + api_key = api_key or "YOUR_API_KEY" + mo.md(f"`{api_key}`") + return api_key, base64, json, mo, os, sb + + +@app.cell +def __(api_key, sb): + substrate = sb.Substrate( + api_key=api_key, + backend="v1", + ) + return substrate, + + +@app.cell +def __(mo): + prompt = mo.ui.text( + placeholder="prompt", + value="A bowl of fruit", + full_width=True, + ).form() + prompt + return prompt, + + +@app.cell +def __(prompt, sb): + image = sb.GenerateImage( + { + "prompt": prompt.value, + } + ) + return image, + + +@app.cell +def __(image, sb): + rmbg = sb.RemoveBackground({ + "image_uri": image.future.image_uri + }) + return rmbg, + + +@app.cell +def __(image, sb): + upscale = sb.UpscaleImage({ + "image_uri": image.future.image_uri + }) + return upscale, + + +@app.cell +def __(image, mo, rmbg, substrate, upscale): + res = substrate.run(image, rmbg, upscale) + viz = substrate.visualize(image, rmbg, upscale) + mo.md(f"[visualize]({viz})") + return res, viz + + +@app.cell +def __(image, mo, res): + image_out = res.get(image) + mo.image(src=image_out.image_uri) + return image_out, + + +@app.cell +def __(mo, res, rmbg): + rmbg_out = res.get(rmbg) + mo.image(src=rmbg_out.image_uri) + return rmbg_out, + + +@app.cell +def __(mo, res, upscale): + upscale_out = res.get(upscale) + mo.download( + data=upscale_out.image_uri, + filename="upscaled.jpeg", + ) + return upscale_out, + + +if __name__ == "__main__": + app.run() diff --git a/examples/notebooks/run-code-chain.py b/examples/notebooks/run-code-chain.py new file mode 100644 index 0000000..8249b3e --- /dev/null +++ b/examples/notebooks/run-code-chain.py @@ -0,0 +1,113 @@ +import marimo + +__generated_with = "0.3.12" +app = marimo.App(width="medium") + + +@app.cell +def __(): + import os + import json + import base64 + import marimo as mo + from substrate import Substrate, GenerateJSON, GenerateText, RunCode, sb + + api_key = os.environ.get("SUBSTRATE_API_KEY") + api_key = api_key or "YOUR_API_KEY" + mo.md(f"`{api_key}`") + return ( + GenerateJSON, + GenerateText, + RunCode, + Substrate, + api_key, + base64, + json, + mo, + os, + sb, + ) + + +@app.cell +def __(Substrate, api_key, os): + user_id = os.environ.get("SUBSTRATE_USER_ID") + ray_api_key = os.environ.get("RAY_API_KEY") + base_url = "http://localhost:8000" + headers = { + "substrate-api-key": ray_api_key, + "x-user-id": user_id, + "x-proxy-path": "/compose", + } + substrate = Substrate( + api_key=api_key, + base_url=base_url, + backend="v1", + additional_headers=headers, + ) + return base_url, headers, ray_api_key, substrate, user_id + + +@app.cell +def __(mo): + question = mo.ui.text( + placeholder="Question", + value="What is the 88th fibonacci number?", + full_width=True, + ).form() + question + return question, + + +@app.cell +def __(GenerateText, RunCode, question): + prompt = f""" + {question.value} + + Think step by step and run Python code to check your work or solve problems before returning an answer. Print the output. The Python runtime does not have network or filesystem access, but does include the entire standard library. Read input from stdin and write output to stdout. Remember to print the output in your code. Wrap the code in your response inside a tag. + """ + text = GenerateText( + { + "prompt": prompt, + "node": "Llama3Instruct70B", + # "node": "Mistral7BInstruct", + } + ) + get_code = RunCode( + { + "code": """import re + import sys + + s = sys.argv[1] + match = re.search(r'(.*?)', s, re.DOTALL) + print(match.group(1) if match else "") + """, + "args": [text.future.text], + } + ) + run_code = RunCode( + { + "code": get_code.future.output + } + ) + return get_code, prompt, run_code, text + + +@app.cell +def __(get_code, run_code, substrate, text): + res = substrate.run(text, get_code, run_code) + viz = substrate.visualize(text, get_code, run_code) + print(viz) + # res = substrate.run(text) + return res, viz + + +@app.cell +def __(json, mo, res, run_code): + print(json.dumps(res.json, indent=2)) + mo.md(res.get(run_code).output) + return + + +if __name__ == "__main__": + app.run() From 11cbaaf9d17b7d79962a9c016472c50ef4287fe8 Mon Sep 17 00:00:00 2001 From: benzguo Date: Wed, 1 May 2024 15:31:33 -0400 Subject: [PATCH 4/5] wip --- examples/notebooks/run-code-chain.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/examples/notebooks/run-code-chain.py b/examples/notebooks/run-code-chain.py index 8249b3e..88c26c4 100644 --- a/examples/notebooks/run-code-chain.py +++ b/examples/notebooks/run-code-chain.py @@ -30,22 +30,12 @@ def __(): @app.cell -def __(Substrate, api_key, os): - user_id = os.environ.get("SUBSTRATE_USER_ID") - ray_api_key = os.environ.get("RAY_API_KEY") - base_url = "http://localhost:8000" - headers = { - "substrate-api-key": ray_api_key, - "x-user-id": user_id, - "x-proxy-path": "/compose", - } +def __(Substrate, api_key): substrate = Substrate( api_key=api_key, - base_url=base_url, backend="v1", - additional_headers=headers, ) - return base_url, headers, ray_api_key, substrate, user_id + return substrate, @app.cell @@ -85,20 +75,15 @@ def __(GenerateText, RunCode, question): "args": [text.future.text], } ) - run_code = RunCode( - { - "code": get_code.future.output - } - ) + run_code = RunCode({"code": get_code.future.output}) return get_code, prompt, run_code, text @app.cell -def __(get_code, run_code, substrate, text): +def __(get_code, mo, run_code, substrate, text): res = substrate.run(text, get_code, run_code) viz = substrate.visualize(text, get_code, run_code) - print(viz) - # res = substrate.run(text) + mo.md(f"[visualize]({viz})") return res, viz From db7d873e9e567bf2d331544641d21452cca7bec0 Mon Sep 17 00:00:00 2001 From: benzguo Date: Wed, 1 May 2024 16:27:19 -0400 Subject: [PATCH 5/5] embed image --- examples/notebooks/gen-image-embed.py | 223 ++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 examples/notebooks/gen-image-embed.py diff --git a/examples/notebooks/gen-image-embed.py b/examples/notebooks/gen-image-embed.py new file mode 100644 index 0000000..11916ef --- /dev/null +++ b/examples/notebooks/gen-image-embed.py @@ -0,0 +1,223 @@ +import marimo + +__generated_with = "0.3.12" +app = marimo.App(width="medium") + + +@app.cell +def __(): + import os + import json + import string + import random + import base64 + import marimo as mo + import substrate as sb + + api_key = os.environ.get("SUBSTRATE_API_KEY") + api_key = api_key or "YOUR_API_KEY" + mo.md(f"`{api_key}`") + characters = string.ascii_letters + random_string = "".join(random.choice(characters) for i in range(3)) + collection_name = f"image_test_a3" + collection_name + return ( + api_key, + base64, + characters, + collection_name, + json, + mo, + os, + random, + random_string, + sb, + string, + ) + + +@app.cell +def __(api_key, sb): + substrate = sb.Substrate( + api_key=api_key, + backend="v1", + ) + return substrate, + + +@app.cell +def __(collection_name, mo, sb, substrate): + # create the vector store + create_vstore = sb.CreateVectorStore( + {"model": "clip", "collection_name": collection_name} + ) + create_res = substrate.run(create_vstore) + mo.tree(create_res.json) + return create_res, create_vstore + + +@app.cell +def __(mo): + prompt = mo.ui.text( + placeholder="prompt", + value="A bowl of fruit", + full_width=True, + ).form() + prompt + return prompt, + + +@app.cell +def __(collection_name, prompt, sb): + image = sb.GenerateImage( + { + "prompt": prompt.value, + } + ) + embed_prompt = sb.EmbedText( + { + "text": prompt.value, + "collection_name": collection_name, + } + ) + embed = sb.EmbedImage( + { + "image_uri": image.future.image_uri, + "collection_name": collection_name, + } + ) + return embed, embed_prompt, image + + +@app.cell +def __(embed, embed_prompt, image, mo, substrate): + res = substrate.run(image, embed, embed_prompt) + mo.tree(res.json) + return res, + + +@app.cell +def __(embed_prompt, res): + prompt_doc_id = res.get(embed_prompt).embedding.doc_id + print(prompt_doc_id) + return prompt_doc_id, + + +@app.cell +def __(embed, res): + image1_doc_id = res.get(embed).embedding.doc_id + print(image1_doc_id) + return image1_doc_id, + + +@app.cell +def __(mo): + prompt2 = mo.ui.text( + placeholder="prompt", + value="A bowl of chocolate", + full_width=True, + ).form() + prompt2 + return prompt2, + + +@app.cell +def __(collection_name, prompt2, sb): + image2 = sb.GenerateImage( + { + "prompt": prompt2.value, + } + ) + embed2 = sb.EmbedImage( + { + "image_uri": image2.future.image_uri, + "collection_name": collection_name, + } + ) + return embed2, image2 + + +@app.cell +def __(embed2, image2, mo, substrate): + res2 = substrate.run(image2, embed2) + mo.tree(res2.json) + return res2, + + +@app.cell +def __(embed2, res2): + image2_doc_id = res2.get(embed2).embedding.doc_id + print(image2_doc_id) + return image2_doc_id, + + +@app.cell +def __(collection_name, image, res, sb): + query = sb.QueryVectorStore( + { + "model": "clip", + "collection_name": collection_name, + "query_image_uris": [res.get(image).image_uri], + "top_k": 100, + "ef_search": 64, + # "query_strings": [prompt.value], + } + ) + return query, + + +@app.cell +def __(query, substrate): + query_res = substrate.run(query) + return query_res, + + +@app.cell +def __(query, query_res): + results = query_res.get(query).results + return results, + + +@app.cell +def __(mo, results): + mo.tree(results) + return + + +@app.cell +def __(image2_doc_id, mo, results): + image2_distance = None + for r in results[0]: + if r.id == image2_doc_id: + image2_distance = r.distance + + mo.tree( + { + "image2_distance": image2_distance, + } + ) + return image2_distance, r + + +@app.cell +def __(image, image2, image2_distance, mo, res, res2): + mo.hstack( + [ + mo.vstack( + [ + mo.image(src=res.get(image).image_uri), + ] + ), + mo.vstack( + [ + mo.image(src=res2.get(image2).image_uri), + mo.md(f"distance: {image2_distance}"), + ] + ), + ] + ) + return + + +if __name__ == "__main__": + app.run()