fix: updated poetry

Makefile

@@ -25,5 +25,5 @@ package_list:
 create_env:
 	conda deactivate -n vortex \
 	&& conda env remove -n vortex -y \
-	&& conda create -n vortex python=3.10 -y \
+	&& conda create -n vortex python=3.11 -y \
 	&& conda activate vortex

main.py

@@ -1,4 +1,2 @@
-from vortex import entrypoint
-
 if __name__ == "__main__":
-    entrypoint()
+    print("Vortex is installed!")

pyproject.toml

@@ -27,7 +27,7 @@ langchainhub = "^0.1.14"
 selenium = "^4.17.2"
 html2text = "^2020.1.16"
 chainlit = "^1.0.200"
-faiss-cpu = "^1.7.4"
+# faiss-cpu = "^1.7.4"
 chromadb = "^0.4.22"
 tiktoken = "^0.5.2"
 pymupdf = "^1.23.19"

requirements.txt

@@ -15,11 +15,11 @@ langchainhub
 selenium
 html2text
 chainlit
-faiss-cpu
+# faiss-cpu
 chromadb
 tiktoken
 pymupdf
 duckduckgo-search
 wikipedia
 mlflow
-sendgrid
+sendgrid

setup.py

@@ -1,10 +1,37 @@
-# import glob
-# import io
-# import os
-# import sys
+import glob
+import io
+import os
+import sys
+
+import setuptools
+from setuptools import find_namespace_packages, find_packages
+
+# Package metadata.
+name = "vortex"
+description = (
+    "A simple util to get a spark and mlflow session objects from an .env file"
+)
+
+package_root = os.path.abspath(os.path.dirname(__file__))
+
+version = {}
+with open(os.path.join(package_root, "version.py")) as fp:
+    exec(fp.read(), version)
+version = version["__version__"]
+
+readme_filename = os.path.join(package_root, "README.md")
+with io.open(readme_filename, encoding="utf-8") as readme_file:
+    readme = readme_file.read()
+
+# Should be one of:
+# 'Development Status :: 3 - Alpha'
+# 'Development Status :: 4 - Beta'
+# 'Development Status :: 5 - Production/Stable'
+release_status = "Development Status :: 3 - Alpha"
+
+with open("requirements.txt") as f:
+    required = f.read().splitlines()
 
-# import setuptools
-# from setuptools import find_namespace_packages, find_packages
 
 # if sys.version_info < (3, 8):
 #     find_namespace_packages()
@@ -13,49 +40,56 @@
 #     sys.exit(1)
 
 
-# def prepare_data_files(directory, extensions):
-#     files = []
-#     for ext in extensions:
-#         files.extend(glob.glob(f"{directory}/*.{ext}"))
-#     return files
-
-
-# data_files_structure = [
-#     (
-#         "vortex",
-#         prepare_data_files(
-#             "vortex",
-#             ["csv", "sql", "txt", "md", "html", "css", "json", "yaml", "faiss", "pkl"],
-#         ),
-#     ),
-# ]
-
-# # Package metadata.
-# name = "vortex"
-# description = (
-#     "A simple util to get a spark and mlflow session objects from an .env file"
-# )
-# # Should be one of:
-# # 'Development Status :: 3 - Alpha'
-# # 'Development Status :: 4 - Beta'
-# # 'Development Status :: 5 - Production/Stable'
-# release_status = "Development Status :: 3 - Alpha"
+def prepare_data_files(directory, extensions):
+    files = []
+    for ext in extensions:
+        files.extend(glob.glob(f"{directory}/*.{ext}"))
+    return files
 
-# with open("requirements.txt") as f:
-#     required = f.read().splitlines()
 
-# # Setup boilerplate below this line.
+data_files_structure = [
+    (
+        "vortex",
+        prepare_data_files(
+            "vortex",
+            ["csv", "sql", "txt", "md", "html", "css", "json", "yaml", "faiss", "pkl"],
+        ),
+    ),
+]
 
-# package_root = os.path.abspath(os.path.dirname(__file__))
 
-# version = {}
-# with open(os.path.join(package_root, "version.py")) as fp:
-#     exec(fp.read(), version)
-# version = version["__version__"]
+setuptools.setup(
+    name="vortex",
+    packages=find_packages(exclude=["vortex_tests"]),
+    install_requires=[
+        "dagster",
+        "dagster-cloud",
+        "boto3",
+        "pandas",
+        "matplotlib",
+        "langchain",
+        "langchain_openai",
+        "openai",
+        "sqlalchemy",
+        "psycopg2-binary",
+        "pyautogen",
+        "bs4",
+        "langchainhub",
+        "langchain-community",
+        "selenium",
+        "html2text",
+        "chainlit",
+        "faiss-cpu",
+        "chromadb",
+        "tiktoken",
+        "pymupdf",
+        # "duckduckgo",
+        "wikipedia",
+        "sendgrid",
+    ],
+    extras_require={"dev": ["dagster-webserver", "pytest"]},
+)
 
-# readme_filename = os.path.join(package_root, "README.md")
-# with io.open(readme_filename, encoding="utf-8") as readme_file:
-#     readme = readme_file.read()
 
 # setuptools.setup(
 #     name=name,
@@ -72,35 +106,11 @@
 #         "Operating System :: OS Independent",
 #         "Development Status :: 3 - Alpha",
 #         "Intended Audience :: Developers",
-#         "Programming Language :: Python",
-#         "Programming Language :: Python :: 3",
-#         "Programming Language :: Python :: 3.8",
-#         "Programming Language :: Python :: 3.9",
-#         "Programming Language :: Python :: 3.10",
-#         "Programming Language :: Python :: 3.11",
+#         "Programming Language :: Python" "Programming Language :: Python :: 3.11",
 #         "Topic :: Software Development :: Libraries :: Python Modules",
 #     ],
-#     # packages=find_packages(),
 #     packages=find_packages(exclude=["vortex_tests"]),
-#     # install_requires=required,
-#     install_requires=[
-#         "dagster",
-#         "dagster-cloud",
-#         "boto3",
-#         "pandas",
-#         "matplotlib",
-#         "langchain",
-#         "langchain_openai",
-#         "openai",
-#         "sqlalchemy",
-#         "psycopg2-binary",
-#         "pyautogen",
-#         "bs4",
-#         "langchainhub",
-#         "langchain-community",
-#         "selenium",
-#         "html2text",
-#     ],
+#     install_requires=required,
 #     extras_require={"dev": ["dagster-webserver", "pytest"]},
 #     python_requires=">=3.8",
 #     include_package_data=True,
@@ -131,38 +141,3 @@
 #         ],
 #     },
 # )
-
-
-from setuptools import find_packages, setup
-
-setup(
-    name="vortex",
-    packages=find_packages(exclude=["vortex_tests"]),
-    install_requires=[
-        "dagster",
-        "dagster-cloud",
-        "boto3",
-        "pandas",
-        "matplotlib",
-        "langchain",
-        "langchain_openai",
-        "openai",
-        "sqlalchemy",
-        "psycopg2-binary",
-        "pyautogen",
-        "bs4",
-        "langchainhub",
-        "langchain-community",
-        "selenium",
-        "html2text",
-        "chainlit",
-        "faiss-cpu",
-        "chromadb",
-        "tiktoken",
-        "pymupdf",
-        # "duckduckgo",
-        "wikipedia",
-        "sendgrid",
-    ],
-    extras_require={"dev": ["dagster-webserver", "pytest"]},
-)

vortex/__init__.py

@@ -4,5 +4,4 @@
 
 
 def entrypoint():
-    # init()
-    ...
+    print("Vortex is installed!")

vortex/flows/DockerFile_dagster

@@ -0,0 +1,20 @@
+# Dagster libraries to run both dagster-webserver and the dagster-daemon. Does not
+# need to have access to any pipeline code.
+
+FROM --platform=linux/amd64 python:3.11
+
+RUN pip install \
+    dagster \
+    dagster-graphql \
+    dagster-webserver \
+    dagster-postgres \
+    dagster-docker
+
+# Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+RUN mkdir -p $DAGSTER_HOME
+
+COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+
+WORKDIR $DAGSTER_HOME

vortex/flows/DockerFile_user_code

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 python:3.11
+
+# Checkout and install dagster libraries needed to run the gRPC server
+# exposing your repository to dagster-webserver and dagster-daemon, and to load the DagsterInstance
+
+RUN pip install \
+    dagster \
+    dagster-postgres \
+    dagster-docker
+
+# Add repository code
+
+WORKDIR /opt/dagster/app
+
+COPY vortex /opt/dagster/app/
+
+# Run dagster gRPC server on port 4000
+
+EXPOSE 4000
+
+# CMD allows this to be overridden from run launchers or executors that want
+# to run other commands against your repository
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "vorte/__init__.py"]