Merge pull request #245 from AkshatBajaj/feature/split-surround

FEATURE: Split surround into cli and library [CON-583]

.gitignore

@@ -4,10 +4,13 @@
 __pycache__/
 
 # Setuptools distribution folder.
-/dist/
+surround-lib/dist/
+surround-cli/dist/
 
 # Distribution / packaging
 build/
+surround-lib/build/
+surround-cli/build/
 
 ## Python egg metadata, regenerated from source files by setuptools.
 **/*.egg-info

README.md

@@ -13,18 +13,6 @@ Surround is a lightweight framework for serving machine learning pipelines in Py
 
 <img src="./docs/source/a2i2_logo.PNG" width="300">
 
-## Installation
-
-### Prerequisites
-- [Python](https://www.python.org/) 3+ (Tested on 3.6.5)
-- [Docker](https://www.docker.com/) (required for running in containers)
-- [Tornado](https://www.tornadoweb.org/en/stable/) (optional, needed if serving via Web)
-
-Use package manager [pip](https://pip.pypa.io/en/stable/) to install the latest (stable) version:
-```
-$ pip3 install surround
-``` 
-
 ## Simple usage
 
 A short explanation is provided in the hello-world example's [README](examples/hello-world/) file.
@@ -55,57 +43,6 @@ if __name__ == "__main__":
     print("Text is '%s'" % data.text)
 ```
 
-## Command Line Usage
-Surround comes with a range of command line tools to help you create and run Surround pipelines.
-
-To get more information on these tools, run the following command:
-```
-$ surround -h
-```
-
-### Generating projects
-For example you can use the sub-command ``init`` to generate a new project:
-```
-$ surround init <path-to-dir> --project-name sample --description "Sample description" --require-web
-```
-
-Where a new folder in `path-to-dir` (current directory if left blank) will be created with the name of the project. In this folder will be a collection of scripts and folders typically needed for a Surround project. For more information on what is generated, see our [Getting Started](https://surround.readthedocs.io/getting-started.html) guide.
-
-### Running projects
-You can then test the genereated pipeline using the `run` sub-command in the root of the project like so:
-```
-$ surround run batchLocal
-```
-
-This will execute the pipeline locally in batch mode. If you want to run the pipeline in a container then use the following:
-```
-$ surround run build
-$ surround run batch
-```
-
-If you would like to serve your pipeline via Web endpoints (`--require-web` is required when generating for this option) then you can use:
-```
-$ surround run web
-```
-Which (by default) will accept input data as JSON via HTTP POST to the endpoint `http://localhost:8080/estimate` in the following format:
-```
-{ "message": "this data will be processed by the pipeline" }
-```
-
-To see a full list of the available tasks just run the following command:
-```
-$ surround run
-```
-
-For more information on different run modes and when/how they should be used see both our [About](https://surround.readthedocs.io/en/latest/about/) and [Getting Started](https://surround.readthedocs.io/en/latest/getting-started/#) pages.
-
-## Surround pipeline architecture
-The following diagram describes how data flows through a Surround pipeline depending on the mode used when running.
-
-<img src="./docs/source/pipeline_flow_diagram.png" width="500">
-
-For a more in-depth description of this diagram, see the [About](https://surround.readthedocs.io/about.html) page on our website.
-
 ## Examples
 
 See the [examples](https://github.com/dstil/surround/tree/master/examples) directory for useful examples on how Surround can be utilized.

codefresh.yml

@@ -12,56 +12,86 @@ steps:
   clone:
     title: "Cloning repository"
     type: "git-clone"
-    repo: "a2i2/surround"
+    repo: "${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}"
     # CF_BRANCH value is auto set when pipeline is triggered
     # Learn more at codefresh.io/docs/docs/codefresh-yaml/variables/
     revision: "${{CF_BRANCH}}"
     git: "github"
     stage: "clone"
 
   test:
-    title: "Run Python tests"
-    type: "freestyle" # Run any command
-    image: "python:3.6.10" # The image in which command will be executed
-    working_directory: "${{clone}}" # Running command where code cloned
-    commands:
-      # Install surround
-      - python3 setup.py install
+    type: parallel
+    steps:
+      test_lib:
+        title: "Test Surround Library"
+        type: "freestyle" # Run any command
+        image: "python:3.6.10" # The image in which command will be executed
+        working_directory: "${{clone}}"
+        commands:
+          # Go inside the surround library
+          - cd surround
 
-      # Run tests
-      - python3 setup.py test
+          # Install surround
+          - python3 setup.py install
 
-      # Run pylint tests
-      - pylint setup.py
-      - find surround/ -iname "*.py" | xargs pylint
-      - find examples/ -iname "*.py" | xargs pylint
+          # Run tests
+          - python3 setup.py test
 
-      # Run examples
-      - ls examples/ | xargs -n 1 -I '{}' python3 examples/'{}'/main.py
-    stage: "test"
+          # Run pylint tests
+          - pip install pylint==2.4.3
+          - pylint setup.py
+          - find surround/ -iname "*.py" | xargs pylint
+        stage: "test"
 
-  release:
-    title: "Release to Pypi"
-    type: "freestyle" # Run any command
-    image: "python:3.6.10" # The image in which command will be executed
-    working_directory: "${{clone}}" # Running command where code cloned
-    commands:
-      # Install required packages
-      - python3 -m pip install --user --upgrade setuptools wheel twine
+      test_cli:
+        title: "Test Surround CLI"
+        type: "freestyle" # Run any command
+        image: "python:3.6.10" # The image in which command will be executed
+        working_directory: "${{clone}}"
+        commands:
+          # Go inside the cli
+          - cd surround_cli
 
-      # Setup Pypi config
-      - echo "[pypi]" > ~/.pypirc
-      - echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
-      - echo "username=${{PYPI_USERNAME}}" >> ~/.pypirc
-      - echo "password=${{PYPI_PASSWORD}}" >> ~/.pypirc
+          # Install surround
+          - python3 setup.py install
 
-      # Build package
-      - VERSION_TAG=$(git tag -l --points-at HEAD) python3 setup.py sdist bdist_wheel
+          # Run tests
+          - python3 setup.py test
 
-      # Upload package for distribution
-      - python3 -m twine upload --repository pypi dist/*
-    when:
-      branch:
-        only:
-          - /([0-9]+)\.([0-9]+)\.([0-9]+)?$/
-    stage: "release"
+          # Run pylint tests
+          - pip install pylint==2.4.3
+          - pylint setup.py
+          - find surround_cli/ -iname "*.py" | xargs pylint
+
+          # Run examples
+          - cd ..
+          - find examples/ -iname "*.py" | xargs pylint
+          - ls examples/ | xargs -n 1 -I '{}' python3 examples/'{}'/main.py
+        stage: "test"
+
+  # TODO: Release on VERSION change
+  # release:
+  #   title: "Release to Pypi"
+  #   type: "freestyle" # Run any command
+  #   image: "python:3.6.10" # The image in which command will be executed
+  #   working_directory: "${{clone}}" # Running command where code cloned
+  #   commands:
+  #     # Install required packages
+  #     - python3 -m pip install --user --upgrade setuptools wheel twine
+
+  #     # Setup Pypi config
+  #     - echo "[pypi]" > ~/.pypirc
+  #     - echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
+  #     - echo "username=${{PYPI_USERNAME}}" >> ~/.pypirc
+  #     - echo "password=${{PYPI_PASSWORD}}" >> ~/.pypirc
+
+  #     # Build package
+  #     - VERSION_TAG=$(git tag -l --points-at HEAD) python3 setup.py sdist bdist_wheel
+
+  #     # Upload package for distribution
+  #     - python3 -m twine upload --repository pypi dist/*
+  #   when:
+  #     branch:
+  #       only:
+  #         - /([0-9]+)\.([0-9]+)\.([0-9]+)?$/
+  #   stage: "release"

examples/runners/runners/web_runner.py

@@ -10,6 +10,10 @@
 logging.basicConfig(level=logging.INFO)
 
 class WebRunner(Runner):
+
+    def load_data(self, mode, config):
+        return None
+
     def run(self, mode=RunMode.BATCH_PREDICT):
         self.assembler.init_assembler()
         self.application = Application(self.assembler)

surround/MANIFEST.in

@@ -0,0 +1,2 @@
+include VERSION
+include surround/*.yaml

surround/README.md

@@ -0,0 +1,19 @@
+<p align="center">
+    <img src="../docs/source/temp_logo_hq.png" width="500">
+</p>
+
+# Surround Library
+
+Surround Python library is the core part of Surround as it provides a flexible way of running a ML pipeline in a variety of situations whether that be from a queue, a http endpoint or from a file system
+
+For more information about Surround library, see [Surround Library](https://surround.readthedocs.io/en/latest/about/#a-python-library)
+
+## Installation
+
+### Prerequisites
+- [Python](https://www.python.org/) 3+ (Tested on 3.6.5)
+
+Use package manager [pip](https://pip.pypa.io/en/stable/) to install the latest (stable) version:
+```
+$ pip3 install surround
+```

surround/VERSION

@@ -0,0 +1 @@
+0.0.14

surround/pylintrc

@@ -0,0 +1,15 @@
+[DISABLE]
+
+disable = missing-docstring,
+    invalid-name,
+    too-few-public-methods,
+    broad-except,
+    no-self-use,
+    inconsistent-return-statements,
+    line-too-long,
+    duplicate-code,
+    abstract-method,
+    attribute-defined-outside-init,
+    relative-beyond-top-level,
+    unused-argument,
+    too-many-arguments

surround/requirements-test.txt

@@ -0,0 +1 @@
+pylint==2.4.3

surround/requirements.txt

@@ -0,0 +1 @@
+pyaml==17.12.1

surround/setup.py

@@ -0,0 +1,30 @@
+"""
+Python package definition for Surround
+"""
+from setuptools import setup
+
+# Collect version from VERSION file
+with open('VERSION') as version_file:
+    VERSION = version_file.read().strip()
+
+with open("requirements.txt") as f:
+    INSTALL_REQUIRES = f.read().split("\n")
+
+with open("requirements-test.txt") as f:
+    TESTS_REQUIRES = f.read().split("\n")
+
+setup(name='surround',
+      version=VERSION,
+      description='Surround is a framework for serving machine learning pipelines in Python.',
+      url='https://github.com/a2i2/surround',
+      author='Scott Barnett',
+      author_email='[email protected]',
+      include_package_data=True,
+      packages=[
+          'surround'
+      ],
+      test_suite='surround.tests',
+      license="BSD-3-Clause License",
+      zip_safe=False,
+      install_requires=INSTALL_REQUIRES,
+      tests_require=TESTS_REQUIRES)