Merge pull request #257 from AkshatBajaj/feature/recent-master

FEATURE: Merge recent master changes to main

surround/CHANGELOG.md

@@ -6,14 +6,30 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
-- Add `license` to setup.py
-
 ### Changed
 
 ### Fixed
 
 ### Limitation
 
+## [0.0.15] - 2020-11-26
+
+### Added
+
+- Added `license` to setup.py
+- Added docker file to the generated Surround projects for `Jupyter` support
+- Added support for creating a versioned output folder for each run that stores the log files.
+
+### Changed
+
+- Print available endpoints information to the console
+- Updated the generated notebook to load data from a runner
+
+### Fixed
+
+- Allow CI to trigger on forked repositories
+- Linting issue in generated project
+
 ## [0.0.14] - 2020-07-01
 
 ### Added

surround/surround/config.py

@@ -2,6 +2,7 @@
 import os
 import functools
 
+from datetime import datetime
 from pathlib import Path
 from collections.abc import Mapping
 from pkg_resources import resource_stream
@@ -101,7 +102,9 @@ def __init__(self, project_root=None, package_path=None, auto_load=False):
             self._storage["project_root"] = project_root
             self._storage["package_path"] = package_path
             self._storage["volume_path"] = volume_path
-            self._storage["output_path"] = os.path.join(project_root, "output")
+
+            now = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+            self._storage["output_path"] = os.path.join(project_root, "output", str(now))
             self._storage["input_path"] = os.path.join(project_root, "input")
             self._storage["models_path"] = os.path.join(project_root, "models")
 

surround/surround/project.py

@@ -27,13 +27,13 @@
             ("{project_name}/__main__.py", "batch_main.py.txt", False, False),
             ("{project_name}/__main__.py", "web_main.py.txt", False, True),
             ("{project_name}/__init__.py", "init.py.txt", False, False),
-            ("notebooks/example.ipynb", "example.ipynb.txt", False, False),
-            ("notebooks/example.ipynb", "example.ipynb.txt", False, True),
+            ("notebooks/data_analysis.ipynb", "data_analysis.ipynb.txt", False, False),
             ("templates/results.html", "results.html.txt", False, False),
             ("templates/results.html", "results.html.txt", False, True),
             ("dodo.py", "dodo.py.txt", False, False),
             ("dodo.py", "web_dodo.py.txt", False, True),
             ("Dockerfile", "Dockerfile.txt", False, False),
+            ("Dockerfile.Notebook", "Dockerfile.Notebook.txt", False, False),
             ("{project_name}/config.yaml", "config.yaml.txt", False, False),
             (".gitignore", ".gitignore.txt", False, False),
             ("VERSION", "VERSION.txt", False, False)

surround_cli/templates/new/Dockerfile.Notebook.txt

@@ -0,0 +1,8 @@
+FROM a2i2/{project_name}:latest
+
+RUN pip3 install jupyter -U && pip3 install jupyterlab
+
+EXPOSE 8888
+
+# TODO: Remove token and password settings for production deployment
+CMD ["jupyter", "lab","--allow-root", "--ip=0.0.0.0", "--no-browser","--NotebookApp.token=''","--NotebookApp.password=''"]

surround_cli/templates/new/data_analysis.ipynb.txt

@@ -0,0 +1,56 @@
+{{
+ "cells": [
+  {{
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {{}},
+   "outputs": [],
+   "source": [
+    "# Magic commands\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "# Set up paths to access project module\n",
+    "import sys\n",
+    "import os\n",
+    "sys.path.append('/app')\n",
+    "os.chdir('/app')"
+   ]
+  }},
+  {{
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {{}},
+   "outputs": [],
+   "source": [
+    "# Package imports\n",
+    "from surround import Config\n",
+    "from {project_name}.file_system_runner import FileSystemRunner\n",
+    "\n",
+    "config = Config(auto_load=True)\n",
+    "data = FileSystemRunner().load_data(None, config)"
+   ]
+  }}
+ ],
+ "metadata": {{
+  "kernelspec": {{
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }},
+  "language_info": {{
+   "codemirror_mode": {{
+    "name": "ipython",
+    "version": 3
+   }},
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }}
+ }},
+ "nbformat": 4,
+ "nbformat_minor": 4
+}}

surround_cli/templates/new/dodo.py.txt

@@ -5,9 +5,6 @@ This module defines the tasks that can be executed using `surround run [task nam
 import os
 import sys
 import subprocess
-import re
-import webbrowser
-import logging
 
 from pathlib import Path
 
@@ -19,7 +16,8 @@ CONFIG = Config(os.path.dirname(__file__))
 DOIT_CONFIG = {{'verbosity':2, 'backend':'sqlite3'}}
 PACKAGE_PATH = os.path.basename(CONFIG["package_path"])
 IMAGE = "%s/%s:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"])
-LOGGER = logging.getLogger(__name__)
+IMAGE_JUPYTER = "%s/%s-jupyter:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"])
+DOCKER_JUPYTER = "Dockerfile.Notebook"
 
 PARAMS = [
     {{
@@ -46,7 +44,7 @@ def task_build():
 def task_remove():
     """Remove the Docker image for the current project"""
     return {{
-        'actions': ['docker rmi %s -f' % IMAGE],
+        'actions': ['docker rmi %s %s -f' % (IMAGE, IMAGE_JUPYTER)],
         'params': PARAMS
     }}
 
@@ -199,107 +197,28 @@ def task_batch_local():
         'params': PARAMS
     }}
 
+def task_build_jupyter():
+    """Build the Docker image for a Jupyter Lab notebook"""
+    return {{
+        'basename': 'buildJupyter',
+        'actions': ['docker build --tag=%s . -f %s' % (IMAGE_JUPYTER, DOCKER_JUPYTER)],
+        'task_dep': ['build'],
+        'params': PARAMS
+    }}
+
 def task_jupyter():
-    """Run a Jupyter notebook in the Docker container"""
-    # Allow for auto reload to be enabled and import modules from project package
-    ipython_config = "c.InteractiveShellApp.extensions.append('autoreload')\n"
-    ipython_config += "c.InteractiveShellApp.exec_lines = "
-    ipython_config += "['%autoreload 2', 'import sys', 'sys.path.append(\\'../\\')']"
-
-    # Build the command for running jupyter
-    command = [
-        "pip install -r /app/requirements.txt",
-        "mkdir /etc/ipython",
-        "echo \"%s\" > /etc/ipython/ipython_config.py" % ipython_config,
-        "/usr/local/bin/start.sh jupyter notebook --NotebookApp.token=''"
+    """Run a Jupyter Lab notebook"""
+    cmd = [
+        "docker",
+        "run",
+        "-itp",
+        "8888:8888",
+        '-w',
+        '/app',
+        "--volume",
+        "\"%s/\":/app" % CONFIG["volume_path"],
+        IMAGE_JUPYTER
     ]
-    command = "; ".join(command)
-
-    def run_command():
-        process = subprocess.Popen(
-            [
-                "docker",
-                "run",
-                "--rm",
-                "--name",
-                "{project_name}_surround_notebook",
-                "--volume",
-                "%s:/app" % CONFIG['volume_path'],
-                "-p",
-                "55910:8888",
-                "--user",
-                "root",
-                "-w",
-                "/app",
-                "jupyter/base-notebook:307ad2bb5fce",
-                "bash",
-                "-c",
-                command
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            encoding='utf-8')
-
-        LOGGER.info("Starting jupyter notbook server...\n")
-
-        # Get the IP address of the container, otherwise use localhost
-        ip_process = subprocess.Popen(
-            ['docker-machine', 'ip'],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            encoding='utf-8')
-        ip_process.wait()
-
-        ip_output = ip_process.stdout.readline().rstrip()
-
-        if re.match(r"^(\d{{1,3}}\.){{3}}\d{{1,3}}$", ip_output):
-            host = ip_output
-        else:
-            host = "localhost"
-
-        # Wait for the notebook server to be up before loading browser
-        while True:
-            line = process.stderr.readline().rstrip()
-            if line and 'Serving notebooks from local directory' in line:
-                break
-
-            if process.poll():
-                LOGGER.error("Failed to start the server, check if its not running somewhere else!")
-
-                # Stop any containers that might be running
-                process = subprocess.Popen(
-                    [
-                        'docker',
-                        'stop',
-                        '{project_name}_surround_notebook'
-                    ],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-                process.wait()
-                return
-
-        # Open the browser automatically
-        webbrowser.open('http://%s:55910/tree' % host, new=2)
-
-        LOGGER.info("Notebook URL: http://%s:55910/tree\n", host)
-        LOGGER.info("Use CTRL+C to stop the server.")
-
-        try:
-            process.wait()
-        except KeyboardInterrupt:
-            pass
-        finally:
-            LOGGER.info("Closing server...")
-            process = subprocess.Popen(
-                [
-                    'docker',
-                    'stop',
-                    '{project_name}_surround_notebook'
-                ],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE)
-            process.wait()
-
     return {{
-        'actions': [run_command]
+        'actions': [" ".join(cmd)],
     }}

surround_cli/templates/new/init.py.txt

@@ -1,3 +1,24 @@
 """
-This file is required for the project to be considered a python package.
+This file is needed to make Python treat this directory as a package
 """
+
+import logging
+import os
+
+from logging.handlers import RotatingFileHandler
+
+from surround import has_config
+
+@has_config
+def setup(config):
+
+    os.makedirs(config["output_path"], exist_ok=True)
+
+    # Initialise logging
+    level = logging.INFO
+    log_file = os.path.join(config["output_path"], "output.log")
+    file_handler = RotatingFileHandler(log_file, maxBytes=1048576, backupCount=5)
+    handlers = [file_handler, logging.StreamHandler()]
+    logging.basicConfig(level=level, handlers=handlers)
+
+setup(config=None)