From af42b3dcccd990cffa7fb5e4c1d56aa7fcce6bdb Mon Sep 17 00:00:00 2001 From: Scott Barnett Date: Sun, 15 Nov 2020 14:37:44 +1100 Subject: [PATCH 1/6] NEW: Add setup function for logging handlers --- surround/config.py | 5 ++++- templates/new/init.py.txt | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/surround/config.py b/surround/config.py index 04ebd328..ec6e25c1 100644 --- a/surround/config.py +++ b/surround/config.py @@ -2,6 +2,7 @@ import os import functools +from datetime import datetime from pathlib import Path from collections.abc import Mapping from pkg_resources import resource_stream @@ -101,7 +102,9 @@ def __init__(self, project_root=None, package_path=None, auto_load=False): self._storage["project_root"] = project_root self._storage["package_path"] = package_path self._storage["volume_path"] = volume_path - self._storage["output_path"] = os.path.join(project_root, "output") + + now = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + self._storage["output_path"] = os.path.join(project_root, "output", str(now)) self._storage["input_path"] = os.path.join(project_root, "input") self._storage["models_path"] = os.path.join(project_root, "models") diff --git a/templates/new/init.py.txt b/templates/new/init.py.txt index 096d31ae..56d18c1e 100644 --- a/templates/new/init.py.txt +++ b/templates/new/init.py.txt @@ -1,3 +1,20 @@ -""" -This file is required for the project to be considered a python package. -""" +import logging +import os + +from logging.handlers import RotatingFileHandler + +from surround import has_config + +@has_config +def setup(config): + + os.makedirs(config["output_path"], exist_ok=True) + + # Initialise logging + level = logging.INFO + log_file = os.path.join(config["output_path"], "output.log") + file_handler = RotatingFileHandler(log_file, maxBytes=1048576, backupCount=5) + handlers = [file_handler, logging.StreamHandler()] + logging.basicConfig(level=level, handlers=handlers) + +setup(config=None) From d496acd0e2b789628ba9a712f9245b770ca29f91 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 23 Nov 2020 11:56:24 +1100 Subject: [PATCH 2/6] QUALITY: Add module doc string to generated init.py --- templates/new/init.py.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/new/init.py.txt b/templates/new/init.py.txt index 56d18c1e..2ff12b2d 100644 --- a/templates/new/init.py.txt +++ b/templates/new/init.py.txt @@ -1,3 +1,7 @@ +""" +This file is needed to make Python treat this directory as a package +""" + import logging import os From 4799e5326464543b55444934f9164103c75cdc8f Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 23 Nov 2020 12:16:59 +1100 Subject: [PATCH 3/6] ENHANCE: Allow CI to trigger on forked repos --- codefresh.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codefresh.yml b/codefresh.yml index 4c9e0f41..c44a2592 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -12,7 +12,7 @@ steps: clone: title: "Cloning repository" type: "git-clone" - repo: "a2i2/surround" + repo: "${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}" # CF_BRANCH value is auto set when pipeline is triggered # Learn more at codefresh.io/docs/docs/codefresh-yaml/variables/ revision: "${{CF_BRANCH}}" From 932fbf24918e04d48d3c61e2ddd4d7160764df66 Mon Sep 17 00:00:00 2001 From: Scott Barnett Date: Sun, 15 Nov 2020 21:00:09 +1100 Subject: [PATCH 4/6] ENHANCE: Add Docker file for Jupyter support --- surround/project.py | 4 +- templates/new/Dockerfile.Notebook.txt | 8 ++ templates/new/data_analysis.ipynb.txt | 56 +++++++++++ templates/new/dodo.py.txt | 129 +++++--------------------- templates/new/example.ipynb.txt | 35 ------- 5 files changed, 90 insertions(+), 142 deletions(-) create mode 100644 templates/new/Dockerfile.Notebook.txt create mode 100644 templates/new/data_analysis.ipynb.txt delete mode 100644 templates/new/example.ipynb.txt diff --git a/surround/project.py b/surround/project.py index d5ee448f..7d99fc23 100644 --- a/surround/project.py +++ b/surround/project.py @@ -27,13 +27,13 @@ ("{project_name}/__main__.py", "batch_main.py.txt", False, False), ("{project_name}/__main__.py", "web_main.py.txt", False, True), ("{project_name}/__init__.py", "init.py.txt", False, False), - ("notebooks/example.ipynb", "example.ipynb.txt", False, False), - ("notebooks/example.ipynb", "example.ipynb.txt", False, True), + ("notebooks/data_analysis.ipynb", "data_analysis.ipynb.txt", False, False), ("templates/results.html", "results.html.txt", False, False), ("templates/results.html", "results.html.txt", False, True), ("dodo.py", "dodo.py.txt", False, False), ("dodo.py", "web_dodo.py.txt", False, True), ("Dockerfile", "Dockerfile.txt", False, False), + ("Dockerfile.Notebook", "Dockerfile.Notebook.txt", False, False), ("{project_name}/config.yaml", "config.yaml.txt", False, False), (".gitignore", ".gitignore.txt", False, False) ] diff --git a/templates/new/Dockerfile.Notebook.txt b/templates/new/Dockerfile.Notebook.txt new file mode 100644 index 00000000..ec60c760 --- /dev/null +++ b/templates/new/Dockerfile.Notebook.txt @@ -0,0 +1,8 @@ +FROM a2i2/{project_name}:latest + +RUN pip3 install jupyter -U && pip3 install jupyterlab + +EXPOSE 8888 + +# TODO: Remove token and password settings for production deployment +CMD ["jupyter", "lab","--allow-root", "--ip=0.0.0.0", "--no-browser","--NotebookApp.token=''","--NotebookApp.password=''"] diff --git a/templates/new/data_analysis.ipynb.txt b/templates/new/data_analysis.ipynb.txt new file mode 100644 index 00000000..1f4ccd67 --- /dev/null +++ b/templates/new/data_analysis.ipynb.txt @@ -0,0 +1,56 @@ +{{ + "cells": [ + {{ + "cell_type": "code", + "execution_count": 1, + "metadata": {{}}, + "outputs": [], + "source": [ + "# Magic commands\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "# Set up paths to access project module\n", + "import sys\n", + "import os\n", + "sys.path.append('/app')\n", + "os.chdir('/app')" + ] + }}, + {{ + "cell_type": "code", + "execution_count": 2, + "metadata": {{}}, + "outputs": [], + "source": [ + "# Package imports\n", + "from surround import Config\n", + "from {project_name}.file_system_runner import FileSystemRunner\n", + "\n", + "config = Config(auto_load=True)\n", + "data = FileSystemRunner().load_data(None, config)" + ] + }} + ], + "metadata": {{ + "kernelspec": {{ + "display_name": "Python 3", + "language": "python", + "name": "python3" + }}, + "language_info": {{ + "codemirror_mode": {{ + "name": "ipython", + "version": 3 + }}, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + }} + }}, + "nbformat": 4, + "nbformat_minor": 4 +}} diff --git a/templates/new/dodo.py.txt b/templates/new/dodo.py.txt index 4dbb6930..07999b35 100644 --- a/templates/new/dodo.py.txt +++ b/templates/new/dodo.py.txt @@ -5,9 +5,6 @@ This module defines the tasks that can be executed using `surround run [task nam import os import sys import subprocess -import re -import webbrowser -import logging from pathlib import Path @@ -19,7 +16,8 @@ CONFIG = Config(os.path.dirname(__file__)) DOIT_CONFIG = {{'verbosity':2, 'backend':'sqlite3'}} PACKAGE_PATH = os.path.basename(CONFIG["package_path"]) IMAGE = "%s/%s:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) -LOGGER = logging.getLogger(__name__) +IMAGE_JUPYTER = "%s/%s-jupyter:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) +DOCKER_JUPYTER = "Dockerfile.Notebook" PARAMS = [ {{ @@ -46,7 +44,7 @@ def task_build(): def task_remove(): """Remove the Docker image for the current project""" return {{ - 'actions': ['docker rmi %s -f' % IMAGE], + 'actions': ['docker rmi %s %s -f' % (IMAGE, IMAGE_JUPYTER)], 'params': PARAMS }} @@ -199,107 +197,28 @@ def task_batch_local(): 'params': PARAMS }} +def task_build_jupyter(): + """Build the Docker image for a Jupyter Lab notebook""" + return {{ + 'basename': 'buildJupyter', + 'actions': ['docker build --tag=%s . -f %s' % (IMAGE_JUPYTER, DOCKER_JUPYTER)], + 'task_dep': ['build'], + 'params': PARAMS + }} + def task_jupyter(): - """Run a Jupyter notebook in the Docker container""" - # Allow for auto reload to be enabled and import modules from project package - ipython_config = "c.InteractiveShellApp.extensions.append('autoreload')\n" - ipython_config += "c.InteractiveShellApp.exec_lines = " - ipython_config += "['%autoreload 2', 'import sys', 'sys.path.append(\\'../\\')']" - - # Build the command for running jupyter - command = [ - "pip install -r /app/requirements.txt", - "mkdir /etc/ipython", - "echo \"%s\" > /etc/ipython/ipython_config.py" % ipython_config, - "/usr/local/bin/start.sh jupyter notebook --NotebookApp.token=''" + """Run a Jupyter Lab notebook""" + cmd = [ + "docker", + "run", + "-itp", + "8888:8888", + '-w', + '/app', + "--volume", + "\"%s/\":/app" % CONFIG["volume_path"], + IMAGE_JUPYTER ] - command = "; ".join(command) - - def run_command(): - process = subprocess.Popen( - [ - "docker", - "run", - "--rm", - "--name", - "{project_name}_surround_notebook", - "--volume", - "%s:/app" % CONFIG['volume_path'], - "-p", - "55910:8888", - "--user", - "root", - "-w", - "/app", - "jupyter/base-notebook:307ad2bb5fce", - "bash", - "-c", - command - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - - LOGGER.info("Starting jupyter notbook server...\n") - - # Get the IP address of the container, otherwise use localhost - ip_process = subprocess.Popen( - ['docker-machine', 'ip'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - ip_process.wait() - - ip_output = ip_process.stdout.readline().rstrip() - - if re.match(r"^(\d{{1,3}}\.){{3}}\d{{1,3}}$", ip_output): - host = ip_output - else: - host = "localhost" - - # Wait for the notebook server to be up before loading browser - while True: - line = process.stderr.readline().rstrip() - if line and 'Serving notebooks from local directory' in line: - break - - if process.poll(): - LOGGER.error("Failed to start the server, check if its not running somewhere else!") - - # Stop any containers that might be running - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return - - # Open the browser automatically - webbrowser.open('http://%s:55910/tree' % host, new=2) - - LOGGER.info("Notebook URL: http://%s:55910/tree\n", host) - LOGGER.info("Use CTRL+C to stop the server.") - - try: - process.wait() - except KeyboardInterrupt: - pass - finally: - LOGGER.info("Closing server...") - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return {{ - 'actions': [run_command] + 'actions': [" ".join(cmd)], }} diff --git a/templates/new/example.ipynb.txt b/templates/new/example.ipynb.txt deleted file mode 100644 index f3745d41..00000000 --- a/templates/new/example.ipynb.txt +++ /dev/null @@ -1,35 +0,0 @@ -{{ - "cells": [ - {{ - "cell_type": "code", - "execution_count": 7, - "metadata": {{}}, - "outputs": [], - "source": [ - "# Example importing code from the Surround project\n", - "from {project_name}.stages import Main" - ] - }} - ], - "metadata": {{ - "kernelspec": {{ - "display_name": "Python 3", - "language": "python", - "name": "python3" - }}, - "language_info": {{ - "codemirror_mode": {{ - "name": "ipython", - "version": 3 - }}, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }} - }}, - "nbformat": 4, - "nbformat_minor": 2 -}} From e4cb341c34048b581a9e09a0cd468f2a8a4e54b7 Mon Sep 17 00:00:00 2001 From: Scott Barnett Date: Thu, 26 Nov 2020 11:43:59 +1100 Subject: [PATCH 5/6] FIX: Add Jupyter update to the web template --- templates/new/web_dodo.py.txt | 126 ++++++---------------------------- 1 file changed, 22 insertions(+), 104 deletions(-) diff --git a/templates/new/web_dodo.py.txt b/templates/new/web_dodo.py.txt index 8e346cd1..a54eeda5 100644 --- a/templates/new/web_dodo.py.txt +++ b/templates/new/web_dodo.py.txt @@ -5,9 +5,6 @@ This module defines the tasks that can be executed using `surround run [task nam import os import sys import subprocess -import re -import webbrowser -import logging from pathlib import Path @@ -19,7 +16,8 @@ CONFIG = Config(os.path.dirname(__file__)) DOIT_CONFIG = {{'verbosity':2, 'backend':'sqlite3'}} PACKAGE_PATH = os.path.basename(CONFIG["package_path"]) IMAGE = "%s/%s:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) -LOGGER = logging.getLogger(__name__) +IMAGE_JUPYTER = "%s/%s-jupyter:%s" % (CONFIG["company"], CONFIG["image"], CONFIG["version"]) +DOCKER_JUPYTER = "Dockerfile.Notebook" PARAMS = [ {{ @@ -246,108 +244,28 @@ def task_web_local(): 'params': PARAMS }} +def task_build_jupyter(): + """Build the Docker image for a Jupyter Lab notebook""" + return {{ + 'basename': 'buildJupyter', + 'actions': ['docker build --tag=%s . -f %s' % (IMAGE_JUPYTER, DOCKER_JUPYTER)], + 'task_dep': ['build'], + 'params': PARAMS + }} def task_jupyter(): - """Run a Jupyter notebook in the Docker container""" - # Allow for auto reload to be enabled and import modules from project package - ipython_config = "c.InteractiveShellApp.extensions.append('autoreload')\n" - ipython_config += "c.InteractiveShellApp.exec_lines = " - ipython_config += "['%autoreload 2', 'import sys', 'sys.path.append(\\'../\\')']" - - # Build the command for running jupyter - command = [ - "pip install -r /app/requirements.txt", - "mkdir /etc/ipython", - "echo \"%s\" > /etc/ipython/ipython_config.py" % ipython_config, - "/usr/local/bin/start.sh jupyter notebook --NotebookApp.token=''" + """Run a Jupyter Lab notebook""" + cmd = [ + "docker", + "run", + "-itp", + "8888:8888", + '-w', + '/app', + "--volume", + "\"%s/\":/app" % CONFIG["volume_path"], + IMAGE_JUPYTER ] - command = "; ".join(command) - - def run_command(): - process = subprocess.Popen( - [ - "docker", - "run", - "--rm", - "--name", - "{project_name}_surround_notebook", - "--volume", - "%s:/app" % CONFIG['volume_path'], - "-p", - "55910:8888", - "--user", - "root", - "-w", - "/app", - "jupyter/base-notebook:307ad2bb5fce", - "bash", - "-c", - command - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - - LOGGER.info("Starting jupyter notbook server...\n") - - # Get the IP address of the container, otherwise use localhost - ip_process = subprocess.Popen( - ['docker-machine', 'ip'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8') - ip_process.wait() - - ip_output = ip_process.stdout.readline().rstrip() - - if re.match(r"^(\d{{1,3}}\.){{3}}\d{{1,3}}$", ip_output): - host = ip_output - else: - host = "localhost" - - # Wait for the notebook server to be up before loading browser - while True: - line = process.stderr.readline().rstrip() - if line and 'Serving notebooks from local directory' in line: - break - - if process.poll(): - LOGGER.error("Failed to start the server, please try again!") - - # Stop any containers that might be running - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return - - # Open the browser automatically - webbrowser.open('http://%s:55910/tree' % host, new=2) - - LOGGER.info("Notebook URL: http://%s:55910/tree\n", host) - LOGGER.info("Use CTRL+C to stop the server.") - - try: - process.wait() - except KeyboardInterrupt: - pass - finally: - LOGGER.info("Closing server...") - process = subprocess.Popen( - [ - 'docker', - 'stop', - '{project_name}_surround_notebook' - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - process.wait() - return {{ - 'actions': [run_command] + 'actions': [" ".join(cmd)], }} From 7a302825b5cd970ed8fd1a4347b042e1ef13de74 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 26 Nov 2020 16:33:05 +1100 Subject: [PATCH 6/6] RELEASE: Version 0.0.15 --- CHANGELOG.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9011133b..ec47cdde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,30 @@ All notable changes to this project will be documented in this file. ### Added -- Add `license` to setup.py - ### Changed ### Fixed ### Limitation +## [0.0.15] - 2020-11-26 + +### Added + +- Added `license` to setup.py +- Added docker file to the generated Surround projects for `Jupyter` support +- Added support for creating a versioned output folder for each run that stores the log files. + +### Changed + +- Print available endpoints information to the console +- Updated the generated notebook to load data from a runner + +### Fixed + +- Allow CI to trigger on forked repositories +- Linting issue in generated project + ## [0.0.14] - 2020-07-01 ### Added