Update release process (#92)

* add strip outputs option

* new release on every commit to master

.github/workflows/release-samples.yml

@@ -1,10 +1,9 @@
-name: Upload Release Asset
+name: Upload Release Assets
 
 on:
   push:
-    # Sequence of patterns matched against refs/tags
-    tags:
-    - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+    branches:
+      - master
 
 jobs:
   build:
@@ -23,32 +22,43 @@ jobs:
         run: |
           python -m pip install --upgrade pip
 
-      - name: Build project
+      - name: Build notebooks
         run: |
-          python resources/package-samples.py notebooks
+          python resources/package-samples.py notebooks --strip-output --outfile notebooks-stripped.zip --notebooks all
+          python resources/package-samples.py notebooks --outfile notebooks-full.zip --notebooks all
 
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
+          tag_name: ${{ github.sha }}
+          release_name: Release ${{ github.sha }}
           draft: false
           prerelease: false
 
-      - name: Upload Release Asset
-        id: upload-release-asset
+      # This pulls from the CREATE RELEASE step above, referencing
+      # it's ID to get its outputs object, which include a `upload_url`.
+      # See this blog post for more info:
+      #   https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+      - name: Upload Stripped Asset
+        id: upload-stripped-asset
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          # This pulls from the CREATE RELEASE step above, referencing
-          # it's ID to get its outputs object, which include a `upload_url`.
-          # See this blog post for more info:
-          #   https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./sample-notebooks.zip
-          asset_name: sample-notebooks.zip
+          asset_path: ./notebooks-stripped.zip
+          asset_name: notebooks-stripped.zip
+          asset_content_type: application/zip
+      - name: Upload Full Asset
+        id: upload-stripped-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./notebooks-full.zip
+          asset_name: notebooks-full.zip
           asset_content_type: application/zip

resources/nb-check.py

@@ -145,12 +145,10 @@ def new_markdown_cell(cell_id: str, content: list[str]) -> dict[str, Any]:
 
     cells = nb.get('cells', [])
 
-    # Remove metadata and outputs
+    # Remove metadata
     for i, cell in enumerate(cells):
         if 'metadata' in cell:
             cell['metadata'] = {}
-        if 'outputs' in cell:
-            cell['outputs'] = []
 
     # Remove empty cells at the end of the notebook
     end = len(cells) - 1

resources/package-samples.py

@@ -7,11 +7,17 @@
 import tomllib
 from zipfile import ZipFile
 
+NOTEBOOK_FILE_NAME = 'notebook.ipynb'
 
-def clear_outputs(path: str) -> str:
+REQUIRED_FILES = [NOTEBOOK_FILE_NAME, 'meta.toml']
+
+
+def strip_outputs(path: str) -> str:
     """Remove outputs from notebook at path."""
+
     with open(path, 'r') as infile:
         nb = json.loads(infile.read())
+
         for cell in nb['cells']:
             if 'metadata' in cell:
                 cell['metadata']['execution'] = {}
@@ -20,9 +26,57 @@ def clear_outputs(path: str) -> str:
             if 'metadata' in nb:
                 if 'singlestore_connection' in nb['metadata']:
                     nb['metadata']['singlestore_connection'] = {}
+
         return json.dumps(nb, indent=2)
 
 
+def get_valid_notebooks(notebooks: str, notebooks_directory: str) -> list[str]:
+    """Return a list of valid notebooks"""
+
+    notebook_names = []
+
+    if notebooks == 'sample':
+        with open(args.toml, 'rb') as f:
+            meta = tomllib.load(f)
+            notebook_names = meta['samples']['display']
+    elif notebooks == 'all':
+        # get all
+        notebook_names = os.listdir(notebooks_directory)
+    else:
+        # comma-separated list
+        notebook_names = list(map(str.strip, notebooks.split(',')))
+
+    valid_notebooks = []
+
+    for notebook_name in notebook_names:
+
+        for required_file in REQUIRED_FILES:
+            path = os.path.join(
+                args.notebooks_directory,
+                notebook_name,
+                required_file,
+            )
+
+            if not os.path.isfile(path):
+                print(
+                    f'error: Required file does not exist: {path}',
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+
+        valid_notebooks.append(notebook_name)
+
+    return valid_notebooks
+
+
+def convert_to_destination_path(path: str) -> str:
+    """Remove 'notebooks' from path"""
+    parts = path.split('/')
+    filtered_parts = list(filter(lambda x: x != 'notebooks', parts))
+
+    return '/'.join(filtered_parts)
+
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(
@@ -31,9 +85,16 @@ def clear_outputs(path: str) -> str:
     )
 
     parser.add_argument(
-        'notebooks_directory', metavar='notebooks-directory',
+        'notebooks_directory',
+        metavar='notebooks-directory',
         help='root `notebooks` directory',
     )
+    parser.add_argument(
+        '--notebooks',
+        help='which notebooks to package',
+        default='all',
+        required=True,
+    )
     parser.add_argument(
         '-t', '--toml',
         help='toml file containing configuration',
@@ -44,53 +105,44 @@ def clear_outputs(path: str) -> str:
         help='name of the output file',
         default='sample-notebooks.zip',
     )
+    parser.add_argument(
+        '-s', '--strip-outputs',
+        help='strip the output cells from the notebooks',
+        default=False,
+        action=argparse.BooleanOptionalAction,
+    )
 
     args = parser.parse_args()
 
-    with open(args.toml, 'rb') as f:
-        meta = tomllib.load(f)
-        files = []
-        for i, name in enumerate(meta['samples']['display']):
+    valid_notebooks = get_valid_notebooks(
+        notebooks=args.notebooks,
+        notebooks_directory=args.notebooks_directory,
+    )
 
-            # Verify the notebook file exists
-            path = os.path.join(
-                args.notebooks_directory,
-                name, 'notebook.ipynb',
-            )
-            if not os.path.isfile(path):
-                print(
-                    f'error: notebook file does not exist at {path}',
-                    file=sys.stderr,
-                )
-                sys.exit(1)
+    with ZipFile(args.outfile, 'w') as out:
+        for notebook_name in valid_notebooks:
+            print(notebook_name)
 
-            # Verify the metadata file exists
-            meta_path = os.path.join(
+            notebook_directory_path = os.path.join(
                 args.notebooks_directory,
-                name, 'meta.toml',
+                notebook_name,
             )
-            if not os.path.isfile(meta_path):
-                print(
-                    f'error: metadata file does not exist at {meta_path}',
-                    file=sys.stderr,
-                )
-                sys.exit(1)
 
-            with open(meta_path, 'rb') as meta_toml:
-                nb_meta = tomllib.load(meta_toml)
+            notebook_path = os.path.join(
+                notebook_directory_path,
+                NOTEBOOK_FILE_NAME,
+            )
 
-            try:
-                files.append(
-                    (
-                        path,
-                        f'{i + 1:02} - {nb_meta["meta"]["title"]}.ipynb.json',
-                    ),
-                )
-            except Exception:
-                print(path, ' =>\n    ', nb_meta)
-                raise
+            # write the whole notebook directory
+            for dirpath, dirs, files in os.walk(notebook_directory_path):
+                for file in files:
+                    source = os.path.join(dirpath, file)
+                    destination = convert_to_destination_path(source)
 
-    with ZipFile(args.outfile, 'w') as out:
-        for path, name in files:
-            print(path, ' => ', name)
-            out.writestr(name, clear_outputs(path))
+                    if source == notebook_path and args.strip_outputs:
+                        # write notebook with stripped output
+                        stripped_nodebook = strip_outputs(notebook_path)
+                        out.writestr(destination, stripped_nodebook)
+                    else:
+                        # write file normally
+                        out.write(source, arcname=destination)