From be2eab5e010dc42fa16274f4df473185398f7641 Mon Sep 17 00:00:00 2001 From: Tim Green Date: Fri, 17 May 2024 15:54:41 +0200 Subject: [PATCH] Update release process (#92) * add strip outputs option * new release on every commit to master --- .github/workflows/release-samples.yml | 42 +++++--- resources/nb-check.py | 4 +- resources/package-samples.py | 136 ++++++++++++++++++-------- 3 files changed, 121 insertions(+), 61 deletions(-) diff --git a/.github/workflows/release-samples.yml b/.github/workflows/release-samples.yml index 1e4560ba..c1613e9e 100644 --- a/.github/workflows/release-samples.yml +++ b/.github/workflows/release-samples.yml @@ -1,10 +1,9 @@ -name: Upload Release Asset +name: Upload Release Assets on: push: - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + branches: + - master jobs: build: @@ -23,9 +22,10 @@ jobs: run: | python -m pip install --upgrade pip - - name: Build project + - name: Build notebooks run: | - python resources/package-samples.py notebooks + python resources/package-samples.py notebooks --strip-output --outfile notebooks-stripped.zip --notebooks all + python resources/package-samples.py notebooks --outfile notebooks-full.zip --notebooks all - name: Create Release id: create_release @@ -33,22 +33,32 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} + tag_name: ${{ github.sha }} + release_name: Release ${{ github.sha }} draft: false prerelease: false - - name: Upload Release Asset - id: upload-release-asset + # This pulls from the CREATE RELEASE step above, referencing + # it's ID to get its outputs object, which include a `upload_url`. + # See this blog post for more info: + # https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + - name: Upload Stripped Asset + id: upload-stripped-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - # This pulls from the CREATE RELEASE step above, referencing - # it's ID to get its outputs object, which include a `upload_url`. - # See this blog post for more info: - # https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./sample-notebooks.zip - asset_name: sample-notebooks.zip + asset_path: ./notebooks-stripped.zip + asset_name: notebooks-stripped.zip + asset_content_type: application/zip + - name: Upload Full Asset + id: upload-stripped-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./notebooks-full.zip + asset_name: notebooks-full.zip asset_content_type: application/zip diff --git a/resources/nb-check.py b/resources/nb-check.py index 80f561c2..80c7830c 100755 --- a/resources/nb-check.py +++ b/resources/nb-check.py @@ -145,12 +145,10 @@ def new_markdown_cell(cell_id: str, content: list[str]) -> dict[str, Any]: cells = nb.get('cells', []) - # Remove metadata and outputs + # Remove metadata for i, cell in enumerate(cells): if 'metadata' in cell: cell['metadata'] = {} - if 'outputs' in cell: - cell['outputs'] = [] # Remove empty cells at the end of the notebook end = len(cells) - 1 diff --git a/resources/package-samples.py b/resources/package-samples.py index ec085efc..1f04a2a8 100755 --- a/resources/package-samples.py +++ b/resources/package-samples.py @@ -7,11 +7,17 @@ import tomllib from zipfile import ZipFile +NOTEBOOK_FILE_NAME = 'notebook.ipynb' -def clear_outputs(path: str) -> str: +REQUIRED_FILES = [NOTEBOOK_FILE_NAME, 'meta.toml'] + + +def strip_outputs(path: str) -> str: """Remove outputs from notebook at path.""" + with open(path, 'r') as infile: nb = json.loads(infile.read()) + for cell in nb['cells']: if 'metadata' in cell: cell['metadata']['execution'] = {} @@ -20,9 +26,57 @@ def clear_outputs(path: str) -> str: if 'metadata' in nb: if 'singlestore_connection' in nb['metadata']: nb['metadata']['singlestore_connection'] = {} + return json.dumps(nb, indent=2) +def get_valid_notebooks(notebooks: str, notebooks_directory: str) -> list[str]: + """Return a list of valid notebooks""" + + notebook_names = [] + + if notebooks == 'sample': + with open(args.toml, 'rb') as f: + meta = tomllib.load(f) + notebook_names = meta['samples']['display'] + elif notebooks == 'all': + # get all + notebook_names = os.listdir(notebooks_directory) + else: + # comma-separated list + notebook_names = list(map(str.strip, notebooks.split(','))) + + valid_notebooks = [] + + for notebook_name in notebook_names: + + for required_file in REQUIRED_FILES: + path = os.path.join( + args.notebooks_directory, + notebook_name, + required_file, + ) + + if not os.path.isfile(path): + print( + f'error: Required file does not exist: {path}', + file=sys.stderr, + ) + sys.exit(1) + + valid_notebooks.append(notebook_name) + + return valid_notebooks + + +def convert_to_destination_path(path: str) -> str: + """Remove 'notebooks' from path""" + parts = path.split('/') + filtered_parts = list(filter(lambda x: x != 'notebooks', parts)) + + return '/'.join(filtered_parts) + + if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -31,9 +85,16 @@ def clear_outputs(path: str) -> str: ) parser.add_argument( - 'notebooks_directory', metavar='notebooks-directory', + 'notebooks_directory', + metavar='notebooks-directory', help='root `notebooks` directory', ) + parser.add_argument( + '--notebooks', + help='which notebooks to package', + default='all', + required=True, + ) parser.add_argument( '-t', '--toml', help='toml file containing configuration', @@ -44,53 +105,44 @@ def clear_outputs(path: str) -> str: help='name of the output file', default='sample-notebooks.zip', ) + parser.add_argument( + '-s', '--strip-outputs', + help='strip the output cells from the notebooks', + default=False, + action=argparse.BooleanOptionalAction, + ) args = parser.parse_args() - with open(args.toml, 'rb') as f: - meta = tomllib.load(f) - files = [] - for i, name in enumerate(meta['samples']['display']): + valid_notebooks = get_valid_notebooks( + notebooks=args.notebooks, + notebooks_directory=args.notebooks_directory, + ) - # Verify the notebook file exists - path = os.path.join( - args.notebooks_directory, - name, 'notebook.ipynb', - ) - if not os.path.isfile(path): - print( - f'error: notebook file does not exist at {path}', - file=sys.stderr, - ) - sys.exit(1) + with ZipFile(args.outfile, 'w') as out: + for notebook_name in valid_notebooks: + print(notebook_name) - # Verify the metadata file exists - meta_path = os.path.join( + notebook_directory_path = os.path.join( args.notebooks_directory, - name, 'meta.toml', + notebook_name, ) - if not os.path.isfile(meta_path): - print( - f'error: metadata file does not exist at {meta_path}', - file=sys.stderr, - ) - sys.exit(1) - with open(meta_path, 'rb') as meta_toml: - nb_meta = tomllib.load(meta_toml) + notebook_path = os.path.join( + notebook_directory_path, + NOTEBOOK_FILE_NAME, + ) - try: - files.append( - ( - path, - f'{i + 1:02} - {nb_meta["meta"]["title"]}.ipynb.json', - ), - ) - except Exception: - print(path, ' =>\n ', nb_meta) - raise + # write the whole notebook directory + for dirpath, dirs, files in os.walk(notebook_directory_path): + for file in files: + source = os.path.join(dirpath, file) + destination = convert_to_destination_path(source) - with ZipFile(args.outfile, 'w') as out: - for path, name in files: - print(path, ' => ', name) - out.writestr(name, clear_outputs(path)) + if source == notebook_path and args.strip_outputs: + # write notebook with stripped output + stripped_nodebook = strip_outputs(notebook_path) + out.writestr(destination, stripped_nodebook) + else: + # write file normally + out.write(source, arcname=destination)