Merge pull request #14 from glue-viz/silicon-build

Modernize MacOS build, use M1 runner and Qt6

.github/workflows/build_stable.yml

@@ -9,6 +9,7 @@ on:
   schedule:
     - cron: '0 3 * * *'
   workflow_dispatch:
+  pull_request:
 
 jobs:
   build_applications:
@@ -17,8 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # os: [macos-11, ubuntu-18.04, windows-2019]
-        os: [macos-11, windows-2019]
+        os: [macos-14, windows-2019]
     steps:
 
     - name: Check glue version to build
@@ -28,15 +28,15 @@ jobs:
     # osx signing based on https://melatonin.dev/blog/how-to-code-sign-and-notarize-macos-audio-plugins-in-ci/
 
     - name: Import Developer ID Application Certificate
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       uses: apple-actions/import-codesign-certs@v1
       with:
         p12-file-base64: ${{ secrets.DEV_ID_APP_CERT }}
         p12-password: ${{ secrets.DEV_ID_APP_PASSWORD }}
         keychain-password: ${{ secrets.KEYCHAIN_PASSWORD }}
 
     - name: Import Mac Installer Distribution Certificate
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       uses: apple-actions/import-codesign-certs@v1
       with:
         p12-file-base64: ${{ secrets.MAC_INST_DIST_CERT }}
@@ -47,44 +47,34 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    # Not all dependencies work correctly on Python 3.9 on Windows yet so we use
-    # Python 3.8 there.
-    - name: Set up Python 3.8
-      if: matrix.os == 'windows-2019'
+    - name: Set up Python 3.11
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
-    - name: Set up Python 3.9
-      if: matrix.os != 'windows-2019'
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-
-    - name: Install development version of pyinstaller
-      if: ${{ matrix.os == 'macos-11' }}
-      run: PYINSTALLER_COMPILE_BOOTLOADER=1 pip install git+https://github.com/pyinstaller/pyinstaller.git
-
-    - name: Install development version of pyinstaller
-      if: ${{ matrix.os != 'macos-11' }}
-      run: pip install git+https://github.com/pyinstaller/pyinstaller.git
+        python-version: 3.11
 
     - name: Install Python dependencies
       run: pip install -r requirements.txt --no-cache-dir
 
+    - name: Uninstall debugpy
+      run: pip uninstall -y debugpy
+
+    - name: Uninstall joblib
+      run: pip uninstall -y joblib
+
     - name: Run pyinstaller
       run: pyinstaller glue_app.spec
 
     # Don't do the following for now since it actually breaks the WWT plugin
     # - name: Remove nested app
-    #   if: ${{ matrix.os == 'macos-11' }}
+    #   if: ${{ matrix.os == 'macos-14' }}
     #   run: rm -rf dist/glue.app/Contents/Frameworks/PyQt5/Qt/lib/QtWebEngineCore.framework/Versions/5/Helpers/QtWebEngineProcess.app
 
     - name: Simple test of MacOS X application
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: dist/glue.app/Contents/MacOS/start_glue --test
 
     - name: Rename MacOS X application
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: mv dist/glue.app dist/"glue ${GITHUB_REF_NAME}.app"
 
     - name: Rename Windows application
@@ -95,48 +85,48 @@ jobs:
     # Build signed DMG for direct distribution
 
     - name: Remove start_glue
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: rm -rf dist/start_glue
     - name: Codesign MacOS X application
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: codesign --entitlements entitlements.plist --force -s "${{ secrets.DEVELOPER_ID_APPLICATION}}" -v dist/"glue ${GITHUB_REF_NAME}.app" --deep --strict --options=runtime --timestamp
     - name: Make DMG
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: hdiutil create -volname "Glue" -srcfolder dist -ov -format UDZO dist/"glue ${GITHUB_REF_NAME}.dmg"
     - name: Notarize app
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: xcrun notarytool submit dist/"glue ${GITHUB_REF_NAME}.dmg" --apple-id ${{ secrets.NOTARIZATION_USERNAME }} --team-id ${{ secrets.TEAM_ID }} --password ${{ secrets.NOTARIZATION_PASSWORD }} --wait
     - name: Staple notarization to dmg
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: xcrun stapler staple dist/"glue ${GITHUB_REF_NAME}.dmg"
 
     # Build signed pkg for potential Mac App Store distribution
 
     - name: Rename MacOS X application
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: mv dist/"glue ${GITHUB_REF_NAME}.app" dist/glueviz.app
     - name: Build MacOS X installer for distribution
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: productbuild --component dist/glueviz.app /Applications/ dist/"glue ${GITHUB_REF_NAME}_unsigned.pkg"
     - name: Sign MacOS X installer for distribution
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: productsign --sign "${{ secrets.MAC_INSTALLER_DISTRIBUTION}}" dist/"glue ${GITHUB_REF_NAME}_unsigned.pkg" dist/"glue ${GITHUB_REF_NAME}.pkg"
     - name: Remove unsigned pkg
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: rm -rf dist/"glue ${GITHUB_REF_NAME}_unsigned.pkg"
 
     # For now validation fails because the entitlements file is missing the sandbox option - but we are leaving
     # it out for now as WWT does not work correctly with it.
     #- name: Validate pkg
-    #  if: ${{ matrix.os == 'macos-11' }}
+    #  if: ${{ matrix.os == 'macos-14' }}
     #  run: xcrun altool --validate-app -f dist/"glue ${GITHUB_REF_NAME}.pkg" --type osx --username ${{ secrets.NOTARIZATION_USERNAME }} --password ${{ secrets.NOTARIZATION_PASSWORD }} --team-id ${{ secrets.TEAM_ID }}
 
     - name: Remove .app
-      if: ${{ matrix.os == 'macos-11' }}
+      if: ${{ matrix.os == 'macos-14' }}
       run: rm -rf dist/"glueviz.app"
 
     - name: Output list of included packages
-      if: matrix.os == 'macos-11'
+      if: matrix.os == 'macos-14'
       run: pip freeze > included-packages.txt
     - name: Install awscli
       run: pip install awscli
@@ -148,7 +138,7 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     - name: Upload package list to S3
-      if: github.event_name != 'pull_request' && matrix.os == 'macos-11'
+      if: github.event_name != 'pull_request' && matrix.os == 'macos-14'
       run: aws s3 cp included-packages.txt s3://glueviz/installers/${GITHUB_REF_NAME}/
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

glue_app.spec

@@ -3,6 +3,8 @@
 import os
 import sys
 
+sys.setrecursionlimit(sys.getrecursionlimit() * 5)
+
 # Version needs to be three integers separated by period, which is
 # the case for stable tags, but otherwise we need to use a dummy version
 
@@ -34,17 +36,23 @@ a = Analysis(
         "vispy",
         "notebook",
         "freetype",
+        "glue_qt",
         "glue_wwt",
         "glue_plotly",
         "glue_statistics",
         "pvextractor",
+        "PyQt6.QtTest",
     ],
     hookspath=["hooks"],
     hooksconfig={
         "matplotlib": {"backends": "all"},
     },
     runtime_hooks=[],
-    excludes=["tkinter", "PyQt5.QtQml"],
+    excludes=[
+        "tkinter",
+        "PyQt5.QtQml",
+        "joblib",
+    ],
     win_no_prefer_redirects=False,
     win_private_assemblies=False,
     cipher=block_cipher,
@@ -53,7 +61,6 @@ a = Analysis(
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
 if onefile:
-
     exe = EXE(
         pyz,
         a.scripts,
@@ -70,7 +77,6 @@ if onefile:
     )
 
 else:
-
     exe = EXE(
         pyz,
         a.scripts,

hooks/hook-glue_qt.py

@@ -0,0 +1,4 @@
+from PyInstaller.utils.hooks import collect_data_files, copy_metadata
+
+datas = collect_data_files("glue_qt", include_py_files=True)
+datas += copy_metadata("glue-qt")

hooks/hook-jupyter_events.py

@@ -0,0 +1,4 @@
+from PyInstaller.utils.hooks import collect_data_files, copy_metadata
+
+datas = collect_data_files("jupyter_events", include_py_files=True)
+datas += copy_metadata("jupyter-events")

requirements.txt

@@ -1,12 +1,14 @@
-ipykernel<6  # Install old version of ipykernel to avoid terminal issues
+pyinstaller
 glue-core
+glue-qt>=0.3.1
 glue-vispy-viewers
-glue-wwt
+glue-wwt>=0.7.2
 glue-plotly
 pvextractor
 pywwt
 notebook
-numpy<1.22
-astropy<5.0
-PyQt5==5.14.2
-PyQtWebEngine==5.14.0
+PyQt6==6.6.*
+PyQt6-Qt6==6.6.*
+PyQt6-WebEngine==6.6.*
+PyQt6-WebEngine-Qt6==6.6.*
+git+https://github.com/vispy/vispy

start_glue.py

@@ -1,25 +1,33 @@
-import os
 import sys
 
 print("start_glue.py called with arguments", sys.argv)
 
+# Exit early if this was launched by multiprocessing
+for arg in sys.argv:
+    if 'multiprocessing' in arg:
+        print('Killed forked process')
+        sys.exit(0)
+
+import os
 import time
 
 from pywwt import qt
 from glue import load_plugins
 from glue.logger import logger
-from glue.app.qt import GlueApplication
+from glue_qt.app import GlueApplication
 
 qt.APP_LIVELINESS_DEADLINE = 60
 
 os.environ["QTWEBENGINE_CHROMIUM_FLAGS"] = "--ignore-gpu-blacklist"
 
-logger.setLevel("INFO")
+if __name__ == "__main__":
 
-load_plugins()
+    if '--debug' in sys.argv or '--test' in sys.argv:
+        logger.setLevel("INFO")
 
-if __name__ == "__main__":
-    if "--debug" in sys.argv:
+    load_plugins()
+
+    if '--debug' in sys.argv:
         import faulthandler
 
         faulthandler.enable()
@@ -43,7 +51,7 @@
 
             # Open a few viewers to test
 
-            from glue.viewers.image.qt import ImageViewer
+            from glue_qt.viewers.image import ImageViewer
 
             ga.new_data_viewer(ImageViewer)