Merge pull request #331 from MetaCell/osb2

Latest from Osb2 branch to gha test branch

.dockerignore

@@ -22,4 +22,6 @@ tox.ini
 /src
 geppetto-client
 .git
-**/__pycache__
+**/__pycache__
+*.log
+workspace

.gitignore

@@ -26,3 +26,4 @@ geppetto-client
 /nwb-explorer
 pyvenv.cfg
 .jupyter-config
+/build

Dockerfile

@@ -1,21 +1,19 @@
-FROM node:13.14 as jsbuild
+FROM node:16 as jsbuild
 
 ENV FOLDER=nwb-explorer
 
 
 WORKDIR $FOLDER/webapp
-COPY webapp/package-lock.json .
+COPY webapp/yarn.lock .
 COPY webapp/package.json .
-RUN npm ci
+RUN yarn install --network-timeout 1000000000
 COPY webapp/ .
-RUN npm run build
+RUN yarn build
 #Remove node_modules, need to keep the geppetto client
-RUN mv node_modules/@geppettoengine .
-RUN rm -Rf node_modules/*
-RUN mv @geppettoengine node_modules
+RUN rm -Rf node_modules
 
 ###
-FROM jupyter/base-notebook:hub-1.1.0
+FROM jupyter/base-notebook:hub-1.5.0
 ENV NB_UID=jovyan
 ENV FOLDER=nwb-explorer
 USER root
@@ -24,29 +22,36 @@ RUN apt-get update -qq &&\
     apt-get install python3-tk vim nano unzip git g++ -qq
 
 COPY --chown=1000:1000 requirements.txt .   
-RUN pip install -r requirements.txt --no-cache-dir
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ 
+    pip install -r requirements.txt
 USER $NB_UID
-COPY  . $FOLDER 
+
+COPY --chown=$NB_UID:$NB_UID . $FOLDER 
 COPY --from=jsbuild --chown=1000:1000 $FOLDER $FOLDER
 
 WORKDIR $FOLDER
-
+RUN mkdir workspace
 
 
 
 # Temporary fix for deprecated api usage on some requirement
 # RUN pip install setuptools==45
 
 USER root
-RUN python utilities/install.py --npm-skip
+
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+    python utilities/install.py --npm-skip
 
 
 RUN rm -rf /var/lib/apt/lists
 # sym link workspace pvc to $FOLDER
 RUN mkdir -p /opt/workspace
 RUN mkdir -p /opt/home
-RUN chown -R $NB_UID .
-RUN chown -R $NB_UID /opt/*
+# clean workspace from tests
+RUN rm -Rf workspace/* 
+RUN chown $NB_UID app.log
+RUN chown $NB_UID /opt/workspace
+RUN chown $NB_UID /opt/home
 RUN ln -s /opt/workspace ./workspace
 RUN ln -s /opt/home ./workspace
 

README.md

@@ -38,7 +38,7 @@ source nwb-explorer/bin/activate
 Or, with conda
 
 ```bash
-conda create -n nwb-explorer python=3.7
+conda create -n nwb-explorer python=3.9
 conda activate nwb-explorer
 ```
 

nwb-extensions/silverlab/silverlab.namespace.yaml

@@ -6,6 +6,7 @@ namespaces:
   - namespace: core
     neurodata_types:
     - LabMetaData
+    - NWBData
   - source: silverlab.ophys.yaml
   - source: silverlab.metadata.yaml
   version: '0.2'

nwb_explorer/__init__.py

@@ -39,7 +39,9 @@ def init_extensions():
             try:
                 with open(init_file) as f:
                     for line in f:
+                        logging.info("evaluating `%s`", line)
                         eval(line)
             except Exception as e:
                 logging.error("Error evaluating extension init file " + init_file, exc_info=True)
+
 init_extensions()

requirements.txt

@@ -1,12 +1,65 @@
+airspeed==0.5.20
+attrs==19.3.0
+cachetools==0.8.0
+cffi==1.14.0
+cryptography==2.9.2
+cycler==0.10.0
+decorator==4.4.2
+defusedxml==0.6.0
+Deprecated==1.2.10
+entrypoints==0.3
+h5py==3.7.0
+idna==2.9
+ipython-genutils==0.2.0
 imageio==2.5.0
-jupyter_geppetto==1.1.1
-nwbwidgets==0.8.0
-pyecore==0.11.6
-pygeppetto==0.8.0
-pynwb==2.0.0
-Pillow==7.1.2
+ipysheet>=0.5.0
+jedi==0.17.0
+Jinja2==2.11.2
+jsonschema==3.2.0
+jupyter_geppetto==1.1.5
+jupyter-client==7.0.6
+jupyter-core==4.9.1
+jupyter-server==1.11.2
+jupyterhub==1.5.0
+jupyterlab==3.2.4
+jupyterthemes==0.20.0
+kiwisolver==1.2.0
+mistune==0.8.4
+multimethod==1.3
+nbconvert==5.6.1
+nbformat==5.0.6
+ordered-set==4.0.2
+pandocfilters==1.4.2
+parso==0.7.0
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow>=7.2.0
+prompt-toolkit==3.0.5
+ptyprocess==0.6.0
+pycparser==2.20
+pyecore==0.11.7
+pygeppetto==0.8.1
+Pygments==2.6.1
+pyparsing==2.4.7
+pyrsistent==0.16.0
+python-dateutil==2.8.1
+python-json-logger==0.1.11
+pytz==2020.1
+PyYAML==5.3.1
+pyzmq==19.0.1
+qtconsole==4.7.5
+QtPy==1.9.0
 quantities==0.12.3
-nose==1.3.7
-redis==2.10.6
-seaborn==0.8.1
-uuid==1.30
+RestrictedPython==5.0
+Send2Trash==1.5.0
+terminado==0.8.3
+tornado==6.1.0
+traitlets==4.3.3
+urllib3==1.25.9
+webencodings==0.5.1
+wrapt==1.12.1
+zipp==3.1.0
+jupyter_geppetto==1.1.5
+nwbwidgets==0.11.3
+pygeppetto==0.8.1
+pynwb==2.5.0

run.py

@@ -14,7 +14,7 @@
     sys.argv.append('--NotebookApp.default_url=/geppetto')
     sys.argv.append("--NotebookApp.token=''")
     sys.argv.append('--library=nwb_explorer')
-
+    
     app = NotebookApp.instance()
     app.initialize(sys.argv)
     app.file_to_run = ''

setup.py

@@ -43,15 +43,18 @@
         'jupyter_geppetto>=1.1.0',
         'bokeh>=0.13.0',
         'holoviews>=1.10.6',
+        'jupyter-core<5',
+        'jupyter-client<8',
+        'jupyter-server<2',
         'msgpack-python==0.5.6',
         'nose>=1.3.7',
         'Pillow>=7.0.0',
         'redis>=2.10.6',
         'seaborn>=0.8.1',
         'uuid>=1.30',
-        'pynwb>=2.0.0',
+        'pynwb>=2.3.0',
         'imageio>=2.5.0',
         'quantities>=0.12.3',
-        'nwbwidgets>=0.8.0'
+        'nwbwidgets>=0.10.0',
     ],
 )

test/utils.py

@@ -53,12 +53,12 @@ def create_nwb_file():
     interface = mod.add(pynwb.TimeSeries(name='t3', data=data, unit='pA', timestamps=timestamps))
     mod.add(pynwb.TimeSeries(name='t4', data=data, unit='UA', timestamps=timestamps))
 
-    nwbfile.add_acquisition(create_image('internal_storaged_image', nwbfile, False))
-    nwbfile.add_acquisition(create_image('external_storaged_image', nwbfile, True))
+    nwbfile.add_acquisition(create_image('internal_storaged_image', False))
+    nwbfile.add_acquisition(create_image('external_storaged_image', True))
 
     return nwbfile
 
-def create_image(name, nwbfile, external_storage):
+def create_image(name, external_storage):
     import imageio
     formats = ['png', 'jpg', 'tiff']
 
@@ -78,7 +78,7 @@ def create_image(name, nwbfile, external_storage):
         return ImageSeries(name=name,
                                external_file=images_uri,
                                timestamps=timestamps,
-                               starting_frame=[0], 
+                               starting_frame=[0, 0, 0], 
                                format='external', 
                                description='Series of images from a simulation of the cerebellum via neuroConstruct')
     else:

utilities/custom.css

@@ -3240,7 +3240,7 @@ div#notebook {
   #header-container {
    display: none !important;
   }
-  
+
   <script>
       MathJax.Hub.Config({
           "HTML-CSS": {

utilities/embed/embed_example.html

@@ -5,16 +5,19 @@
     <title>NWB explorer embedding example</title>
     <script>
 
-        function updateFile(){
+        function updateFile(fileName){
             console.log("Update called");
             let frame = document.getElementById("nwb-explorer");
-            let fileName = "https://github.com/OpenSourceBrain/NWBShowcase/raw/master/NWB/time_series_data.nwb";
-            frame.contentWindow.postMessage(fileName, '*');
+            frame.contentWindow.postMessage({ type: 'LOAD_RESOURCE', payload: fileName }, '*');
         }
+
+
     </script>
 </head>
 
 <body>
-    <iframe id="nwb-explorer" src="http://localhost:8888/geppetto" style="width:100%; min-height: 800px" onLoad="updateFile();"></iframe>
+    <button onclick="updateFile('https://github.com/OpenSourceBrain/NWBShowcase/raw/master/FergusonEtAl2015/FergusonEtAl2015.nwb');">Load NWB file</button>
+    <iframe id="nwb-explorer" src="http://localhost:8081/geppetto" style="width:100%; height: 100vh" onLoad="updateFile('https://github.com/OpenSourceBrain/NWBShowcase/raw/master/NWB/time_series_data.nwb');"></iframe>
+
 </body>
 </html>

utilities/install.py

@@ -117,7 +117,7 @@ def main(branch=branch, skipNpm=False, skipTest=False, development=False):
 
     cprint("Installing notebook theme")
     # Generate a default config if not already present
-    execute(cmd=['jupyter', 'notebook', '--generate-config'])
+    execute(cmd=['jupyter', 'notebook', '--generate-config', '-y'])
     from jupyter_core import paths
     config_dir = paths.jupyter_config_dir()
     print('Jupyter configuration dir is {}'.format(config_dir))
@@ -137,8 +137,8 @@ def main(branch=branch, skipNpm=False, skipTest=False, development=False):
 
     cprint("Installing client packages")
     if not skipNpm:
-        execute(cmd=['npm', 'install' if development else 'ci'], cwd=WEBAPP_DIR)
-        execute(cmd=['npm', 'run', 'build-dev' if development else 'build'], cwd=WEBAPP_DIR)
+        execute(cmd=['yarn', 'install'], cwd=WEBAPP_DIR)
+        execute(cmd=['yarn',  'build-dev' if development else 'build'], cwd=WEBAPP_DIR)
 
 
 def steps():