Merge branch 'develop' into feature/sessions

# Conflicts:
#	idelib/dataset.py

.github/workflows/publish-to-pypi.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build-n-publish:
     name: Build and publish Python 🐍 distributions 📦 to PyPI
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@master

README.md

@@ -70,31 +70,37 @@ Each `Channel` object has a `subchannels` member, which is a list of
 
 The below table lists current conventions for channels across all enDAQ sensors:
 
-| (Abbreviated) Product No. | Description                                                                       | Example Product Nos.    |
-|--------------------------:|-----------------------------------------------------------------------------------|-------------------------|
-| S-D                       | enDAQ S-series devices with single digital accelerometers                         | S3-D16, S4-D40          |
-| S-DD                      | enDAQ S-series devices with dual digital accelerometers                           | S1-D100D40, S2-D25D16   |
-| S-ED                      | enDAQ S-series devices with an analog electrocapacitive and digital accelerometer | S5-E25D40, S4-E100D40   |
-| S-RD                      | enDAQ S-series devices with an analog piezoresistive and digital accelerometer    | S4-R500D40, S5-R2000D40 |
-| SSX                       | Midé Slam Stick X data recorders                                                  | SSX                     |
-| SSC                       | Midé Slam Stick C data recorders                                                  | SSC                     |
-| SSS                       | Midé Slam Stick S data recorders                                                  | SSS                     |
+| (Abbreviated) Product No. | Description                                                                    | Example Product Nos.    |
+|--------------------------:|--------------------------------------------------------------------------------|-------------------------|
+|                       S-D | enDAQ S-series devices with a single digital accelerometer                     | S3-D16, S4-D40          |
+|                      S-DD | enDAQ S-series devices with dual digital accelerometers                        | S1-D100D40, S2-D25D16   |
+|                      S-ED | enDAQ S-series devices with an analog piezoelectric and digital accelerometer  | S5-E25D40, S4-E100D40   |
+|                      S-RD | enDAQ S-series devices with an analog piezoresistive and digital accelerometer | S4-R500D40, S5-R2000D40 |
+|                       W-D | enDAQ W-series devices with a single digital accelerometer                     | W5-D40                  |
+|                      W-ED | enDAQ W-series devices with an analog piezoelectric and digital accelerometer  | W8-E100D40, W8-E2000D40 |
+|                      W-RD | enDAQ W-series devices with an analog piezoresistive and digital accelerometer | W8-R500D40, W8-R2000D40 |
+|                       SSX | Midé Slam Stick X data recorders                                               | SSX                     |
+|                       SSC | Midé Slam Stick C data recorders                                               | SSC                     |
+|                       SSS | Midé Slam Stick S data recorders                                               | SSS                     |
 
 The below table lists channel ID numbers used in a recording file based on the 
 recording device’s product number (device series numbers and accelerometer 
 sensitivity ranges are omitted when applicable to all such devices):
 
-| Sensor                | Channel | Valid Devices                      | Suchannels                                   |
-|----------------------:|:--------|------------------------------------|----------------------------------------------|
-| Main Accelerometer    | 8       | S-RD, S-ED, SSS, SSX               | X-, Y-, Z-axis Acceleration                  |
-| 16/200g Accelerometer | 32      | S-DD, SSX, SSS, SSC, S-D16, S-D200 | X-, Y-, Z-axis Acceleration                  |
-| 8/40g Accelerometer   | 80      | S-RD, S-DD, S-ED, S-D40, S-D8      | X-, Y-, Z-axis Acceleration                  |
-| IMU Gyroscope         | 47      | All<sup>1</sup>                    | X-, Y-, Z-axis Rotation                      |
-| Absolute Orientation  | 65      | All<sup>1</sup>                    | X-, Y-, Z-, W-axis Quaternion; Acc           |
-| Relative Orientation  | 70      | All<sup>1</sup>                    | X-, Y-, Z-, W-axis Quaternion                |
-| MPL3115               | 36      | All<sup>1</sup>                    | Pressure, Temperature <sup>2</sup>           |
-| MS8607                | 59      | All<sup>1</sup>                    | Pressure, Temperature, Humidity <sup>3</sup> |
-| SI1133                | 76      | All<sup>1</sup>                    | Lux, UV                                      |
+|                  Sensor | Channel | Valid Devices                           | Suchannels                         |
+|------------------------:|:--------|-----------------------------------------|------------------------------------|
+|      Main Accelerometer | 8       | S-RD, S-ED, SSS, SSX                    | X-, Y-, Z-axis Acceleration        |
+|   16/200g Accelerometer | 32      | S-DD, SSX, SSS, SSC, S-D16, S-D200      | X-, Y-, Z-axis Acceleration        |
+|     8/40g Accelerometer | 80      | S-RD, S-DD, S-ED, S-D40, S-D8           | X-, Y-, Z-axis Acceleration        |
+|           IMU Gyroscope | 47      | All<sup>1</sup>                         | X-, Y-, Z-axis Rotation            |
+|    Absolute Orientation | 65      | All<sup>1</sup>                         | X-, Y-, Z-, W-axis Quaternion; Acc |
+|    Relative Orientation | 70      | All<sup>1</sup>                         | X-, Y-, Z-, W-axis Quaternion      |
+|                 MPL3115 | 36      | S-D16, All<sup>1</sup> before Mid-2023  | Pressure, Temperature <sup>2</sup> |
+|         MS8607 Internal | 20      | All<sup>1</sup> after Mid-2023          | Pressure, Temperature, Humidity    |
+|      MS8607 Control Pad | 59      | All<sup>1</sup>                         | Pressure, Temperature, Humidity    |
+|                  SI1133 | 76      | All<sup>1</sup>                         | Lux, UV                            |
+| BMI270/BMG250 Gyroscope | 84      | All<sup>1</sup> after Mid-2023          | X-, Y-, Z-axis Rotation            |
+|             CAM-M8Q GPS | 88      | W-D, W-ED, W-RD                         | Latitude, Longitude, Time, Speed   |
 
 <sup>1</sup> excluding early SSC/SSS/SSX models
 
@@ -126,7 +132,7 @@ For channels, this will be a (c+1)-by-n array, where n is the number of samples
 recorded and c is the number of subchannels; `eventarray[1:]` indexes the 
 samples, `eventarray[0]` indexes the respective timestamps.
 
-### Getting metadta
+### Getting metadata
 
 `Dataset` makes available some basic metadata. Some useful pieces of information 
 are stored directly as members:

idelib/__init__.py

@@ -5,12 +5,13 @@
 """
 
 __author__ = "David Randall Stokes"
-__copyright__ = "Copyright (c) 2022 Midé Technology"
+__copyright__ = "Copyright (c) 2023 Midé Technology"
 
 __maintainer__ = "Midé Technology"
 __email__ = "[email protected]"
 
-__version__ = (3, 2, 4)
+__version__ = '3.2.9'
+
 __status__ = "Production/Stable"
 
 from .importer import importFile

idelib/dataset.py

@@ -27,7 +27,7 @@
 #    they are ignored, causing problems with other calculations.
 #
 # TODO: Consider an EventArray subclass for subchannels to reduce the number
-#    of conditionals evaluated, and/or see about making parent Channels' 
+#    of conditionals evaluated, and/or see about making parent Channels'
 #    EventArrays flat.
 #   
 # TODO: Nice discontinuity handing. This will probably be conveyed as events 
@@ -52,9 +52,7 @@
 from threading import Lock
 import warnings
 
-from functools import partial
 import os.path
-import random
 import struct
 import sys
 from time import sleep
@@ -70,16 +68,13 @@
 SCHEMA_FILE = 'mide_ide.xml'
 
 #===============================================================================
-# DEBUGGING: XXX: Remove later!
+#
 #===============================================================================
 
 import logging
 logger = logging.getLogger('idelib')
 logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s")
 
-
-# __DEBUG__ = False
-
 __DEBUG__ = str(os.environ.get('MIDE_DEV', 0)) == '1'
 
 if __DEBUG__:
@@ -91,6 +86,13 @@
 # logger.info("Loaded python-ebml from %s" % os.path.abspath(ebml.__file__))
 
 
+def mapRange(x, in_min, in_max, out_min, out_max):
+    """ Given a value `x` between `in_min` and `in_max`, get the equivalent
+        value relative to `out_min` and `out_max`.
+    """
+    return ((x - in_min + 0.0) * (out_max - out_min) /
+            (in_max - in_min) + out_min)
+
 #===============================================================================
 # Mix-In Classes
 #===============================================================================
@@ -1951,8 +1953,17 @@ def getEventIndexBefore(self, t):
             block = self._data[blockIdx]
         except IndexError:
             block = self._data[-1]
-        return int(block.indexRange[0] + \
-                   ((t - block.startTime) / self._getBlockSampleTime(blockIdx)))
+
+        if t > block.endTime:
+            # Time falls within a gap between blocks
+            return block.indexRange[-1]
+
+        return int(mapRange(t,
+                            block.startTime, block.endTime,
+                            block.indexRange[0], block.indexRange[1]-1))
+
+        # return int((block.indexRange[0] +
+        #            ((t - block.startTime) / self._getBlockSampleTime(blockIdx))))
 
 
     def getEventIndexNear(self, t):
@@ -1993,15 +2004,20 @@ def getRangeIndices(self, startTime, endTime):
             return startIdx, endIdx
 
         if startTime is None or startTime <= self._data[0].startTime:
-            startIdx = startBlockIdx = 0
-            startBlock = self._data[0]
+            startIdx = 0
         else:
             startBlockIdx = self._getBlockIndexWithTime(startTime)
             startBlock = self._data[startBlockIdx]
-            startIdx = int(startBlock.indexRange[0] + \
-                           ((startTime - startBlock.startTime) / \
-                            self._getBlockSampleTime(startBlockIdx)) + 1)
-
+
+            if startTime < startBlock.endTime:
+                # Start time is within a block (the usual case)
+                startIdx = int((startBlock.indexRange[0] +
+                               ((startTime - startBlock.startTime) /
+                                self._getBlockSampleTime(startBlockIdx)) + 1))
+            else:
+                # Start time falls within a gap between blocks
+                startIdx = startBlock.indexRange[1]
+
         if endTime is None:
             endIdx = self._data[-1].indexRange[1]
         elif endTime <= self._data[0].startTime:
@@ -2173,6 +2189,8 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0,
             :return: A structured array of data block statistics (min, mean,
                 and max, respectively).
         """
+        if not self._data:
+            return None
 
         startBlock, endBlock = self._getBlockRange(startTime, endTime)
         shape = (3, max(1, len(self._npType)) + int(times), endBlock - startBlock)
@@ -2227,10 +2245,14 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0,
                     xform.inplace(out[m, :, :], out=out[m, :, :])
 
         # iterate through the arrayMinMeanMaxes specific to each subchannel
-        for i in range(int(times), out.shape[1]):
-            # swap mins and maxes if a (negative) transform has made min > max
-            if out[0, i, 0] > out[2, i, 0]:
-                out[:, i] = np.flipud(out[:, i])
+        try:
+            for i in range(int(times), out.shape[1]):
+                # swap mins and maxes if a (negative) transform has made min > max
+                if out[0, i, 0] > out[2, i, 0]:
+                    out[:, i] = np.flipud(out[:, i])
+        except IndexError:
+            logger.warning('Channel contains no data')
+            return None
 
         return out
 
@@ -2279,7 +2301,7 @@ def getRangeMinMeanMax(self, startTime=None, endTime=None, subchannel=None,
         stats = self.arrayMinMeanMax(startTime, endTime, times=False,
                                      display=display, iterator=iterator)
 
-        if stats.size == 0:
+        if stats is None or stats.size == 0:
             return None
         if self.hasSubchannels and subchannel is not None:
             return (
@@ -2524,6 +2546,9 @@ def getMean(self, startTime=None, endTime=None, display=False, iterator=iter):
         means = self.arrayMinMeanMax(startTime, endTime, times=False,
                                      display=display, iterator=iterator)[1]
 
+        if means is None:
+            return None
+
         mean = np.mean(np.average(means, weights=[d.numSamples for d in self._data], axis=-1))
 
         if startTime is None and endTime is None:

idelib/matfile.py

@@ -449,7 +449,8 @@ def startArray(self, name, cols, rows=1, arrayNumber=0,
         self.packStr(sanitizeName(name))
 
         # Write the start of the 'PR' element; the size will be filled in later.
-        self._write(struct.pack('II', self.arrayDType, self.rowFormatter.size * rows))
+        size = min(self.rowFormatter.size * rows, 2 ** 32 - 1)
+        self._write(struct.pack('II', self.arrayDType, size))
         self.prSize = self.stream.tell() - 4
 
 

idelib/schemata/mide_ide.xml

@@ -94,6 +94,8 @@
             <StringElement name="BootloaderRevStr" id="0x521F" multiple="0" minver="1">Bootloader revision string.</StringElement>
             <UIntegerElement name="BootloaderRev" id="0x5220" multiple="0" minver="1">Incrementing bootloader revision level</UIntegerElement>
             <MasterElement name="HasEsp32" id="0x4D70" multiple="0" minver="1">If present, the device has an ESP32 installed</MasterElement>
+            <UIntegerElement name="WispiApiLevel" id="0x4D71" multiple="0" minver="1">WiSpi API Level for network co-processor</UIntegerElement>
+            <StringElement name="NcpAppFwVer" id="0x4D72" multiple="0" minver="1">Network Co-Processor Application Firmware Version</StringElement>
             <UIntegerElement name="KeyRev" id="0x5231" multiple="0" minver="1">Indicates the version of the key table, 0 if not present</UIntegerElement>
             <MasterElement name="SerialCommandInterface" id="0x5230" multiple="0" minver="2" mandatory="0">Present in the device's DEVINFO if it supports control via serial</MasterElement>
             <UIntegerElement name="FileCommandInterface" id="0x5233" multiple="0" minver="2" mandatory="0">Element in the device's DEVINFO indicating it supports control via the COMMAND and RESPONSE files. 0 is unsupported, 1 is supported; assumed to be 1 if not present.</UIntegerElement>

setup.py

@@ -1,7 +1,21 @@
+import codecs
+import os
 import setuptools
 
-with open('README.md', 'r', encoding='utf-8') as fh:
-    long_description = fh.read()
+def read(rel_path):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with codecs.open(os.path.join(here, rel_path), 'r') as fp:
+        return fp.read()
+
+
+def get_version(rel_path):
+    for line in read(rel_path).splitlines():
+        if line.startswith('__version__'):
+            delim = '"' if '"' in line else "'"
+            return line.split(delim)[1]
+    else:
+        raise RuntimeError("Unable to find version string.")
+
 
 INSTALL_REQUIRES = [
     'numpy',
@@ -29,18 +43,17 @@
 
 setuptools.setup(
         name='idelib',
-        version='3.2.7',
+        version=get_version('idelib/__init__.py'),
         author='Mide Technology',
         author_email='[email protected]',
         description='Python API for accessing IDE data recordings',
-        long_description=long_description,
+        long_description=read('README.md'),
         long_description_content_type='text/markdown',
         url='https://github.com/MideTechnology/idelib',
         license='MIT',
         classifiers=['Development Status :: 5 - Production/Stable',
                      'License :: OSI Approved :: MIT License',
                      'Natural Language :: English',
-                     'Programming Language :: Python :: 3.5',
                      'Programming Language :: Python :: 3.6',
                      'Programming Language :: Python :: 3.7',
                      'Programming Language :: Python :: 3.8',