From ea05c57d3a9f5f33140d99f95d0faeb20bc63ae5 Mon Sep 17 00:00:00 2001 From: David Randall Stokes Date: Wed, 17 May 2023 16:32:15 -0400 Subject: [PATCH 01/13] Added note about Jupyter/Colab rendering DataFrames (#220) * Added note about Jupyter/Colab rendering DataFrames * Removed deprecated method * Removed deprecated type --- docs/endaq/ide_usage.rst | 5 +++++ endaq/batch/analyzer.py | 4 ++-- endaq/plot/dashboards.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/endaq/ide_usage.rst b/docs/endaq/ide_usage.rst index 775d88ae..324392ed 100644 --- a/docs/endaq/ide_usage.rst +++ b/docs/endaq/ide_usage.rst @@ -43,6 +43,11 @@ Summarizing IDE files: :py:func:`endaq.ide.get_channel_table()` Once an IDE file has been loaded, :py:func:`~endaq.ide.get_channel_table()` will retrieve basic summary information about its contents. +Some environments, such as `Jupyter Notebook `_ or `Colab `_, +will automatically render and display the channel table data. From inside the standard Python interactive interpreter, +use `get_channel_table(doc).data` to display the raw information, or to access the table's contents directly as a +Pandas `DataFrame`. + .. code:: python3 get_channel_table(doc) diff --git a/endaq/batch/analyzer.py b/endaq/batch/analyzer.py index 7abd5099..ddee04ee 100644 --- a/endaq/batch/analyzer.py +++ b/endaq/batch/analyzer.py @@ -112,8 +112,8 @@ def to_pandas(self, time_mode="datetime"): timedelta=(pd.TimedeltaIndex, "TimedeltaIndex"), datetime=(pd.DatetimeIndex, "DatetimeIndex"), seconds=( - (pd.Float64Index, pd.Int64Index, pd.UInt64Index, pd.RangeIndex), - "{Float64/Int64/UInt64/Range}Index", + (pd.Index, pd.RangeIndex), + "{Numeric/Range}Index", ), ) if not isinstance(self.data.index, expected_index_types[time_mode][0]): diff --git a/endaq/plot/dashboards.py b/endaq/plot/dashboards.py index 59339ca7..c077d63b 100644 --- a/endaq/plot/dashboards.py +++ b/endaq/plot/dashboards.py @@ -206,7 +206,7 @@ def rolling_enveloped_dashboard( # Loop through each of the sub-channels, and their respective '0-height rectangle mask' - for subchannel_name, cur_min_max_equal in min_max_equal[channel_data.columns].iteritems(): + for subchannel_name, cur_min_max_equal in min_max_equal[channel_data.columns].items(): traces = [] cur_color = colorway[subplot_num % len(colorway)] @@ -396,7 +396,7 @@ def rolling_metric_dashboard(channel_df_dict: dict, desired_num_points: int = 25 else: time = channel_data.index - for c_i, (subchannel_name, subchannel_data) in enumerate(channel_data.iteritems()): + for c_i, (subchannel_name, subchannel_data) in enumerate(channel_data.items()): for metric in rolling_metrics_to_plot: if n == 0: data = subchannel_data.values From f4025b0391659d45a0764c76c662c848a5d8e61f Mon Sep 17 00:00:00 2001 From: Peter Scheidler Date: Tue, 27 Jun 2023 17:10:50 -0400 Subject: [PATCH 02/13] Feature/requirements update (#221) Updates to requirements and setup to require IDELib >= 3.2.8, to avoid potential issues for new endaq units Updating tests to remove set-output Removing support for Python 3.7 due to MacOS test bugs --- .github/workflows/docs-tests.yml | 19 +++++++++++++++---- .github/workflows/publish-to-pypi.yml | 2 +- .github/workflows/unit-tests.yml | 24 ++++++++++++++++-------- docs/requirements.txt | 2 +- endaq/batch/analyzer.py | 5 +---- requirements.txt | 2 +- setup.py | 3 +-- 7 files changed, 36 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml index 646a854c..c0d6bc03 100644 --- a/.github/workflows/docs-tests.yml +++ b/.github/workflows/docs-tests.yml @@ -1,7 +1,16 @@ name: Docs test on push and PR -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - main + - development + pull_request: + branches: + - main + - development + workflow_dispatch: jobs: @@ -20,7 +29,7 @@ jobs: steps: - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.9" @@ -35,15 +44,17 @@ jobs: - name: Get pip cache location id: pip-cache - run: echo "::set-output name=dir::$(pip cache dir)" + shell: bash + run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: Get date id: date + shell: bash run: > python -c "import datetime; now = datetime.datetime.now(); - print(f'::set-output name=text::{now.year}/{now.month}-part{1 + now.day // 8}')" + print(f'text={now.year}/{now.month}-part{1 + now.day // 8}')" >> $GITHUB_OUTPUT - name: Load pip cache uses: actions/cache@v3 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index df229806..0b66c0c5 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@master - name: Set up Python 3.8 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: 3.8 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7624f581..992df2a8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,7 +1,16 @@ name: Unit test on push and PR -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - main + - development + pull_request: + branches: + - main + - development + workflow_dispatch: jobs: @@ -12,10 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] - exclude: - - os: ubuntu-latest - python-version: '3.6' + python-version: ['3.8', '3.9', '3.10', '3.11'] env: OS: ${{ matrix.os }} @@ -26,7 +32,7 @@ jobs: steps: - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -38,15 +44,17 @@ jobs: - name: Get pip Cache Location id: pip-cache - run: echo "::set-output name=dir::$(pip cache dir)" + shell: bash + run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: Get date id: date + shell: bash run: > python -c "import datetime; now = datetime.datetime.now(); - print(f'::set-output name=text::{now.year}/{now.month}-part{1 + now.day // 8}')" + print(f'text={now.year}/{now.month}-part{1 + now.day // 8}')" >> $GITHUB_OUTPUT - name: Load pip Cache uses: actions/cache@v3 diff --git a/docs/requirements.txt b/docs/requirements.txt index 48b8cc00..f6e4e536 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,4 +9,4 @@ sphinxcontrib-serializinghtml==1.1.5 pydata-sphinx-theme==0.7.1 sphinx-plotly-directive==0.1.3 nbsphinx==0.8.8 -ipython==8.1.0 +ipython==8.10 diff --git a/endaq/batch/analyzer.py b/endaq/batch/analyzer.py index ddee04ee..bd611313 100644 --- a/endaq/batch/analyzer.py +++ b/endaq/batch/analyzer.py @@ -6,10 +6,7 @@ import sys import warnings -if sys.version_info[:2] >= (3, 8): - from functools import cached_property -else: - from backports.cached_property import cached_property +from functools import cached_property import numpy as np import pandas as pd diff --git a/requirements.txt b/requirements.txt index cff989a9..c6f9a4a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ backports.cached-property; python_version<'3.8' ebmlite>=3.2.0 -idelib>=3.2.3 +idelib>=3.2.8 jinja2 numpy>=1.19.5 pandas>=1.3 diff --git a/setup.py b/setup.py index 779bf225..a4c1ddb2 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def get_version(rel_path): INSTALL_REQUIRES = [ "backports.cached-property; python_version<'3.8'", "ebmlite>=3.2.0", - "idelib>=3.2.3", + "idelib>=3.2.8", "jinja2", "numpy>=1.19.5", "pandas>=1.3", @@ -68,7 +68,6 @@ def get_version(rel_path): classifiers=['Development Status :: 4 - Beta', 'License :: OSI Approved :: MIT License', 'Natural Language :: English', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', From 11efc31e0528145464751cb377cc7f934963169a Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Wed, 29 May 2024 14:29:20 -0400 Subject: [PATCH 03/13] update ipython to 3.10 --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 48b8cc00..501b0684 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,4 +9,4 @@ sphinxcontrib-serializinghtml==1.1.5 pydata-sphinx-theme==0.7.1 sphinx-plotly-directive==0.1.3 nbsphinx==0.8.8 -ipython==8.1.0 +ipython>=8.10 From fe7be3beba0e2b4b5fc07b47f6f21630d3f0d1fa Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Thu, 30 May 2024 09:53:27 -0400 Subject: [PATCH 04/13] update sphinx in docs + refactor endaq.calc.integrate to use culmulative_trapezoid --- docs/requirements.txt | 2 +- endaq/calc/integrate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f6e4e536..baa9f312 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx==4.2.0 +Sphinx>=5.0.2 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.0 diff --git a/endaq/calc/integrate.py b/endaq/calc/integrate.py index c2fcd36e..30daf397 100644 --- a/endaq/calc/integrate.py +++ b/endaq/calc/integrate.py @@ -19,7 +19,7 @@ def _integrate( dt = utils.sample_spacing(df) result = df.apply( - functools.partial(scipy.integrate.cumtrapz, dx=dt, initial=0), + functools.partial(scipy.integrate.cumulative_trapezoid, dx=dt, initial=0), axis=0, raw=True, ) From 21ad1f05d9c32fb908d834848f2e6d82c4cb617e Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Thu, 30 May 2024 10:36:17 -0400 Subject: [PATCH 05/13] refactor utcfromtimestamp + add python 3.12 tests --- .github/workflows/unit-tests.yml | 2 +- endaq/ide/files.py | 6 +++--- endaq/ide/info.py | 2 +- setup.py | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b7ce55c5..2fbb73e7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] env: OS: ${{ matrix.os }} diff --git a/endaq/ide/files.py b/endaq/ide/files.py index eed982ed..9344d0eb 100644 --- a/endaq/ide/files.py +++ b/endaq/ide/files.py @@ -11,7 +11,7 @@ import typing from typing import Union, Tuple -from datetime import datetime +from datetime import datetime, timezone import os from pathlib import Path import tempfile @@ -249,7 +249,7 @@ def get_doc(name=None, filename=None, url=None, parsed=True, start=0, end=None, session_start = doc.lastSession.utcStartTime if session_start: - session_start = datetime.utcfromtimestamp(session_start) + session_start = datetime.fromtimestamp(session_start, tz=timezone.utc) if start: read_kwargs['startTime'] = parse_time(start, session_start) @@ -303,7 +303,7 @@ def extract_time(doc, out, start=0, end=None, channels=None, **kwargs): session_start = doc.lastSession.utcStartTime if session_start: - session_start = datetime.utcfromtimestamp(session_start) + session_start = datetime.fromtimestamp(session_start, tz=timezone.utc) if start: kwargs['startTime'] = parse_time(start, session_start) diff --git a/endaq/ide/info.py b/endaq/ide/info.py index 6a9ffcc0..94380d80 100644 --- a/endaq/ide/info.py +++ b/endaq/ide/info.py @@ -179,7 +179,7 @@ def get_channel_table(dataset: typing.Union[idelib.dataset.Dataset, list], data = source.getSession(session) if data.session.utcStartTime: - session_start = datetime.datetime.utcfromtimestamp(data.session.utcStartTime) + session_start = datetime.datetime.fromtimestamp(data.session.utcStartTime, tz=datetime.timezone.utc) start = parse_time(start, session_start) end = parse_time(end, session_start) diff --git a/setup.py b/setup.py index a4c1ddb2..a19b21ba 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ def get_version(rel_path): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Scientific/Engineering', ], keywords='ebml binary ide mide endaq', From 44ba3a328eacc35e1c3f4a996c4df4c55b3ca2da Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:37:42 -0400 Subject: [PATCH 06/13] fix stack deprecation warning --- endaq/batch/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index 3596fc93..3993c122 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -208,7 +208,7 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): # Format results result = ( - aligned_peak_data.stack() + aligned_peak_data.stack(future_stack=True) .stack() .reorder_levels(["axis", "peak time", "peak offset"]) ) From 1c798ddec4948268aecf79c4531cb07417d98e28 Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:54:43 -0400 Subject: [PATCH 07/13] fix future_stack error in core --- endaq/batch/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index 3993c122..150e2090 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -7,6 +7,7 @@ from functools import partial import warnings import os +import sys import numpy as np import pandas as pd @@ -206,9 +207,13 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): ) ) + # minor version of python 3.x + minor_version = sys.version_info.minor + # Format results + # Use new implementation of future_stack if Python version >= 3.9 result = ( - aligned_peak_data.stack(future_stack=True) + aligned_peak_data.stack(future_stack=minor_version>=9) .stack() .reorder_levels(["axis", "peak time", "peak offset"]) ) From ae27e1b8c82da8358282593e385609ff5663eb44 Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:01:12 -0400 Subject: [PATCH 08/13] try fixing future stack again --- endaq/batch/core.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index 150e2090..0d70c823 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -212,13 +212,11 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): # Format results # Use new implementation of future_stack if Python version >= 3.9 - result = ( - aligned_peak_data.stack(future_stack=minor_version>=9) - .stack() - .reorder_levels(["axis", "peak time", "peak offset"]) - ) + levels = ["axis", "peak time", "peak offset"] + if minor_version < 9: + return aligned_peak_data.stack().stack().recorder_levels(levels) - return result + return aligned_peak_data.stack(future_stack=True).stack().reorder_levels(levels) def _make_vc_curves(ch_data_cache: analyzer.CalcCache): From 4b4d73ffb4415d45fab964b6f1dee05864eb9f08 Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:07:53 -0400 Subject: [PATCH 09/13] fix typo --- endaq/batch/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index 0d70c823..b34e0ca1 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -214,7 +214,7 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): # Use new implementation of future_stack if Python version >= 3.9 levels = ["axis", "peak time", "peak offset"] if minor_version < 9: - return aligned_peak_data.stack().stack().recorder_levels(levels) + return aligned_peak_data.stack().stack().reorder_levels(levels) return aligned_peak_data.stack(future_stack=True).stack().reorder_levels(levels) From ffad7aacd447d3a48560964345495d0a08470012 Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:53:20 -0400 Subject: [PATCH 10/13] check for python<3.9 Co-authored-by: David Randall Stokes --- endaq/batch/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index b34e0ca1..3fd512c4 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -213,7 +213,7 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): # Format results # Use new implementation of future_stack if Python version >= 3.9 levels = ["axis", "peak time", "peak offset"] - if minor_version < 9: + if sys.version_info< (3, 9): return aligned_peak_data.stack().stack().reorder_levels(levels) return aligned_peak_data.stack(future_stack=True).stack().reorder_levels(levels) From fd491ef1177a62fe54a9af7339d4d4faf972537b Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:53:57 -0400 Subject: [PATCH 11/13] rm unused var --- endaq/batch/core.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index 3fd512c4..bf80b1d4 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -207,13 +207,10 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): ) ) - # minor version of python 3.x - minor_version = sys.version_info.minor - # Format results # Use new implementation of future_stack if Python version >= 3.9 levels = ["axis", "peak time", "peak offset"] - if sys.version_info< (3, 9): + if sys.version_info < (3, 9): return aligned_peak_data.stack().stack().reorder_levels(levels) return aligned_peak_data.stack(future_stack=True).stack().reorder_levels(levels) From 08c620acc1e3ba78520218619d8446eb0955954e Mon Sep 17 00:00:00 2001 From: Jayden Personnat <89941534+jaydenpersonnat@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:28:29 -0400 Subject: [PATCH 12/13] retry tests --- endaq/batch/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/endaq/batch/core.py b/endaq/batch/core.py index bf80b1d4..d2a04944 100644 --- a/endaq/batch/core.py +++ b/endaq/batch/core.py @@ -208,6 +208,7 @@ def _make_peak_windows(ch_data_cache: analyzer.CalcCache, margin_len): ) # Format results + # Use new implementation of future_stack if Python version >= 3.9 levels = ["axis", "peak time", "peak offset"] if sys.version_info < (3, 9): From 010836459e30fb8031869faf96b54131129284f9 Mon Sep 17 00:00:00 2001 From: David Stokes Date: Wed, 26 Jun 2024 17:10:39 -0400 Subject: [PATCH 13/13] README update, version bump (1.5.3) --- README.md | 2 +- endaq/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e3396dfe..e8b8f27a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![unit tests](https://github.com/midetechnology/endaq-python/actions/workflows/unit-tests.yml/badge.svg) [![codecov](https://codecov.io/gh/MideTechnology/endaq-python/branch/develop/graph/badge.svg)](https://codecov.io/gh/MideTechnology/endaq-python) +[![PyPI Latest Release](https://img.shields.io/pypi/v/endaq-python.svg)](https://pypi.org/project/endaq-python/) ![example workflow](https://github.com/MideTechnology/endaq-python/actions/workflows/unit-tests.yml/badge.svg) # `endaq-python`: A comprehensive, user-centric Python API for working with enDAQ data and devices diff --git a/endaq/__init__.py b/endaq/__init__.py index 9928c763..c35abdbf 100644 --- a/endaq/__init__.py +++ b/endaq/__init__.py @@ -5,4 +5,4 @@ import endaq.batch -__version__ = "1.5.2" +__version__ = "1.5.3"