From 0642212efcbc9f68fb9b4007181d1097a398d1df Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Mon, 26 Apr 2021 20:50:51 +0200 Subject: [PATCH 1/5] Packages/doc/releasenotes_template.rst: Update it --- Packages/doc/releasenotes_template.rst | 57 +++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/Packages/doc/releasenotes_template.rst b/Packages/doc/releasenotes_template.rst index 9430c02e3a..0547bd065b 100644 --- a/Packages/doc/releasenotes_template.rst +++ b/Packages/doc/releasenotes_template.rst @@ -1,6 +1,48 @@ Release X.Y =========== +Controls +-------- + +All added, removed or renamed controls of the main GUIs are listed. These lists are intended to help upgrading the JSON +configuration files. Controls, like GroupBox'es, which can not be read/written with the configuration code are not included. + +DA\_Ephys +~~~~~~~~~ + +Added +^^^^^ + +Removed +^^^^^^^ + +Renamed +^^^^^^^ + +Databrowser +~~~~~~~~~~~ + +Added +^^^^^ + +Removed +^^^^^^^ + +Renamed +^^^^^^^ + +Wavebuilder +~~~~~~~~~~~ + +Added +^^^^^ + +Removed +^^^^^^^ + +Renamed +^^^^^^^ + Sweep Formula ------------- @@ -21,8 +63,8 @@ DA\_Ephys --------- -ExperimentConfig ----------------- +JSON Configuration +------------------ Downsample @@ -37,6 +79,10 @@ Foreign Function interface -------------------------- +Pulse Average Plot +------------------ + + General ------- @@ -101,10 +147,6 @@ Internal -------- -Tango ------ - - Tests ----- @@ -113,5 +155,8 @@ Async Framework --------------- +Logging +------- + Installer --------- From 0790a743df6c98a30a0042f2d583476baf16cae6 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 30 Apr 2021 22:21:02 +0200 Subject: [PATCH 2/5] tools/create-changelog.sh: Add list of changed controls We want to also mention the new/removed/renamed controls in the release notes so that people can update their configs. --- tools/create-changelog.sh | 2 ++ tools/get-changed-controls.sh | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tools/get-changed-controls.sh diff --git a/tools/create-changelog.sh b/tools/create-changelog.sh index d9e9380785..4e984e8093 100755 --- a/tools/create-changelog.sh +++ b/tools/create-changelog.sh @@ -33,3 +33,5 @@ fmt="(%h) %<(80,trunc)%s%w(0,0,10)%+b" old_tag=$(git describe --tags --abbrev=0 --match "Release_*") git --git-dir=$git_dir log --submodule=diff --no-merges --pretty="$fmt" $old_tag..HEAD > changelog.txt + +$top_level/tools/get-changed-controls.sh >> changelog.txt diff --git a/tools/get-changed-controls.sh b/tools/get-changed-controls.sh new file mode 100644 index 0000000000..1ae690e1fe --- /dev/null +++ b/tools/get-changed-controls.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +set -e +# set -x + +old_tag=$(git describe --tags --abbrev=0 --match "Release_*") + +top_level=$(git rev-parse --show-toplevel) + +if [ -z "$top_level" ] +then + echo "This is not a git repository" + exit 1 +fi + +for path in $(ls $top_level/Packages/MIES/*Macro.ipf) +do + file=$(basename $path) + + for revision in HEAD $old_tag + do + output=controls_${file}_${revision} + echo $output + git show $revision:Packages/MIES/$file | grep -i -P \ + "\b(Button|Checkbox|PopupMenu|ValDisplay|SetVariable|Chart|Slider|Tab|GroupBox|Titlebox|ListBox|CustomControl)\b.*" | \ + cut -f 1 -d "," | cut -f 2 -d " " | sort | uniq > $output + done + + diff --unified=0 controls_${file}_${old_tag} controls_${file}_HEAD > controls_${file}_diff || true + + echo "" + echo "Changed controls for ${file}" + echo "" + cat controls_${file}_diff +done From 25dd4641f5c4707e40157c0b22221ee503464814 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 30 Apr 2021 23:33:45 +0200 Subject: [PATCH 3/5] tools/create-changelog.sh: List commits which change LABNOTEBOOK_VERSION --- tools/create-changelog.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/create-changelog.sh b/tools/create-changelog.sh index 4e984e8093..92770c086e 100755 --- a/tools/create-changelog.sh +++ b/tools/create-changelog.sh @@ -35,3 +35,9 @@ old_tag=$(git describe --tags --abbrev=0 --match "Release_*") git --git-dir=$git_dir log --submodule=diff --no-merges --pretty="$fmt" $old_tag..HEAD > changelog.txt $top_level/tools/get-changed-controls.sh >> changelog.txt + +echo "" >> changelog.txt +echo "Commits which have LABNOTEBOOK_VERSION in their diff" >> changelog.txt +echo "" >> changelog.txt + +git log --regexp-ignore-case --pretty="$fmt" -G LABNOTEBOOK_VERSION ${old_tag}.. $top_level/Packages/MIES/MIES_Constants.ipf >> changelog.txt From 041c828da6955df7c119ad76f9933f390ab143ae Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Sun, 2 May 2021 17:03:51 +0200 Subject: [PATCH 4/5] Packages/doc/releasenotes.rst: Add release notes for 2.2 --- Packages/doc/releasenotes.rst | 346 ++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) diff --git a/Packages/doc/releasenotes.rst b/Packages/doc/releasenotes.rst index ca44688309..8ba0dfc6b0 100644 --- a/Packages/doc/releasenotes.rst +++ b/Packages/doc/releasenotes.rst @@ -3,6 +3,352 @@ Release notes .. toctree:: +Release 2.2 +=========== + +This version is the first version of MIES with support for upcoming Igor Pro 9. And it will also be the last with +support for Igor Pro 8. + +Controls +-------- + +All added, removed or renamed controls of the main GUIs are listed here. These lists are intended to help upgrading the JSON +configuration files manually. Controls which can not be read/written with the configuration code, like GroupBox'es, are not included. + +DA\_Ephys +~~~~~~~~~ + +Added +^^^^^ + +None + +Removed +^^^^^^^ + +- ``Check_Settings_Append`` +- ``setvar_Settings_DecMethodFac`` +- ``title_hardware_Follow`` +- ``title_hardware_Release`` + +Renamed +^^^^^^^ + +None + +Databrowser +~~~~~~~~~~~ + +Added +^^^^^ + +- ``check_BrowserSettings_DS`` +- ``check_pulseAver_ShowImage`` +- ``check_pulseAver_drawXZeroLine`` +- ``check_pulseAver_fixedPulseLength`` +- ``check_pulseAver_hideFailedPulses`` +- ``check_pulseAver_searchFailedPulses`` +- ``check_pulseAver_showTraces`` +- ``popup_pulseAver_colorscales`` +- ``popup_pulseAver_pulseSortOrder`` +- ``setvar_pulseAver_failedPulses_level`` +- ``setvar_pulseAver_numberOfSpikes`` +- ``setvar_pulseAver_vert_scale_bar`` + +Removed +^^^^^^^ + +None + +Renamed +^^^^^^^ + +- ``check_pulseAver_indTraces`` → ``check_pulseAver_indPulses`` +- ``check_pulseAver_zeroTrac`` → ``check_pulseAver_zero`` +- ``setvar_pulseAver_fallbackLength`` → ``setvar_pulseAver_overridePulseLength`` + +Wavebuilder +~~~~~~~~~~~ + +Added +^^^^^ + +None + +Removed +^^^^^^^ + +None + +Renamed +^^^^^^^ + +None + +Sweep Formula +------------- + +- Add ``area`` operation +- Fix most cases of ignored minus signs in formulas +- Make it more user friendly on errors by not just asserting out + +AnalysisBrowser +--------------- + +None + +DataBrowser +----------- + +Dashboard +~~~~~~~~~ + +- Display the headstages for each sweep +- Add support for multipatch seq analysis functions +- Add "enable checkbox" for the dashboard +- Show the dashboard result message as tooltip +- Add support for all sweeps even without having an analysis function attached to the stimulus set + +DataBrowser/SweepBrowser +------------------------ + +- Fix restoring from backup so that it works again and also restore all sweeps and not only the displayed ones +- The axes' locations are now more predictable for zooming in/out via mouse wheel +- Don't display data from TP during DAQ channels +- Always create backup waves when splitting the sweep +- Speedup plotting of many traces by grouping AppendToGraph calls. This is mostly noticable in Igor Pro 8, less in + Igor Pro 9. +- Make overlay sweeps with headstage removal faster +- Select the current sweep when enabling overlay sweeps +- Add new trace popup for convenient stimulus set opening in the wavebuilder +- Fix displaying TTL data with only "TP during DAQ" data +- Use a different marker for the first headstage. ``+`` can be easily hidden by axis ticks, but ``#`` not + +SweepBrowser +------------ + +- Add dashboard support +- Show the sweep number in the sweep control and not the index into the list of sweeps + +DA\_Ephys +--------- + +- Closing the DAEphys panel that is running TP now waits for the TP analysis to finish before closing +- Ignore TP during DAQ channels for locked indexing +- Prevent sweep skipping from crossing stimulus set borders +- Handle relocking a device with data better. We now don't start at sweep zero but at the next sweep after the last acquired +- Remove decimation factor control +- Make rerunning the TP with the same settings faster +- Don't disable dDAQ and oodDAQ checkboxes during data aquisition anymore +- Don't allow dialogs during background function execution, this should prevent Igor Pro crashes +- Add new trace popup for convenient stimulus set opening +- Warn users about permanent data loss when using sweep rollback +- Fix TP during DAQ channels with TTL channels +- Fix auto pipette offset taking "Apply on mode switch" into account +- Remove the "Enable async acquisition" checkbox from the settings panel. Enable async from the async tab of the DA_Ephys panel. +- Change sweep rollback to move the data to an archive folder instead of deleting it + +JSON Configuration +------------------ + +- Remove the old experiment configuration. The new JSON based configuration now completely replaces it. +- Add menu option to open all JSON configuration files in a notebook +- Add entry "Sweep Rollback allowed" defaulting to false. This means users who are configuring MIES must explicitly + allow sweep rollback. This was done to make the chance of misuse smaller + +Downsample +---------- + +None + +Analysis Functions +------------------ + +- SetControlInEvent: Warn when trying to set a control which can not be set +- Add ``SC_SpikeControl`` +- Convert ReachTargetVoltage to V3 format +- Use correct labnotebook prefix for headstage active entry in MSQ_DAScale +- Make MSQ_FastRheoEst/MSQ_DAScale/MSQ_SpikeControl/ReachTargetVoltage compatible with locked indexing +- Add option to query the autobias target voltage from the user in ReachTargetVoltage +- Make execution faster by not trying to redo baseline QC when it was already done + +Pulse Average Plot +------------------ + +- Completely rework it for better performance +- Add documentation for time align code +- Add support for failed spike detection. Failed spikes can be highlighted or hidden in the PA plot. +- Allow one PA plot per browser window +- Add X/Y scale bars with zoom support +- Make averaging faster by adding support for incremental updates and parallelizing the calculations +- Make adding new sweeps during data acquisition much faster +- Nicify GUI controls and add more help entries +- Rework the time alignment code for faster execution +- Allow displaying deconvoluted pulses without visualized average wave +- Add image plot mode with external subwindows for the color scales and the option to + choose the pulse sorting order +- Add option ``Draw X zero line`` to draw a line crossing the plots at ``x == 0`` +- Add option to always use a fixed pulse length +- Read pulse starting times from the labnotebook if present and only fallback to calculating them +- Add Tests +- Ignore spikes which are narrower than 0.2 ms + +Foreign Function interface +-------------------------- + +- Add ``FFI_GetAvailableMessageFilters`` to query all available subscriber message filters + +General +------- + +- Add a menu option for resetting the package settings to their default +- Adapt ``Check Installation`` for Igor Pro 9 +- Make querying the labnotebok faster by adding support for incremental updates for the cache waves +- An early version of MIES user documentation +- Make the Igor Pro version check much more user friendly to use. We now don't bug out on old Igor Pro + versions anymore but display a dialog and allow direct download of the new version. + This also includes up-to-date links in the documentation. +- Sweep Rollback: Fix and avoid deleting the wrong sweeps. In some cases deleted sweeps can be reconstructed via + ``RecreateSweepWaveFromBackupAndLBN``. Please create an issue if you need help with that. +- Enhance the user experience when old NIDAQ-mx XOP versions are used with MIES +- Make unlocked indexing work with TP during DAQ +- Add menu option for opening the package settings in a notebook +- Add sub sub epoch information for pulse train pulses +- Fix POST_SET_EVENT/PRE_SET_EVENT determination in DC_PlaceDataInDAQDataWave for headstages in special cases. +- Add option to upload the MIES and ZeroMQ log files + +ITC XOP 2 +--------- + +None + +ZeroMQ XOP +---------- + +- Add support for logging in JSONL-format on disk. This is used for debugging and performance gathering. Enabled by default. +- Add support for publisher/subscriber sockets + +MCC XOP +------- + +None + +MIESUtils XOP +------------- + +None + +Labnotebook +----------- + +- Set the exact same timestamps for all entries added with one call. Previously these could differ in the + sub-millisecond range. +- Store timestamps with enough resolution in textual labnotebook, so instead of 3.6968e+09 we now store 3696770484.463 +- Fix storing the wrong value for the alarm checkbox for the asynchronous tab +- With asynchronous acquisition unused channels don't result in empty labnotebook entries anymore + +New numerical keys +~~~~~~~~~~~~~~~~~~ + +- ``Autobias %``: Autobias percentage as set in DAEphys +- ``Autobias Interval``: Autobias interval as set in DAEphys +- ``Acquisition State``: Add new standard entry which defines at which point during data acquisition an entry was + added. See also :ref:`File MIES_AcquisitionStateHandling.ipf` +- ``Skip Sweeps``: Store the number of performed sweep skips +- ``PSQ_FMT_LBN_DA_OPMODE``: Operation mode for ``PSQ_DAScale`` analysis function +- ``PSQ_FMT_LBN_TARGETV``: Target voltage baseline +- New entries for ``SC_SpikeControl``, see :ref:`File MIES_AnalysisFunctions_MultiPatchSeq.ipf` +- New entries for ``PSQ_Chirp``, see :ref:`File MIES_AnalysisFunctions_PatchSeq.ipf` + +New textual keys +~~~~~~~~~~~~~~~~ + +- ``Igor Pro build``: Igor Pro build revision +- ``JSON config file: stimset nwb file path``: Stimulus set path from the configuration file +- New entries for ``SC_SpikeControl``, see :ref:`File MIES_AnalysisFunctions_MultiPatchSeq.ipf` +- New entries for ``PSQ_Chirp``, see :ref:`File MIES_AnalysisFunctions_PatchSeq.ipf` + +Changed numerical entries +~~~~~~~~~~~~~~~~~~~~~~~~~ + +None + +Changed textual entries +~~~~~~~~~~~~~~~~~~~~~~~ + +None + +NWB/IPNWB +--------- + +- Add the Igor Pro build version to the ``generated_by`` dataset from ndx-MIES +- Make it faster by only saving the filled rows for the labnotebook and TP storage waves +- Skip sweeps on export which don't have a config wave +- Store the MIES log file in the same place as the Igor Pro history + +File format +~~~~~~~~~~~ + +None + +Pressure Control +---------------- + +- We now enforce that NI hardware uses differential setup for the AI channels + +WaveBuilder +----------- + +- Use WP and SegWvType with double precision for new waves. We keep the existing single precision waves + as we want to generate the exact stimulus sets again. +- Jump to the selected analysis function if possible when "Open" is pressed +- Make selecting epochs with the mouse work with flipping enabled +- Add help entry for delta modes +- Warn on too long explicit delta value list + +Work Sequencing Engine +---------------------- + +- Remove existing file first when exporting into NWB +- Publish QC results from background functions via ZeroMQ publisher socket + +Internal +-------- + +- Enhance error reporting in ``ASSERT_TS`` with outputting the stacktrace in IP9 +- Make NumericWaveToList faster by using wfprintf +- Make ``GetAllDevices`` faster +- Fix various corner cases in ``RA_SkipSweeps`` +- Unify naming for HardwareDataWave/ITCDataWave to DAQDataWave +- Minor fixes for Igor Pro 9 +- Make compiling slightly faster by not compiling the background watchter panel by default +- Add debug visualization for epochs in the Databrowser/Sweepbrowser +- Fix GetSweepSettings for the textual labnotebook +- Add ``GetActiveChannels`` which allow to determine which channels were active for a given sweep + +Tests +----- + +- Cleanup initialization code and add ``RunWithOpts`` +- Add basic dashboard testing + +Async Framework +--------------- + +- Add tracking jobs by ``workload`` parameters. This allows to check if all jobs of a given workload are finished. + +Logging +------- + +- Add support for logging in JSONL-format on disk. This is used for debugging and performance gathering. +- Store assertions, Igor starting/stopping/quitting/compiling, NWB export and device/pressure locking + +Installer +--------- + +- Skip vc_redist installation as non-admin +- Use latest 2019 vc_redist package + Release 2.1 =========== From bdd985a721312cafcd70d4f3ee1d0efbfbd538b8 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Sun, 2 May 2021 18:37:07 +0200 Subject: [PATCH 5/5] WB_OpenStimulusSetInWaveBuilder: Handle non existing stimsets better When we try showing a stimulus set we might realize that it does not exist. Bug introduced in b5bf561f (Add new trace popup for convenient stimulus set opening, 2021-02-18). We can not do anything when we are in the databrowser. But we can try loading the stimulus set when we are in the sweepbrowser. Close #932 --- Packages/MIES/MIES_AnalysisBrowser.ipf | 13 +++++++ .../MIES_AnalysisBrowser_SweepBrowser.ipf | 12 ++++++ Packages/MIES/MIES_WaveBuilderPanel.ipf | 38 ++++++++++++++++--- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/Packages/MIES/MIES_AnalysisBrowser.ipf b/Packages/MIES/MIES_AnalysisBrowser.ipf index 82f65c829b..1d9ba6224c 100644 --- a/Packages/MIES/MIES_AnalysisBrowser.ipf +++ b/Packages/MIES/MIES_AnalysisBrowser.ipf @@ -1452,6 +1452,19 @@ static Function AB_LoadFromFile(AB_LoadType, [sweepBrowserDFR]) return oneValidLoad End +Function AB_LoadStimsetForSweep(string device, variable index, variable sweep) + + string dataFolder, discLocation, fileType + + WAVE/T map = GetAnalysisBrowserMap() + + dataFolder = map[index][%DataFolder] + discLocation = map[index][%DiscLocation] + fileType = map[index][%FileType] + + return AB_LoadStimsetFromFile(discLocation, dataFolder, fileType, device, sweep, overwrite = 0) +End + // @brief common ASSERT statements for AB_LoadSweepFromFile and AB_LoadStimsetFromFile static Function AB_LoadFromFileASSERT(discLocation, dataFolder, fileType, device, sweep, overwrite) string discLocation, dataFolder, fileType, device diff --git a/Packages/MIES/MIES_AnalysisBrowser_SweepBrowser.ipf b/Packages/MIES/MIES_AnalysisBrowser_SweepBrowser.ipf index 073a4f28b3..f130bf84db 100644 --- a/Packages/MIES/MIES_AnalysisBrowser_SweepBrowser.ipf +++ b/Packages/MIES/MIES_AnalysisBrowser_SweepBrowser.ipf @@ -21,6 +21,18 @@ Function/DF SB_GetSweepBrowserFolder(win) return BSP_GetFolder(win, MIES_BSP_PANEL_FOLDER) End +Function SB_TranslateSBMapIndexToABMapIndex(string win, variable sbIndex) + + WAVE/T sweepMap = SB_GetSweepBrowserMapFromGraph(win) + WAVE/T analysisMap = GetAnalysisBrowserMap() + + // now search the DataFolder from sweepMap in the analyisMap + FindValue/TXOP=4/RMD=[][FindDimLabel(analysisMap, COLS, "DataFolder")]/TEXT=(sweepMap[sbIndex][%DataFolder]) analysisMap + ASSERT(V_Value >= 0, "Inconsistent AnalysisBrowserMap and SweepBrowserMap") + + return V_row +End + static Function/DF SB_GetSweepDataPathFromIndex(sweepBrowserDFR, mapIndex) DFREF sweepBrowserDFR variable mapIndex diff --git a/Packages/MIES/MIES_WaveBuilderPanel.ipf b/Packages/MIES/MIES_WaveBuilderPanel.ipf index be04ae21d6..a437e209be 100644 --- a/Packages/MIES/MIES_WaveBuilderPanel.ipf +++ b/Packages/MIES/MIES_WaveBuilderPanel.ipf @@ -38,8 +38,8 @@ static Constant FROM_WAVE_TO_PANEL = 0x2 Function WB_OpenStimulusSetInWaveBuilder() - string graph, trace, extPanel, waveBuilder - variable sweepNo, headstage + string graph, trace, extPanel, waveBuilder, stimset, device + variable sweepNo, headstage, abIndex, sbIndex GetLastUserMenuInfo graph = S_graphName @@ -57,16 +57,44 @@ Function WB_OpenStimulusSetInWaveBuilder() headstage = str2num(TUD_GetUserData(graph, trace, "headstage")) WAVE/T textualValues = $TUD_GetUserData(graph, trace, "textualValues") - WAVE/T/Z stimset = GetLastSetting(textualValues, sweepNo, STIM_WAVE_NAME_KEY, DATA_ACQUISITION_MODE) + WAVE/T/Z stimsetLBN = GetLastSetting(textualValues, sweepNo, STIM_WAVE_NAME_KEY, DATA_ACQUISITION_MODE) - if(!WaveExists(stimset)) + if(!WaveExists(stimsetLBN) || IsNaN(headstage)) printf "Context menu option \"%s\" could not find the stimulus set of the trace %s.\r", S_Value, trace ControlWindowToFront() return NaN endif + stimset = stimsetLBN[headstage] + + WAVE/Z stimsetWave = WB_CreateAndGetStimSet(stimset) + + if(!WaveExists(stimsetWave)) + if(BSP_IsDataBrowser(graph)) + printf "Context menu option \"%s\" could not be find the stimulus set %s.", S_Value, stimset + ControlWindowToFront() + return NaN + else + // we might need to load the stimset + WAVE traceWave = $TUD_GetUserData(graph, trace, "fullPath") + DFREF sweepDataDFR = GetWavesDataFolderDFR(traceWave) + sbIndex = SB_GetIndexFromSweepDataPath(graph, sweepDataDFR) + + DFREF sweepBrowserDFR = SB_GetSweepBrowserFolder(graph) + WAVE/T sweepMap = GetSweepBrowserMap(sweepBrowserDFR) + + abIndex = SB_TranslateSBMapIndexToABMapIndex(graph, sbIndex) + device = sweepMap[sbIndex][%Device] + if(AB_LoadStimsetForSweep(device, abIndex, sweepNo)) + printf "Context menu option \"%s\" could not load the stimulus set %s.", S_Value, stimset + ControlWindowToFront() + return NaN + endif + endif + endif + waveBuilder = WBP_CreateWaveBuilderPanel() - PGC_SetAndActivateControl(waveBuilder, "popup_WaveBuilder_SetList", str = stimset[headstage]) + PGC_SetAndActivateControl(waveBuilder, "popup_WaveBuilder_SetList", str = stimset) End Function/S WBP_CreateWaveBuilderPanel()