Merge branch 'master' into Issue-#238_FlyingFoxRecognizer

.editorconfig

@@ -96,7 +96,7 @@ csharp_style_var_when_type_is_apparent = true:none
 csharp_style_var_elsewhere = true:none
 # Expression-bodied members
 # https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-code-style-settings-reference#expression_bodied_members
-csharp_style_expression_bodied_methods = when_on_single_line:suggestion
+csharp_style_expression_bodied_methods = when_on_single_line:hint
 csharp_style_expression_bodied_constructors = when_on_single_line:suggestion
 csharp_style_expression_bodied_operators = true:warning
 csharp_style_expression_bodied_properties = true:warning

.gitattributes

@@ -44,15 +44,13 @@ docs/**/*.jpg filter=lfs diff=lfs merge=lfs -text
 *.dll     binary
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.mp3 filter=lfs diff=lfs merge=lfs -text
-
 # all binaries in lib should be tracked by git-lfs
 lib/**/*.exe    filter=lfs diff=lfs merge=lfs -text
 lib/**/*.dll    filter=lfs diff=lfs merge=lfs -text
 lib/audio-utils/macosx/**/*        filter=lfs diff=lfs merge=lfs -text
-
 # assets in tests/Fixtures should generally be in lfs
 tests/Fixtures/**/*.zip filter=lfs diff=lfs merge=lfs -text
 tests/Fixtures/**/*.csv filter=lfs diff=lfs merge=lfs -text
 tests/Fixtures/**/*.bin filter=lfs diff=lfs merge=lfs -text
 tests/Fixtures/**/*.json filter=lfs diff=lfs merge=lfs -text
-tests/Fixtures/**/*.png filter=lfs diff=lfs merge=lfs -text
+tests/Fixtures/**/*.png filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.md

@@ -17,21 +17,64 @@ A clear and concise description of what you expected to happen.
 
 ## Diagnostics
 
-- I Confirm I have followed [the instructions for reporting a bug](https://github.com/QutEcoacoustics/audio-analysis/blob/master/docs/bug_report.md)
-- I have attached a log <!-- Drag and drop the log here -->
-- I have answered the following questions:
-  <!-- e.g. Windows 10 Version 1809 -->  
-  - **Operating system:** 
-  <!-- e.g. 18.05.3.6 -->  
-  - **AP.exe version:**
-  <!-- e.g. N/A if not available or not needed, or instructions on how to get the test data -->  
-  - **The data used can be found:**
-  - **The command I used:**
-  - **The config file I used:**  <!-- Drag and drop the config here -->
-
-
-### Screenshots
-If applicable, add screenshots to help explain your problem.
+<!-- Try to answer as many questions as possible -->
+
+### Instructions
+
+- [x] I have followed the [instructions for reporting a bug](https://github.com/QutEcoacoustics/audio-analysis/blob/master/docs/bug_report.md)
+
+### I have attached a log:
+
+<!-- Information about log files can be found here: -->
+<!-- https://github.com/QutEcoacoustics/audio-analysis/blob/master/docs/logs.md#log-files -->
+<!-- 👇 Drag and drop the log into the empty line below -->
+
+<!-- 👆 -->
+
+### The **operating system** I'm using is:  
+
+<!-- e.g. Windows 10 Version 1809 -->  
+
+
+### The version of **AnalysisPrograms.exe** I'm using is:
+
+<!-- e.g. 18.05.3.6 -->
+
+
+### The data used can be found:
+
+<!-- Describe or provide the data relevant to this bug below -->
+<!-- If the data is sensitive or too large to easily be shared -->
+<!-- then describe the shape of it (size, duration, count, format, etc..) -->
+<!-- and we'll try to use simulated data. If we need more information -->
+<!-- then we'll contact you privately. -->
+
+
+
+<!-- 👇 Optionally, drag and drop **SMALL** files into the empty line below-->
+
+<!-- 👆 -->
+
+### The command I used:
+
+<!-- 👇 Copy and paste the command you used in the empty line below -->
+```
+
+```
+<!-- 👆 -->
+
+### The config file I used:
+
+<!-- 👇 Drag and drop the log into the empty line below -->
+
+<!-- 👆 -->
+
+
+## Screenshots
+
+<!-- If applicable, add screenshots to help explain your problem. Drag and drop into the space below. -->
+
 
 ## Additional Details
-Add any other context about the problem here.
+
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,37 @@
+---
+name: Ask a question
+about: I'd just like to ask a question
+---
+
+## My question
+
+<!-- Put your question here -->
+
+
+## Diagnostics
+
+<!-- Answer any questions you think are relevant -->
+
+- I have attached a log:
+<!-- Information about log files can be found here: https://github.com/QutEcoacoustics/audio-analysis/blob/master/docs/logs.md#log-files -->
+<!-- 👇 Drag and drop the log into the empty line below -->
+
+<!-- 👆 -->
+
+- The **Operating system** I'm using:             <!-- e.g. Windows 10 Version 1809 -->  
+
+- The version of **AnalysisPrograms.exe**:        <!-- e.g. 18.05.3.6 -->  
+
+- **The data used can be found:**                 <!-- describe or provide the data relevant to this question -->
+
+- **The command I used:**
+<!-- 👇 Copy and paste the command you used in the empty line below -->
+```
+
+```
+<!-- 👆 -->
+
+- **The config file I used:**  <!-- Drag and drop the config here -->
+<!-- 👇 Drag and drop the log into the empty line below -->
+
+<!-- 👆 -->

AudioAnalysis.sln.DotSettings

@@ -280,8 +280,12 @@
 	<s:Boolean x:Key="/Default/ReSpeller/UserDictionaries/=en_005Fus/Words/=parallelized/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/ReSpeller/UserDictionaries/=en_005Fus/Words/=spectrums/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/ReSpeller/UserDictionaries/=en_005Fus/Words/=Truskinger/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Bioacoustics/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Chromeless/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=dedupe/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=ffmpeg/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=ffprobe/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=flac/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Redirector/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=spectrograms/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Submatrix/@EntryIndexedValue">True</s:Boolean>

CONTRIBUTING.md

@@ -29,7 +29,9 @@ Required Software
 The required software for developing new code (not running the program) includes:
 
 - A Windows OS (we're working on making this Linux compatible)
-- Visual Studio 2017
+- Visual Studio 2019
+  - If you're at a university that has an Office365 Subscription you can download software from https://azureforeducation.microsoft.com/devtools
+  - The [community edition](https://visualstudio.microsoft.com/vs/) of Visual Studio should work fine and is totally free
 - [Optional] Resharper Ulitmate (Academic License)
   - Install these plugins (_ReSharper_ menu > _Extension Manager_)
     - ReSpeller Free

NuGet.Config

@@ -4,5 +4,6 @@
     <add key="All" value="(Aggregate source)" />
     <add key="SqLiteFileSystem Packages" value="https://ci.appveyor.com/nuget/sqlitefilesystem-dcfwbf0p9du0" />
     <add key="natemcmaster Packages" value="https://www.myget.org/F/natemcmaster" />
+    <add key="SixLabors Packages (myget)" value="https://www.myget.org/F/sixlabors" />
   </packageSources>
 </configuration>

azure-pipelines.yml

@@ -0,0 +1,23 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:  none
+
+jobs:
+- job: Linux
+  pool:
+    vmImage: 'ubuntu-18.04'
+  steps:
+  - script: echo hello from Linux
+- job: macOS
+  pool:
+    vmImage: 'macOS-10.14'
+  steps:
+  - script: echo hello from macOS
+- job: Windows
+  pool:
+    vmImage: 'windows-2019'
+  steps:
+  - script: echo hello from Windows

docs/developing.md

@@ -0,0 +1 @@
+See [Contributing](https://github.com/QutEcoacoustics/audio-analysis/blob/master/CONTRIBUTING.md)

docs/faq.md

@@ -33,6 +33,120 @@ mind the target audience. You're in the right ballpark if:
 
 More than likely if you're stuck we can help 😊.
 
+## Why do we analyze data in 1-minute chunks?
+
+There are a couple reasons, but mainly, because when we first started doing this,
+computers were far less efficient than they are now. Computers are fundamentally
+limited by the amount of data they can work with at one time; in technical terms,
+the amount of data that can fit into main memory (RAM). By breaking the data into 
+smaller chunks, we could _stream_ the data through the analysis and our overall
+analysis speed was greatly improved.
+
+We could have chosen any size from 30-seconds to 5-minutes, but
+one-minute blocks also had nice temporal features (they compose well with data
+at different resolutions) and are still detailed enough to answer questions in
+our multi-day analyses.
+
+Today it seems to be the de facto standard to analyze data in one-minute blocks.
+We suggest that it is still a good default for most use cases:
+
+- Computers don't have the same limitations as they did when we started, but small
+  blocks of data allow for parallel analysis that effectively utilizes all CPU cores
+- While computer's are getting better, we're also doing more complex analyses. In
+  parallel we can use a large amount of RAM and most of the computer's CPU(s) for
+  the quickest analysis
+- One-minute blocks still retain the nice temporally composable attributes detailed
+  above.
+- And since one-minute blocks seem to be a defacto standard it does (by happenstance)
+  provide common ground for comparing data
+
+## What effect does chunk size have on data?
+
+For acoustic event recognition, typically only boundary effects are 
+affected by chunk-size choice.
+That is, if an acoustic event occurs and is clipped by
+by either the start or end of the one-minute chunk, then it is now only a "partial vocalisation".
+A typical event recognizer may not detect such "partial vocalisations".
+
+For acoustic indices, from a theoretical point of view, chunk-size has the same
+kinds of issues as the choice of FFT frame length in speech processing. Because an FFT
+assumes signal stationarity, one chooses a frame length over which the spectral
+content in the signal of interest is approximately constant. In the case of
+acoustic indices, one chooses an index calculation duration which captures
+a sufficient amount of data for the acoustic feature that you are interested in.
+The longer the analysis duration the more you blur or average out features of
+interest. However, if you choose too short an interval in then the calculated
+index may be dominated by "noise"---that is, features that are not of interest.
+We find this is particularly the case with the ACI index; one-minute seems to be
+an appropriate duration for the indices that are typically calculated.
+
+## Can I change the chunk size?
+
+The [config files](https://github.com/QutEcoacoustics/audio-analysis/tree/master/src/AnalysisConfigFiles) for most of our analyses contain these common settings:
+
+### Chunk-size:
+
+```yaml
+# SegmentDuration: units=seconds;
+# Long duration recordings are cut into short segments for more 
+# efficient processing. Default segment length = 60 seconds.
+SegmentDuration: 60
+```
+Here, `SegmentDuration` is our name for the chunk-size. If, for example, you wanted
+to process data in 5 minute chunks, you could change the configuration to
+`SegmentDuration: 300`.
+
+### Chunk-overlap:
+
+```yaml
+# SegmentOverlap: units=seconds;
+SegmentOverlap: 0
+```
+
+If you're doing event recognition and you're concerned about boundary effects,
+you could change the overlap to `SegmentOverlap: 10`, which would ensure every
+`SegmentDuration`-sized-chunk (typically one-minute in size) would be cut with
+an extra, trailing, 10-second buffer.
+
+Note: we rarely change this setting and setting too much overlap may produce
+duplicate events.
+
+### Index Calculation Duration (for the indices analysis only):
+
+For acoustic indices in particular, their calculation resolution depends on a
+second setting that is limited by chunk-size (`SegmentDuration`):
+
+```yaml
+# IndexCalculationDuration: units=seconds (default=60 seconds)
+# The Timespan (in seconds) over which summary and spectral indices are calculated
+# This value MUST not exceed value of SegmentDuration.
+# Default value = 60 seconds, however can be reduced down to 0.1 seconds for higher resolution.
+# IndexCalculationDuration should divide SegmentDuration with MODULO zero
+IndexCalculationDuration: 60.0
+```
+
+If you wanted indices calculated over a duration longer than one-minute, you
+could change the `SegmentDuration` and the `IndexCalculation` duration to higher
+values: 
+
+```yaml
+SegmentDuration: 300
+IndexCalculationDuration: 300
+```
+
+However, we suggest that there are better methods for calculating low-resolution
+indices. A method we often use is to calculate indices at a 60.0 seconds resolution
+and aggregate the values into lower resolution chunks. The aggregation method
+can provide some interesting choices:
+
+- We've seen the maximum, median, or minimum value for a block of indices
+    chosen (and sometimes all 3).
+    - though be cautious when using a mean function, it can skew the value of
+        logarithmic indices
+- And we've seen a block of indices flattened into a larger feature vector and
+    fed to a machine learning or clustering algorithm
+
+
 ## We collect metrics/statistics; what information is collected and how is it used?
 
 **NOTE: this is an upcoming feature and has not been released yet**

docs/installing.md

@@ -49,7 +49,7 @@ The following additional dependencies are required for Unix machines:
 - sox
 - shntool
 - mp3splt
-- libav-tools
+- libav-tools (on some distros only, not needed in Ubuntu 18)
 - [Optional] [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/setup/installing-powershell-core-on-linux?view=powershell-6)
 
 Note all AP.exe commands must be prefixed with `mono ` to be used on Linux.

src/AcousticWorkbench/AcousticWorkbench.csproj

@@ -60,8 +60,8 @@
     <Reference Include="System.Net" />
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Runtime.Serialization" />
-    <Reference Include="System.ValueTuple, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\System.ValueTuple.4.4.0-preview2-25405-01\lib\netstandard1.0\System.ValueTuple.dll</HintPath>
+    <Reference Include="System.ValueTuple, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.ValueTuple.4.5.0\lib\netstandard1.0\System.ValueTuple.dll</HintPath>
     </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />

src/AcousticWorkbench/packages.config

@@ -7,5 +7,5 @@
   <package id="StyleCop.Analyzers" version="1.1.1-beta.61" targetFramework="net452" developmentDependency="true" />
   <package id="StyleCop.Analyzers.Unstable" version="1.1.1.61" targetFramework="net452" developmentDependency="true" />
   <package id="System.Net.Http" version="4.3.3" targetFramework="net452" />
-  <package id="System.ValueTuple" version="4.4.0-preview2-25405-01" targetFramework="net452" />
+  <package id="System.ValueTuple" version="4.5.0" targetFramework="net452" />
 </packages>