From 2ddcb070ec240d8c0032fa06a5bd2b4155300b37 Mon Sep 17 00:00:00 2001
From: "Luke.Humphreys" <lhumphreys67@gmail.com>
Date: Thu, 24 Oct 2019 19:58:34 +0100
Subject: [PATCH 1/3] Convert existing test.bsh into a regression test

The existing test.bash script has been converted to a regression test
that:
  - Verifies that in default mode (GCOV) we calcualte the correct
    coverage for a simple c binary
  - Verifies that in lcov mode  we correctly parse the .info file for a
    simple c binary

Additional tests can be created by copying the "test-src/simple"
directory and:
  - Updating the source file & Makefile with the new test code
  - Updating the expected.json with the expected coverage.
    NOTE: If gcov and lcov provide different results, this can be
          configured in the config.sh by specifying separate
          expected.json files
  - [optional] Providing a config.sh file in the directory to configure
    optional flags. (See valid options listed at the top of
    test-utils/testDir.sh)
  - Adding the directory to test.bash
---
 .gitignore                    |   7 +
 .travis.yml                   |   5 +
 test-src/simple/Makefile      |   8 +
 test-src/simple/expected.json |   6 +
 test-src/simple/foo.c         |  16 ++
 test-utils/check_output.py    |  69 ++++++++
 test-utils/testDir.sh         | 324 ++++++++++++++++++++++++++++++++++
 test.bash                     |  56 +++---
 8 files changed, 464 insertions(+), 27 deletions(-)
 create mode 100644 test-src/simple/Makefile
 create mode 100644 test-src/simple/expected.json
 create mode 100644 test-src/simple/foo.c
 create mode 100755 test-utils/check_output.py
 create mode 100755 test-utils/testDir.sh

diff --git a/.gitignore b/.gitignore
index 9e4f131..17f14c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,13 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+*_test_output
+*.gcov
+output.json
+*.gcno
+*.gcda
+*.info
+
 
 # Translations
 *.mo
diff --git a/.travis.yml b/.travis.yml
index 1181d7f..6d69b60 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,11 @@ python:
     - "3.4"
     - "3.5"
 
+addons:
+  apt:
+    packages:
+      - lcov
+
 install:
     - python setup.py --quiet install
 
diff --git a/test-src/simple/Makefile b/test-src/simple/Makefile
new file mode 100644
index 0000000..532d868
--- /dev/null
+++ b/test-src/simple/Makefile
@@ -0,0 +1,8 @@
+foo: foo.c
+	gcc --coverage -o foo foo.c
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json
+
+test: foo
+	./foo
diff --git a/test-src/simple/expected.json b/test-src/simple/expected.json
new file mode 100644
index 0000000..3c923b0
--- /dev/null
+++ b/test-src/simple/expected.json
@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null],
+        "source_digest": "500a03d2484de8cc2b9ed06615d7f02a"}]
+}
diff --git a/test-src/simple/foo.c b/test-src/simple/foo.c
new file mode 100644
index 0000000..9462b55
--- /dev/null
+++ b/test-src/simple/foo.c
@@ -0,0 +1,16 @@
+int main() {
+    int a = 1;
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}
diff --git a/test-utils/check_output.py b/test-utils/check_output.py
new file mode 100755
index 0000000..23b6610
--- /dev/null
+++ b/test-utils/check_output.py
@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import json
+import sys
+
+expectedJson="expected.json"
+if sys.argv[1] == "-e":
+    expectedJson = sys.argv[2]
+
+
+expected=json.loads(open(expectedJson).read())
+output=json.loads(open("output.json").read())
+
+if 'source_files' not in expected:
+    print ("expected.json does not contain a 'source_files' array")
+    print(expected)
+    exit (1)
+
+if 'source_files' not in output:
+    print ("output.json does not contain a 'source_files' array")
+    print(output)
+    exit (1)
+
+expected_files={}
+output_files={}
+
+for f in expected['source_files']:
+    expected_files[f['name']] = f
+for f in output['source_files']:
+    output_files[f['name']] = f
+
+#
+# Check for missing files
+#
+for f in expected_files:
+    if f not in output_files:
+        print ("File '%s' is missing from output.json" %f)
+        exit(1)
+#
+# Check for rogue files
+#
+for f in output_files:
+    if f not in expected_files:
+        print ("File '%s' is reported in output.json, but was not expected!" %f)
+        exit(1)
+
+#
+# Finally check each file has the expected output...
+#
+for f in expected_files:
+    exp_file = expected_files[f]
+    out_file = output_files[f]
+
+    exp_lines=exp_file['coverage']
+    out_lines=out_file['coverage']
+    if len(exp_lines) != len(out_lines):
+        print ("File '%s': expected %d lines, but actually has %s" %(f, len(exp_lines), len(out_lines)))
+        exit(1)
+
+    for i in range(len(out_lines)):
+        if exp_lines[i] != out_lines[i]:
+            print ("File '%s':%d, expected %s hits, but actually has %s" %(f, i+1, exp_lines[i], out_lines[i]))
+            exit(1)
+
+
+
diff --git a/test-utils/testDir.sh b/test-utils/testDir.sh
new file mode 100755
index 0000000..4b3bcae
--- /dev/null
+++ b/test-utils/testDir.sh
@@ -0,0 +1,324 @@
+#!/bin/bash
+
+# START: Valid config.sh options
+#    The following variables can be set in the directory's config.sh file
+
+# Set if we expect the GCOV test for a known reason
+gcov_fail_reason=""
+
+# Set if we expect the LCOV test for a known reason
+lcov_fail_reason=""
+
+# Modes to test in (default "gcov lcov")
+#
+modes="gcov lcov"
+
+
+# Set to override the build directory (default is the test's root directory).
+# This is where "make" will be invoked.
+# Path should relative to the test's root directory
+build_dir=""
+
+# Set to provide additional flags to cpp-coveralls when in gcov mode
+gcov_coveralls_flags=""
+
+# Set to provide additional flags to cpp-coveralls when in lcov mode
+lcov_coveralls_flags=""
+
+# Set to provide additional flags lcov on every invocation
+lcov_flags=""
+
+# Set to override the directory in which lcov will be invoked
+# (default is the tests's root directory)
+# Path should relative to the test's root directory
+lcov_dir=""
+
+# Set to override the directory in which cpp-coveralls will be invoked
+# for gcov (default is the tests's root directory)
+# Path should relative to the test's root directory
+gcov_coveralls_dir=""
+
+# Set to override the directory in which cpp-coveralls will be invoked
+# for lcov (default is the tests's root directory)
+# Path should relative to the test's root directory
+lcov_coveralls_dir=""
+
+# Override to change the file that contains the expected output for the gcov run
+gcov_expected_output="expected.json"
+
+# Override to change the file that contains the expected output for the lcov run
+lcov_expected_output="expected.json"
+
+#
+# END: Valid config.sh options
+#
+
+# Internal counter of failed tests
+failed_tests=0
+exit_on_fail="no"
+
+
+function Usage {
+    echo "testDir.sh [-e/ --exit-on-fail] test-src/<dir to test>"
+    echo ""
+    echo "Options:"
+    echo "   -e / --exit-on-fail: Stop execution at the first unexpected failure."
+}
+
+#
+#  Parse the command line arguments
+#
+#  Arguments:
+#     @: The set of arguments parsed to this script
+#
+function ParseArguments {
+    if [[ "$1" == "-e" || "$1" == "--exit-on-fail" ]]; then
+        exit_on_fail="yes"
+        shift
+    fi
+
+    test_dir=$1
+
+    if [[ "$test_dir" == "" || ! -d "$test_dir" ]]; then
+        echo "Cannot find test directory: $test_dir"
+        Usage
+        exit 1
+    fi
+}
+
+function SetupEnvironment {
+    if [ -z "$TRAVIS_JOB_ID" ]; then
+        export COVERALLS_REPO_TOKEN="fake testing token"
+    fi
+
+    cd $test_dir || exit
+
+    if [[ -e config.sh ]]; then
+        source config.sh
+    fi
+
+    if [[ ! -e $gcov_expected_output ]]; then
+        echo "Cannot test $PWD:"
+        echo "    Configured GCOV output file not found ($gcov_expected_output)"
+        exit 1
+    fi
+
+    if [[ ! -e $lcov_expected_output ]]; then
+        echo "Cannot test $PWD:"
+        echo "    Configured LCOV output file not found ($lcov_expected_output)"
+        exit 1
+    fi
+
+    if [[ "$build_dir" != "" && ! -d "$build_dir" ]]; then
+        echo "Cannot test $PWD:"
+        echo "    No such build directory (build_dir): $build_dir"
+        exit 1
+    fi
+
+    if [[ "$gcov_coveralls_dir" != "" && ! -d "$gcov_coveralls_dir" ]]; then
+        echo "Cannot test $PWD:"
+        echo "    No such directory (gcov_coveralls_dir): $gcov_coveralls_dir"
+        exit 1
+    fi
+
+    if [[ "$lcov_coveralls_dir" != "" && ! -d "$lcov_coveralls_dir" ]]; then
+        echo "Cannot test $PWD:"
+        echo "    No such directory (lcov_coveralls_dir): $lcov_coveralls_dir"
+        exit 1
+    fi
+
+    if [[ "$lcov_dir" != "" && ! -d "$lcov_dir" ]]; then
+        echo "Cannot test $PWD:"
+        echo "    No such directory (lcov_dir): $lcov_dir"
+        exit 1
+    fi
+
+    if [[ ! -e "$PWD/$build_dir/Makefile" ]]; then
+        echo "$PWD/$build_dir does not contain a valid Makefile!"
+        Usage
+        exit 1
+    fi
+}
+
+#
+#  Check the coveralls result matches the directory's expected.json
+#
+#  Arguments:
+#     1: log           - The logfile to append check result
+#     2: expected_json - The filename that contains the json block with the
+#                        expected coverage output
+#     3: fail_reason   - Set if we expect this to test fail, with a reason text
+#
+function CheckResult {
+    log=$1
+    expected_json=$2
+    fail_reason=$3
+
+    result=$(check_output.py -e $expected_json)
+    failed="no"
+    if [[ "$result" != "" ]]; then
+        if [[ "$fail_reason" != "" ]]; then
+            echo "FAIL (as expected: $fail_reason)" | tee -a $log
+        else
+            failed="yes"
+            echo "FAIL" | tee -a $log
+            echo "$result" | sed -e 's/^/   /' | tee -a $log
+            echo "   Expected:" | tee -a $log
+            cat $expected_json | sed -e 's/^/        /' | tee -a $log
+            echo "    Actual Output:" | tee -a $log
+            cat output.json | sed -e 's/^/        /' | tee -a $log
+            echo
+        fi
+    else
+        if [[ "$fail_reason" != "" ]]; then
+            echo "ERROR: PASS (should have failed: $fail_reason)" | tee -a $log
+            failed="yes"
+        else
+            echo "PASS" | tee -a $log
+        fi
+    fi
+
+    if [[ "$failed" == "yes" ]]; then
+        let "failed_tests+=1"
+        if [[ "$exit_on_fail" == "yes" ]]; then
+            exit 1
+        fi
+    fi
+}
+
+function Make {
+    log_file=$1
+    shift
+
+    if [[ "$build_dir" != "" ]]; then
+        pushd $build_dir >> $log_file
+    fi
+
+    echo ">> make $@" >> $log_file
+    make $@  >> $log_file 2>&1 || Exit "make command failed!" $log_file
+
+    if [[ "$build_dir" != "" ]]; then
+        popd >> $log_file
+    fi
+}
+
+function CppCoverals {
+    log_file=$1
+    coveralls_dir=$2
+    shift
+    shift
+
+    if [[ "$coveralls_dir" != "" ]]; then
+        pushd $coveralls_dir >> $log_file
+    fi
+
+    echo ">> cpp-coveralls $@" >> $log_file
+    cpp-coveralls $@ >> $logfile 2>&1 || Exit "cpp-coveralls failed" "$log_file"
+
+    if [[ "$coveralls_dir" != "" ]]; then
+        popd >> $log_file
+    fi
+}
+
+function LCov {
+    log_file=$1
+    shift
+
+    if [[ "$lcov_dir" != "" ]]; then
+        pushd $lcov_dir >> $log_file
+    fi
+
+    echo ">> lcov $lcov_flags $@" >> $log_file
+    lcov $lcov_flags $@ >> $log_file 2>&1 || Exit "LCov invocation failed" "$logfile"
+
+    if [[ "$lcov_dir" != "" ]]; then
+        popd >> $log_file
+    fi
+}
+
+function Exit {
+    reason=$1
+    logfile=$2
+    echo "FATAL_ERROR: $reason"
+    echo "Test output:"
+    cat $logfile | sed -e 's/^/    /'
+    exit 1
+}
+
+
+#
+#  Test the directory by asking coveralls to do a full gcov run
+#
+#  Arguments: None
+#
+function CheckGCOV {
+    echo -n ">> Checking $PWD using GCOV..."
+
+    logfile="$PWD/gcov_test_output"
+    CleanUp  > $logfile 2>&1
+
+    Make $logfile test
+
+    CppCoverals $logfile "$gcov_coveralls_dir" --dump $PWD/output.json --verbose  $gcov_coveralls_flags
+
+    CheckResult $logfile "$gcov_expected_output" "$gcov_fail_reason"
+
+    CleanUp
+}
+
+#
+#  Test the directory by manually running lcov, and asking coveralls to parse the output
+#
+#  Arguments: None
+#
+function CheckLCOV {
+    echo -n ">> Checking $PWD using LCOV..."
+
+    logfile="$PWD/lcov_test_output"
+    CleanUp  > $logfile 2>&1
+
+    Make $logfile prepare_test
+
+    LCov $logfile --capture -d . -i --output-file="$PWD/lcov_baseline.info"
+
+    Make $logfile test
+
+    LCov $logfile --capture -d . --output-file="$PWD/lcov_test_run.info"
+
+    LCov $logfile $lcov_flags -a $PWD/lcov_baseline.info -a $PWD/lcov_test_run.info -o "$PWD/lcov.info"
+
+    CppCoverals $logfile "$lcov_coveralls_dir" --dump $PWD/output.json --no-gcov --lcov-file="$PWD/lcov.info" $lcov_coveralls_flags
+
+    CheckResult $logfile "$lcov_expected_output" "$lcov_fail_reason"
+
+    CleanUp
+}
+
+#
+#  Clean down the current directory, ready to start a new test
+#
+#  Arguments: None
+#
+function CleanUp {
+    Make /dev/null clean
+    rm -f *.gcda *.gcno lcov*.info output.json > /dev/null
+    find . -name "*.gcov" | xargs rm -f > /dev/null
+}
+
+ParseArguments $@
+
+SetupEnvironment
+
+for mode in $modes; do
+    if [[ "$mode" == "gcov" ]]; then
+        CheckGCOV
+    elif [[ "$mode" == "lcov" ]]; then
+        CheckLCOV
+    else
+        Exit /dev/stdout "Unknown test mode requested: $mode"
+    fi
+done
+
+
+
+exit $failed_tests
diff --git a/test.bash b/test.bash
index 2c1661b..cd8bccd 100755
--- a/test.bash
+++ b/test.bash
@@ -1,34 +1,36 @@
-#!/bin/bash -ex
+#!/bin/bash
 #
 # Run system test.
 
-coveralls --help
-
-cat > foo.c <<'EOF'
-int main() {
-    int a = 1;
-    if(a == 2) {
-        a = 3;
-        /* LCOV_EXCL_START */
-        a = 4;
-        a = 5;
-        /* LCOV_EXCL_STOP */
-        a = 6;
-    }
-    if(a == 7) {
-        a = 8;
-        a = 9; /* LCOV_EXCL_LINE */
-    }
-    return 0;
+export PATH="$PATH:$PWD/test-utils"
+
+exit_on_fail="no"
+
+function ParseArguments {
+    while [[ "$1" != "" ]]; do
+        if [[ "$1" == "-e" || "$1" == "--exit-on-fail" ]]; then
+            exit_on_fail="yes"
+        fi
+        shift
+    done
+}
+
+failed_tests=0
+function TestFailed {
+    let "failed_tests+=1"
+    if [[ "$exit_on_fail" == "yes" ]]; then
+        echo "(-e): Exiting due to first failure"
+        exit 1
+    fi
 }
-EOF
-gcc -coverage -o foo foo.c
-./foo
 
-if [ -z "$TRAVIS_JOB_ID" ]; then
-    export COVERALLS_REPO_TOKEN="fake testing token"
+ParseArguments $@
+
+testDir.sh $@ test-src/simple || TestFailed
+
+if [[ $failed_tests -gt 0 ]]; then
+    echo ""
+    echo "FAIL: $failed_tests unexpected failures!"
 fi
 
-coveralls --verbose --encodings utf-8 latin-1 foobar | \
-    grep '\[1, 1, 1, 0, None, None, None, None, 0, None, 1, 0, None, None, 1, None\]'
-rm -f foo foo.c* foo.gc*
+exit $failed_tests

From 927887ea9a90025610ad338b7228a95f8ff9aeb7 Mon Sep 17 00:00:00 2001
From: "Luke.Humphreys" <lhumphreys67@gmail.com>
Date: Fri, 25 Oct 2019 08:39:31 +0100
Subject: [PATCH 2/3] Additional test cases:

   - Simple static library, which requires to .gcov files to be merged
     in the GCOV workflow, and for multiple records to be merged in the
     LCOV workflow
   - Simple template library, which requires to .gcov files to be merged
     in the GCOV workflow, and for multiple records to be merged in the
     LCOV workflow
   - Out of tree test, where --root is used to specify a separate source
     directory as the base file path
   - Missing files test - which ensures that files are reported as
     missing, even if it hasn't been linked into any test
   - File / directory exlcusion patterns (gcov only)
   - File / directory explicit inclusions (gcov only)
---
 test-src/excluded_files/Makefile              | 19 ++++++++++++++
 .../another_extra_lib/another_toy.cpp         |  5 ++++
 .../another_extra_lib/another_toy.h           |  1 +
 .../excluded_files/another_extra_lib/toy.h    |  1 +
 test-src/excluded_files/config.sh             | 10 ++++++++
 test-src/excluded_files/expected.json         |  6 +++++
 test-src/excluded_files/extra_lib/toy.cpp     |  5 ++++
 test-src/excluded_files/extra_lib/toy.h       |  1 +
 test-src/excluded_files/foo.cpp               | 25 +++++++++++++++++++
 test-src/excluded_files/part_unused.cpp       |  9 +++++++
 test-src/excluded_files/part_unused.h         |  7 ++++++
 test-src/included_files/Makefile              | 19 ++++++++++++++
 .../another_extra_lib/another_toy.cpp         |  5 ++++
 .../another_extra_lib/another_toy.h           |  1 +
 .../included_files/another_extra_lib/toy.h    |  1 +
 test-src/included_files/config.sh             | 11 ++++++++
 test-src/included_files/expected.json         |  6 +++++
 test-src/included_files/extra_lib/toy.cpp     |  5 ++++
 test-src/included_files/extra_lib/toy.h       |  1 +
 test-src/included_files/foo.cpp               | 25 +++++++++++++++++++
 test-src/included_files/part_unused.cpp       |  9 +++++++
 test-src/included_files/part_unused.h         |  7 ++++++
 test-src/missing_files/Makefile               | 12 +++++++++
 test-src/missing_files/config.sh              | 15 +++++++++++
 test-src/missing_files/expected_gcov.json     | 12 +++++++++
 test-src/missing_files/expected_lcov.json     | 12 +++++++++
 test-src/missing_files/foo.c                  | 16 ++++++++++++
 test-src/missing_files/unused.cpp             |  7 ++++++
 test-src/missing_files/unused.h               |  5 ++++
 test-src/out_of_tree/build_tree/Makefile      | 23 +++++++++++++++++
 test-src/out_of_tree/config.sh                | 20 +++++++++++++++
 test-src/out_of_tree/expected.json            | 15 +++++++++++
 test-src/out_of_tree/source_tree/A/a.cpp      |  9 +++++++
 test-src/out_of_tree/source_tree/B/b.cpp      |  9 +++++++
 test-src/out_of_tree/source_tree/llib/lib.cpp |  9 +++++++
 test-src/out_of_tree/source_tree/llib/lib.h   |  8 ++++++
 test-src/simple/Makefile                      |  4 ++-
 test-src/static_lib/A/a.cpp                   |  9 +++++++
 test-src/static_lib/B/b.cpp                   |  9 +++++++
 test-src/static_lib/Makefile                  | 20 +++++++++++++++
 test-src/static_lib/config.sh                 |  1 +
 test-src/static_lib/expected.json             | 15 +++++++++++
 .../expected_lcov_doubt_counts.json           | 15 +++++++++++
 test-src/static_lib/llib/lib.cpp              |  9 +++++++
 test-src/static_lib/llib/lib.h                |  8 ++++++
 test-src/template_includes/A.cpp              |  6 +++++
 test-src/template_includes/B.cpp              |  6 +++++
 test-src/template_includes/Makefile           | 14 +++++++++++
 test-src/template_includes/common_header.h    | 10 ++++++++
 test-src/template_includes/config.sh          | 10 ++++++++
 test-src/template_includes/expected.json      | 12 +++++++++
 test-utils/testDir.sh                         |  2 +-
 test.bash                                     |  6 +++++
 53 files changed, 505 insertions(+), 2 deletions(-)
 create mode 100644 test-src/excluded_files/Makefile
 create mode 100644 test-src/excluded_files/another_extra_lib/another_toy.cpp
 create mode 100644 test-src/excluded_files/another_extra_lib/another_toy.h
 create mode 100644 test-src/excluded_files/another_extra_lib/toy.h
 create mode 100644 test-src/excluded_files/config.sh
 create mode 100644 test-src/excluded_files/expected.json
 create mode 100644 test-src/excluded_files/extra_lib/toy.cpp
 create mode 100644 test-src/excluded_files/extra_lib/toy.h
 create mode 100644 test-src/excluded_files/foo.cpp
 create mode 100644 test-src/excluded_files/part_unused.cpp
 create mode 100644 test-src/excluded_files/part_unused.h
 create mode 100644 test-src/included_files/Makefile
 create mode 100644 test-src/included_files/another_extra_lib/another_toy.cpp
 create mode 100644 test-src/included_files/another_extra_lib/another_toy.h
 create mode 100644 test-src/included_files/another_extra_lib/toy.h
 create mode 100644 test-src/included_files/config.sh
 create mode 100644 test-src/included_files/expected.json
 create mode 100644 test-src/included_files/extra_lib/toy.cpp
 create mode 100644 test-src/included_files/extra_lib/toy.h
 create mode 100644 test-src/included_files/foo.cpp
 create mode 100644 test-src/included_files/part_unused.cpp
 create mode 100644 test-src/included_files/part_unused.h
 create mode 100644 test-src/missing_files/Makefile
 create mode 100644 test-src/missing_files/config.sh
 create mode 100644 test-src/missing_files/expected_gcov.json
 create mode 100644 test-src/missing_files/expected_lcov.json
 create mode 100644 test-src/missing_files/foo.c
 create mode 100644 test-src/missing_files/unused.cpp
 create mode 100644 test-src/missing_files/unused.h
 create mode 100644 test-src/out_of_tree/build_tree/Makefile
 create mode 100644 test-src/out_of_tree/config.sh
 create mode 100644 test-src/out_of_tree/expected.json
 create mode 100644 test-src/out_of_tree/source_tree/A/a.cpp
 create mode 100644 test-src/out_of_tree/source_tree/B/b.cpp
 create mode 100644 test-src/out_of_tree/source_tree/llib/lib.cpp
 create mode 100644 test-src/out_of_tree/source_tree/llib/lib.h
 create mode 100644 test-src/static_lib/A/a.cpp
 create mode 100644 test-src/static_lib/B/b.cpp
 create mode 100644 test-src/static_lib/Makefile
 create mode 100644 test-src/static_lib/config.sh
 create mode 100644 test-src/static_lib/expected.json
 create mode 100644 test-src/static_lib/expected_lcov_doubt_counts.json
 create mode 100644 test-src/static_lib/llib/lib.cpp
 create mode 100644 test-src/static_lib/llib/lib.h
 create mode 100644 test-src/template_includes/A.cpp
 create mode 100644 test-src/template_includes/B.cpp
 create mode 100644 test-src/template_includes/Makefile
 create mode 100644 test-src/template_includes/common_header.h
 create mode 100644 test-src/template_includes/config.sh
 create mode 100644 test-src/template_includes/expected.json

diff --git a/test-src/excluded_files/Makefile b/test-src/excluded_files/Makefile
new file mode 100644
index 0000000..8a4347d
--- /dev/null
+++ b/test-src/excluded_files/Makefile
@@ -0,0 +1,19 @@
+part_unused.o: part_unused.h part_unused.cpp
+	g++ --coverage -c -o part_unused.o part_unused.cpp
+
+toy.o: extra_lib/toy.cpp extra_lib/toy.h
+	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+
+a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
+	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+
+foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
+	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
+
+prepare_test: foo
+
+test: foo
+	./foo
diff --git a/test-src/excluded_files/another_extra_lib/another_toy.cpp b/test-src/excluded_files/another_extra_lib/another_toy.cpp
new file mode 100644
index 0000000..d73a010
--- /dev/null
+++ b/test-src/excluded_files/another_extra_lib/another_toy.cpp
@@ -0,0 +1,5 @@
+#include "another_toy.h"
+
+int a_toy(int& i) {
+    i*=2;
+}
diff --git a/test-src/excluded_files/another_extra_lib/another_toy.h b/test-src/excluded_files/another_extra_lib/another_toy.h
new file mode 100644
index 0000000..3786902
--- /dev/null
+++ b/test-src/excluded_files/another_extra_lib/another_toy.h
@@ -0,0 +1 @@
+int a_toy(int& i);
diff --git a/test-src/excluded_files/another_extra_lib/toy.h b/test-src/excluded_files/another_extra_lib/toy.h
new file mode 100644
index 0000000..5afec0b
--- /dev/null
+++ b/test-src/excluded_files/another_extra_lib/toy.h
@@ -0,0 +1 @@
+int toy(int& i);
diff --git a/test-src/excluded_files/config.sh b/test-src/excluded_files/config.sh
new file mode 100644
index 0000000..26b1045
--- /dev/null
+++ b/test-src/excluded_files/config.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+#
+# Only run in gcov mdoe: exclusions are irrelevant when processing lcov files,
+# these should already been done when manually assembling the lcov file...
+#
+modes="gcov"
+
+gcov_coveralls_flags="-e part_unused.cpp -e part_unused.h -e extra_lib -E .*another_extra_lib.*"
+
diff --git a/test-src/excluded_files/expected.json b/test-src/excluded_files/expected.json
new file mode 100644
index 0000000..be04517
--- /dev/null
+++ b/test-src/excluded_files/expected.json
@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.cpp",
+        "coverage": [null, null, null, null, 1, 1, 1, null, 1, 1, 1, 1, 0,  null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }]
+}
diff --git a/test-src/excluded_files/extra_lib/toy.cpp b/test-src/excluded_files/extra_lib/toy.cpp
new file mode 100644
index 0000000..fc22df9
--- /dev/null
+++ b/test-src/excluded_files/extra_lib/toy.cpp
@@ -0,0 +1,5 @@
+#include "toy.h"
+
+int toy(int& i) {
+    i*=2;
+}
diff --git a/test-src/excluded_files/extra_lib/toy.h b/test-src/excluded_files/extra_lib/toy.h
new file mode 100644
index 0000000..5afec0b
--- /dev/null
+++ b/test-src/excluded_files/extra_lib/toy.h
@@ -0,0 +1 @@
+int toy(int& i);
diff --git a/test-src/excluded_files/foo.cpp b/test-src/excluded_files/foo.cpp
new file mode 100644
index 0000000..e23eba5
--- /dev/null
+++ b/test-src/excluded_files/foo.cpp
@@ -0,0 +1,25 @@
+#include "part_unused.h"
+#include "extra_lib/toy.h"
+#include "another_extra_lib/another_toy.h"
+
+int main() {
+    int a = 1;
+    g(a);
+    Inline i;
+    i.inline_g(a);
+    toy(a);
+    a_toy(a);
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}
diff --git a/test-src/excluded_files/part_unused.cpp b/test-src/excluded_files/part_unused.cpp
new file mode 100644
index 0000000..ca539cb
--- /dev/null
+++ b/test-src/excluded_files/part_unused.cpp
@@ -0,0 +1,9 @@
+#include "part_unused.h"
+
+void unused_f(int& i) {
+    i*=2;
+}
+
+void g(int& i) {
+    i*=2;
+}
diff --git a/test-src/excluded_files/part_unused.h b/test-src/excluded_files/part_unused.h
new file mode 100644
index 0000000..1509d30
--- /dev/null
+++ b/test-src/excluded_files/part_unused.h
@@ -0,0 +1,7 @@
+void unused_f(int& i);
+void g(int& i);
+
+struct Inline {
+    void inline_unused_f(int& i) { i*=2; }
+    void inline_g(int& i) { i*=2; }
+};
diff --git a/test-src/included_files/Makefile b/test-src/included_files/Makefile
new file mode 100644
index 0000000..8a4347d
--- /dev/null
+++ b/test-src/included_files/Makefile
@@ -0,0 +1,19 @@
+part_unused.o: part_unused.h part_unused.cpp
+	g++ --coverage -c -o part_unused.o part_unused.cpp
+
+toy.o: extra_lib/toy.cpp extra_lib/toy.h
+	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+
+a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
+	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+
+foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
+	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
+
+prepare_test: foo
+
+test: foo
+	./foo
diff --git a/test-src/included_files/another_extra_lib/another_toy.cpp b/test-src/included_files/another_extra_lib/another_toy.cpp
new file mode 100644
index 0000000..d73a010
--- /dev/null
+++ b/test-src/included_files/another_extra_lib/another_toy.cpp
@@ -0,0 +1,5 @@
+#include "another_toy.h"
+
+int a_toy(int& i) {
+    i*=2;
+}
diff --git a/test-src/included_files/another_extra_lib/another_toy.h b/test-src/included_files/another_extra_lib/another_toy.h
new file mode 100644
index 0000000..3786902
--- /dev/null
+++ b/test-src/included_files/another_extra_lib/another_toy.h
@@ -0,0 +1 @@
+int a_toy(int& i);
diff --git a/test-src/included_files/another_extra_lib/toy.h b/test-src/included_files/another_extra_lib/toy.h
new file mode 100644
index 0000000..5afec0b
--- /dev/null
+++ b/test-src/included_files/another_extra_lib/toy.h
@@ -0,0 +1 @@
+int toy(int& i);
diff --git a/test-src/included_files/config.sh b/test-src/included_files/config.sh
new file mode 100644
index 0000000..c2fd462
--- /dev/null
+++ b/test-src/included_files/config.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#
+# Only run in gcov mdoe: explicit inclusions are irrelevant when processing
+# lcov files, these should already been done when manually assembling the lcov
+# file...
+#
+modes="gcov"
+
+gcov_coveralls_flags="-i foo.cpp"
+
diff --git a/test-src/included_files/expected.json b/test-src/included_files/expected.json
new file mode 100644
index 0000000..be04517
--- /dev/null
+++ b/test-src/included_files/expected.json
@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.cpp",
+        "coverage": [null, null, null, null, 1, 1, 1, null, 1, 1, 1, 1, 0,  null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }]
+}
diff --git a/test-src/included_files/extra_lib/toy.cpp b/test-src/included_files/extra_lib/toy.cpp
new file mode 100644
index 0000000..fc22df9
--- /dev/null
+++ b/test-src/included_files/extra_lib/toy.cpp
@@ -0,0 +1,5 @@
+#include "toy.h"
+
+int toy(int& i) {
+    i*=2;
+}
diff --git a/test-src/included_files/extra_lib/toy.h b/test-src/included_files/extra_lib/toy.h
new file mode 100644
index 0000000..5afec0b
--- /dev/null
+++ b/test-src/included_files/extra_lib/toy.h
@@ -0,0 +1 @@
+int toy(int& i);
diff --git a/test-src/included_files/foo.cpp b/test-src/included_files/foo.cpp
new file mode 100644
index 0000000..e23eba5
--- /dev/null
+++ b/test-src/included_files/foo.cpp
@@ -0,0 +1,25 @@
+#include "part_unused.h"
+#include "extra_lib/toy.h"
+#include "another_extra_lib/another_toy.h"
+
+int main() {
+    int a = 1;
+    g(a);
+    Inline i;
+    i.inline_g(a);
+    toy(a);
+    a_toy(a);
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}
diff --git a/test-src/included_files/part_unused.cpp b/test-src/included_files/part_unused.cpp
new file mode 100644
index 0000000..ca539cb
--- /dev/null
+++ b/test-src/included_files/part_unused.cpp
@@ -0,0 +1,9 @@
+#include "part_unused.h"
+
+void unused_f(int& i) {
+    i*=2;
+}
+
+void g(int& i) {
+    i*=2;
+}
diff --git a/test-src/included_files/part_unused.h b/test-src/included_files/part_unused.h
new file mode 100644
index 0000000..1509d30
--- /dev/null
+++ b/test-src/included_files/part_unused.h
@@ -0,0 +1,7 @@
+void unused_f(int& i);
+void g(int& i);
+
+struct Inline {
+    void inline_unused_f(int& i) { i*=2; }
+    void inline_g(int& i) { i*=2; }
+};
diff --git a/test-src/missing_files/Makefile b/test-src/missing_files/Makefile
new file mode 100644
index 0000000..dc37fb9
--- /dev/null
+++ b/test-src/missing_files/Makefile
@@ -0,0 +1,12 @@
+unused.o: unused.h unused.cpp
+	gcc --coverage -c -o unused.o unused.cpp
+foo: foo.c unused.o
+	gcc --coverage -o foo foo.c
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json unused.o
+
+prepare_test: foo
+
+test: foo
+	./foo
diff --git a/test-src/missing_files/config.sh b/test-src/missing_files/config.sh
new file mode 100644
index 0000000..d4e4fb0
--- /dev/null
+++ b/test-src/missing_files/config.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Override to change the file that contains the expected output for the gcov run
+gcov_expected_output="expected_gcov.json"
+
+# Override to change the file that contains the expected output for the lcov run
+lcov_expected_output="expected_lcov.json"
+
+gcc_version=$(g++ -v 2>&1 | awk '/gcc version/ {print $3}')
+if [[ "$gcc_version" < "5.5" ]]; then
+    echo "LEGACY G++ detected: skipping GCOV missing files test"
+    modes="lcov"
+else
+    modes="gcov lcov"
+fi
diff --git a/test-src/missing_files/expected_gcov.json b/test-src/missing_files/expected_gcov.json
new file mode 100644
index 0000000..bff7cb0
--- /dev/null
+++ b/test-src/missing_files/expected_gcov.json
@@ -0,0 +1,12 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }, {
+        "name": "unused.h",
+        "coverage": [null, null, null, 0, null]
+    }, {
+        "name": "unused.cpp",
+        "coverage": [null, null, 0, null, 0, 0, null]
+    }]
+}
diff --git a/test-src/missing_files/expected_lcov.json b/test-src/missing_files/expected_lcov.json
new file mode 100644
index 0000000..2c2a4c6
--- /dev/null
+++ b/test-src/missing_files/expected_lcov.json
@@ -0,0 +1,12 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }, {
+        "name": "unused.h",
+        "coverage": [null, null, null, 0, null]
+    }, {
+        "name": "unused.cpp",
+        "coverage": [null, null, 0, null, 0, 0, 0]
+    }]
+}
diff --git a/test-src/missing_files/foo.c b/test-src/missing_files/foo.c
new file mode 100644
index 0000000..9462b55
--- /dev/null
+++ b/test-src/missing_files/foo.c
@@ -0,0 +1,16 @@
+int main() {
+    int a = 1;
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}
diff --git a/test-src/missing_files/unused.cpp b/test-src/missing_files/unused.cpp
new file mode 100644
index 0000000..84467f1
--- /dev/null
+++ b/test-src/missing_files/unused.cpp
@@ -0,0 +1,7 @@
+#include "unused.h"
+
+void unused_f(int& i) {
+    Inline in;
+    in.infline_f(i);
+    i*=2;
+}
diff --git a/test-src/missing_files/unused.h b/test-src/missing_files/unused.h
new file mode 100644
index 0000000..cbfef43
--- /dev/null
+++ b/test-src/missing_files/unused.h
@@ -0,0 +1,5 @@
+void unused_f(int& i);
+
+struct Inline {
+    void infline_f(int& i) { i*=2; }
+};
diff --git a/test-src/out_of_tree/build_tree/Makefile b/test-src/out_of_tree/build_tree/Makefile
new file mode 100644
index 0000000..77e7cda
--- /dev/null
+++ b/test-src/out_of_tree/build_tree/Makefile
@@ -0,0 +1,23 @@
+SOURCE_TREE=../source_tree
+INCLUDE_FLAGS=-I$(SOURCE_TREE)/llib
+
+lib.o: $(SOURCE_TREE)/llib/lib.cpp $(SOURCE_TREE)/llib/lib.h
+	g++ --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
+
+lib.a: lib.o
+	ar cr $@ $+
+
+a: $(SOURCE_TREE)/A/a.cpp lib.a
+	g++ --coverage $(INCLUDE_FLAGS) -o a $+
+
+b: $(SOURCE_TREE)/B/b.cpp lib.a
+	g++ --coverage $(INCLUDE_FLAGS) -o b $+
+
+prepare_test: a b
+
+test: prepare_test
+	./a
+	./b
+
+clean:
+	rm -f lib.o lib.a a b a.gc* b.gc* lib.gc* lcov.info
diff --git a/test-src/out_of_tree/config.sh b/test-src/out_of_tree/config.sh
new file mode 100644
index 0000000..b5402f7
--- /dev/null
+++ b/test-src/out_of_tree/config.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Set if we expect the GCOV test for a known reason
+gcov_fail_reason=""
+
+# We must invoke "make" in the build tree
+build_dir="build_tree"
+
+
+
+# lcov must be invoked in the build-tree, which means we must:
+#    -> tell lcov where the source tree is 
+#    -> tell coveralls to strip the "../source_tree prefix off of
+#       the paths
+lcov_dir="build_tree"
+lcov_coveralls_dir="build_tree"
+lcov_coveralls_flags="--root .."
+lcov_flags="-b ../source_tree"
+
+
diff --git a/test-src/out_of_tree/expected.json b/test-src/out_of_tree/expected.json
new file mode 100644
index 0000000..c64de5f
--- /dev/null
+++ b/test-src/out_of_tree/expected.json
@@ -0,0 +1,15 @@
+{
+    "source_files": [{
+        "name": "source_tree/llib/lib.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, 1]
+    }, {
+        "name": "source_tree/llib/lib.h",
+        "coverage": [null, null, null, null, null, 1, 1,  null]
+    }, {
+        "name": "source_tree/A/a.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }, {
+        "name": "source_tree/B/b.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }]
+}
diff --git a/test-src/out_of_tree/source_tree/A/a.cpp b/test-src/out_of_tree/source_tree/A/a.cpp
new file mode 100644
index 0000000..6109e32
--- /dev/null
+++ b/test-src/out_of_tree/source_tree/A/a.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+int main() {
+    int ret = 0;
+    double_it(ret);
+    inline_double id;
+    id.f(ret);
+    return ret;
+}
diff --git a/test-src/out_of_tree/source_tree/B/b.cpp b/test-src/out_of_tree/source_tree/B/b.cpp
new file mode 100644
index 0000000..68e354b
--- /dev/null
+++ b/test-src/out_of_tree/source_tree/B/b.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+int main() {
+    int ret = 0;
+    tripple_it(ret);
+    inline_double id;
+    id.g(ret);
+    return ret;
+}
diff --git a/test-src/out_of_tree/source_tree/llib/lib.cpp b/test-src/out_of_tree/source_tree/llib/lib.cpp
new file mode 100644
index 0000000..06a1479
--- /dev/null
+++ b/test-src/out_of_tree/source_tree/llib/lib.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+void double_it(int& i) {
+    i*=2;
+}
+
+void tripple_it(int& i) {
+    i*=3;
+}
diff --git a/test-src/out_of_tree/source_tree/llib/lib.h b/test-src/out_of_tree/source_tree/llib/lib.h
new file mode 100644
index 0000000..1e73f48
--- /dev/null
+++ b/test-src/out_of_tree/source_tree/llib/lib.h
@@ -0,0 +1,8 @@
+void double_it(int& i);
+
+void tripple_it(int& i);
+
+struct inline_double {
+    void f(int& i) { i*=2; }
+    void g(int& i) { i*=2; }
+};
diff --git a/test-src/simple/Makefile b/test-src/simple/Makefile
index 532d868..d7628b7 100644
--- a/test-src/simple/Makefile
+++ b/test-src/simple/Makefile
@@ -4,5 +4,7 @@ foo: foo.c
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json
 
-test: foo
+prepare_test: foo
+
+test: prepare_test
 	./foo
diff --git a/test-src/static_lib/A/a.cpp b/test-src/static_lib/A/a.cpp
new file mode 100644
index 0000000..6109e32
--- /dev/null
+++ b/test-src/static_lib/A/a.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+int main() {
+    int ret = 0;
+    double_it(ret);
+    inline_double id;
+    id.f(ret);
+    return ret;
+}
diff --git a/test-src/static_lib/B/b.cpp b/test-src/static_lib/B/b.cpp
new file mode 100644
index 0000000..68e354b
--- /dev/null
+++ b/test-src/static_lib/B/b.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+int main() {
+    int ret = 0;
+    tripple_it(ret);
+    inline_double id;
+    id.g(ret);
+    return ret;
+}
diff --git a/test-src/static_lib/Makefile b/test-src/static_lib/Makefile
new file mode 100644
index 0000000..ec9ae8e
--- /dev/null
+++ b/test-src/static_lib/Makefile
@@ -0,0 +1,20 @@
+llib/lib.o: llib/lib.cpp llib/lib.h
+	g++ --coverage -c -o $@ llib/lib.cpp
+
+llib/lib.a: llib/lib.o
+	ar cr $@ $+
+
+A/a: A/a.cpp llib/lib.a
+	cd A && g++ --coverage -I../llib -o a a.cpp ../llib/lib.a
+
+B/b: B/b.cpp llib/lib.a
+	cd B && g++ --coverage -I../llib -o b b.cpp ../llib/lib.a
+
+prepare_test: A/a B/b
+
+test: prepare_test
+	cd A && ./a
+	cd B && ./b
+
+clean:
+	rm -f llib/lib.o llib/lib.a A/a B/b A/a.gc* B/b.gc* llib/lib.*gc* lcov.info
diff --git a/test-src/static_lib/config.sh b/test-src/static_lib/config.sh
new file mode 100644
index 0000000..a9bf588
--- /dev/null
+++ b/test-src/static_lib/config.sh
@@ -0,0 +1 @@
+#!/bin/bash
diff --git a/test-src/static_lib/expected.json b/test-src/static_lib/expected.json
new file mode 100644
index 0000000..06ebeee
--- /dev/null
+++ b/test-src/static_lib/expected.json
@@ -0,0 +1,15 @@
+{
+    "source_files": [{
+        "name": "llib/lib.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, 1]
+    }, {
+        "name": "llib/lib.h",
+        "coverage": [null, null, null, null, null, 1, 1,  null]
+    }, {
+        "name": "A/a.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }, {
+        "name": "B/b.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }]
+}
diff --git a/test-src/static_lib/expected_lcov_doubt_counts.json b/test-src/static_lib/expected_lcov_doubt_counts.json
new file mode 100644
index 0000000..30abdbc
--- /dev/null
+++ b/test-src/static_lib/expected_lcov_doubt_counts.json
@@ -0,0 +1,15 @@
+{
+    "source_files": [{
+        "name": "llib/lib.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, 1]
+    }, {
+        "name": "llib/lib.h",
+        "coverage": [null, null, null, null, null,  2, 2, null]
+    }, {
+        "name": "A/a.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }, {
+        "name": "B/b.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }]
+}
diff --git a/test-src/static_lib/llib/lib.cpp b/test-src/static_lib/llib/lib.cpp
new file mode 100644
index 0000000..06a1479
--- /dev/null
+++ b/test-src/static_lib/llib/lib.cpp
@@ -0,0 +1,9 @@
+#include "lib.h"
+
+void double_it(int& i) {
+    i*=2;
+}
+
+void tripple_it(int& i) {
+    i*=3;
+}
diff --git a/test-src/static_lib/llib/lib.h b/test-src/static_lib/llib/lib.h
new file mode 100644
index 0000000..1e73f48
--- /dev/null
+++ b/test-src/static_lib/llib/lib.h
@@ -0,0 +1,8 @@
+void double_it(int& i);
+
+void tripple_it(int& i);
+
+struct inline_double {
+    void f(int& i) { i*=2; }
+    void g(int& i) { i*=2; }
+};
diff --git a/test-src/template_includes/A.cpp b/test-src/template_includes/A.cpp
new file mode 100644
index 0000000..9a12929
--- /dev/null
+++ b/test-src/template_includes/A.cpp
@@ -0,0 +1,6 @@
+#include "common_header.h"
+
+int main() {
+    Util<int> u;
+    return u.Method_A(0);
+}
diff --git a/test-src/template_includes/B.cpp b/test-src/template_includes/B.cpp
new file mode 100644
index 0000000..7b5584c
--- /dev/null
+++ b/test-src/template_includes/B.cpp
@@ -0,0 +1,6 @@
+#include "common_header.h"
+
+int main() {
+    Util<char> u;
+    return (int) u.Method_B(0);
+}
diff --git a/test-src/template_includes/Makefile b/test-src/template_includes/Makefile
new file mode 100644
index 0000000..156264b
--- /dev/null
+++ b/test-src/template_includes/Makefile
@@ -0,0 +1,14 @@
+A: A.cpp common_header.h
+	g++ --coverage -o A A.cpp
+
+B: B.cpp common_header.h
+	g++ --coverage -o B B.cpp
+
+clean:
+	rm -f B A *.gcno *.gcda output *.gcov output.json *.gch
+
+prepare_test: A B
+
+test: prepare_test
+	./A
+	./B
diff --git a/test-src/template_includes/common_header.h b/test-src/template_includes/common_header.h
new file mode 100644
index 0000000..b3000fb
--- /dev/null
+++ b/test-src/template_includes/common_header.h
@@ -0,0 +1,10 @@
+template<class T>
+struct Util {
+    T Method_A(const T& i) {
+        return 2*i;
+    }
+
+    T Method_B(const T& i) {
+        return 3*i;
+    }
+};
diff --git a/test-src/template_includes/config.sh b/test-src/template_includes/config.sh
new file mode 100644
index 0000000..472c6b8
--- /dev/null
+++ b/test-src/template_includes/config.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Set if we expect the GCOV test for a known reason
+gcov_fail_reason=""
+
+# Set if we expect the LCOV test for a known reason
+lcov_fail_reason=""
+
+# Set to provide additional flags to cpp-coveralls when in lcov mode
+lcov_coveralls_flags=""
diff --git a/test-src/template_includes/expected.json b/test-src/template_includes/expected.json
new file mode 100644
index 0000000..003bed0
--- /dev/null
+++ b/test-src/template_includes/expected.json
@@ -0,0 +1,12 @@
+{
+    "source_files": [{
+            "name": "A.cpp",
+            "coverage": [null, null, 1, null, 1, null]
+        }, {
+            "name": "B.cpp",
+            "coverage": [null, null, 1, null, 1, null]
+        }, {
+            "name": "common_header.h",
+            "coverage": [null, null, 1, 1, null, null, 1, 1, null, null]
+        }]
+}
diff --git a/test-utils/testDir.sh b/test-utils/testDir.sh
index 4b3bcae..b35c895 100755
--- a/test-utils/testDir.sh
+++ b/test-utils/testDir.sh
@@ -259,7 +259,7 @@ function CheckGCOV {
 
     Make $logfile test
 
-    CppCoverals $logfile "$gcov_coveralls_dir" --dump $PWD/output.json --verbose  $gcov_coveralls_flags
+    CppCoverals $logfile "$gcov_coveralls_dir" --gcov-options '\-lp' --dump $PWD/output.json --verbose  $gcov_coveralls_flags
 
     CheckResult $logfile "$gcov_expected_output" "$gcov_fail_reason"
 
diff --git a/test.bash b/test.bash
index cd8bccd..77622af 100755
--- a/test.bash
+++ b/test.bash
@@ -27,6 +27,12 @@ function TestFailed {
 ParseArguments $@
 
 testDir.sh $@ test-src/simple || TestFailed
+testDir.sh $@ test-src/included_files || TestFailed
+testDir.sh $@ test-src/excluded_files || TestFailed
+testDir.sh $@ test-src/static_lib || TestFailed
+testDir.sh $@ test-src/out_of_tree  || TestFailed
+testDir.sh $@ test-src/missing_files  || TestFailed
+testDir.sh $@ test-src/template_includes/ || TestFailed
 
 if [[ $failed_tests -gt 0 ]]; then
     echo ""

From b4c120accafc065d5bcad0431b3603da2e9b81f6 Mon Sep 17 00:00:00 2001
From: "Luke.Humphreys" <lhumphreys67@gmail.com>
Date: Sun, 27 Oct 2019 16:15:16 +0000
Subject: [PATCH 3/3] Run the regresion tests over multiple gcc versions

Whilst investigating issues with dodgey coverage being reported to
coveralls, it was found that the issue was dependent on the version of
g++/gcov used. Extending the existing test suite to multiple gcc
versions shows this up.

The issue is that in version 8, gcov changed the format of its output,
which makes the assumptions made in the current parser invalid. Tools
such as LCOV have already migrated to use gcov's new JSON format to work
around this issue.

For now the affected tests are marked as known failure to prevent a
failing build.
---
 .gitignore                               |  2 +-
 .travis.yml                              | 15 ++++-
 test-src/excluded_files/Makefile         |  8 +--
 test-src/included_files/Makefile         |  8 +--
 test-src/missing_files/Makefile          |  4 +-
 test-src/missing_files/config.sh         |  4 +-
 test-src/out_of_tree/build_tree/Makefile |  6 +-
 test-src/simple/Makefile                 |  2 +-
 test-src/static_lib/Makefile             |  6 +-
 test-src/static_lib/config.sh            |  6 ++
 test-src/template_includes/A.cpp         |  2 +
 test-src/template_includes/B.cpp         |  2 +
 test-src/template_includes/Makefile      |  4 +-
 test-src/template_includes/config.sh     |  7 ++-
 test-src/template_includes/expected.json |  4 +-
 test-utils/testDir.sh                    | 77 ++++++++++++++++--------
 16 files changed, 107 insertions(+), 50 deletions(-)

diff --git a/.gitignore b/.gitignore
index 17f14c3..5e33d71 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
-*_test_output
+*_test_output*
 *.gcov
 output.json
 *.gcno
diff --git a/.travis.yml b/.travis.yml
index 6d69b60..1957f93 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,12 +9,25 @@ python:
 
 addons:
   apt:
+    sources:
+      - ubuntu-toolchain-r-test
     packages:
       - lcov
+      - g++-4.8
+      - g++-5
+      - g++-6
+      - g++-7
+      - g++-8
+      - g++-9
+      - g++-9
+# Required for latest lcov build - TODO: remove the need for this...
+      - libperlio-gzip-perl
+      - libjson-perl
 
 install:
     - python setup.py --quiet install
 
 script:
     - nosetests
-    - ./test.bash
+    - git clone https://github.com/linux-test-project/lcov.git
+    - PATH="$PWD/lcov/bin:$PATH" ./test.bash --gcc-version 4.8 --gcc-version 5 --gcc-version 6 --gcc-version 7 --gcc-version 8 --gcc-version 9
diff --git a/test-src/excluded_files/Makefile b/test-src/excluded_files/Makefile
index 8a4347d..bedb866 100644
--- a/test-src/excluded_files/Makefile
+++ b/test-src/excluded_files/Makefile
@@ -1,14 +1,14 @@
 part_unused.o: part_unused.h part_unused.cpp
-	g++ --coverage -c -o part_unused.o part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
 
 toy.o: extra_lib/toy.cpp extra_lib/toy.h
-	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
 
 a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
-	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
 
 foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
-	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
diff --git a/test-src/included_files/Makefile b/test-src/included_files/Makefile
index 8a4347d..bedb866 100644
--- a/test-src/included_files/Makefile
+++ b/test-src/included_files/Makefile
@@ -1,14 +1,14 @@
 part_unused.o: part_unused.h part_unused.cpp
-	g++ --coverage -c -o part_unused.o part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
 
 toy.o: extra_lib/toy.cpp extra_lib/toy.h
-	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
 
 a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
-	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
 
 foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
-	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
diff --git a/test-src/missing_files/Makefile b/test-src/missing_files/Makefile
index dc37fb9..5780c23 100644
--- a/test-src/missing_files/Makefile
+++ b/test-src/missing_files/Makefile
@@ -1,7 +1,7 @@
 unused.o: unused.h unused.cpp
-	gcc --coverage -c -o unused.o unused.cpp
+	$(CXX) --coverage -c -o unused.o unused.cpp
 foo: foo.c unused.o
-	gcc --coverage -o foo foo.c
+	$(CXX) --coverage -o foo foo.c
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json unused.o
diff --git a/test-src/missing_files/config.sh b/test-src/missing_files/config.sh
index d4e4fb0..4b08666 100644
--- a/test-src/missing_files/config.sh
+++ b/test-src/missing_files/config.sh
@@ -6,9 +6,9 @@ gcov_expected_output="expected_gcov.json"
 # Override to change the file that contains the expected output for the lcov run
 lcov_expected_output="expected_lcov.json"
 
-gcc_version=$(g++ -v 2>&1 | awk '/gcc version/ {print $3}')
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
 if [[ "$gcc_version" < "5.5" ]]; then
-    echo "LEGACY G++ detected: skipping GCOV missing files test"
+    echo "LEGACY G++ detected ($gcc_version: $CXX): skipping GCOV missing files test"
     modes="lcov"
 else
     modes="gcov lcov"
diff --git a/test-src/out_of_tree/build_tree/Makefile b/test-src/out_of_tree/build_tree/Makefile
index 77e7cda..1598ab4 100644
--- a/test-src/out_of_tree/build_tree/Makefile
+++ b/test-src/out_of_tree/build_tree/Makefile
@@ -2,16 +2,16 @@ SOURCE_TREE=../source_tree
 INCLUDE_FLAGS=-I$(SOURCE_TREE)/llib
 
 lib.o: $(SOURCE_TREE)/llib/lib.cpp $(SOURCE_TREE)/llib/lib.h
-	g++ --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
+	$(CXX) --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
 
 lib.a: lib.o
 	ar cr $@ $+
 
 a: $(SOURCE_TREE)/A/a.cpp lib.a
-	g++ --coverage $(INCLUDE_FLAGS) -o a $+
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o a $+
 
 b: $(SOURCE_TREE)/B/b.cpp lib.a
-	g++ --coverage $(INCLUDE_FLAGS) -o b $+
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o b $+
 
 prepare_test: a b
 
diff --git a/test-src/simple/Makefile b/test-src/simple/Makefile
index d7628b7..7b983c3 100644
--- a/test-src/simple/Makefile
+++ b/test-src/simple/Makefile
@@ -1,5 +1,5 @@
 foo: foo.c
-	gcc --coverage -o foo foo.c
+	$(CXX) --coverage -o foo foo.c
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json
diff --git a/test-src/static_lib/Makefile b/test-src/static_lib/Makefile
index ec9ae8e..a432f1c 100644
--- a/test-src/static_lib/Makefile
+++ b/test-src/static_lib/Makefile
@@ -1,14 +1,14 @@
 llib/lib.o: llib/lib.cpp llib/lib.h
-	g++ --coverage -c -o $@ llib/lib.cpp
+	$(CXX) --coverage -c -o $@ llib/lib.cpp
 
 llib/lib.a: llib/lib.o
 	ar cr $@ $+
 
 A/a: A/a.cpp llib/lib.a
-	cd A && g++ --coverage -I../llib -o a a.cpp ../llib/lib.a
+	cd A && $(CXX) --coverage -I../llib -o a a.cpp ../llib/lib.a
 
 B/b: B/b.cpp llib/lib.a
-	cd B && g++ --coverage -I../llib -o b b.cpp ../llib/lib.a
+	cd B && $(CXX) --coverage -I../llib -o b b.cpp ../llib/lib.a
 
 prepare_test: A/a B/b
 
diff --git a/test-src/static_lib/config.sh b/test-src/static_lib/config.sh
index a9bf588..c607139 100644
--- a/test-src/static_lib/config.sh
+++ b/test-src/static_lib/config.sh
@@ -1 +1,7 @@
 #!/bin/bash
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
+if [[ "$gcc_version" < "8" ]]; then
+    gcov_fail_reason=""
+else
+    gcov_fail_reason="cpp-coveralls parser does not support modern gcov's output"
+fi
diff --git a/test-src/template_includes/A.cpp b/test-src/template_includes/A.cpp
index 9a12929..37c1a3f 100644
--- a/test-src/template_includes/A.cpp
+++ b/test-src/template_includes/A.cpp
@@ -1,5 +1,7 @@
 #include "common_header.h"
 
+template class Util <int>;
+template class Util <char>;
 int main() {
     Util<int> u;
     return u.Method_A(0);
diff --git a/test-src/template_includes/B.cpp b/test-src/template_includes/B.cpp
index 7b5584c..580f772 100644
--- a/test-src/template_includes/B.cpp
+++ b/test-src/template_includes/B.cpp
@@ -1,5 +1,7 @@
 #include "common_header.h"
 
+template class Util <int>;
+template class Util <char>;
 int main() {
     Util<char> u;
     return (int) u.Method_B(0);
diff --git a/test-src/template_includes/Makefile b/test-src/template_includes/Makefile
index 156264b..970f9ce 100644
--- a/test-src/template_includes/Makefile
+++ b/test-src/template_includes/Makefile
@@ -1,8 +1,8 @@
 A: A.cpp common_header.h
-	g++ --coverage -o A A.cpp
+	$(CXX) --coverage -o A A.cpp
 
 B: B.cpp common_header.h
-	g++ --coverage -o B B.cpp
+	$(CXX) --coverage -o B B.cpp
 
 clean:
 	rm -f B A *.gcno *.gcda output *.gcov output.json *.gch
diff --git a/test-src/template_includes/config.sh b/test-src/template_includes/config.sh
index 472c6b8..c18da7a 100644
--- a/test-src/template_includes/config.sh
+++ b/test-src/template_includes/config.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
 # Set if we expect the GCOV test for a known reason
-gcov_fail_reason=""
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
+if [[ "$gcc_version" < "8" ]]; then
+    gcov_fail_reason=""
+else
+    gcov_fail_reason="cpp-coveralls parser does not support modern gcov's output"
+fi
 
 # Set if we expect the LCOV test for a known reason
 lcov_fail_reason=""
diff --git a/test-src/template_includes/expected.json b/test-src/template_includes/expected.json
index 003bed0..3c740ec 100644
--- a/test-src/template_includes/expected.json
+++ b/test-src/template_includes/expected.json
@@ -1,10 +1,10 @@
 {
     "source_files": [{
             "name": "A.cpp",
-            "coverage": [null, null, 1, null, 1, null]
+            "coverage": [null, null, null, null, 1, null, 1, null]
         }, {
             "name": "B.cpp",
-            "coverage": [null, null, 1, null, 1, null]
+            "coverage": [null, null, null, null, 1, null, 1, null]
         }, {
             "name": "common_header.h",
             "coverage": [null, null, 1, 1, null, null, 1, 1, null, null]
diff --git a/test-utils/testDir.sh b/test-utils/testDir.sh
index b35c895..7594afc 100755
--- a/test-utils/testDir.sh
+++ b/test-utils/testDir.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+gcc_versions=""
+default_gcc_versions="4.8 5 6 7 8 9"
+
 # START: Valid config.sh options
 #    The following variables can be set in the directory's config.sh file
 
@@ -13,7 +16,6 @@ lcov_fail_reason=""
 #
 modes="gcov lcov"
 
-
 # Set to override the build directory (default is the test's root directory).
 # This is where "make" will be invoked.
 # Path should relative to the test's root directory
@@ -59,10 +61,13 @@ exit_on_fail="no"
 
 
 function Usage {
-    echo "testDir.sh [-e/ --exit-on-fail] test-src/<dir to test>"
+    echo "testDir.sh [-e/ --exit-on-fail] [--gcc-version <version>] test-src/<dir to test>"
     echo ""
     echo "Options:"
     echo "   -e / --exit-on-fail: Stop execution at the first unexpected failure."
+    echo "   --gcc_version      : Version of gcc to test (e.g 5 for g++-5 / gcov-5)"
+    echo "                        (May be specified multiple times)"
+    echo "                        By default the following versions are tested: $default_gcc_versions"
 }
 
 #
@@ -72,21 +77,45 @@ function Usage {
 #     @: The set of arguments parsed to this script
 #
 function ParseArguments {
-    if [[ "$1" == "-e" || "$1" == "--exit-on-fail" ]]; then
-        exit_on_fail="yes"
-        shift
+    done="no"
+    while [[ "$done" != "yes" ]]; do
+        if [[ "$1" == "-e" || "$1" == "--exit-on-fail" ]]; then
+            exit_on_fail="yes"
+            shift
+        elif [[ "$1" == "--gcc-version" ]]; then
+            gcc_versions+=" $2"
+            shift
+            shift
+        else
+            done="yes"
+        fi
+    done
+
+    if [[ "$gcc_versions" == "" ]]; then
+        gcc_versions=$default_gcc_versions
     fi
 
+
     test_dir=$1
 
     if [[ "$test_dir" == "" || ! -d "$test_dir" ]]; then
         echo "Cannot find test directory: $test_dir"
         Usage
         exit 1
+    else
+        # hack to get the absolute path
+        pushd $test_dir > /dev/null || exit 1
+        test_dir=$PWD
+        popd > /dev/null || exit 1
     fi
 }
 
 function SetupEnvironment {
+    gcc_version=$1
+
+    export CXX="g++-$gcc_version"
+    export GCOV="gcov-$gcc_version"
+
     if [ -z "$TRAVIS_JOB_ID" ]; then
         export COVERALLS_REPO_TOKEN="fake testing token"
     fi
@@ -252,14 +281,14 @@ function Exit {
 #  Arguments: None
 #
 function CheckGCOV {
-    echo -n ">> Checking $PWD using GCOV..."
+    echo -n ">> Checking $PWD using GCOV (CXX:$CXX, GCOV:$GCOV)..."
 
-    logfile="$PWD/gcov_test_output"
+    logfile="$PWD/gcov_test_output_$gcc_version"
     CleanUp  > $logfile 2>&1
 
     Make $logfile test
 
-    CppCoverals $logfile "$gcov_coveralls_dir" --gcov-options '\-lp' --dump $PWD/output.json --verbose  $gcov_coveralls_flags
+    CppCoverals $logfile "$gcov_coveralls_dir" --gcov $GCOV --gcov-options '\-lp' --dump $PWD/output.json --verbose  $gcov_coveralls_flags
 
     CheckResult $logfile "$gcov_expected_output" "$gcov_fail_reason"
 
@@ -272,20 +301,20 @@ function CheckGCOV {
 #  Arguments: None
 #
 function CheckLCOV {
-    echo -n ">> Checking $PWD using LCOV..."
+    echo -n ">> Checking $PWD using LCOV (CXX:$CXX, GCOV:$GCOV)..."
 
-    logfile="$PWD/lcov_test_output"
+    logfile="$PWD/lcov_test_output_$gcc_version"
     CleanUp  > $logfile 2>&1
 
     Make $logfile prepare_test
 
-    LCov $logfile --capture -d . -i --output-file="$PWD/lcov_baseline.info"
+    LCov $logfile --gcov-tool "$GCOV" --capture -d . -i --output-file="$PWD/lcov_baseline.info"
 
     Make $logfile test
 
-    LCov $logfile --capture -d . --output-file="$PWD/lcov_test_run.info"
+    LCov $logfile --gcov-tool "$GCOV" --capture -d . --output-file="$PWD/lcov_test_run.info"
 
-    LCov $logfile $lcov_flags -a $PWD/lcov_baseline.info -a $PWD/lcov_test_run.info -o "$PWD/lcov.info"
+    LCov $logfile --gcov-tool "$GCOV" $lcov_flags -a $PWD/lcov_baseline.info -a $PWD/lcov_test_run.info -o "$PWD/lcov.info"
 
     CppCoverals $logfile "$lcov_coveralls_dir" --dump $PWD/output.json --no-gcov --lcov-file="$PWD/lcov.info" $lcov_coveralls_flags
 
@@ -307,18 +336,18 @@ function CleanUp {
 
 ParseArguments $@
 
-SetupEnvironment
+for v in $gcc_versions; do
+    SetupEnvironment $v
 
-for mode in $modes; do
-    if [[ "$mode" == "gcov" ]]; then
-        CheckGCOV
-    elif [[ "$mode" == "lcov" ]]; then
-        CheckLCOV
-    else
-        Exit /dev/stdout "Unknown test mode requested: $mode"
-    fi
+    for mode in $modes; do
+        if [[ "$mode" == "gcov" ]]; then
+            CheckGCOV
+        elif [[ "$mode" == "lcov" ]]; then
+            CheckLCOV
+        else
+            Exit /dev/stdout "Unknown test mode requested: $mode"
+        fi
+    done
 done
 
-
-
 exit $failed_tests