Add a set of regression tests for common GCOV and LCOV workflows

.gitignore

@@ -25,6 +25,13 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+*_test_output*
+*.gcov
+output.json
+*.gcno
+*.gcda
+*.info
+
 
 # Translations
 *.mo

.travis.yml

@@ -7,9 +7,27 @@ python:
     - "3.4"
     - "3.5"
 
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - lcov
+      - g++-4.8
+      - g++-5
+      - g++-6
+      - g++-7
+      - g++-8
+      - g++-9
+      - g++-9
+# Required for latest lcov build - TODO: remove the need for this...
+      - libperlio-gzip-perl
+      - libjson-perl
+
 install:
     - python setup.py --quiet install
 
 script:
     - nosetests
-    - ./test.bash
+    - git clone https://github.com/linux-test-project/lcov.git
+    - PATH="$PWD/lcov/bin:$PATH" ./test.bash --gcc-version 4.8 --gcc-version 5 --gcc-version 6 --gcc-version 7 --gcc-version 8 --gcc-version 9

test-src/excluded_files/Makefile

@@ -0,0 +1,19 @@
+part_unused.o: part_unused.h part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
+
+toy.o: extra_lib/toy.cpp extra_lib/toy.h
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
+
+a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+
+foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
+
+prepare_test: foo
+
+test: foo
+	./foo

test-src/excluded_files/another_extra_lib/another_toy.cpp

@@ -0,0 +1,5 @@
+#include "another_toy.h"
+
+int a_toy(int& i) {
+    i*=2;
+}

test-src/excluded_files/another_extra_lib/another_toy.h

@@ -0,0 +1 @@
+int a_toy(int& i);

test-src/excluded_files/another_extra_lib/toy.h

@@ -0,0 +1 @@
+int toy(int& i);

test-src/excluded_files/config.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+#
+# Only run in gcov mdoe: exclusions are irrelevant when processing lcov files,
+# these should already been done when manually assembling the lcov file...
+#
+modes="gcov"
+
+gcov_coveralls_flags="-e part_unused.cpp -e part_unused.h -e extra_lib -E .*another_extra_lib.*"
+

test-src/excluded_files/expected.json

@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.cpp",
+        "coverage": [null, null, null, null, 1, 1, 1, null, 1, 1, 1, 1, 0,  null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }]
+}

test-src/excluded_files/extra_lib/toy.cpp

@@ -0,0 +1,5 @@
+#include "toy.h"
+
+int toy(int& i) {
+    i*=2;
+}

test-src/excluded_files/extra_lib/toy.h

@@ -0,0 +1 @@
+int toy(int& i);

test-src/excluded_files/foo.cpp

@@ -0,0 +1,25 @@
+#include "part_unused.h"
+#include "extra_lib/toy.h"
+#include "another_extra_lib/another_toy.h"
+
+int main() {
+    int a = 1;
+    g(a);
+    Inline i;
+    i.inline_g(a);
+    toy(a);
+    a_toy(a);
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}

test-src/excluded_files/part_unused.cpp

@@ -0,0 +1,9 @@
+#include "part_unused.h"
+
+void unused_f(int& i) {
+    i*=2;
+}
+
+void g(int& i) {
+    i*=2;
+}

test-src/excluded_files/part_unused.h

@@ -0,0 +1,7 @@
+void unused_f(int& i);
+void g(int& i);
+
+struct Inline {
+    void inline_unused_f(int& i) { i*=2; }
+    void inline_g(int& i) { i*=2; }
+};

test-src/included_files/Makefile

@@ -0,0 +1,19 @@
+part_unused.o: part_unused.h part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
+
+toy.o: extra_lib/toy.cpp extra_lib/toy.h
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
+
+a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+
+foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o
+
+prepare_test: foo
+
+test: foo
+	./foo

test-src/included_files/another_extra_lib/another_toy.cpp

@@ -0,0 +1,5 @@
+#include "another_toy.h"
+
+int a_toy(int& i) {
+    i*=2;
+}

test-src/included_files/another_extra_lib/another_toy.h

@@ -0,0 +1 @@
+int a_toy(int& i);

test-src/included_files/another_extra_lib/toy.h

@@ -0,0 +1 @@
+int toy(int& i);

test-src/included_files/config.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#
+# Only run in gcov mdoe: explicit inclusions are irrelevant when processing
+# lcov files, these should already been done when manually assembling the lcov
+# file...
+#
+modes="gcov"
+
+gcov_coveralls_flags="-i foo.cpp"
+

test-src/included_files/expected.json

@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.cpp",
+        "coverage": [null, null, null, null, 1, 1, 1, null, 1, 1, 1, 1, 0,  null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }]
+}

test-src/included_files/extra_lib/toy.cpp

@@ -0,0 +1,5 @@
+#include "toy.h"
+
+int toy(int& i) {
+    i*=2;
+}

test-src/included_files/extra_lib/toy.h

@@ -0,0 +1 @@
+int toy(int& i);

test-src/included_files/foo.cpp

@@ -0,0 +1,25 @@
+#include "part_unused.h"
+#include "extra_lib/toy.h"
+#include "another_extra_lib/another_toy.h"
+
+int main() {
+    int a = 1;
+    g(a);
+    Inline i;
+    i.inline_g(a);
+    toy(a);
+    a_toy(a);
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}

test-src/included_files/part_unused.cpp

@@ -0,0 +1,9 @@
+#include "part_unused.h"
+
+void unused_f(int& i) {
+    i*=2;
+}
+
+void g(int& i) {
+    i*=2;
+}

test-src/included_files/part_unused.h

@@ -0,0 +1,7 @@
+void unused_f(int& i);
+void g(int& i);
+
+struct Inline {
+    void inline_unused_f(int& i) { i*=2; }
+    void inline_g(int& i) { i*=2; }
+};

test-src/missing_files/Makefile

@@ -0,0 +1,12 @@
+unused.o: unused.h unused.cpp
+	$(CXX) --coverage -c -o unused.o unused.cpp
+foo: foo.c unused.o
+	$(CXX) --coverage -o foo foo.c
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json unused.o
+
+prepare_test: foo
+
+test: foo
+	./foo

test-src/missing_files/config.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Override to change the file that contains the expected output for the gcov run
+gcov_expected_output="expected_gcov.json"
+
+# Override to change the file that contains the expected output for the lcov run
+lcov_expected_output="expected_lcov.json"
+
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
+if [[ "$gcc_version" < "5.5" ]]; then
+    echo "LEGACY G++ detected ($gcc_version: $CXX): skipping GCOV missing files test"
+    modes="lcov"
+else
+    modes="gcov lcov"
+fi

test-src/missing_files/expected_gcov.json

@@ -0,0 +1,12 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }, {
+        "name": "unused.h",
+        "coverage": [null, null, null, 0, null]
+    }, {
+        "name": "unused.cpp",
+        "coverage": [null, null, 0, null, 0, 0, null]
+    }]
+}

test-src/missing_files/expected_lcov.json

@@ -0,0 +1,12 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null]
+    }, {
+        "name": "unused.h",
+        "coverage": [null, null, null, 0, null]
+    }, {
+        "name": "unused.cpp",
+        "coverage": [null, null, 0, null, 0, 0, 0]
+    }]
+}

test-src/missing_files/foo.c

@@ -0,0 +1,16 @@
+int main() {
+    int a = 1;
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}

test-src/missing_files/unused.cpp

@@ -0,0 +1,7 @@
+#include "unused.h"
+
+void unused_f(int& i) {
+    Inline in;
+    in.infline_f(i);
+    i*=2;
+}

test-src/missing_files/unused.h

@@ -0,0 +1,5 @@
+void unused_f(int& i);
+
+struct Inline {
+    void infline_f(int& i) { i*=2; }
+};

test-src/out_of_tree/build_tree/Makefile

@@ -0,0 +1,23 @@
+SOURCE_TREE=../source_tree
+INCLUDE_FLAGS=-I$(SOURCE_TREE)/llib
+
+lib.o: $(SOURCE_TREE)/llib/lib.cpp $(SOURCE_TREE)/llib/lib.h
+	$(CXX) --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
+
+lib.a: lib.o
+	ar cr $@ $+
+
+a: $(SOURCE_TREE)/A/a.cpp lib.a
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o a $+
+
+b: $(SOURCE_TREE)/B/b.cpp lib.a
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o b $+
+
+prepare_test: a b
+
+test: prepare_test
+	./a
+	./b
+
+clean:
+	rm -f lib.o lib.a a b a.gc* b.gc* lib.gc* lcov.info

test-src/out_of_tree/config.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Set if we expect the GCOV test for a known reason
+gcov_fail_reason=""
+
+# We must invoke "make" in the build tree
+build_dir="build_tree"
+
+
+
+# lcov must be invoked in the build-tree, which means we must:
+#    -> tell lcov where the source tree is 
+#    -> tell coveralls to strip the "../source_tree prefix off of
+#       the paths
+lcov_dir="build_tree"
+lcov_coveralls_dir="build_tree"
+lcov_coveralls_flags="--root .."
+lcov_flags="-b ../source_tree"
+
+

test-src/out_of_tree/expected.json

@@ -0,0 +1,15 @@
+{
+    "source_files": [{
+        "name": "source_tree/llib/lib.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, 1]
+    }, {
+        "name": "source_tree/llib/lib.h",
+        "coverage": [null, null, null, null, null, 1, 1,  null]
+    }, {
+        "name": "source_tree/A/a.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }, {
+        "name": "source_tree/B/b.cpp",
+        "coverage": [null, null, 1, 1, 1, null, 1, 1, null]
+    }]
+}