Stable/Unstable tests.

This MR divides tests into stable, which are mandatory for any MR to be merged, and unstable, which are still in development and are allowed to fail. It is of course desirable to have a maximum of tests in the stable category.

One problem preventing parallel execution of stable/unstable tests is that `ctest` does not allow output to a custom XML file. The output goes to `Testing/YYYYMMDD-HHMM/Test.xml`, and the minute granularity is not very helpful. So stable tests are done first in stage `test` and unstable are done in a further stage, `test-unstable`. Thus, if on the expedite line, a MR may be merged as soon as the `test` stage is done.

.gitlab-ci.yml

@@ -6,12 +6,13 @@
 stages:
   - build
   - test
+  - test-unstable
 
 image: tallfurryman/kstars-ci:0.7
 
 variables:
   CCACHE_BASEDIR: "$CI_PROJECT_DIR"
-  QT_TEST_TIMEOUT_FUNCTION: "600000"
+  QT_TEST_TIMEOUT_FUNCTION: "600"
   QT_QPA_PLATFORM: "eglfs"
   # In order to be cached successfully, the ccache folder must reside *inside* $CI_PROJECT_DIR
   CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
@@ -27,6 +28,8 @@ cache:
   #   - always
 
 .build_recipe: &build_recipe
+  - add-apt-repository --remove ppa:mutlaqja/indinightly
+  - add-apt-repository ppa:mutlaqja/ppa
   - apt update
   - apt -y --no-install-recommends install libindi1 libindi-dev libindi-data indi-bin xplanet gsc phd2 libstellarsolver libstellarsolver-dev
   - mkdir -p kstars-build
@@ -47,13 +50,15 @@ build:
   after_script:
     - ccache -s
 
-# Run the full validation in one step
+# Run the full validation in one step, stable tests that must not fail
 # The artifacts take far too much time to propagate from one step to the other
 # The cache is unreliable, and only works on the same runner if there is no shared cache - use it for ccache instead
 # Consolidate runner with build packages and build
-build-and-test:
+# We have less than 10% unstability on stable tests, so we allow 2 attempts on the full check
+build-and-test-stable:
   stage: test
   interruptible: true
+  retry: 2
   only:
     - merge_requests
   variables:
@@ -62,14 +67,38 @@ build-and-test:
     - *build_recipe
   script:
     - rm -rf Testing
-    - xvfb-run ctest -T test -V --output-on-failure --no-compress-output || true
-    - saxon-xslt -u $(find ./Testing -name Test.xml) https://raw.githubusercontent.com/rpavlik/jenkins-ctest-plugin/master/ctest-to-junit.xsl > ./junit_result.xml
+    - xvfb-run ctest -T test -L stable --output-on-failure
   after_script:
+    - pwd
+    - saxon-xslt -u $(find . -name Test.xml) https://raw.githubusercontent.com/rpavlik/jenkins-ctest-plugin/master/ctest-to-junit.xsl > ./junit_result.stable.xml
     - ccache -s
   artifacts:
     reports:
       junit:
-        kstars-build/junit_result.xml
+        kstars-build/junit_result.stable.xml
+
+# Run the full validation in one step, unstable tests still in development
+build-and-test-unstable:
+  stage: test-unstable
+  interruptible: true
+  allow_failure: true
+  only:
+    - merge_requests
+  variables:
+    BUILD_TESTING: "ON"
+  before_script:
+    - *build_recipe
+  script:
+    - rm -rf Testing
+    - xvfb-run ctest -T test -LE stable --output-on-failure --no-compress-output
+  after_script:
+    - pwd
+    - saxon-xslt -u $(find . -name Test.xml) https://raw.githubusercontent.com/rpavlik/jenkins-ctest-plugin/master/ctest-to-junit.xsl > ./junit_result.unstable.xml
+    - ccache -s
+  artifacts:
+    reports:
+      junit:
+        kstars-build/junit_result.unstable.xml
 
 build-appimage:
   stage: test

Tests/auxiliary/CMakeLists.txt

@@ -17,36 +17,45 @@ SET( KSParserTests_SRCS testcsvparser.cpp testfwparser.cpp )
 ADD_EXECUTABLE( testcsvparser testcsvparser.cpp )
 TARGET_LINK_LIBRARIES( testcsvparser ${TEST_LIBRARIES})
 ADD_TEST( NAME CSVParserTest COMMAND testcsvparser )
+SET_TESTS_PROPERTIES( CSVParserTest PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testfwparser testfwparser.cpp )
 TARGET_LINK_LIBRARIES( testfwparser ${TEST_LIBRARIES})
 ADD_TEST( NAME FixedWidthParserTest COMMAND testfwparser )
+SET_TESTS_PROPERTIES( FixedWidthParserTest PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testdms testdms.cpp )
 TARGET_LINK_LIBRARIES( testdms ${TEST_LIBRARIES})
 ADD_TEST( NAME DMSTest COMMAND testdms )
+SET_TESTS_PROPERTIES( DMSTest PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testcachingdms testcachingdms.cpp )
 TARGET_LINK_LIBRARIES( testcachingdms ${TEST_LIBRARIES})
 ADD_TEST( NAME TestCachingDms COMMAND testcachingdms )
+SET_TESTS_PROPERTIES( TestCachingDms PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testcolorscheme testcolorscheme.cpp )
 TARGET_LINK_LIBRARIES( testcolorscheme ${TEST_LIBRARIES})
 ADD_TEST( NAME TestColorscheme COMMAND testcolorscheme )
+SET_TESTS_PROPERTIES( TestColorscheme PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testbinhelper testbinhelper.cpp )
 TARGET_LINK_LIBRARIES( testbinhelper ${TEST_LIBRARIES})
 ADD_TEST( NAME TestBinHelper COMMAND testbinhelper )
+SET_TESTS_PROPERTIES( TestBinHelper PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testfov testfov.cpp )
 TARGET_LINK_LIBRARIES( testfov ${TEST_LIBRARIES})
 ADD_TEST( NAME TestFOV COMMAND testfov )
+SET_TESTS_PROPERTIES( TestFOV PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testgeolocation testgeolocation.cpp )
 TARGET_LINK_LIBRARIES( testgeolocation ${TEST_LIBRARIES})
 ADD_TEST( NAME TestGeolocation COMMAND testgeolocation )
+SET_TESTS_PROPERTIES( TestGeolocation PROPERTIES LABELS "stable")
 
 ADD_EXECUTABLE( testksuserdb testksuserdb.cpp )
 TARGET_LINK_LIBRARIES( testksuserdb ${TEST_LIBRARIES})
 ADD_TEST( NAME TestKSUserDB COMMAND testksuserdb )
+SET_TESTS_PROPERTIES( TestKSUserDB PROPERTIES LABELS "stable")
 

Tests/auxiliary/testksuserdb.cpp

@@ -24,7 +24,8 @@ void TestKSUserDB::initTestCase()
     QVERIFY(QStandardPaths::isTestModeEnabled());
 
     // Remove the user folder that may eventually exist
-    QDir(KSPaths::writableLocation(QStandardPaths::GenericDataLocation)).removeRecursively();
+    QWARN(qPrintable("Removing " + KSPaths::writableLocation(QStandardPaths::GenericDataLocation)));
+    QVERIFY(QDir(KSPaths::writableLocation(QStandardPaths::GenericDataLocation)).removeRecursively());
     QVERIFY(!QDir(KSPaths::writableLocation(QStandardPaths::GenericDataLocation)).exists());
 }
 

Tests/fitsviewer/CMakeLists.txt

@@ -1,7 +1,6 @@
 if (StellarSolver_FOUND)
 ADD_EXECUTABLE( testfitsdata testfitsdata.cpp )
 TARGET_LINK_LIBRARIES( testfitsdata ${TEST_LIBRARIES})
-ADD_TEST( NAME FitsDataTest COMMAND testfitsdata )
 ADD_CUSTOM_COMMAND( TARGET testfitsdata POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy
             ${CMAKE_CURRENT_SOURCE_DIR}/m47_sim_stars.fits

Tests/focus/CMakeLists.txt

@@ -3,4 +3,5 @@ SET( FocusTests_SRCS testfocus.cpp  )
 ADD_EXECUTABLE( testfocus testfocus.cpp )
 TARGET_LINK_LIBRARIES( testfocus ${TEST_LIBRARIES})
 ADD_TEST( NAME FocusTest COMMAND testfocus )
+SET_TESTS_PROPERTIES( FocusTest PROPERTIES LABELS "stable")
 

Tests/internalguide/CMakeLists.txt

@@ -5,9 +5,11 @@ IF (INDI_FOUND)
   ADD_EXECUTABLE( testguidestars testguidestars.cpp )
   TARGET_LINK_LIBRARIES( testguidestars ${TEST_LIBRARIES})
   ADD_TEST( NAME GuideStarsTest COMMAND testguidestars )
+  SET_TESTS_PROPERTIES( GuideStarsTest PROPERTIES LABELS "stable")
 ENDIF ()
 
 ADD_EXECUTABLE( teststarcorrespondence teststarcorrespondence.cpp )
 TARGET_LINK_LIBRARIES( teststarcorrespondence ${TEST_LIBRARIES})
 ADD_TEST( NAME StarCorrespondenceTest COMMAND teststarcorrespondence )
+SET_TESTS_PROPERTIES( StarCorrespondenceTest PROPERTIES LABELS "stable")
 

Tests/kstars_ui/CMakeLists.txt

@@ -23,33 +23,41 @@ INCLUDE_DIRECTORIES(${INDI_INCLUDE_DIR})
 SET(KSTARS_UI_EKOS_SRC ${KSTARS_UI_TESTS_SRC} test_ekos_wizard.cpp)
 SET(KSTARS_UI_EKOS_LIBS ${TEST_LIBRARIES} ${CFITSIO_LIBRARIES} ${INDI_CLIENT_LIBRARIES} ${NOVA_LIBRARIES} z)
 
+ADD_TEST(ui_xterm xterm)
+
 # FIXME: this particular test crashes when done on the EGLFS platform
 # Disabling until we find the reason, other tests do cover this scenario
 
 # ADD_EXECUTABLE(test_ekos ${KSTARS_UI_EKOS_SRC} test_ekos.cpp)
 # TARGET_LINK_LIBRARIES(test_ekos ${KSTARS_UI_EKOS_LIBS})
 # ADD_TEST(NAME TestEkosProfiles COMMAND test_ekos)
+# SET_TESTS_PROPERTIES( TestEkosProfiles PROPERTIES LABELS "stable;ui")
 
 ADD_EXECUTABLE(test_ekos_simulator ${KSTARS_UI_EKOS_SRC} test_ekos_simulator.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_simulator ${KSTARS_UI_EKOS_LIBS})
 ADD_TEST(NAME TestEkosSimulator COMMAND test_ekos_simulator)
+# SET_TESTS_PROPERTIES( TestEkosSimulator PROPERTIES LABELS "stable;ui")
 
 ADD_EXECUTABLE(test_ekos_focus ${KSTARS_UI_EKOS_SRC} test_ekos_focus.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_focus ${KSTARS_UI_EKOS_LIBS})
 ADD_TEST(NAME TestEkosFocus COMMAND test_ekos_focus)
+# SET_TESTS_PROPERTIES( TestEkosFocus PROPERTIES LABELS "stable;ui")
 
 ADD_EXECUTABLE(test_ekos_capture ${KSTARS_UI_EKOS_SRC} test_ekos_capture.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_capture ${KSTARS_UI_EKOS_LIBS})
 ADD_TEST(NAME TestEkosCapture COMMAND test_ekos_capture)
+SET_TESTS_PROPERTIES( TestEkosCapture PROPERTIES LABELS "stable;ui" )
 
 ADD_EXECUTABLE(test_ekos_capture_count ${KSTARS_UI_EKOS_SRC} test_ekos_capture_helper.cpp test_ekos_capture_count.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_capture_count ${KSTARS_UI_EKOS_LIBS})
 ADD_TEST(NAME TestEkosCaptureCount COMMAND test_ekos_capture_count)
+#SET_TESTS_PROPERTIES( TestEkosCaptureCount PROPERTIES LABELS "stable;ui" TIMEOUT 1200 )
 
 ADD_EXECUTABLE(test_ekos_meridianflip ${KSTARS_UI_EKOS_SRC} test_ekos_capture_helper.cpp test_ekos_meridianflip_base.cpp test_ekos_meridianflip.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_meridianflip ${KSTARS_UI_EKOS_LIBS})
 # excluded to avoid CI timelimit failure
 # ADD_TEST(NAME TestEkosMeridianFlip COMMAND test_ekos_meridianflip)
+# SET_TESTS_PROPERTIES( TestEkosMeridianFlip PROPERTIES LABELS "astrometry;ui" )
 ADD_CUSTOM_COMMAND( TARGET test_ekos_meridianflip POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy
             ${CMAKE_CURRENT_SOURCE_DIR}/phd2_Simulators_mf.PHDGuidingV2
@@ -59,6 +67,7 @@ ADD_EXECUTABLE(test_ekos_meridianflip_specials ${KSTARS_UI_EKOS_SRC} test_ekos_c
 TARGET_LINK_LIBRARIES(test_ekos_meridianflip_specials ${KSTARS_UI_EKOS_LIBS})
 # excluded to avoid CI timelimit failure
 # ADD_TEST(NAME TestEkosMeridianFlipSpecials COMMAND test_ekos_meridianflip_specials)
+# SET_TESTS_PROPERTIES( TestEkosMeridianFlipSpecials PROPERTIES LABELS "unstable;ui" TIMEOUT 600 )
 
 ADD_EXECUTABLE(test_ekos_scheduler ${KSTARS_UI_EKOS_SRC} test_ekos_scheduler.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_scheduler ${KSTARS_UI_EKOS_LIBS})
@@ -71,6 +80,7 @@ ADD_CUSTOM_COMMAND( TARGET test_ekos_scheduler POST_BUILD
             ${CMAKE_CURRENT_SOURCE_DIR}/../scheduler/1x1s_RGBLumRGB.esq
             ${CMAKE_CURRENT_BINARY_DIR}/1x1s_RGBLumRGB.esq)
 ADD_TEST(NAME TestEkosScheduler COMMAND test_ekos_scheduler)
+SET_TESTS_PROPERTIES( TestEkosScheduler PROPERTIES LABELS "stable;ui" TIMEOUT 120 )
 
 ADD_EXECUTABLE(test_artificial_horizon ${KSTARS_UI_EKOS_SRC} test_artificial_horizon.cpp)
 TARGET_LINK_LIBRARIES(test_artificial_horizon ${KSTARS_UI_EKOS_LIBS})
@@ -83,10 +93,12 @@ ADD_CUSTOM_COMMAND( TARGET test_ekos_guide POST_BUILD
             ${CMAKE_CURRENT_SOURCE_DIR}/phd2_Simulators.PHDGuidingV2
             ${CMAKE_CURRENT_BINARY_DIR}/.PHDGuidingV2)
 ADD_TEST(NAME TestEkosGuide COMMAND test_ekos_guide)
+SET_TESTS_PROPERTIES( TestEkosGuide PROPERTIES LABELS "stable;ui" TIMEOUT 600 )
 
 ADD_EXECUTABLE(test_ekos_mount ${KSTARS_UI_EKOS_SRC} test_ekos_mount.cpp)
 TARGET_LINK_LIBRARIES(test_ekos_mount ${KSTARS_UI_EKOS_LIBS})
 ADD_TEST(NAME TestEkosMount COMMAND test_ekos_mount)
+SET_TESTS_PROPERTIES( TestEkosMount PROPERTIES LABELS "no-xvfb;ui" TIMEOUT 600 )
 
 ELSE ()
 

Tests/kstars_ui/kstars_ui_tests.cpp

@@ -41,6 +41,37 @@
 #include <ctime>
 #include <unistd.h>
 
+QSystemTrayIcon * KStarsUiTests::m_Notifier { nullptr };
+void KStarsUiTests::notifierBegin()
+{
+    if (!m_Notifier)
+        m_Notifier = new QSystemTrayIcon(QIcon::fromTheme("kstars_stars"), Ekos::Manager::Instance());
+    if (m_Notifier)
+        KStarsUiTests::m_Notifier->show();
+}
+
+void KStarsUiTests::notifierHide()
+{
+    if (m_Notifier)
+        m_Notifier->hide();
+}
+
+void KStarsUiTests::notifierMessage(QString title, QString message)
+{
+    qDebug() << message.replace('\n',' ');
+    if (m_Notifier)
+        m_Notifier->showMessage(title, message, QIcon());
+}
+
+void KStarsUiTests::notifierEnd()
+{
+    if (m_Notifier)
+    {
+        delete m_Notifier;
+        m_Notifier = nullptr;
+    }
+}
+
 // We want to launch the application before running our tests
 // Thus we want to explicitly call QApplication::exec(), and run our tests in parallel of the event loop
 // We then reimplement QTEST_MAIN(KStarsUiTests);
@@ -101,7 +132,7 @@ void execute_tests()
     QCoreApplication *app = QApplication::instance();
 
     // Limit execution duration
-    QTimer::singleShot(9 * 60 * 1000, app, &QCoreApplication::quit);
+    QTimer::singleShot(60 * 60 * 1000, app, &QCoreApplication::quit);
 
     app->exec();
 

Tests/kstars_ui/kstars_ui_tests.h

@@ -18,9 +18,16 @@
 #include <QTimer>
 #include <QApplication>
 #include <QtTest>
+#include <QSystemTrayIcon>
 
 class KStars;
 
+// Helper for on-screen test messages
+#define KTELL_BEGIN() do { KStarsUiTests::notifierBegin(); } while(false)
+#define KTELL_HIDE() do { KStarsUiTests::notifierHide(); } while(false)
+#define KTELL_END() do { KStarsUiTests::notifierEnd(); } while(false)
+#define KTELL(message) do { KStarsUiTests::notifierMessage(__FUNCTION__, message); } while(false)
+
 // We need to call a set of preliminary operations for each UI test, such as the KStars Wizard setup.
 // We also need QtCreator to detect our tests - that application scans class definitions for qExec calls.
 
@@ -44,17 +51,19 @@ extern void execute_tests();
         for (int i = 0; i < argc; i++) \
             if (!strcmp("-functions", argv[i])) \
                 return QTest::qExec(&tc, argc, argv); \
-        QApplication app(argc, argv); \
+        QApplication* app = new QApplication(argc, argv); \
         prepare_tests(); \
         int failure = 0; \
-        QTimer::singleShot(1000, &app, [&] { \
-            qDebug("Starting tests..."); \
+        QTimer::singleShot(1000, app, [&] { \
+            qDebug("Starting wizard..."); \
             failure |= run_wizards(argc, argv); \
             if (!failure) { \
-                failure |= QTest::qExec(&tc, argc, argv); \
+                KTELL_BEGIN(); \
+                failure |= QTest::qExec(&tc, app->arguments()); \
+                KTELL_END(); \
             } \
             qDebug("Tests are done."); \
-            app.quit(); \
+            app->quit(); \
         }); \
         execute_tests(); \
         return failure; }
@@ -71,6 +80,13 @@ class KStarsUiTests : public QObject
 public:
     explicit KStarsUiTests(QObject *parent = nullptr): QObject(parent) {};
 
+public:
+    static QSystemTrayIcon * m_Notifier;
+    static void notifierBegin();
+    static void notifierHide();
+    static void notifierMessage(QString, QString);
+    static void notifierEnd();
+
 private slots:
 
     /** @brief Members "initTestCase" and "cleanupTestCase" trigger when the framework processes this class.

Tests/kstars_ui/test_ekos.h

@@ -72,7 +72,6 @@
     name->setCurrentText(lookup); \
     QCOMPARE(name->currentText(), lookup); } while(false)
 
-
 class TestEkos: public QObject
 {
     Q_OBJECT

Tests/kstars_ui/test_ekos_capture.cpp

@@ -197,7 +197,8 @@ void TestEkosCapture::testCaptureSingle()
 
     // Verify a FITS file was created
     QTRY_VERIFY_WITH_TIMEOUT(searchFITS(QDir(destination.path())).count() == 1, 1000);
-    QVERIFY(searchFITS(QDir(destination.path())).contains("Light_001.fits"));
+    QVERIFY(searchFITS(QDir(destination.path()))[0].startsWith("Light_"));
+    QVERIFY(searchFITS(QDir(destination.path()))[0].endsWith("_001.fits"));
 
     // Reset sequence state - this makes a confirmation dialog appear
     volatile bool dialogValidated = false;
@@ -221,8 +222,10 @@ void TestEkosCapture::testCaptureSingle()
 
     // Verify an additional FITS file was created - asynchronously eventually
     QTRY_VERIFY_WITH_TIMEOUT(searchFITS(QDir(destination.path())).count() == 2, 2000);
-    QVERIFY(searchFITS(QDir(destination.path())).contains("Light_001.fits"));
-    QVERIFY(searchFITS(QDir(destination.path())).contains("Light_002.fits"));
+    QVERIFY(searchFITS(QDir(destination.path()))[0].startsWith("Light_"));
+    QVERIFY(searchFITS(QDir(destination.path()))[0].endsWith("_001.fits"));
+    QVERIFY(searchFITS(QDir(destination.path()))[1].startsWith("Light_"));
+    QVERIFY(searchFITS(QDir(destination.path()))[1].endsWith("_002.fits"));
 
     // TODO: test storage options
 }
@@ -272,7 +275,7 @@ void TestEkosCapture::testCaptureMultiple()
     // Capture again
     KTRY_CAPTURE_CLICK(startB);
     QTRY_VERIFY_WITH_TIMEOUT(!startB->icon().name().compare("media-playback-stop"), 500);
-    QTRY_VERIFY_WITH_TIMEOUT(!startB->icon().name().compare("media-playback-start"), duration * 1.2);
+    QTRY_VERIFY_WITH_TIMEOUT(!startB->icon().name().compare("media-playback-start"), duration * 2);
 
     // Verify the proper number of additional FITS file were created again
     QTRY_VERIFY_WITH_TIMEOUT(searchFITS(QDir(destination.path())).count() == 2 * count, 1000);

Tests/kstars_ui/test_ekos_capture.h

@@ -23,6 +23,15 @@
 #include <QCheckBox>
 #include <QtTest>
 
+/** @brief Helper to show the Capture tab
+ */
+#define KTRY_CAPTURE_SHOW() do { \
+    QTRY_VERIFY_WITH_TIMEOUT(Ekos::Manager::Instance()->captureModule() != nullptr, 5000); \
+    KTRY_EKOS_GADGET(QTabWidget, toolsWidget); \
+    toolsWidget->setCurrentWidget(Ekos::Manager::Instance()->captureModule()); \
+    QTRY_COMPARE_WITH_TIMEOUT(toolsWidget->currentWidget(), Ekos::Manager::Instance()->captureModule(), 5000); \
+    QTRY_VERIFY_WITH_TIMEOUT(!Ekos::Manager::Instance()->captureModule()->camera().isEmpty(), 5000); } while (false)
+
 /** @brief Helper to retrieve a gadget in the Capture tab specifically.
  * @param klass is the class of the gadget to look for.
  * @param name is the gadget name to look for in the UI configuration.