XWIKI-21916: Allow to ignore errors happening during wcag validation …

…in integration tests

  * Provide a new parameter wcagStopOnError which default is true, to
    allow ignoring wcag validation errors

(cherry picked from commit f554c87)

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-docker/src/main/java/org/xwiki/test/docker/internal/junit5/UITestTestConfigurationResolver.java

@@ -68,6 +68,8 @@ public class UITestTestConfigurationResolver
 
     private static final String WCAG_PROPERTY = "xwiki.test.ui.wcag";
 
+    private static final String WCAG_STOP_ON_ERROR_PROPERTY = "xwiki.test.ui.wcagStopOnError";
+
     private static final String PROPERTIES_PREFIX_PROPERTY = "xwiki.test.ui.properties.";
 
     private static final String PROFILES_PROPERTY = "xwiki.test.ui.profiles";
@@ -96,6 +98,7 @@ public TestConfiguration resolve(UITest uiTestAnnotation)
         configuration.setJDBCDriverVersion(resolveJDBCDriverVersion(uiTestAnnotation.jdbcDriverVersion()));
         configuration.setVNC(resolveVNC(uiTestAnnotation.vnc()));
         configuration.setWCAG(resolveWCAG(uiTestAnnotation.wcag()));
+        configuration.setWCAGStopOnError(resolveWCAGStopOnError(uiTestAnnotation.wcagStopOnError()));
         configuration.setProperties(resolveProperties(uiTestAnnotation.properties()));
         configuration.setExtraJARs(resolveExtraJARs(uiTestAnnotation.extraJARs()));
         configuration.setResolveExtraJARs(resolveResolveExtraJARs(uiTestAnnotation.resolveExtraJARs()));
@@ -229,6 +232,11 @@ private boolean resolveWCAG(boolean wcag)
         return resolve(wcag, WCAG_PROPERTY);
     }
 
+    private boolean resolveWCAGStopOnError(boolean wcagStopOnError)
+    {
+        return resolve(wcagStopOnError, WCAG_STOP_ON_ERROR_PROPERTY);
+    }
+
     private boolean resolveOffice(boolean office)
     {
         return resolve(office, OFFICE_PROPERTY);

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-docker/src/main/java/org/xwiki/test/docker/internal/junit5/XWikiDockerExtension.java

@@ -408,8 +408,10 @@ private BrowserWebDriverContainer startBrowser(TestConfiguration testConfigurati
             testConfiguration.getServletEngine().getInternalPort()));
 
         // Setup the wcag validation context.
-        testContext.getUtil().getWCAGUtils().setupWCAGValidation(testConfiguration.isWCAG(),
-            extensionContext.getTestClass().get().getName());
+        testContext.getUtil().getWCAGUtils().setupWCAGValidation(
+            testConfiguration.isWCAG(),
+            extensionContext.getTestClass().get().getName(),
+            testConfiguration.shouldWCAGStopOnError());
 
 
         // - the one used by RestTestUtils, i.e. outside of any container

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-docker/src/main/java/org/xwiki/test/docker/junit5/TestConfiguration.java

@@ -64,6 +64,8 @@ public class TestConfiguration
 
     private boolean wcag;
 
+    private boolean wcagStopOnError;
+
     private Properties properties;
 
     private List<ExtensionOverride> extensionOverrides;
@@ -105,6 +107,7 @@ public void merge(TestConfiguration testConfiguration) throws DockerTestExceptio
         mergeJDBCDriverVersion(testConfiguration.getJDBCDriverVersion());
         mergeVNC(testConfiguration.vnc());
         mergeWCAG(testConfiguration.isWCAG());
+        mergeWCAGStopOnError(testConfiguration.shouldWCAGStopOnError());
         mergeProperties(testConfiguration.getProperties());
         mergeExtraJARs(testConfiguration.getExtraJARs());
         mergeResolveExtraJARs(testConfiguration.isResolveExtraJARs());
@@ -244,6 +247,11 @@ private void mergeWCAG(boolean wcag)
         this.wcag = isWCAG() || wcag;
     }
 
+    private void mergeWCAGStopOnError(boolean wcagStopOnError)
+    {
+        this.wcagStopOnError = shouldWCAGStopOnError() || wcagStopOnError;
+    }
+
     private void mergeOffice(boolean office)
     {
         if (!isOffice() && office) {
@@ -513,6 +521,24 @@ public void setWCAG(boolean wcag)
         this.wcag = wcag;
     }
 
+    /**
+     * @return {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
+     * @since 16.1.0
+     */
+    public boolean shouldWCAGStopOnError()
+    {
+        return this.wcagStopOnError;
+    }
+
+    /**
+     * @param wcagStopOnError {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
+     * @since 16.1.0
+     */
+    public void setWCAGStopOnError(boolean wcagStopOnError)
+    {
+        this.wcagStopOnError = wcagStopOnError;
+    }
+
     /**
      * @return the list of configuration properties to use when generating the XWiki configuration files such as as
      * {@code xwiki.properties} (check {@code xwiki.properties.vm} to find the list of supported properties)

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-docker/src/main/java/org/xwiki/test/docker/junit5/UITest.java

@@ -124,6 +124,12 @@
      */
     boolean wcag() default false;
 
+    /**
+     * @return {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
+     * @since 16.1.0
+     */
+    boolean wcagStopOnError() default true;
+
     /**
      * @return the list of configuration properties to use when generating the XWiki configuration files such as
      *         {@code xwiki.properties} with Velocity (check {@code xwiki.properties.vm} to find the list of supported

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-ui/src/main/java/org/xwiki/test/ui/WCAGContext.java

@@ -259,6 +259,8 @@ String getIncompleteReport()
 
     private String testMethodName;
 
+    private boolean stopOnError = true;
+
     /**
      * Sets the current test class name. This name is the string representation of the TestUI class in which the
      * current wcag validation happens.
@@ -451,6 +453,22 @@ public boolean isWCAGEnabled()
         return this.wcagEnabled;
     }
 
+    /**
+     * @param stopOnError {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
+     */
+    public void setWCAGStopOnError(boolean stopOnError)
+    {
+        this.stopOnError = stopOnError;
+    }
+
+    /**
+     * @return {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
+     */
+    public boolean shouldWCAGStopOnError()
+    {
+        return this.stopOnError;
+    }
+
     /**
      * @return any of the validations found an incomplete check.
      */

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-ui/src/main/java/org/xwiki/test/ui/WCAGUtils.java

@@ -52,10 +52,12 @@ public class WCAGUtils
     /**
      * @param wcag true if WCAG tests must be executed, false otherwise
      * @param testClassName the PO class name to use for logging and reporting
+     * @param stopOnError {@code false} if WCAG validation should ignore errors, {@code true} otherwise.
      */
-    public void setupWCAGValidation(boolean wcag, String testClassName)
+    public void setupWCAGValidation(boolean wcag, String testClassName, boolean stopOnError)
     {
         this.wcagContext.setWCAGEnabled(wcag);
+        this.wcagContext.setWCAGStopOnError(stopOnError);
         if (wcag) {
             LOGGER.info("WCAG validation is enabled.");
             wcagContext.setTestClassName(testClassName);

xwiki-platform-core/xwiki-platform-test/xwiki-platform-test-ui/src/main/java/org/xwiki/test/ui/po/BasePage.java

@@ -687,23 +687,31 @@ public void validateWCAG(WCAGContext wcagContext, boolean checkCache)
             return;
         }
 
-        long startTime = System.currentTimeMillis();
-        // Run WCAG tests on the current UI page if the current URL + PO class name are not in the cache, or if checking
-        // the cache is disabled.
-        if (!checkCache || wcagContext.isNotCached(this.getPageURL(), this.getClass().getName())) {
-            XWikiWebDriver driver = this.getDriver();
-            AxeBuilder axeBuilder = wcagContext.getAxeBuilder();
-            Results axeResult = axeBuilder.analyze(driver);
-            wcagContext.addWCAGResults(driver.getCurrentUrl(), this.getClass().getName(), axeResult);
-            long stopTime = System.currentTimeMillis();
-            long deltaTime = stopTime - startTime;
-            LOGGER.debug("[{} : {}] WCAG Validation on this element took [{}] ms.",
-                this.getPageURL(), this.getClass().getName(), deltaTime);
-            wcagContext.addWCAGTime(deltaTime);
-        } else {
-            // If the identifying pair is already in the cache, don't perform accessibility validation.
-            LOGGER.debug("[{} : {}] This combination of URL:class was already WCAG-checked.",
-                this.getPageURL(), this.getClass().getName());
+        try {
+            long startTime = System.currentTimeMillis();
+            // Run WCAG tests on the current UI page if the current URL + PO class name are not in the cache, or if checking
+            // the cache is disabled.
+            if (!checkCache || wcagContext.isNotCached(this.getPageURL(), this.getClass().getName())) {
+                XWikiWebDriver driver = this.getDriver();
+                AxeBuilder axeBuilder = wcagContext.getAxeBuilder();
+                Results axeResult = axeBuilder.analyze(driver);
+                wcagContext.addWCAGResults(driver.getCurrentUrl(), this.getClass().getName(), axeResult);
+                long stopTime = System.currentTimeMillis();
+                long deltaTime = stopTime - startTime;
+                LOGGER.debug("[{} : {}] WCAG Validation on this element took [{}] ms.",
+                    this.getPageURL(), this.getClass().getName(), deltaTime);
+                wcagContext.addWCAGTime(deltaTime);
+            } else {
+                // If the identifying pair is already in the cache, don't perform accessibility validation.
+                LOGGER.debug("[{} : {}] This combination of URL:class was already WCAG-checked.",
+                    this.getPageURL(), this.getClass().getName());
+            }
+        } catch (Exception e) {
+            if (wcagContext.shouldWCAGStopOnError()) {
+                throw e;
+            } else {
+                LOGGER.debug("Error during WCAG execution, but ignored thanks to wcagStopOnError flag: ", e);
+            }
         }
     }