PMM-13425 tests fixes

README.md

@@ -78,7 +78,7 @@ Execute command in the Project Root folder
                                 (e.g. Data from exporters is displayed at those dashboards)
     @dbaas                  DB as a Service functionality tests
     @grafana-pr             Executed in Github Actions for PRs in percona-platform/grafana repository
-    @ia                     Integrated Alerting functionality tests
+    @alerting-fb                     Integrated Alerting functionality tests
     @instances              Remote Instances addition functionality 
                                 and checking that data appears from exporters
     @inventory              Inventory functionality, removing nodes, services, etc.

docker-compose-clickhouse.yml

@@ -31,7 +31,7 @@ services:
         condition: service_healthy
     environment:
       - PMM_AGENT_SETUP=1
-      - PMM_AGENT_SERVER_ADDRESS=pmm-server-external-clickhouse
+      - PMM_AGENT_SERVER_ADDRESS=pmm-server-external-clickhouse:8443
       - PMM_AGENT_SERVER_USERNAME=admin
       - PMM_AGENT_SERVER_PASSWORD=admin
       - PMM_AGENT_PORTS_MIN=41000
@@ -43,9 +43,9 @@ services:
       - PMM_AGENT_SETUP_NODE_TYPE=container
     restart: on-failure
 
-  mysql5.7:
-    container_name: mysql5.7
-    image: mysql:5.7
+  ps8:
+    container_name: ps8
+    image: percona:8
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     environment:

docker-compose-disconnect.yml

@@ -55,9 +55,9 @@ services:
       - server-network
       - mysql-network
 
-  mysql5.7:
-    container_name: mysql-disconnect-5.7
-    image: mysql:5.7
+  ps8:
+    container_name: ps8
+    image: percona:8.0
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     environment:

package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "lint:tests": "eslint --ext .js --fix tests/",
     "e2e": "npx codeceptjs run-multiple parallel --debug --steps --reporter mocha-multi -c pr.codecept.js --grep '(?=.*)^(?!.*@not-ui-pipeline)^(?!.*@not-pr-pipeline)'",
-    "e2e:ia": "npx codeceptjs run -c pr.codecept.js --grep '@ia'",
+    "e2e:ia": "npx codeceptjs run -c pr.codecept.js --grep '@alerting-fb'",
     "e2e:stt": "npx codeceptjs run -c pr.codecept.js --grep '@stt'",
     "e2e:dbaas": "npx codeceptjs run -c pr.codecept.js --grep '@dbaas'",
     "e2e:grafana-pr": "npx codeceptjs run -c pr.codecept.js --grep '@grafana-pr'"

tests/QAN/externalClickhouse_test.js

@@ -5,12 +5,12 @@ Feature('External Clickhouse Tests');
 // Address of PMM with external clickhouse created with docker compose.
 const pmmServerPort = '8081';
 const basePmmUrl = `http://127.0.0.1:${pmmServerPort}/`;
-const dockerVersion = process.env.DOCKER_VERSION || 'perconalab/pmm-server:dev-latest';
+const dockerVersion = process.env.DOCKER_VERSION || 'perconalab/pmm-server:3-dev-latest';
 
 BeforeSuite(async ({ I }) => {
   await I.verifyCommand(`PMM_SERVER_IMAGE=${dockerVersion} docker-compose -f docker-compose-clickhouse.yml up -d`);
   await I.wait(30);
-  await I.verifyCommand('docker exec pmm-client-clickhouse pmm-admin add mysql --username=root --password=7B*53@lCdflR --query-source=perfschema mysql5.7 mysql5.7:3306');
+  await I.verifyCommand('docker exec pmm-client-clickhouse pmm-admin add mysql --username=root --password=7B*53@lCdflR --query-source=perfschema ps8 ps8:3306');
   await I.wait(60);
 });
 

tests/configuration/verifyPMMServerDisconnect_test.js

@@ -4,15 +4,16 @@ Feature('Pmm Server stability');
 const pmmServerPort = '8180';
 const basePmmUrl = `http://127.0.0.1:${pmmServerPort}/`;
 let clientServerNetwork = 'pmm-ui-tests_server-network';
+const serverImage = process.env.DOCKER_VERSION || 'perconalab/pmm-server:3-dev-latest';
 
 BeforeSuite(async ({ I }) => {
-  await I.verifyCommand(`PMM_SERVER_IMAGE=${process.env.DOCKER_VERSION} docker compose -f docker-compose-disconnect.yml up -d pmm-server-disconnect`);
-  await I.asyncWaitFor(async () => await I.verifyCommand(`echo $(curl -s -o /dev/null -w '%{http_code}' 127.0.0.1:${pmmServerPort}/ping)`) === '200', 100);
+  await I.verifyCommand(`PMM_SERVER_IMAGE=${serverImage} docker compose -f docker-compose-disconnect.yml up -d pmm-server-disconnect`);
+  await I.asyncWaitFor(async () => await I.verifyCommand(`echo $(curl -s -o /dev/null -w '%{http_code}' 127.0.0.1:${pmmServerPort}/ping)`) === '200', 60);
   await I.verifyCommand('docker compose -f docker-compose-disconnect.yml up -d pmm-client');
-  await I.verifyCommand('docker compose -f docker-compose-disconnect.yml up -d mysql5.7');
+  await I.verifyCommand('docker compose -f docker-compose-disconnect.yml up -d ps8');
   clientServerNetwork = await I.verifyCommand('docker inspect pmm-client-disconnect -f \'{{range $k, $v := .NetworkSettings.Networks}}{{printf "%s\\n" $k}}{{end}}\' | grep -o \'.*server-network\'');
-  await I.asyncWaitFor(async () => await I.verifyCommand('echo $(docker container logs mysql-disconnect-5.7 2>&1 | grep "Server hostname (bind-address)")') !== '', 100);
-  await I.verifyCommand('docker exec pmm-client-disconnect pmm-admin add mysql --username=root --password=7B*53@lCdflR --host=mysql-disconnect-5.7 --port=3306 --query-source=perfschema mysql-disconnect-5.7');
+  I.wait(20);
+  await I.verifyCommand('docker exec pmm-client-disconnect pmm-admin add mysql --username=root --password=7B*53@lCdflR --host=ps8 --port=3306 --query-source=perfschema ps8');
   // wait for the data to be scraped from db
   I.wait(60);
 });
@@ -42,7 +43,7 @@ Scenario(
     await I.amOnPage(withCustomBaseUrl(`${dashboardPage.mySQLInstanceOverview.clearUrl}?orgId=1&from=now-3m&to=now-1m`));
     await dashboardPage.waitForDashboardOpened();
     await dashboardPage.expandEachDashboardRow();
-    await dashboardPage.verifyThereAreNoGraphsWithoutData(1);
+    await dashboardPage.verifyThereAreNoGraphsWithoutData(2);
     I.dontSeeElement(dashboardPage.fields.metricPanelNa('Services panel'));
   },
 );
@@ -62,7 +63,7 @@ Scenario(
     await I.amOnPage(withCustomBaseUrl(`${dashboardPage.mySQLInstanceOverview.clearUrl}?orgId=1&from=now-3m&to=now-1m`));
     await dashboardPage.waitForDashboardOpened();
     await dashboardPage.expandEachDashboardRow();
-    await dashboardPage.verifyThereAreNoGraphsWithoutData(1);
+    await dashboardPage.verifyThereAreNoGraphsWithoutData(2);
     I.dontSeeElement(dashboardPage.fields.metricPanelNa('Services panel'));
   },
 );

tests/configuration/verifyPMMSettingsPageFunctionality_test.js

@@ -74,7 +74,7 @@ Scenario.skip(
 ).retry(2);
 
 Scenario(
-  'PMM-T532 PMM-T533 PMM-T536 - Verify user can disable/enable IA in Settings @ia @settings',
+  'PMM-T532 PMM-T533 PMM-T536 - Verify user can disable/enable IA in Settings @alerting-fb @settings',
   async ({
     I, pmmSettingsPage, settingsAPI, adminPage,
   }) => {

tests/dashboards/verifyGCRemoteInstance_test.js

@@ -88,7 +88,7 @@ Data(instances).Scenario(
 
     const instanceDetails = getInstance(instance);
 
-    I.wait(10);
+    I.wait(30);
     if (instanceType === 'mysql') {
       I.amOnPage(dashboardPage.mySQLInstanceOverview.url);
     }
@@ -98,10 +98,10 @@ Data(instances).Scenario(
     }
 
     dashboardPage.waitForDashboardOpened();
-    await adminPage.applyTimeRange('Last 12 hours');
+    await adminPage.applyTimeRange('Last 5 minutes');
     await dashboardPage.applyFilter('Service Name', instanceDetails.serviceName);
     await dashboardPage.expandEachDashboardRow();
-    await dashboardPage.verifyThereAreNoGraphsWithoutData(1);
+    await dashboardPage.verifyThereAreNoGraphsWithoutData(6);
   },
 ).retry(2);
 
@@ -136,11 +136,8 @@ Data(instances).Scenario(
 
     const instanceDetails = getInstance(instance);
 
-    I.wait(10);
-    const response = await grafanaAPI.checkMetricExist(metric, { type: 'service_name', value: instanceDetails.serviceName });
-    const result = JSON.stringify(response.data.data.result);
-
-    assert.ok(response.data.data.result.length !== 0, `Metrics ${metric} from ${instanceDetails.serviceName} should be available but got empty ${result}`);
+    I.wait(30);
+    await grafanaAPI.checkMetricExist(metric, { type: 'service_name', value: instanceDetails.serviceName });
   },
 );
 

tests/ia/README.md

@@ -29,10 +29,10 @@ Shut down webhook server command: `docker-compose -f docker-compose-webhook.yml
 
 ### Command to execute all Integrated Alerting tests:
 
-`npx codeceptjs run -c pr.codecept.js --grep '@ia'`
+`npx codeceptjs run -c pr.codecept.js --grep '@alerting-fb'`
 
 ### Execute Integrated Alerting tests in Jenkins
 * Job name - `pmm2-ui-tests`
-* tag - `@ia`
+* tag - `@alerting-fb`
 * run tagged day - `yes`
 * clients not needed

tests/ia/alertRules_test.js

@@ -21,14 +21,15 @@ Feature('Alerting: Alert rules');
 
 Before(async ({ I }) => {
   await I.Authorize();
+  await rulesAPI.removeAllAlertRules();
 });
 
 After(async () => {
   await rulesAPI.removeAllAlertRules();
 });
 
-Scenario.skip(
-  'PMM-T1384 Verify empty alert rules list @ia @grafana-pr',
+Scenario(
+  'PMM-T1384 Verify empty alert rules list @alerting-fb @grafana-pr',
   async ({ I, alertRulesPage }) => {
     alertRulesPage.openAlertRulesTab();
     I.waitForText(alertRulesPage.messages.noRulesFound, 10, alertRulesPage.elements.noRules);
@@ -40,8 +41,8 @@ Scenario.skip(
   },
 ).retry(0);
 
-Scenario.skip(
-  'PMM-T1385 Verify alert rules elements @ia @grafana-pr',
+Scenario(
+  'PMM-T1385 Verify alert rules elements @alerting-fb @grafana-pr',
   async ({ I, alertRulesPage, rulesAPI }) => {
     const ruleName = 'testRule';
     const ruleFolder = 'PostgreSQL';
@@ -71,25 +72,25 @@ Scenario.skip(
   },
 );
 
-Scenario.skip(
-  'PMM-T1392 Verify fields dynamically change value when template is changed @ia @grafana-pr',
+Scenario(
+  'PMM-T1392 Verify fields dynamically change value when template is changed @alerting-fb @grafana-pr',
   async ({ I, alertRulesPage }) => {
     // TODO: https://jira.percona.com/browse/PMM-10860 name doesn't change
     alertRulesPage.openAlertRulesTab();
     I.click(alertRulesPage.buttons.newAlertRule);
     I.waitForElement(alertRulesPage.fields.templatesLoader);
-    alertRulesPage.searchAndSelectResult('template', 'PostgreSQL down');
+    await alertRulesPage.searchAndSelectResult('template', 'PostgreSQL down');
     I.waitForValue(alertRulesPage.fields.inputField('duration'), '60s');
     I.seeTextEquals('Critical', alertRulesPage.fields.dropdownValue('severity'));
-    alertRulesPage.searchAndSelectResult('template', 'MySQL restarted');
+    await alertRulesPage.searchAndSelectResult('template', 'MySQL restarted');
     I.waitForValue(alertRulesPage.fields.inputField('threshold'), 300);
     I.waitForValue(alertRulesPage.fields.inputField('duration'), '60s');
     I.seeTextEquals('Warning', alertRulesPage.fields.dropdownValue('severity'));
   },
 );
 
-Scenario.skip(
-  'PMM-T1420 Verify user can create Percona templated alert @ia @alerting-fb',
+Scenario(
+  'PMM-T1420 Verify user can create Percona templated alert @alerting-fb',
   async ({ I, alertRulesPage, rulesAPI }) => {
     const rule = page.rules[15];
     const newRule = page.rules[0];
@@ -110,7 +111,7 @@ Scenario.skip(
 
 // TODO: unskip in scope of https://perconadev.atlassian.net/browse/PMM-12938
 Scenario.skip(
-  'PMM-T2282 Verfied Alerting is able to monitor for "PMM Agent Down" @ia @alerting-fb',
+  'PMM-T2282 Verfied Alerting is able to monitor for "PMM Agent Down" @alerting-fb',
   async ({ I, alertRulesPage, rulesAPI }) => {
     const rule = page.rules[29];
     const newRule = page.rules[30];
@@ -137,8 +138,8 @@ Scenario.skip(
 );
 
 // TODO: check ovf failure
-Scenario.skip(
-  'PMM-T1430 Verify user can edit Percona templated alert @ia @not-ovf @alerting-fb',
+Scenario(
+  'PMM-T1430 Verify user can edit Percona templated alert @alerting-fb @not-ovf',
   async ({
     I, alertRulesPage, rulesAPI,
   }) => {
@@ -163,8 +164,8 @@ Scenario.skip(
   },
 );
 
-Scenario.skip(
-  'PMM-T1433 Verify user can delete Percona templated alert @ia @alerting-fb',
+Scenario(
+  'PMM-T1433 Verify user can delete Percona templated alert @alerting-fb',
   async ({
     I, alertRulesPage, rulesAPI, iaCommon,
   }) => {
@@ -190,7 +191,7 @@ Scenario.skip(
 // nightly candidate
 // FIXME: flaky test fix and unskip
 Scenario.skip(
-  'PMM-T1434 Verify validation errors when creating new alert rule @ia @grafana-pr',
+  'PMM-T1434 Verify validation errors when creating new alert rule @alerting-fb @grafana-pr',
   async ({
     I, alertRulesPage,
   }) => {

tests/ia/alerts_test.js

@@ -33,16 +33,16 @@ BeforeSuite(async ({ I, rulesAPI }) => {
   await rulesAPI.createAlertRule({ ruleName }, ruleFolder);
 
   // Preparation steps for checking Alert via webhook server
-  await I.verifyCommand('docker-compose -f docker-compose-webhook.yml up -d');
+  await I.verifyCommand('docker compose -f docker-compose-webhook.yml up -d');
 });
 
 AfterSuite(async ({ rulesAPI, I }) => {
   await rulesAPI.removeAllAlertRules();
-  await I.verifyCommand('docker-compose -f docker-compose-webhook.yml stop');
+  await I.verifyCommand('docker compose -f docker-compose-webhook.yml stop');
 });
 
 Scenario.skip(
-  'PMM-T1482 Verify fired alert @ia',
+  'PMM-T1482 Verify fired alert @alerting-fb',
   async ({ I, alertsPage, alertsAPI }) => {
     await alertsAPI.waitForAlerts(24, 1);
     await I.amOnPage(alertsPage.url);
@@ -60,7 +60,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T1494 PMM-T1495 Verify fired alert in Pager Duty and Webhook @ia',
+  'PMM-T1494 PMM-T1495 Verify fired alert in Pager Duty and Webhook @alerting-fb',
   async ({ I, alertsAPI, rulesAPI }) => {
     const file = './testdata/ia/scripts/alert.txt';
     const alertUID = await rulesAPI.getAlertUID(ruleName, ruleFolder);
@@ -76,7 +76,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T1496 PMM-T1497 Verify it is possible to silence and unsilence alert @ia',
+  'PMM-T1496 PMM-T1497 Verify it is possible to silence and unsilence alert @alerting-fb',
   async ({
     I, alertsPage, alertmanagerAPI,
   }) => {
@@ -94,7 +94,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T1498 Verify firing alerts dissappear when the condition is fixed @ia',
+  'PMM-T1498 Verify firing alerts dissappear when the condition is fixed @alerting-fb',
   async ({
     I, alertsPage, alertRulesPage,
   }) => {
@@ -113,7 +113,7 @@ Scenario.skip(
 
 // FIXME: Skip until https://jira.percona.com/browse/PMM-11130 is fixed
 Scenario.skip(
-  'PMM-T659 Verify alerts are deleted after deleting rules @ia',
+  'PMM-T659 Verify alerts are deleted after deleting rules @alerting-fb',
   async ({ I, alertsPage, rulesAPI }) => {
     // Deleting rules
     await rulesAPI.removeAllAlertRules();
@@ -124,7 +124,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T1499 Verify an alert with non-existing filter (label) does not show up in list @ia',
+  'PMM-T1499 Verify an alert with non-existing filter (label) does not show up in list @alerting-fb',
   async ({ I, alertsPage, rulesAPI }) => {
     await rulesAPI.removeAllAlertRules();
     const wrongFilterRule = {
@@ -146,7 +146,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T564 Verify fired alert severity colors @ia',
+  'PMM-T564 Verify fired alert severity colors @alerting-fb',
   async ({
     I, alertsPage, rulesAPI, alertsAPI,
   }) => {
@@ -172,7 +172,7 @@ Scenario.skip(
 );
 
 Scenario.skip(
-  'PMM-T1467 Verify empty Fired alerts list @ia',
+  'PMM-T1467 Verify empty Fired alerts list @alerting-fb',
   async ({ I, alertsPage, rulesAPI }) => {
     await rulesAPI.removeAllAlertRules();
     I.amOnPage(alertsPage.url);