build: assign region at runtime in e2e tests (#13513)

* build: assign region at runtime in e2e tests

* forgot that

* use different salt

* try this

* try that

* windows $%@#T%@#%

codebuild_specs/e2e_workflow_base.yml

@@ -6,7 +6,6 @@ env:
     AWS_DEFAULT_REGION: us-east-1
     AWS_REGION: us-east-1
     CDK_DEFAULT_REGION: us-east-1
-    CLI_REGION: us-east-1
     AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out'
     AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64'
 

codebuild_specs/run_e2e_tests_windows.yml

@@ -2,7 +2,6 @@ version: 0.2
 env:
   shell: powershell.exe
   variables:
-    CLI_REGION: us-east-1
     TEST_SUITE: src/__tests__/auth_2a.test.ts|src/__tests__/auth_2b.test.ts|src/__tests__/auth_2d.test.ts|src/__tests__/auth_2f.test.ts
     CI: true
     CIRCLECI: true

codebuild_specs/scripts-windows/run-e2e-windows.sh

@@ -13,6 +13,9 @@ export NODE_OPTIONS=--max-old-space-size=5120
 source .circleci/local_publish_helpers_codebuild.sh
 source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh
 
+export CLI_REGION=$(yarn ts-node ./scripts/select-region-for-e2e-test.ts)
+echo "Test will run in $CLI_REGION"
+
 # source $BASH_ENV
 
 amplify version

scripts/select-region-for-e2e-test.ts

@@ -0,0 +1,36 @@
+import { AWS_REGIONS_TO_RUN_TESTS } from './cci-utils';
+import * as fs from 'fs-extra';
+import path from 'path';
+
+/**
+ * This script prints region assignment for an e2e test job.
+ * The algorithm takes input via environment variable - CODEBUILD_BATCH_BUILD_IDENTIFIER
+ * and computes deterministic region assignment by looking at position
+ * of build job in 'wait_for_ids.json' file.
+ * In order to reshuffle regions 'offset' constant below should be modified.
+ *
+ * If region is already assigned, i.e. CLI_REGION environment variable is set this script is pass-through.
+ */
+
+// if region is specified by job honor it.
+let selectedRegion = process.env.CLI_REGION;
+if (!selectedRegion) {
+  const jobId = process.env.CODEBUILD_BATCH_BUILD_IDENTIFIER;
+  if (!jobId) {
+    throw Error('CODEBUILD_BATCH_BUILD_IDENTIFIER environment variable must be set');
+  }
+  // Offset should be changed if we want re-shuffle regions.
+  const offset = 0;
+  const waitForIdsFilePath = path.join('.', 'codebuild_specs', 'wait_for_ids.json');
+  const jobIds = JSON.parse(fs.readFileSync(waitForIdsFilePath, 'utf-8')) as Array<string>;
+  let jobPosition = jobIds.indexOf(jobId);
+  if (jobPosition < 0) {
+    // this should never happen if PR checks pass, but just in case fall back to first region.
+    jobPosition = 0;
+  }
+  const regionIndex = (jobPosition + offset) % AWS_REGIONS_TO_RUN_TESTS.length;
+
+  selectedRegion = AWS_REGIONS_TO_RUN_TESTS[regionIndex];
+}
+
+console.log(selectedRegion);

scripts/split-e2e-tests-codebuild.ts

@@ -2,7 +2,6 @@ import * as glob from 'glob';
 import * as fs from 'fs-extra';
 import { join } from 'path';
 import * as yaml from 'js-yaml';
-import { AWS_REGIONS_TO_RUN_TESTS as regions } from './cci-utils';
 import { REPO_ROOT } from './cci-utils';
 import { FORCE_REGION_MAP, getOldJobNameWithoutSuffixes, loadTestTimings, USE_PARENT_ACCOUNT } from './cci-utils';
 import { migrationFromV10Tests, migrationFromV12Tests, migrationFromV8Tests } from './split-e2e-test-filters';
@@ -146,17 +145,15 @@ type ConfigBase = {
 const MAX_WORKERS = 3;
 type OS_TYPE = 'w' | 'l';
 type CandidateJob = {
-  region: string;
+  region?: string;
   os: OS_TYPE;
   executor: string;
   tests: string[];
   useParentAccount: boolean;
   disableCoverage: boolean;
 };
 const createRandomJob = (os: OS_TYPE): CandidateJob => {
-  const region = regions[Math.floor(Math.random() * regions.length)];
   return {
-    region,
     os,
     executor: os === 'l' ? 'l_large' : 'w_medium',
     tests: [],
@@ -262,7 +259,11 @@ const splitTestsV3 = (
         formattedJob.env.variables['compute-type'] = 'BUILD_GENERAL1_SMALL';
       }
       formattedJob.env.variables.TEST_SUITE = job.tests.join('|');
-      formattedJob.env.variables.CLI_REGION = job.region;
+      if (job.region) {
+        // Jobs with forced region are assigned one explicitly.
+        // Otherwise, region is assigned at runtime by select-region-for-e2e-test.ts script.
+        formattedJob.env.variables.CLI_REGION = job.region;
+      }
       if (job.useParentAccount) {
         formattedJob.env.variables.USE_PARENT_ACCOUNT = 1;
       }
@@ -283,7 +284,11 @@ const splitTestsV3 = (
       };
       formattedJob.env.variables = {};
       formattedJob.env.variables.TEST_SUITE = job.tests.join('|');
-      formattedJob.env.variables.CLI_REGION = job.region;
+      if (job.region) {
+        // Jobs with forced region are assigned one explicitly.
+        // Otherwise, region is assigned at runtime by select-region-for-e2e-test.ts script.
+        formattedJob.env.variables.CLI_REGION = job.region;
+      }
       if (job.useParentAccount) {
         formattedJob.env.variables.USE_PARENT_ACCOUNT = 1;
       }

shared-scripts.sh

@@ -336,6 +336,9 @@ function _runE2ETestsLinux {
     echo RUN E2E Tests Linux
     _loadE2ECache
     _install_packaged_cli_linux
+    # select region
+    export CLI_REGION=$(yarn ts-node ./scripts/select-region-for-e2e-test.ts)
+    echo "Test will run in $CLI_REGION"
     # verify installation
     which amplify
     amplify version