SNOW-1789759: NodeJS Support GCS region specific endpoint

lib/file_transfer_agent/gcs_util.js

@@ -69,8 +69,15 @@ function GCSUtil(httpclient, filestream) {
         }
       });
 
-      const storage = new Storage({ interceptors_: interceptors });
-
+      //TODO: SNOW-1789759 hardcoded region will be replaced in the future
+      const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
+      let endPoint = null;
+      if (stageInfo['endPoint']) {
+        endPoint = stageInfo['endPoint'];
+      } else if (isRegionalUrlEnabled) {
+        endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
+      }
+      const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
       client = { gcsToken: gcsToken, gcsClient: storage };
     } else {
       client = null;

lib/file_transfer_agent/s3_util.js

@@ -54,12 +54,19 @@ function S3Util(connectionConfig, s3, filestream) {
   this.createClient = function (stageInfo, useAccelerateEndpoint) {
     const stageCredentials = stageInfo['creds'];
     const securityToken = stageCredentials['AWS_TOKEN'];
+    const isRegionalUrlEnabled = stageInfo.useRegionalUrl || stageInfo.useS3RegionalUrl;
+
     // if GS sends us an endpoint, it's likely for FIPS. Use it.
     let endPoint = null;
     if (stageInfo['endPoint']) {
-      endPoint = 'https://' + stageInfo['endPoint'];
+      endPoint = `https://${stageInfo['endPoint']}`;
+    } else {
+      if (stageInfo.region && isRegionalUrlEnabled) {
+        const domainSuffixForRegionalUrl = (stageInfo.region).toLowerCase().startsWith('cn-') ? 'amazonaws.com.cn' : 'amazonaws.com';
+        endPoint = `https://s3.${stageInfo.region}.${domainSuffixForRegionalUrl}`;
+      }
     }
-
+  
     const config = {
       apiVersion: '2006-03-01',
       region: stageInfo['region'],
@@ -80,7 +87,7 @@ function S3Util(connectionConfig, s3, filestream) {
       }
     }
     if (proxy) {
-      const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION);
+      const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || SNOWFLAKE_S3_DESTINATION);
       config.requestHandler = new NodeHttpHandler({
         httpAgent: proxyAgent,
         httpsAgent: proxyAgent

test/unit/file_transfer_agent/gcs_test.js

@@ -64,6 +64,82 @@ describe('GCS client', function () {
     GCS = new SnowflakeGCSUtil(httpclient, filestream);
   });
 
+  describe('GCS client endpoint testing', async function () {
+    const testCases = [
+      {
+        name: 'when the useRegionalURL is only enabled',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: true,
+          region: 'mockLocation',
+        },
+        result: 'https://storage.mocklocation.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'me-central2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2 (mixed case)',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'ME-cEntRal2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2 (uppercase)',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'ME-CENTRAL2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the endPoint is specified',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: false,
+          region: 'ME-cEntRal1'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+      {
+        name: 'when both the endPoint and the useRegionalUrl are specified',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: true,
+          region: 'ME-cEntRal1'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+      {
+        name: 'when both the endPoint is specified and the region is me-central2',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: true,
+          region: 'ME-CENTRAL2'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+    ];
+
+    testCases.forEach(({ name, stageInfo, result }) => {
+      it(name, () => {
+        const client = GCS.createClient({ ...stageInfo, ...meta.stageInfo, creds: { GCS_ACCESS_TOKEN: 'mockToken' } });
+        assert.strictEqual(client.gcsClient.apiEndpoint, result);
+      } );
+
+    });
+  });
+
   it('extract bucket name and path', async function () {
     const GCS = new SnowflakeGCSUtil();
 

test/unit/file_transfer_agent/s3_test.js

@@ -44,7 +44,7 @@ describe('S3 client', function () {
 
   before(function () {
     mock('s3', {
-      S3: function () {
+      S3: function (config) {
         function S3() {
           this.getObject = function () {
             function getObject() {
@@ -57,6 +57,8 @@ describe('S3 client', function () {
 
             return new getObject;
           };
+
+          this.config = config;
           this.putObject = function () {
             function putObject() {
               this.then = function (callback) {
@@ -82,6 +84,59 @@ describe('S3 client', function () {
     AWS = new SnowflakeS3Util(noProxyConnectionConfig, s3, filesystem);
   });
 
+  describe('AWS client endpoint testing', async function () {
+    const originalStageInfo = meta.stageInfo;
+    const testCases = [
+      {
+        name: 'when useS3RegionalURL is only enabled',
+        stageInfo: {
+          ...originalStageInfo,
+          useS3RegionalUrl: true,
+          endPoint: null,
+        },
+        result: null
+      },
+      {
+        name: 'when useS3RegionalURL and is enabled and domain starts with cn',
+        stageInfo: {
+          ...originalStageInfo,
+          useS3RegionalUrl: true,
+          endPoint: null,
+          region: 'cn-mockLocation'
+        },
+        result: 'https://s3.cn-mockLocation.amazonaws.com.cn'
+      },
+      {
+        name: 'when endPoint is enabled',
+        stageInfo: {
+          ...originalStageInfo,
+          endPoint: 's3.endpoint',
+          useS3RegionalUrl: false
+        },
+        result: 'https://s3.endpoint'
+      },
+      {
+        name: 'when both endPoint and useS3PReiongalUrl is valid',
+        stageInfo: {
+          ...originalStageInfo,
+          endPoint: 's3.endpoint',
+          useS3RegionalUrl: true,
+
+        },
+        result: 'https://s3.endpoint'
+      },
+    ];
+
+    testCases.forEach(({ name, stageInfo, result }) => {
+      it(name, () => {
+        const client = AWS.createClient(stageInfo);
+        assert.strictEqual(client.config.endpoint, result);
+      } );
+
+    });
+  });
+
+
   it('extract bucket name and path', async function () {
     let result = extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/');
     assert.strictEqual(result.bucketName, 'sfc-eng-regression');