HADOOP-19352. Hadoop OSS Connector adds support for V4 signatures.

hadoop-project/pom.xml

@@ -1663,7 +1663,7 @@
       <dependency>
         <groupId>com.aliyun.oss</groupId>
         <artifactId>aliyun-sdk-oss</artifactId>
-        <version>3.13.2</version>
+        <version>3.18.1</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.httpcomponents</groupId>

hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java

@@ -73,6 +73,7 @@
 import java.util.ListIterator;
 import java.util.NoSuchElementException;
 import java.util.stream.Collectors;
+import com.aliyun.oss.common.comm.SignVersion;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 
@@ -113,6 +114,16 @@ public void initialize(URI uri, Configuration conf, String user,
         conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/"
             + VersionInfo.getVersion());
 
+    String region = conf.get(REGION_KEY, "");
+    String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT);
+    if ("V4".equalsIgnoreCase(signatureVersion)) {
+      clientConf.setSignatureVersion(SignVersion.V4);
+      if (StringUtils.isEmpty(region)) {
+        LOG.error("Signature version is V4 ,but region is empty.");
+        throw new IllegalArgumentException("SignVersion is V4 but region is empty");
+      }
+    }
+
     String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
     int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
     if (StringUtils.isNotEmpty(proxyHost)) {
@@ -171,6 +182,11 @@ public void initialize(URI uri, Configuration conf, String user,
       statistics.incrementWriteOps(1);
     }
 
+    if (StringUtils.isNotEmpty(region)) {
+      ossClient.setRegion(region);
+      LOG.debug("ossClient setRegion {}", region);
+    }
+
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
     int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
     if (listVersion < 1 || listVersion > 2) {

hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java

@@ -211,4 +211,19 @@ private Constants() {
   public static final String LIST_VERSION = "fs.oss.list.version";
 
   public static final int DEFAULT_LIST_VERSION = 2;
+
+  /**
+   * OSS signature version.
+   */
+  public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion";
+
+  /**
+   * OSS signature version DEFAULT {@value}
+   */
+  public static final String SIGNATURE_VERSION_DEFAULT = "V1";
+
+  /**
+   * OSS region {@value}
+   */
+  public static final String REGION_KEY = "fs.oss.region";
 }

hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeNotNull;
+import static org.junit.Assume.assumeTrue;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY;
+
+/**
+ * Tests Aliyun OSS system.
+ */
+public class ITAliyunOSSSignatureV4 {
+  private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class);
+  private Configuration conf;
+  private URI testURI;
+  private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr");
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    String bucketUri = conf.get("test.fs.oss.name");
+    LOG.debug("bucketUri={}", bucketUri);
+    testURI = URI.create(bucketUri);
+  }
+
+  @Test
+  public void testV4() throws IOException {
+    conf.set(SIGNATURE_VERSION_KEY, "V4");
+    conf.set(REGION_KEY, "cn-hongkong");
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    fs.initialize(testURI, conf);
+    assumeNotNull(fs);
+
+    createFile(fs, testFile, true, dataset(256, 0, 255));
+    FileStatus status = fs.getFileStatus(testFile);
+    fs.delete(testFile);
+    fs.close();
+  }
+
+  @Test
+  public void testDefaultSignatureVersion() throws IOException {
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    fs.initialize(testURI, conf);
+    assumeNotNull(fs);
+
+    Path testFile = new Path("/test/atestr");
+    createFile(fs, testFile, true, dataset(256, 0, 255));
+    FileStatus status = fs.getFileStatus(testFile);
+    fs.delete(testFile);
+    fs.close();
+  }
+
+  @Test
+  public void testV4WithoutRegion() throws IOException {
+    conf.set(SIGNATURE_VERSION_KEY, "V4");
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    try {
+      fs.initialize(testURI, conf);
+    } catch (IllegalArgumentException e) {
+      LOG.warn("use V4 , but do not set region, get exception={}", e);
+      assertEquals("se V4 , but do not set region", e.getMessage(), "SignVersion is V4 but region is empty");
+    }
+  }
+}

hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties

@@ -21,3 +21,6 @@ log4j.threshold=ALL
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Log all oss classes
+log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG