HADOOP-19272. S3A: AWS SDK 2.25.53 warnings logged by transfer manager (

#7048)


Disables all logging below error in the AWS SDK Transfer Manager.

This is done in ClientManagerImpl construction so is automatically done
during S3A FS initialization.

ITests verify that
* It is possible to restore the warning log. This verifies the validity of
  the test suite, and will identify when an SDK update fixes this regression.
* Constructing an S3A FS instance will disable the logging.

The log manipulation code is lifted from Cloudstore, where it was used to
dynamically enable logging. It uses reflection to load the Log4J binding;
all uses of the API catch and swallow exceptions.
This is needed to avoid failures when running against different log backends

This is an emergency fix -we could come up with a better design for
the reflection based code using the new DynMethods classes.
But this is based on working code, which is always good.

Contributed by Steve Loughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AwsSdkWorkarounds.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.s3a.impl.logging.LogControl;
+import org.apache.hadoop.fs.s3a.impl.logging.LogControllerFactory;
+
+/**
+ * This class exists to support workarounds for parts of the AWS SDK
+ * which have caused problems.
+ */
+public final class AwsSdkWorkarounds {
+
+  /**
+   * Transfer manager log name. See HADOOP-19272.
+   * {@value}.
+   */
+  public static final String TRANSFER_MANAGER =
+      "software.amazon.awssdk.transfer.s3.S3TransferManager";
+
+  private AwsSdkWorkarounds() {
+  }
+
+  /**
+   * Prepare logging before creating AWS clients.
+   * @return true if the log tuning operation took place.
+   */
+  public static boolean prepareLogging() {
+    return LogControllerFactory.createController().
+        setLogLevel(TRANSFER_MANAGER, LogControl.LogLevel.ERROR);
+  }
+
+  /**
+   * Restore all noisy logs to INFO.
+   * @return true if the restoration operation took place.
+   */
+  @VisibleForTesting
+  static boolean restoreNoisyLogging() {
+    return LogControllerFactory.createController().
+        setLogLevel(TRANSFER_MANAGER, LogControl.LogLevel.INFO);
+  }
+}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java

@@ -90,7 +90,10 @@ public class ClientManagerImpl implements ClientManager {
 
   /**
    * Constructor.
+   * <p>
    * This does not create any clients.
+   * <p>
+   * It does disable noisy logging from the S3 Transfer Manager.
    * @param clientFactory client factory to invoke
    * @param clientCreationParameters creation parameters.
    * @param durationTrackerFactory duration tracker.
@@ -105,6 +108,9 @@ public ClientManagerImpl(
     this.s3Client = new LazyAutoCloseableReference<>(createS3Client());
     this.s3AsyncClient = new LazyAutoCloseableReference<>(createAyncClient());
     this.transferManager = new LazyAutoCloseableReference<>(createTransferManager());
+
+    // fix up SDK logging.
+    AwsSdkWorkarounds.prepareLogging();
   }
 
   /**

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/Log4JController.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl.logging;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * Something to control logging levels in Log4j.
+ * <p>
+ * Package private to avoid any direct instantiation.
+ * <p>
+ * Important: this must never be instantiated exception through
+ * reflection code which can catch and swallow exceptions related
+ * to not finding Log4J on the classpath.
+ * The Hadoop libraries can and are used with other logging
+ * back ends and we MUST NOT break that.
+ */
+class Log4JController extends LogControl {
+
+  /**
+   * Set the log4J level, ignoring all exceptions raised.
+   * {@inheritDoc}
+   */
+  @Override
+  protected boolean setLevel(final String logName, final LogLevel level) {
+    try {
+      Logger logger = Logger.getLogger(logName);
+      logger.setLevel(Level.toLevel(level.getLog4Jname()));
+      return true;
+    } catch (Exception ignored) {
+      // ignored.
+      return false;
+    }
+  }
+}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/LogControl.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl.logging;
+
+/**
+ * class to assist reflection-based control of logger back ends.
+ * <p>
+ * An instance of LogControl is able to control the log levels of
+ * loggers for log libraries such as Log4j, yet can be created in
+ * code designed to support multiple back end loggers behind
+ * SLF4J.
+ */
+public abstract class LogControl {
+
+  /**
+   * Enumeration of log levels.
+   * <p>
+   * The list is in descending order.
+   */
+  public enum LogLevel {
+    ALL("ALL"),
+    FATAL("FATAL"),
+    ERROR("ERROR"),
+    WARN("WARN"),
+    INFO("INFO"),
+    DEBUG("DEBUG"),
+    TRACE("TRACE"),
+    OFF("OFF");
+
+    /**
+     * Level name as used in Log4J.
+     */
+    private final String log4Jname;
+
+    LogLevel(final String log4Jname) {
+      this.log4Jname = log4Jname;
+    }
+
+    /**
+     * Get the log4j name of this level.
+     * @return the log name for use in configuring Log4J.
+     */
+    public String getLog4Jname() {
+      return log4Jname;
+    }
+  }
+
+  /**
+   * Sets a log level for a class/package.
+   * @param log log to set
+   * @param level level to set
+   * @return true if the log was set
+   */
+  public final boolean setLogLevel(String log, LogLevel level) {
+    try {
+      return setLevel(log, level);
+    } catch (Exception ignored) {
+      // ignored.
+      return false;
+    }
+
+  }
+
+
+  /**
+   * Sets a log level for a class/package.
+   * Exceptions may be raised; they will be caught in
+   * {@link #setLogLevel(String, LogLevel)} and ignored.
+   * @param log log to set
+   * @param level level to set
+   * @return true if the log was set
+   * @throws Exception any problem loading/updating the log
+   */
+  protected abstract boolean setLevel(String log, LogLevel level) throws Exception;
+
+}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/LogControllerFactory.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl.logging;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+/**
+ * Factory for creating controllers.
+ * <p>
+ * It currently only supports Log4J as a back end.
+ */
+public final class LogControllerFactory {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LogControllerFactory.class);
+
+  /**
+   * Log once: when there are logging issues, logging lots just
+   * makes it worse.
+   */
+  private static final LogExactlyOnce LOG_ONCE = new LogExactlyOnce(LOG);
+
+  /**
+   * Class name of log controller implementation to be loaded
+   * through reflection.
+   * {@value}.
+   */
+  private static final String LOG4J_CONTROLLER =
+      "org.apache.hadoop.fs.s3a.impl.logging.Log4JController";
+
+  private LogControllerFactory() {
+  }
+
+  /**
+   * Create a controller. Failure to load is logged at debug
+   * and null is returned.
+   * @param classname name of controller to load and create.
+   * @return the instantiated controller or null if it failed to load
+   */
+  public static LogControl createController(String classname) {
+    try {
+      Class<?> clazz = Class.forName(classname);
+      return (LogControl) clazz.newInstance();
+    } catch (Exception e) {
+      LOG_ONCE.debug("Failed to create controller {}: {}", classname, e, e);
+      return null;
+    }
+  }
+
+  /**
+   * Create a Log4J controller.
+   * @return the instantiated controller or null if the class can't be instantiated.
+   */
+  public static LogControl createLog4JController() {
+    return createController(LOG4J_CONTROLLER);
+  }
+
+  /**
+   * Create a controller, Log4j or falling back to a stub implementation.
+   * @return the instantiated controller or empty() if the class can't be instantiated.
+   */
+  public static LogControl createController() {
+    final LogControl controller = createLog4JController();
+    return controller != null
+        ? controller
+        : new StubLogControl();
+  }
+
+  /**
+   * Stub controller which always reports false.
+   */
+  private static final class StubLogControl extends LogControl {
+
+    @Override
+    protected boolean setLevel(final String log, final LogLevel level) {
+      return false;
+
+    }
+  }
+}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/package-info.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains reflection-based code to manipulate logging
+ * levels in external libraries.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.fs.s3a.impl.logging;
+
+import org.apache.hadoop.classification.InterfaceAudience;