From b5b62261497c25d64fbfb9936cba852520ebd3e4 Mon Sep 17 00:00:00 2001 From: Bodor Laszlo Date: Thu, 25 Apr 2024 12:19:33 +0200 Subject: [PATCH 1/3] TEZ-4555: Fail fast in LocalClient if the dirs (log, local) haven't been created (#348). (Laszlo Bodor, reviewed by Ayush Saxena) --- .../java/org/apache/tez/client/LocalClient.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java index f6a626e64c..bf1cef4c45 100644 --- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java +++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java @@ -331,8 +331,17 @@ public void run() { // Prepare Environment Path logDir = new Path(userDir, "localmode-log-dir"); Path localDir = new Path(userDir, "localmode-local-dir"); - localFs.mkdirs(logDir); - localFs.mkdirs(localDir); + + // fail fast if the local directories (on the paths that were used on HDFS) cannot be created + // in this case, user might want to choose a different staging path, which works on the local FS too + if (!localFs.mkdirs(logDir)) { + throw new IOException( + "Unable to create log directory, try to create it manually for further insights: " + logDir); + } + if (!localFs.mkdirs(localDir)) { + throw new IOException( + "Unable to create local directory, try to create it manually for further insights: " + localDir); + } UserGroupInformation.setConfiguration(conf); // Add session specific credentials to the AM credentials. From a77f8ef9f8d8753c125c9fb57dc47cdaace09198 Mon Sep 17 00:00:00 2001 From: jteagles Date: Wed, 1 May 2024 10:02:54 -0500 Subject: [PATCH 2/3] TEZ-4553. Default task scheduler to DagAwareTaskScheduler to avoid hang in TEZ-3535 (#347) Co-authored-by: Jonathan Eagles --- .../src/main/java/org/apache/tez/dag/api/TezConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 3dc6fe4745..7e86853631 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -905,7 +905,7 @@ public TezConfiguration(boolean loadDefaults) { @ConfigurationProperty public static final String TEZ_AM_YARN_SCHEDULER_CLASS = TEZ_AM_PREFIX + "yarn.scheduler.class"; public static final String TEZ_AM_YARN_SCHEDULER_CLASS_DEFAULT = - "org.apache.tez.dag.app.rm.YarnTaskSchedulerService"; + "org.apache.tez.dag.app.rm.DagAwareYarnTaskScheduler"; /** * Int value. The AM waits this amount of time when the first DAG is submitted but not all the services are ready. From 906059adb1f196c07cade0b3abdac4142f4e819f Mon Sep 17 00:00:00 2001 From: jteagles Date: Fri, 3 May 2024 13:51:25 -0500 Subject: [PATCH 3/3] TEZ-4558. Update build setup maven version and enforcer minimum to correct minimum (#350) Co-authored-by: Jonathan Eagles --- BUILDING.txt | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index a917bfa1a2..57c9cd13c0 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ https://cwiki.apache.org/confluence/display/TEZ Requirements: * JDK 1.8+ -* Maven 3.1 or later +* Maven 3.6.3 or later * Findbugs 2.0.2 or later (if running findbugs) * ProtocolBuffer 3.21.1 * Internet connection for first build (to fetch all dependencies) diff --git a/pom.xml b/pom.xml index 159bbae58f..67fa9ec369 100644 --- a/pom.xml +++ b/pom.xml @@ -1058,7 +1058,7 @@ - [3.0.2,) + [3.6.3,) [${maven.compiler.target},)