[WIP] Add Manifest Stats in snapshot summary.

.baseline/checkstyle/checkstyle.xml

@@ -414,6 +414,9 @@
             <property name="format" value="@Test\(.*expected.*\)"/>
             <property name="message" value="Prefer using Assertions.assertThatThrownBy(...).isInstanceOf(...) instead."/>
         </module>
+        <module name="IllegalToken">
+            <property name="tokens" value="LITERAL_ASSERT"/>
+        </module>
         <module name="IllegalImport">
             <property name="id" value="BanExpectedExceptionUsage"/>
             <property name="illegalClasses" value="org.junit.rules.ExpectedException"/>

.github/ISSUE_TEMPLATE/iceberg_improvement.yml

@@ -50,4 +50,12 @@ body:
         - Hive
         - Other
     validations:
-      required: false
+      required: false
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages contributions. Would you or another member of your organization be willing to contribute this improvement/feature to the Apache Iceberg codebase?
+      options:
+        - label: I can contribute this improvement/feature independently
+        - label: I would be willing to contribute this improvement/feature with guidance from the Iceberg community
+        - label: I cannot contribute this improvement/feature at this time

.github/workflows/delta-conversion-ci.yml

@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -100,7 +100,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/flink-ci.yml

@@ -73,15 +73,8 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
-        flink: ['1.17', '1.18', '1.19']
-        exclude:
-          # Flink 1.17 does not support Java 17.
-          - jvm: 17
-            flink: '1.17'
-          # Flink 1.17 does not support Java 21.
-          - jvm: 21
-            flink: '1.17'
+        jvm: [11, 17, 21]
+        flink: ['1.18', '1.19', '1.20']
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/hive-ci.yml

@@ -69,7 +69,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/java-ci.yml

@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
@@ -107,7 +107,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4

.github/workflows/publish-snapshot.yml

@@ -37,7 +37,7 @@ jobs:
       - uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 8
+          java-version: 11
       - run: |
           ./gradlew printVersion
           ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

.github/workflows/spark-ci.yml

@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17, 21]
+        jvm: [11, 17, 21]
         spark: ['3.3', '3.4', '3.5']
         scala: ['2.12', '2.13']
         exclude:

.palantir/revapi.yml

@@ -1056,6 +1056,87 @@ acceptedBreaks:
     - code: "java.method.removed"
       old: "method org.apache.iceberg.DataFiles.Builder org.apache.iceberg.DataFiles.Builder::withEqualityFieldIds(java.util.List<java.lang.Integer>)"
       justification: "Deprecations for 1.6.0 release"
+  "1.6.0":
+    org.apache.iceberg:iceberg-common:
+    - code: "java.method.removed"
+      old: "method <T> org.apache.iceberg.common.DynFields.StaticField<T> org.apache.iceberg.common.DynFields.Builder::buildStaticChecked()\
+        \ throws java.lang.NoSuchFieldException"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method java.lang.Class<? extends C> org.apache.iceberg.common.DynConstructors.Ctor<C>::getConstructedClass()"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynConstructors.Builder org.apache.iceberg.common.DynConstructors.Builder::hiddenImpl(java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynMethods.Builder org.apache.iceberg.common.DynMethods.Builder::ctorImpl(java.lang.Class<?>,\
+        \ java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynMethods.Builder org.apache.iceberg.common.DynMethods.Builder::ctorImpl(java.lang.String,\
+        \ java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.visibilityReduced"
+      old: "method <R> R org.apache.iceberg.common.DynMethods.UnboundMethod::invokeChecked(java.lang.Object,\
+        \ java.lang.Object[]) throws java.lang.Exception"
+      new: "method <R> R org.apache.iceberg.common.DynMethods.UnboundMethod::invokeChecked(java.lang.Object,\
+        \ java.lang.Object[]) throws java.lang.Exception"
+      justification: "Reduced visibility and scoped to package"
+    org.apache.iceberg:iceberg-core:
+    - code: "java.class.removed"
+      old: "enum org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus"
+      justification: "Removing deprecated code"
+    - code: "java.method.addedToInterface"
+      new: "method org.apache.iceberg.metrics.CounterResult org.apache.iceberg.metrics.CommitMetricsResult::totalDataManifestFiles()"
+      justification: "Added new parameters for manifest stats"
+    - code: "java.method.addedToInterface"
+      new: "method org.apache.iceberg.metrics.CounterResult org.apache.iceberg.metrics.CommitMetricsResult::totalDeleteManifestFiles()"
+      justification: "Added new parameters for manifest stats"
+    - code: "java.method.removed"
+      old: "method java.lang.String org.apache.iceberg.FileScanTaskParser::toJson(org.apache.iceberg.FileScanTask)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.FileScanTask org.apache.iceberg.FileScanTaskParser::fromJson(java.lang.String,\
+        \ boolean)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.ContentCache.CacheEntry org.apache.iceberg.io.ContentCache::get(java.lang.String,\
+        \ java.util.function.Function<java.lang.String, org.apache.iceberg.io.ContentCache.FileContent>)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.ContentCache.CacheEntry org.apache.iceberg.io.ContentCache::getIfPresent(java.lang.String)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.InputFile org.apache.iceberg.io.ContentCache::tryCache(org.apache.iceberg.io.FileIO,\
+        \ java.lang.String, long)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseOverwriteFiles"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseReplacePartitions"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseRewriteManifests"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.StreamingDelete"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, java.lang.String, java.lang.String, java.lang.String,\
+        \ java.lang.String, java.lang.String)"
+      justification: "Removing deprecated code"
+    - code: "java.method.returnTypeChanged"
+      old: "method org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus org.apache.iceberg.BaseMetastoreTableOperations::checkCommitStatus(java.lang.String,\
+        \ org.apache.iceberg.TableMetadata)"
+      new: "method org.apache.iceberg.BaseMetastoreOperations.CommitStatus org.apache.iceberg.BaseMetastoreTableOperations::checkCommitStatus(java.lang.String,\
+        \ org.apache.iceberg.TableMetadata)"
+      justification: "Removing deprecated code"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

README.md

@@ -51,7 +51,7 @@ Community discussions happen primarily on the [dev mailing list][dev-list] or on
 
 ### Building
 
-Iceberg is built using Gradle with Java 8, 11, 17, or 21.
+Iceberg is built using Gradle with Java 11, 17, or 21.
 
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`

aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java

@@ -18,6 +18,8 @@
  */
 package org.apache.iceberg.aliyun.oss.mock;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import com.aliyun.oss.OSSErrorCode;
 import com.aliyun.oss.model.Bucket;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -137,7 +139,9 @@ ObjectMetadata putObject(
       Map<String, String> userMetaData)
       throws IOException {
     File bucketDir = new File(root, bucketName);
-    assert bucketDir.exists() || bucketDir.mkdirs();
+    assertThat(bucketDir)
+        .satisfiesAnyOf(
+            bucket -> assertThat(bucket).exists(), bucket -> assertThat(bucket.mkdirs()).isTrue());
 
     File dataFile = new File(bucketDir, fileName + DATA_FILE);
     File metaFile = new File(bucketDir, fileName + META_FILE);
@@ -170,17 +174,21 @@ ObjectMetadata putObject(
 
   void deleteObject(String bucketName, String filename) {
     File bucketDir = new File(root, bucketName);
-    assert bucketDir.exists();
+    assertThat(bucketDir).exists();
 
     File dataFile = new File(bucketDir, filename + DATA_FILE);
     File metaFile = new File(bucketDir, filename + META_FILE);
-    assert !dataFile.exists() || dataFile.delete();
-    assert !metaFile.exists() || metaFile.delete();
+    assertThat(dataFile)
+        .satisfiesAnyOf(
+            file -> assertThat(file).doesNotExist(), file -> assertThat(file.delete()).isTrue());
+    assertThat(metaFile)
+        .satisfiesAnyOf(
+            file -> assertThat(file).doesNotExist(), file -> assertThat(file.delete()).isTrue());
   }
 
   ObjectMetadata getObjectMetadata(String bucketName, String filename) throws IOException {
     File bucketDir = new File(root, bucketName);
-    assert bucketDir.exists();
+    assertThat(bucketDir).exists();
 
     File dataFile = new File(bucketDir, filename + DATA_FILE);
     if (!dataFile.exists()) {

api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java

@@ -122,4 +122,15 @@ public interface UpdatePartitionSpec extends PendingUpdate<PartitionSpec> {
    *     change conflicts with other additions, removals, or renames.
    */
   UpdatePartitionSpec renameField(String name, String newName);
+
+  /**
+   * Sets that the new partition spec will be NOT set as the default partition spec for the table,
+   * the default behavior is to do so.
+   *
+   * @return this for method chaining
+   */
+  default UpdatePartitionSpec addNonDefaultSpec() {
+    throw new UnsupportedOperationException(
+        this.getClass().getName() + " doesn't implement addNonDefaultSpec()");
+  };
 }

api/src/main/java/org/apache/iceberg/io/SupportsRecoveryOperations.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.io;
+
+/**
+ * This interface is intended as an extension for FileIO implementations to provide additional
+ * best-effort recovery operations that can be useful for repairing corrupted tables where there are
+ * reachable files missing from disk. (e.g. a live manifest points to data file entry which no
+ * longer exists on disk)
+ */
+public interface SupportsRecoveryOperations {
+
+  /**
+   * Perform a best-effort recovery of a file at a given path
+   *
+   * @param path Absolute path of file to attempt recovery for
+   * @return true if recovery was successful, false otherwise
+   */
+  boolean recoverFile(String path);
+}

api/src/main/java/org/apache/iceberg/types/Comparators.java

@@ -173,6 +173,10 @@ public static Comparator<CharSequence> charSequences() {
     return CharSeqComparator.INSTANCE;
   }
 
+  public static Comparator<CharSequence> filePath() {
+    return FilePathComparator.INSTANCE;
+  }
+
   private static class NullsFirst<T> implements Comparator<T> {
     private static final NullsFirst<?> INSTANCE = new NullsFirst<>();
 
@@ -351,4 +355,41 @@ public int compare(CharSequence s1, CharSequence s2) {
       return Integer.compare(s1.length(), s2.length());
     }
   }
+
+  private static class FilePathComparator implements Comparator<CharSequence> {
+    private static final FilePathComparator INSTANCE = new FilePathComparator();
+
+    private FilePathComparator() {}
+
+    @Override
+    public int compare(CharSequence s1, CharSequence s2) {
+      if (s1 == s2) {
+        return 0;
+      }
+      int count = s1.length();
+
+      int cmp = Integer.compare(count, s2.length());
+      if (cmp != 0) {
+        return cmp;
+      }
+
+      if (s1 instanceof String && s2 instanceof String) {
+        cmp = Integer.compare(s1.hashCode(), s2.hashCode());
+        if (cmp != 0) {
+          return cmp;
+        }
+      }
+      // File paths inside a delete file normally have more identical chars at the beginning. For
+      // example, a typical
+      // path is like "s3:/bucket/db/table/data/partition/00000-0-[uuid]-00001.parquet".
+      // The uuid is where the difference starts. So it's faster to find the first diff backward.
+      for (int i = count - 1; i >= 0; i--) {
+        cmp = Character.compare(s1.charAt(i), s2.charAt(i));
+        if (cmp != 0) {
+          return cmp;
+        }
+      }
+      return 0;
+    }
+  }
 }