Skip to content

Commit

Permalink
fix comment
Browse files Browse the repository at this point in the history
  • Loading branch information
leaves12138 committed Apr 15, 2024
1 parent 3e8e8bf commit 105012c
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 248 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ public RawFileSplitRead newBatchRawFileRead() {
schema,
valueType,
FileFormatDiscover.of(options),
pathFactory());
pathFactory(),
options.fileIndexReadEnabled());
}

public KeyValueFileReaderFactory.Builder newReaderFactoryBuilder() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ public FileIndexFileReader(
.filter(name -> name.endsWith(DataFilePathFactory.INDEX_PATH_SUFFIX))
.collect(Collectors.toList());
if (!indexFiles.isEmpty()) {
if (indexFiles.size() > 1) {
throw new RuntimeException(
"Found more than one index file for one data file: "
+ String.join(" and ", indexFiles));
}
// go to file index check
try (FileIndexPredicate predicate =
new FileIndexPredicate(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import javax.annotation.Nullable;

import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
Expand All @@ -39,7 +40,7 @@
import java.util.Map;

/** Index file writer. */
public final class FileIndexWriter {
public final class FileIndexWriter implements Closeable {

public static final FileIndexResult EMPTY_RESULT = FileIndexResult.of(null, null);

Expand Down Expand Up @@ -93,6 +94,7 @@ public void write(InternalRow row) {
indexMaintainers.forEach(indexMaintainer -> indexMaintainer.write(row));
}

@Override
public void close() throws IOException {
Map<String, Map<String, byte[]>> indexMaps = new HashMap<>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,6 @@ public class FileRecordReader implements RecordReader<InternalRow> {
@Nullable private final PartitionInfo partitionInfo;
@Nullable private final CastFieldGetter[] castMapping;

public FileRecordReader(
FormatReaderFactory readerFactory,
FormatReaderFactory.Context context,
@Nullable int[] indexMapping,
@Nullable CastFieldGetter[] castMapping,
@Nullable PartitionInfo partitionInfo)
throws IOException {
try {
this.reader = readerFactory.createReader(context);
} catch (Exception e) {
FileUtils.checkExists(context.fileIO(), context.filePath());
throw e;
}
this.indexMapping = indexMapping;
this.partitionInfo = partitionInfo;
this.castMapping = castMapping;
}

@Nullable
@Override
public RecordReader.RecordIterator<InternalRow> readBatch() throws IOException {
Expand Down Expand Up @@ -89,6 +71,24 @@ public RecordReader.RecordIterator<InternalRow> readBatch() throws IOException {
return iterator;
}

public FileRecordReader(
FormatReaderFactory readerFactory,
FormatReaderFactory.Context context,
@Nullable int[] indexMapping,
@Nullable CastFieldGetter[] castMapping,
@Nullable PartitionInfo partitionInfo)
throws IOException {
try {
this.reader = readerFactory.createReader(context);
} catch (Exception e) {
FileUtils.checkExists(context.fileIO(), context.filePath());
throw e;
}
this.indexMapping = indexMapping;
this.partitionInfo = partitionInfo;
this.castMapping = castMapping;
}

@Override
public void close() throws IOException {
reader.close();
Expand Down

This file was deleted.

Loading

0 comments on commit 105012c

Please sign in to comment.