diff --git a/core/src/main/java/org/apache/iceberg/ManifestReader.java b/core/src/main/java/org/apache/iceberg/ManifestReader.java index d7917eabb10c..f2ef5cb0deff 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/ManifestReader.java @@ -278,6 +278,9 @@ private CloseableIterable> open(Schema projection) { if (projection.findField(DataFile.RECORD_COUNT.fieldId()) == null) { fields.add(DataFile.RECORD_COUNT); } + if (projection.findField(DataFile.FIRST_ROW_ID.fieldId()) == null) { + fields.add(DataFile.FIRST_ROW_ID); + } fields.add(MetadataColumns.ROW_POSITION); CloseableIterable> reader = diff --git a/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java b/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java index 411d401075d6..ad6ef605420a 100644 --- a/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java @@ -53,6 +53,17 @@ private PartitionUtil() {} // use java.util.HashMap because partition data may contain null values Map idToConstant = Maps.newHashMap(); + // add first_row_id as _row_id + if (task.file().firstRowId() != null) { + idToConstant.put( + MetadataColumns.ROW_ID.fieldId(), + convertConstant.apply(Types.LongType.get(), task.file().firstRowId())); + } + + idToConstant.put( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), + convertConstant.apply(Types.LongType.get(), task.file().fileSequenceNumber())); + // add _file idToConstant.put( MetadataColumns.FILE_PATH.fieldId(), diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageAssignment.java b/core/src/test/java/org/apache/iceberg/TestRowLineageAssignment.java index 404e083f48d3..38bea41d7ddc 100644 --- a/core/src/test/java/org/apache/iceberg/TestRowLineageAssignment.java +++ b/core/src/test/java/org/apache/iceberg/TestRowLineageAssignment.java @@ -685,6 +685,9 @@ private static void checkDataFileAssignment( try (ManifestReader reader = ManifestFiles.read(manifest, table.io(), table.specs())) { + // test that the first_row_id column is always scanned, even if not requested + reader.select(BaseScan.SCAN_COLUMNS); + for (DataFile file : reader) { assertThat(file.content()).isEqualTo(FileContent.DATA); if (index < firstRowIds.length) { diff --git a/core/src/test/java/org/apache/iceberg/data/DataTest.java b/core/src/test/java/org/apache/iceberg/data/DataTest.java index cc788e2ec078..f28022cc792e 100644 --- a/core/src/test/java/org/apache/iceberg/data/DataTest.java +++ b/core/src/test/java/org/apache/iceberg/data/DataTest.java @@ -28,9 +28,11 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -53,6 +55,14 @@ public abstract class DataTest { + private static final long FIRST_ROW_ID = 2_000L; + protected static final Map ID_TO_CONSTANT = + Map.of( + MetadataColumns.ROW_ID.fieldId(), + FIRST_ROW_ID, + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), + 34L); + protected abstract void writeAndValidate(Schema schema) throws IOException; protected void writeAndValidate(Schema schema, List data) throws IOException { @@ -139,6 +149,10 @@ protected boolean supportsGeospatial() { return false; } + protected boolean supportsRowLineage() { + return false; + } + @ParameterizedTest @FieldSource("SIMPLE_TYPES") public void testTypeSchema(Type type) throws IOException { @@ -599,4 +613,38 @@ public void testWriteNullValueForRequiredType() throws Exception { () -> writeAndValidate(schema, ImmutableList.of(genericRecord))); } } + + @Test + public void testRowLineage() throws Exception { + Assumptions.assumeThat(supportsRowLineage()) + .as("Row Lineage support is not implemented") + .isTrue(); + + Schema schema = + new Schema( + required(1, "id", LongType.get()), + required(2, "data", Types.StringType.get()), + MetadataColumns.ROW_ID, + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER); + + GenericRecord record = GenericRecord.create(schema); + + writeAndValidate( + schema, + List.of( + record.copy(Map.of("id", 1L, "data", "a")), + record.copy(Map.of("id", 2L, "data", "b")), + record.copy( + Map.of( + "id", + 3L, + "data", + "c", + "_row_id", + 1_000L, + "_last_updated_sequence_number", + 33L)), + record.copy(Map.of("id", 4L, "data", "d", "_row_id", 1_001L)), + record.copy(Map.of("id", 5L, "data", "e")))); + } } diff --git a/data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java b/data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java index e05afb998828..fc8d47680b0f 100644 --- a/data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java +++ b/data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.variants.Variant; @@ -31,12 +32,39 @@ public class DataTestHelpers { private DataTestHelpers() {} public static void assertEquals(Types.StructType struct, Record expected, Record actual) { + assertEquals(struct, expected, actual, null, -1); + } + + public static void assertEquals( + Types.StructType struct, + Record expected, + Record actual, + Map idToConstant, + int pos) { Types.StructType expectedType = expected.struct(); for (Types.NestedField field : struct.fields()) { Types.NestedField expectedField = expectedType.field(field.fieldId()); + Object expectedValue; if (expectedField != null) { - assertEquals( - field.type(), expected.getField(expectedField.name()), actual.getField(field.name())); + int id = expectedField.fieldId(); + if (id == MetadataColumns.ROW_ID.fieldId()) { + expectedValue = expected.getField(expectedField.name()); + if (expectedValue == null && idToConstant != null) { + expectedValue = (Long) idToConstant.get(id) + pos; + } + + } else if (id == MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()) { + expectedValue = expected.getField(expectedField.name()); + if (expectedValue == null && idToConstant != null) { + expectedValue = idToConstant.get(id); + } + + } else { + expectedValue = expected.getField(expectedField.name()); + } + + assertEquals(field.type(), expectedValue, actual.getField(field.name())); + } else { assertEquals( field.type(), diff --git a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java index c663ad228c5c..1b5917e97296 100644 --- a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java +++ b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java @@ -36,8 +36,10 @@ import org.apache.iceberg.data.DataTestHelpers; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; @@ -61,6 +63,11 @@ protected boolean supportsTimestampNanos() { return true; } + @Override + protected boolean supportsRowLineage() { + return true; + } + @Override protected void writeAndValidate(Schema schema) throws IOException { writeAndValidate(schema, schema); @@ -80,11 +87,10 @@ protected void writeAndValidate(Schema writeSchema, Schema expectedSchema) throw private void writeAndValidate(Schema writeSchema, Schema expectedSchema, List expected) throws IOException { - File testFile = File.createTempFile("junit", null, temp.toFile()); - assertThat(testFile.delete()).isTrue(); + OutputFile output = new InMemoryOutputFile(); try (FileAppender appender = - Parquet.write(Files.localOutput(testFile)) + Parquet.write(output) .schema(writeSchema) .createWriterFunc(GenericParquetWriter::create) .build()) { @@ -93,30 +99,34 @@ private void writeAndValidate(Schema writeSchema, Schema expectedSchema, List rows; try (CloseableIterable reader = - Parquet.read(Files.localInput(testFile)) + Parquet.read(output.toInputFile()) .project(expectedSchema) .createReaderFunc( - fileSchema -> GenericParquetReaders.buildReader(expectedSchema, fileSchema)) + fileSchema -> + GenericParquetReaders.buildReader(expectedSchema, fileSchema, ID_TO_CONSTANT)) .build()) { rows = Lists.newArrayList(reader); } - for (int i = 0; i < expected.size(); i += 1) { - DataTestHelpers.assertEquals(expectedSchema.asStruct(), expected.get(i), rows.get(i)); + for (int pos = 0; pos < expected.size(); pos += 1) { + DataTestHelpers.assertEquals( + expectedSchema.asStruct(), expected.get(pos), rows.get(pos), ID_TO_CONSTANT, pos); } // test reuseContainers try (CloseableIterable reader = - Parquet.read(Files.localInput(testFile)) + Parquet.read(output.toInputFile()) .project(expectedSchema) .reuseContainers() .createReaderFunc( - fileSchema -> GenericParquetReaders.buildReader(expectedSchema, fileSchema)) + fileSchema -> + GenericParquetReaders.buildReader(expectedSchema, fileSchema, ID_TO_CONSTANT)) .build()) { - int index = 0; + int pos = 0; for (Record actualRecord : reader) { - DataTestHelpers.assertEquals(expectedSchema.asStruct(), expected.get(index), actualRecord); - index += 1; + DataTestHelpers.assertEquals( + expectedSchema.asStruct(), expected.get(pos), actualRecord, ID_TO_CONSTANT, pos); + pos += 1; } } } diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java index 4fa2d37a6235..8f2957e1c60d 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.parquet.ParquetSchemaUtil; import org.apache.iceberg.parquet.ParquetValueReader; @@ -77,7 +76,7 @@ protected ParquetValueReader createReader( } protected abstract ParquetValueReader createStructReader( - List types, List> fieldReaders, Types.StructType structType); + List> fieldReaders, Types.StructType structType); protected abstract ParquetValueReader fixedReader(ColumnDescriptor desc); @@ -110,7 +109,6 @@ public ParquetValueReader struct( // the expected struct is ignored because nested fields are never found when the List> newFields = Lists.newArrayListWithExpectedSize(fieldReaders.size()); - List types = Lists.newArrayListWithExpectedSize(fieldReaders.size()); List fields = struct.getFields(); for (int i = 0; i < fields.size(); i += 1) { ParquetValueReader fieldReader = fieldReaders.get(i); @@ -118,11 +116,10 @@ public ParquetValueReader struct( Type fieldType = fields.get(i); int fieldD = type().getMaxDefinitionLevel(path(fieldType.getName())) - 1; newFields.add(ParquetValueReaders.option(fieldType, fieldD, fieldReader)); - types.add(fieldType); } } - return createStructReader(types, newFields, expected); + return createStructReader(newFields, expected); } } @@ -225,10 +222,12 @@ public ParquetValueReader message( @Override public ParquetValueReader struct( Types.StructType expected, GroupType struct, List> fieldReaders) { + if (null == expected) { + return createStructReader(ImmutableList.of(), null); + } + // match the expected struct's order Map> readersById = Maps.newHashMap(); - Map typesById = Maps.newHashMap(); - Map maxDefinitionLevelsById = Maps.newHashMap(); List fields = struct.getFields(); for (int i = 0; i < fields.size(); i += 1) { ParquetValueReader fieldReader = fieldReaders.get(i); @@ -237,55 +236,37 @@ public ParquetValueReader struct( int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; int id = fieldType.getId().intValue(); readersById.put(id, ParquetValueReaders.option(fieldType, fieldD, fieldReader)); - typesById.put(id, fieldType); - if (idToConstant.containsKey(id)) { - maxDefinitionLevelsById.put(id, fieldD); - } } } - List expectedFields = - expected != null ? expected.fields() : ImmutableList.of(); + int constantDefinitionLevel = type.getMaxDefinitionLevel(currentPath()); + List expectedFields = expected.fields(); List> reorderedFields = Lists.newArrayListWithExpectedSize(expectedFields.size()); - List types = Lists.newArrayListWithExpectedSize(expectedFields.size()); - // Defaulting to parent max definition level - int defaultMaxDefinitionLevel = type.getMaxDefinitionLevel(currentPath()); + for (Types.NestedField field : expectedFields) { int id = field.fieldId(); - ParquetValueReader reader = readersById.get(id); - if (idToConstant.containsKey(id)) { - // containsKey is used because the constant may be null - int fieldMaxDefinitionLevel = - maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel); - reorderedFields.add( - ParquetValueReaders.constant(idToConstant.get(id), fieldMaxDefinitionLevel)); - types.add(null); - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { - reorderedFields.add(ParquetValueReaders.position()); - types.add(null); - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { - reorderedFields.add(ParquetValueReaders.constant(false)); - types.add(null); - } else if (reader != null) { - reorderedFields.add(reader); - types.add(typesById.get(id)); - } else if (field.initialDefault() != null) { - reorderedFields.add( - ParquetValueReaders.constant( - convertConstant(field.type(), field.initialDefault()), - maxDefinitionLevelsById.getOrDefault(id, defaultMaxDefinitionLevel))); - types.add(typesById.get(id)); - } else if (field.isOptional()) { - reorderedFields.add(ParquetValueReaders.nulls()); - types.add(null); - } else { - throw new IllegalArgumentException( - String.format("Missing required field: %s", field.name())); - } + ParquetValueReader reader = + ParquetValueReaders.replaceWithMetadataReader( + id, readersById.get(id), idToConstant, constantDefinitionLevel); + reorderedFields.add(defaultReader(field, reader, constantDefinitionLevel)); + } + + return createStructReader(reorderedFields, expected); + } + + private ParquetValueReader defaultReader( + Types.NestedField field, ParquetValueReader reader, int constantDL) { + if (reader != null) { + return reader; + } else if (field.initialDefault() != null) { + return ParquetValueReaders.constant( + convertConstant(field.type(), field.initialDefault()), constantDL); + } else if (field.isOptional()) { + return ParquetValueReaders.nulls(); } - return createStructReader(types, reorderedFields, expected); + throw new IllegalArgumentException(String.format("Missing required field: %s", field.name())); } @Override diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/GenericParquetReaders.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/GenericParquetReaders.java index e12f379b36bb..182412cfb54c 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/GenericParquetReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/GenericParquetReaders.java @@ -59,12 +59,24 @@ public static ParquetValueReader buildReader( return INSTANCE.createReader(expectedSchema, fileSchema, idToConstant); } - @Override + /** + * Create a struct reader. + * + * @deprecated will be removed in 1.10.0; use {@link #createStructReader(List, StructType)} + * instead. + */ + @Deprecated protected ParquetValueReader createStructReader( List types, List> fieldReaders, StructType structType) { return ParquetValueReaders.recordReader(fieldReaders, structType); } + @Override + protected ParquetValueReader createStructReader( + List> fieldReaders, StructType structType) { + return ParquetValueReaders.recordReader(fieldReaders, structType); + } + @Override protected ParquetValueReader fixedReader(ColumnDescriptor desc) { return new GenericParquetReaders.FixedReader(desc); diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/InternalReader.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/InternalReader.java index 05613eb1de16..692a9857cf77 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/InternalReader.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/InternalReader.java @@ -47,13 +47,26 @@ public static ParquetValueReader create( return (ParquetValueReader) INSTANCE.createReader(expectedSchema, fileSchema, idToConstant); } - @Override + /** + * Create a struct reader. + * + * @deprecated will be removed in 1.10.0; use {@link #createStructReader(List, StructType)} + * instead. + */ + @Deprecated @SuppressWarnings("unchecked") protected ParquetValueReader createStructReader( List types, List> fieldReaders, StructType structType) { return (ParquetValueReader) ParquetValueReaders.recordReader(fieldReaders, structType); } + @Override + @SuppressWarnings("unchecked") + protected ParquetValueReader createStructReader( + List> fieldReaders, StructType structType) { + return (ParquetValueReader) ParquetValueReaders.recordReader(fieldReaders, structType); + } + @Override protected ParquetValueReader fixedReader(ColumnDescriptor desc) { return new ParquetValueReaders.BytesReader(desc); diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReaders.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReaders.java index 63aac8006e2d..e91db8282e60 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReaders.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -161,6 +162,25 @@ public static ParquetValueReader position() { return new PositionReader(); } + @SuppressWarnings("unchecked") + public static ParquetValueReader rowIds(Long baseRowId, ParquetValueReader idReader) { + if (baseRowId != null) { + return new RowIdReader(baseRowId, (ParquetValueReader) idReader); + } else { + return ParquetValueReaders.nulls(); + } + } + + @SuppressWarnings("unchecked") + public static ParquetValueReader lastUpdated( + Long baseRowId, Long fileLastUpdated, ParquetValueReader seqReader) { + if (fileLastUpdated != null && baseRowId != null) { + return new LastUpdatedSeqReader(fileLastUpdated, (ParquetValueReader) seqReader); + } else { + return ParquetValueReaders.nulls(); + } + } + public static ParquetValueReader uuids(ColumnDescriptor desc) { return new UUIDReader(desc); } @@ -174,6 +194,27 @@ public static ParquetValueReader recordReader( return new RecordReader(readers, struct); } + public static ParquetValueReader replaceWithMetadataReader( + int id, ParquetValueReader reader, Map idToConstant, int constantDL) { + if (id == MetadataColumns.ROW_ID.fieldId()) { + Long baseRowId = (Long) idToConstant.get(id); + return ParquetValueReaders.rowIds(baseRowId, reader); + } else if (id == MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()) { + Long baseRowId = (Long) idToConstant.get(id); + Long fileSeqNumber = (Long) idToConstant.get(id); + return ParquetValueReaders.lastUpdated(baseRowId, fileSeqNumber, reader); + } else if (idToConstant.containsKey(id)) { + // containsKey is used because the constant may be null + return ParquetValueReaders.constant(idToConstant.get(id), constantDL); + } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { + return ParquetValueReaders.position(); + } else if (id == MetadataColumns.IS_DELETED.fieldId()) { + return ParquetValueReaders.constant(false, constantDL); + } + + return reader; + } + private static class NullReader implements ParquetValueReader { private static final NullReader INSTANCE = new NullReader<>(); private static final ImmutableList> COLUMNS = ImmutableList.of(); @@ -237,36 +278,9 @@ private static class ConstantReader implements ParquetValueReader { this.children = NullReader.COLUMNS; } - ConstantReader(C constantValue, int definitionLevel) { + ConstantReader(C constantValue, int parentDl) { this.constantValue = constantValue; - this.column = - new TripleIterator