semistable

apache · RussellSpitzer · Jul 30, 2025 · Jun 10, 2025 · Jun 10, 2025 · Jun 18, 2025
commit c375e99d0e35b1255d179f51e4449c60a478791c
diff --git a/...a/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java b/...a/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java
@@ -30,6 +30,7 @@
 import org.apache.iceberg.arrow.vectorized.NullabilityHolder;
 import org.apache.iceberg.parquet.ParquetUtil;
 import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.values.ValuesReader;
 import org.apache.parquet.column.values.plain.PlainValuesReader;
 
 public final class VectorizedParquetDefinitionLevelReader
@@ -236,7 +237,7 @@ protected void nextRleBatch(
         int idx,
         int numValues,
         byte[] byteArray) {
-      setNextNValuesInVector(typeWidth, nullabilityHolder, valuesReader, idx, vector, numValues);
+      setNextNValuesInVector(nullabilityHolder, valuesReader, idx, vector, numValues, this);
     }
 
     @Override
@@ -265,6 +266,9 @@ protected void nextPackedBatch(
 
     protected abstract void nextVal(
         FieldVector vector, int idx, VectorizedValuesReader valuesReader, Mode mode);
+
+    public abstract void nextVals(
+            FieldVector vector, int rowId, VectorizedValuesReader valuesReader, int total);
   }
 
   class LongReader extends NumericBaseReader {
@@ -292,6 +296,11 @@ protected void nextDictEncodedVal(
             .setLong((long) idx * typeWidth, dict.decodeToLong(reader.readInteger()));
       }
     }
+
+    @Override
+    public void nextVals(FieldVector vector, int rowId, VectorizedValuesReader valuesReader, int total) {
+      valuesReader.readLongs(total, vector, rowId);
+    }
   }
 
   class DoubleReader extends NumericBaseReader {
@@ -319,6 +328,11 @@ protected void nextDictEncodedVal(
             .setDouble((long) idx * typeWidth, dict.decodeToDouble(reader.readInteger()));
       }
     }
+
+    @Override
+    public void nextVals(FieldVector vector, int rowId, VectorizedValuesReader valuesReader, int total) {
+      valuesReader.readDoubles(total, vector, rowId);
+    }
   }
 
   class FloatReader extends NumericBaseReader {
@@ -346,6 +360,11 @@ protected void nextDictEncodedVal(
             .setFloat((long) idx * typeWidth, dict.decodeToFloat(reader.readInteger()));
       }
     }
+
+    @Override
+    public void nextVals(FieldVector vector, int rowId, VectorizedValuesReader valuesReader, int total) {
+      valuesReader.readFloats(total, vector, rowId);
+    }
   }
 
   class IntegerReader extends NumericBaseReader {
@@ -375,6 +394,11 @@ protected void nextDictEncodedVal(
             .setInt((long) idx * typeWidth, dict.decodeToInt(reader.readInteger()));
       }
     }
+
+    @Override
+    public void nextVals(FieldVector vector, int rowId, VectorizedValuesReader valuesReader, int total) {
+      valuesReader.readIntegers(total, vector, rowId);
+    }
   }
 
   abstract class BaseReader extends CommonReader {
@@ -662,16 +686,15 @@ private void setNulls(
 
   @SuppressWarnings({"all"})
   private void setNextNValuesInVector(
-      int typeWidth,
       NullabilityHolder nullabilityHolder,
       VectorizedValuesReader valuesReader,
       int bufferIdx,
       FieldVector vector,
-      int numValues) {
+      int numValues,
+      NumericBaseReader reader) {
     ArrowBuf validityBuffer = vector.getValidityBuffer();
     if (currentValue == maxDefLevel) {
-      // TODO read the correct type not just hard-coded longs here
-      valuesReader.readFloats(numValues, vector, bufferIdx);
+      reader.nextVals(vector, bufferIdx, valuesReader, numValues);
       nullabilityHolder.setNotNulls(bufferIdx, numValues);
       if (setArrowValidityVector) {
         for (int i = 0; i < numValues; i++) {

diff --git a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java
@@ -175,7 +175,6 @@ public abstract static class RandomDataGenerator<T>
 
     private final Random random;
     private final float nullPercentage;
-    private int currentInt = 1;
 
     protected RandomDataGenerator(long seed) {
       this(seed, DEFAULT_NULL_PERCENTAGE);
@@ -290,10 +289,6 @@ public Object primitive(Type.PrimitiveType primitive) {
           } else {
             return EPOCH.plus((long) result, NANOS).toLocalDateTime();
           }
-        case INTEGER:
-          return currentInt++;
-        case LONG:
-          return (long)currentInt++;
         default:
           return result;
       }

diff --git a/...est/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetVectorizedReads.java b/...est/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetVectorizedReads.java
@@ -300,11 +300,11 @@ public void testSupportedReadsForParquetV2() throws Exception {
     // (i.e. decimals > 8 bytes)
     Schema schema =
         new Schema(
-            optional(102, "float_data", Types.FloatType.get())
-//            optional(103, "double_data", Types.DoubleType.get()),
-//            optional(104, "decimal_data", Types.DecimalType.of(25, 5)),
-//            optional(105, "int_data", Types.IntegerType.get()),
-//            optional(106, "long_data", Types.LongType.get())
+            optional(102, "float_data", Types.FloatType.get()),
+            optional(103, "double_data", Types.DoubleType.get()),
+            optional(104, "decimal_data", Types.DecimalType.of(25, 5)),
+            optional(105, "int_data", Types.IntegerType.get()),
+            optional(106, "long_data", Types.LongType.get())
         );
 
     File dataFile = File.createTempFile("junit", null, temp.toFile());
@@ -319,8 +319,7 @@ public void testSupportedReadsForParquetV2() throws Exception {
 
   @Test
   public void testUnsupportedReadsForParquetV2() throws Exception {
-    // Longs, ints, string types etc. use delta encoding and which are not supported for vectorized
-    // reads
+    // Some types use delta encoding and which are not supported for vectorized reads
     Schema schema = new Schema(SUPPORTED_PRIMITIVES.fields());
     File dataFile = File.createTempFile("junit", null, temp.toFile());
     assertThat(dataFile.delete()).as("Delete should succeed").isTrue();