fix

amogh-jahagirdar · amogh-jahagirdar · commit a0e2ce15fd90 · 2025-04-28T19:25:47.000-06:00
diff --git a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java
@@ -73,7 +73,8 @@ public static Iterable<Record> generateDictionaryEncodableRecords(
 
   public static Iterable<Record> generateDictionaryEncodableRecords(
       Schema schema, int numRecords, long seed, float nullPercentage) {
-    return generateIcebergGenerics(schema, numRecords, () -> new DictionaryEncodedGenerator(seed));
+    return generateIcebergGenerics(
+        schema, numRecords, () -> new DictionaryEncodedGenerator(seed, nullPercentage));
   }
 
   private static Iterable<Record> generateIcebergGenerics(
@@ -131,7 +132,7 @@ private static class DictionaryEncodedGenerator extends RandomRecordGenerator {
       super(seed);
     }
 
-    DictionaryEncodedGenerator(long seed, long nullPercentage) {
+    DictionaryEncodedGenerator(long seed, float nullPercentage) {
       super(seed, nullPercentage);
     }
 
@@ -218,8 +219,7 @@ public Object list(Types.ListType list, Supplier<Object> elementResult) {
 
       List<Object> result = Lists.newArrayListWithExpectedSize(numElements);
       for (int i = 0; i < numElements; i += 1) {
-        // return null 5% of the time when the value is optional
-        if (list.isElementOptional() && random.nextInt(20) == 1) {
+        if (list.isElementOptional() && isNull()) {
           result.add(null);
         } else {
           result.add(elementResult.get());
@@ -251,8 +251,7 @@ public Object map(Types.MapType map, Supplier<Object> keyResult, Supplier<Object
 
         keySet.add(key);
 
-        // return null 5% of the time when the value is optional
-        if (map.isValueOptional() && random.nextInt(20) == 1) {
+        if (map.isValueOptional() && isNull()) {
           result.put(key, null);
         } else {
           result.put(key, valueResult.get());
diff --git a/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java b/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java
@@ -36,6 +36,7 @@
 import java.nio.ByteOrder;
 import java.nio.file.Files;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
@@ -263,9 +264,37 @@ public void testRandomData() throws IOException {
 
     append.commit();
 
-    Set<Record> records = Sets.newHashSet(IcebergGenerics.read(table).build());
+    RecordComparator comparator = new RecordComparator();
+    List<Record> records = Lists.newArrayList(IcebergGenerics.read(table).build());
+
+    expected.sort(comparator);
+    records.sort(comparator);
     assertThat(records).as("Should produce correct number of records").hasSameSizeAs(expected);
-    assertThat(records).as("Random record set should match").isEqualTo(Sets.newHashSet(expected));
+    assertThat(records).as("Random record set should match").isEqualTo(expected);
+  }
+
+  // Compares by ID, then puts null data first, then lexicographically by data
+  private static class RecordComparator implements Comparator<Record> {
+
+    @Override
+    public int compare(Record r1, Record r2) {
+      long idComparison = r1.get(0, Long.class) - r2.get(0, Long.class);
+      if (idComparison < 0) {
+        return -1;
+      } else if (idComparison > 0) {
+        return 1;
+      }
+
+      if (r1.get(1, String.class) == null) {
+        return -1;
+      }
+
+      if (r2.get(1, String.class) == null) {
+        return 1;
+      }
+
+      return r1.get(1, String.class).compareTo(r2.get(1, String.class));
+    }
   }
 
   @TestTemplate
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/RandomData.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/RandomData.java
@@ -241,9 +241,15 @@ protected Object randomValue(Type.PrimitiveType primitive, Random rand) {
 
   private static class SparkRandomDataGenerator extends TypeUtil.CustomOrderSchemaVisitor<Object> {
     private final Random random;
+    private final float nullPercentage;
 
     private SparkRandomDataGenerator(long seed) {
+      this(seed, DEFAULT_NULL_PERCENTAGE);
+    }
+
+    private SparkRandomDataGenerator(long seed, float nullPercentage) {
       this.random = new Random(seed);
+      this.nullPercentage = nullPercentage;
     }
 
     @Override
@@ -265,12 +271,16 @@ public InternalRow struct(Types.StructType struct, Iterable<Object> fieldResults
     @Override
     public Object field(Types.NestedField field, Supplier<Object> fieldResult) {
       // return null 5% of the time when the value is optional
-      if (field.isOptional() && random.nextInt(20) == 1) {
+      if (field.isOptional() && isNull()) {
         return null;
       }
       return fieldResult.get();
     }
 
+    private boolean isNull() {
+      return random.nextFloat() < nullPercentage;
+    }
+
     @Override
     public GenericArrayData list(Types.ListType list, Supplier<Object> elementResult) {
       int numElements = random.nextInt(20);
@@ -279,7 +289,7 @@ public GenericArrayData list(Types.ListType list, Supplier<Object> elementResult
 
       for (int i = 0; i < numElements; i += 1) {
         // return null 5% of the time when the value is optional
-        if (list.isElementOptional() && random.nextInt(20) == 1) {
+        if (list.isElementOptional() && isNull()) {
           arr[i] = null;
         } else {
           arr[i] = elementResult.get();
@@ -310,8 +320,7 @@ public Object map(Types.MapType map, Supplier<Object> keyResult, Supplier<Object
         keySet.add(key);
 
         keysArr[i] = key;
-        // return null 5% of the time when the value is optional
-        if (map.isValueOptional() && random.nextInt(20) == 1) {
+        if (map.isValueOptional() && isNull()) {
           valuesArr[i] = null;
         } else {
           valuesArr[i] = valueResult.get();
