Extended index types to include temporal columns: Date, Time, and DateTime.

lwhite1 · lwhite1 · commit 528ff286567e · 2016-10-31T10:59:24.000-04:00
Minor fixes for LongIndex
diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.github.lwhite1</groupId>
   <artifactId>tablesaw</artifactId>
   <name>Tablesaw</name>
-  <version>0.7.6.4</version>
+  <version>0.7.6.5-SNAPSHOT</version>
   <description>High-performance Java Dataframe with integrated columnar storage</description>
   <url>https://jtablesaw.wordpress.com</url>
   <developers>
diff --git a/src/main/java/com/github/lwhite1/tablesaw/columns/DateColumnUtils.java b/src/main/java/com/github/lwhite1/tablesaw/columns/DateColumnUtils.java
@@ -1,6 +1,7 @@
 package com.github.lwhite1.tablesaw.columns;
 
 import com.github.lwhite1.tablesaw.api.DateColumn;
+import com.github.lwhite1.tablesaw.filtering.IntBiPredicate;
 import com.github.lwhite1.tablesaw.filtering.IntPredicate;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 
@@ -16,4 +17,13 @@ public interface DateColumnUtils extends Column, Iterable<LocalDate> {
   IntPredicate isMissing = i -> i == DateColumn.MISSING_VALUE;
 
   IntPredicate isNotMissing = i -> i != DateColumn.MISSING_VALUE;
+
+  IntBiPredicate isGreaterThan = (valueToTest, valueToCompareAgainst) -> valueToTest > valueToCompareAgainst;
+  IntBiPredicate isGreaterThanOrEqualTo = (valueToTest, valueToCompareAgainst) -> valueToTest >= valueToCompareAgainst;
+
+  IntBiPredicate isLessThan = (valueToTest, valueToCompareAgainst) -> valueToTest < valueToCompareAgainst;
+  IntBiPredicate isLessThanOrEqualTo = (valueToTest, valueToCompareAgainst) -> valueToTest <= valueToCompareAgainst;
+
+  IntBiPredicate isEqualTo = (valueToTest, valueToCompareAgainst) -> valueToTest == valueToCompareAgainst;
+
 }
diff --git a/src/main/java/com/github/lwhite1/tablesaw/index/DateIndex.java b/src/main/java/com/github/lwhite1/tablesaw/index/DateIndex.java
@@ -0,0 +1,44 @@
+package com.github.lwhite1.tablesaw.index;
+
+import com.github.lwhite1.tablesaw.api.DateColumn;
+import com.github.lwhite1.tablesaw.columns.packeddata.PackedLocalDate;
+import com.github.lwhite1.tablesaw.util.Selection;
+
+import java.time.LocalDate;
+
+/**
+ * An index for four-byte integer and Date columns
+ */
+public class DateIndex {
+
+  private final IntIndex index;
+
+  public DateIndex(DateColumn column) {
+    index = new IntIndex(column);
+  }
+
+  /**
+   * Returns a bitmap containing row numbers of all cells matching the given int
+   *
+   * @param value This is a 'key' from the index perspective, meaning it is a value from the standpoint of the column
+   */
+  public Selection get(LocalDate value) {
+    return index.get(PackedLocalDate.pack(value));
+  }
+
+  public Selection atLeast(LocalDate value) {
+    return index.atLeast(PackedLocalDate.pack(value));
+  }
+
+  public Selection greaterThan(LocalDate value) {
+    return index.greaterThan(PackedLocalDate.pack(value));
+  }
+
+  public Selection atMost(LocalDate value) {
+    return index.atMost(PackedLocalDate.pack(value));
+  }
+
+  public Selection lessThan(LocalDate value) {
+    return index.lessThan(PackedLocalDate.pack(value));
+  }
+}
diff --git a/src/main/java/com/github/lwhite1/tablesaw/index/DateTimeIndex.java b/src/main/java/com/github/lwhite1/tablesaw/index/DateTimeIndex.java
@@ -0,0 +1,44 @@
+package com.github.lwhite1.tablesaw.index;
+
+import com.github.lwhite1.tablesaw.api.DateTimeColumn;
+import com.github.lwhite1.tablesaw.columns.packeddata.PackedLocalDateTime;
+import com.github.lwhite1.tablesaw.util.Selection;
+
+import java.time.LocalDateTime;
+
+/**
+ * An index for four-byte integer and Date columns
+ */
+public class DateTimeIndex {
+
+  private final LongIndex index;
+
+  public DateTimeIndex(DateTimeColumn column) {
+    index = new LongIndex(column);
+  }
+
+  /**
+   * Returns a bitmap containing row numbers of all cells matching the given int
+   *
+   * @param value This is a 'key' from the index perspective, meaning it is a value from the standpoint of the column
+   */
+  public Selection get(LocalDateTime value) {
+    return index.get(PackedLocalDateTime.pack(value));
+  }
+
+  public Selection atLeast(LocalDateTime value) {
+    return index.atLeast(PackedLocalDateTime.pack(value));
+  }
+
+  public Selection greaterThan(LocalDateTime value) {
+    return index.greaterThan(PackedLocalDateTime.pack(value));
+  }
+
+  public Selection atMost(LocalDateTime value) {
+    return index.atMost(PackedLocalDateTime.pack(value));
+  }
+
+  public Selection lessThan(LocalDateTime value) {
+    return index.lessThan(PackedLocalDateTime.pack(value));
+  }
+}
diff --git a/src/main/java/com/github/lwhite1/tablesaw/index/IntIndex.java b/src/main/java/com/github/lwhite1/tablesaw/index/IntIndex.java
@@ -1,6 +1,8 @@
 package com.github.lwhite1.tablesaw.index;
 
+import com.github.lwhite1.tablesaw.api.DateColumn;
 import com.github.lwhite1.tablesaw.api.IntColumn;
+import com.github.lwhite1.tablesaw.api.TimeColumn;
 import com.github.lwhite1.tablesaw.util.BitmapBackedSelection;
 import com.github.lwhite1.tablesaw.util.Selection;
 import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap;
@@ -35,6 +37,42 @@ public IntIndex(IntColumn column) {
     index = new Int2ObjectAVLTreeMap<>(tempMap);
   }
 
+  public IntIndex(DateColumn column) {
+    int sizeEstimate = Integer.min(1_000_000, column.size() / 100);
+    Int2ObjectOpenHashMap<IntArrayList> tempMap = new Int2ObjectOpenHashMap<>(sizeEstimate);
+    for (int i = 0; i < column.size(); i++) {
+      int value = column.getInt(i);
+      IntArrayList recordIds = tempMap.get(value);
+      if (recordIds == null) {
+        recordIds = new IntArrayList();
+        recordIds.add(i);
+        tempMap.trim();
+        tempMap.put(value, recordIds);
+      } else {
+        recordIds.add(i);
+      }
+    }
+    index = new Int2ObjectAVLTreeMap<>(tempMap);
+  }
+
+  public IntIndex(TimeColumn column) {
+    int sizeEstimate = Integer.min(1_000_000, column.size() / 100);
+    Int2ObjectOpenHashMap<IntArrayList> tempMap = new Int2ObjectOpenHashMap<>(sizeEstimate);
+    for (int i = 0; i < column.size(); i++) {
+      int value = column.getInt(i);
+      IntArrayList recordIds = tempMap.get(value);
+      if (recordIds == null) {
+        recordIds = new IntArrayList();
+        recordIds.add(i);
+        tempMap.trim();
+        tempMap.put(value, recordIds);
+      } else {
+        recordIds.add(i);
+      }
+    }
+    index = new Int2ObjectAVLTreeMap<>(tempMap);
+  }
+
   private final static Comparator<int[]> intArrayComparator = new Comparator<int[]>() {
     public int compare(int[] a, int[] b) {
       return Integer.compare(a[1], b[1]);
diff --git a/src/main/java/com/github/lwhite1/tablesaw/index/LongIndex.java b/src/main/java/com/github/lwhite1/tablesaw/index/LongIndex.java
@@ -1,5 +1,6 @@
 package com.github.lwhite1.tablesaw.index;
 
+import com.github.lwhite1.tablesaw.api.DateTimeColumn;
 import com.github.lwhite1.tablesaw.api.LongColumn;
 import com.github.lwhite1.tablesaw.util.BitmapBackedSelection;
 import com.github.lwhite1.tablesaw.util.Selection;
@@ -34,6 +35,24 @@ public LongIndex(LongColumn column) {
     index = new Long2ObjectAVLTreeMap<>(tempMap);
   }
 
+  public LongIndex(DateTimeColumn column) {
+    int sizeEstimate = Integer.min(1_000_000, column.size() / 100);
+    Long2ObjectOpenHashMap<IntArrayList> tempMap = new Long2ObjectOpenHashMap<>(sizeEstimate);
+    for (int i = 0; i < column.size(); i++) {
+      long value = column.getLong(i);
+      IntArrayList recordIds = tempMap.get(value);
+      if (recordIds == null) {
+        recordIds = new IntArrayList();
+        recordIds.add(i);
+        tempMap.trim();
+        tempMap.put(value, recordIds);
+      } else {
+        recordIds.add(i);
+      }
+    }
+    index = new Long2ObjectAVLTreeMap<>(tempMap);
+  }
+
   /**
    * Returns a bitmap containing row numbers of all cells matching the given long
    *
@@ -46,7 +65,7 @@ public Selection get(long value) {
     return selection;
   }
 
-  public Selection atLeast(int value) {
+  public Selection atLeast(long value) {
     Selection selection = new BitmapBackedSelection();
     Long2ObjectSortedMap<IntArrayList> tail = index.tailMap(value);
     for (IntArrayList keys : tail.values()) {
@@ -55,7 +74,7 @@ public Selection atLeast(int value) {
     return selection;
   }
 
-  public Selection greaterThan(int value) {
+  public Selection greaterThan(long value) {
     Selection selection = new BitmapBackedSelection();
     Long2ObjectSortedMap<IntArrayList> tail = index.tailMap(value + 1);
     for (IntArrayList keys : tail.values()) {
@@ -64,7 +83,7 @@ public Selection greaterThan(int value) {
     return selection;
   }
 
-  public Selection atMost(int value) {
+  public Selection atMost(long value) {
     Selection selection = new BitmapBackedSelection();
     Long2ObjectSortedMap<IntArrayList> head = index.headMap(value + 1);  // we add 1 to get values equal to the arg
     for (IntArrayList keys : head.values()) {
@@ -73,7 +92,7 @@ public Selection atMost(int value) {
     return selection;
   }
 
-  public Selection lessThan(int value) {
+  public Selection lessThan(long value) {
     Selection selection = new BitmapBackedSelection();
     Long2ObjectSortedMap<IntArrayList> head = index.headMap(value);  // we add 1 to get values equal to the arg
     for (IntArrayList keys : head.values()) {
diff --git a/src/main/java/com/github/lwhite1/tablesaw/index/TimeIndex.java b/src/main/java/com/github/lwhite1/tablesaw/index/TimeIndex.java
@@ -0,0 +1,44 @@
+package com.github.lwhite1.tablesaw.index;
+
+import com.github.lwhite1.tablesaw.api.TimeColumn;
+import com.github.lwhite1.tablesaw.columns.packeddata.PackedLocalTime;
+import com.github.lwhite1.tablesaw.util.Selection;
+
+import java.time.LocalTime;
+
+/**
+ * An index for four-byte integer and Date columns
+ */
+public class TimeIndex {
+
+  private final IntIndex index;
+
+  public TimeIndex(TimeColumn column) {
+    index = new IntIndex(column);
+  }
+
+  /**
+   * Returns a bitmap containing row numbers of all cells matching the given int
+   *
+   * @param value This is a 'key' from the index perspective, meaning it is a value from the standpoint of the column
+   */
+  public Selection get(LocalTime value) {
+    return index.get(PackedLocalTime.pack(value));
+  }
+
+  public Selection atLeast(LocalTime value) {
+    return index.atLeast(PackedLocalTime.pack(value));
+  }
+
+  public Selection greaterThan(LocalTime value) {
+    return index.greaterThan(PackedLocalTime.pack(value));
+  }
+
+  public Selection atMost(LocalTime value) {
+    return index.atMost(PackedLocalTime.pack(value));
+  }
+
+  public Selection lessThan(LocalTime value) {
+    return index.lessThan(PackedLocalTime.pack(value));
+  }
+}
diff --git a/src/test/java/com/github/lwhite1/tablesaw/index/IntIndexTest.java b/src/test/java/com/github/lwhite1/tablesaw/index/IntIndexTest.java
@@ -2,13 +2,17 @@
 
 import com.github.lwhite1.tablesaw.api.ColumnType;
 import com.github.lwhite1.tablesaw.api.Table;
+import com.github.lwhite1.tablesaw.columns.DateColumnUtils;
 import com.github.lwhite1.tablesaw.columns.IntColumnUtils;
+import com.github.lwhite1.tablesaw.columns.packeddata.PackedLocalDate;
 import com.github.lwhite1.tablesaw.io.csv.CsvReader;
 import com.github.lwhite1.tablesaw.util.Selection;
 import com.google.common.base.Stopwatch;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.time.LocalDate;
+
 import static com.github.lwhite1.tablesaw.api.ColumnType.*;
 import static org.junit.Assert.assertEquals;
 
@@ -24,13 +28,15 @@ public class IntIndexTest {
   };
 
   private IntIndex index;
+  private DateIndex dateIndex;
   private Table table;
 
   @Before
   public void setUp() throws Exception {
     Stopwatch stopwatch = Stopwatch.createStarted();
     table = CsvReader.read(types, "data/BushApproval.csv");
     index = new IntIndex(table.intColumn("approval"));
+    dateIndex = new DateIndex(table.dateColumn("date"));
   }
 
   @Test
@@ -40,13 +46,31 @@ public void testGet() {
     assertEquals(fromCol, fromIdx);
   }
 
+  @Test
+  public void testGet2() {
+    LocalDate date = LocalDate.of(2001, 12, 12);
+    int packedDate = PackedLocalDate.pack(date);
+    Selection fromCol = table.dateColumn("date").select(DateColumnUtils.isEqualTo, packedDate);
+    Selection fromIdx = dateIndex.get(date);
+    assertEquals(fromCol, fromIdx);
+  }
+
   @Test
   public void testGTE() {
     Selection fromCol = table.intColumn("approval").select(IntColumnUtils.isGreaterThanOrEqualTo, 71);
     Selection fromIdx = index.atLeast(71);
     assertEquals(fromCol, fromIdx);
   }
 
+  @Test
+  public void testGTE2() {
+    LocalDate date = LocalDate.of(2001, 12, 12);
+    int packedDate = PackedLocalDate.pack(date);
+    Selection fromCol = table.dateColumn("date").select(DateColumnUtils.isGreaterThanOrEqualTo, packedDate);
+    Selection fromIdx = dateIndex.atLeast(date);
+    assertEquals(fromCol, fromIdx);
+  }
+
   @Test
   public void testLTE() {
     Selection fromCol = table.intColumn("approval").select(IntColumnUtils.isLessThanOrEqualTo, 71);
