packaging up aging work related to cleanup on some ML models and their tests

lwhite1 · lwhite1 · commit 45492305a602 · 2016-09-30T16:38:35.000-04:00
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ With Tablesaw, you can manipulate half a billion rows on a laptop and over 2 bil
     <dependency>
         <groupId>com.github.lwhite1</groupId>
         <artifactId>tablesaw</artifactId>
-        <version>0.7.6</version>
+        <version>0.7.6.3</version>
     </dependency>
 
 ### Documentation and support:
diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.github.lwhite1</groupId>
   <artifactId>tablesaw</artifactId>
   <name>Tablesaw</name>
-  <version>0.7.6</version>
+  <version>0.7.6.3</version>
   <description>High-performance Java Dataframe with integrated columnar storage</description>
   <url>https://jtablesaw.wordpress.com</url>
   <developers>
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.github.lwhite1</groupId>
     <artifactId>tablesaw</artifactId>
-    <version>0.7.6</version>
+    <version>0.7.6.3</version>
     <name>Tablesaw</name>
     <description>High-performance Java Dataframe with integrated columnar storage</description>
     <url>https://jtablesaw.wordpress.com</url>
diff --git a/src/main/java/com/github/lwhite1/tablesaw/api/BooleanColumn.java b/src/main/java/com/github/lwhite1/tablesaw/api/BooleanColumn.java
@@ -26,8 +26,7 @@
 import java.util.Iterator;
 import java.util.Map;
 
-import static com.github.lwhite1.tablesaw.columns.BooleanColumnUtils.isMissing;
-import static com.github.lwhite1.tablesaw.columns.BooleanColumnUtils.isNotMissing;
+import static com.github.lwhite1.tablesaw.columns.BooleanColumnUtils.*;
 
 /**
  * A column in a base table that contains float values
@@ -460,6 +459,15 @@ public int[] toIntArray() {
     return output;
   }
 
+  public IntColumn toIntColumn() {
+    IntColumn intColumn = IntColumn.create(this.name() + ": ints", size());
+    ByteArrayList data = data();
+    for (int i = 0; i < size(); i++) {
+      intColumn.add(data.getByte(i));
+    }
+    return intColumn;
+  }
+
   static class BooleanColumnIterator implements Iterator<Boolean> {
 
     final ByteIterator iterator;
diff --git a/src/main/java/com/github/lwhite1/tablesaw/api/ml/association/FrequentItemset.java b/src/main/java/com/github/lwhite1/tablesaw/api/ml/association/FrequentItemset.java
@@ -96,6 +96,36 @@ public FrequentItemset(IntColumn sets, CategoryColumn items, double support) {
     this.model = new FPGrowth(itemsets, support);
   }
 
+  public FrequentItemset(ShortColumn sets, CategoryColumn items, double support) {
+
+    labelMap = items.dictionaryMap().keyToValueMap();
+    Table temp = Table.create("temp");
+    temp.addColumn(sets.copy());
+    IntColumn encodedItems = items.toIntColumn();
+    encodedItems.setName(items.name());   // Needs t
+    temp.addColumn(encodedItems);
+    temp.sortAscendingOn(sets.name(), items.name());
+
+    ViewGroup baskets = temp.splitOn(temp.column(0));
+
+    this.setCount = baskets.size();
+
+    int[][] itemsets = new int[setCount][];
+    int basketIndex = 0;
+    for (TemporaryView basket : baskets) {
+      IntRBTreeSet set = new IntRBTreeSet(basket.intColumn(1).data());
+      int itemIndex = 0;
+      itemsets[basketIndex] = new int[set.size()];
+      for (int item : set) {
+        itemsets[basketIndex][itemIndex] = item;
+        itemIndex++;
+      }
+      basketIndex++;
+    }
+
+    this.model = new FPGrowth(itemsets, support);
+  }
+
   public FrequentItemset(ShortColumn sets, ShortColumn items, double support) {
 
     Table temp = Table.create("temp");
diff --git a/src/main/java/com/github/lwhite1/tablesaw/api/ml/classification/AbstractClassifier.java b/src/main/java/com/github/lwhite1/tablesaw/api/ml/classification/AbstractClassifier.java
@@ -9,6 +9,7 @@ public abstract class AbstractClassifier {
 
   abstract int predictFromModel(double[] data);
 
+
   void populateMatrix(int[] labels, ConfusionMatrix confusion, NumericColumn[] predictors) {
     double[] data = new double[predictors.length];
     for (int row = 0; row < predictors[0].size(); row++) {
diff --git a/src/main/java/com/github/lwhite1/tablesaw/table/Relation.java b/src/main/java/com/github/lwhite1/tablesaw/table/Relation.java
@@ -241,6 +241,9 @@ default NumericColumn numericColumn(int columnIndex) {
     if (c.type() == ColumnType.CATEGORY) {
       CategoryColumn categoryColumn = (CategoryColumn) c;
       return categoryColumn.toIntColumn();
+    }  else if (c.type() == ColumnType.BOOLEAN) {
+      BooleanColumn booleanColumn = (BooleanColumn) c;
+      return booleanColumn.toIntColumn();
     }
     return (NumericColumn) column(columnIndex);
   }
@@ -250,6 +253,9 @@ default NumericColumn numericColumn(String columnName) {
     if (c.type() == ColumnType.CATEGORY) {
       CategoryColumn categoryColumn = (CategoryColumn) c;
       return categoryColumn.toIntColumn();
+    } else if (c.type() == ColumnType.BOOLEAN) {
+      BooleanColumn booleanColumn = (BooleanColumn) c;
+      return booleanColumn.toIntColumn();
     }
     return (NumericColumn) column(columnName);
   }
diff --git a/src/main/java/com/github/lwhite1/tablesaw/util/DictionaryMap.java b/src/main/java/com/github/lwhite1/tablesaw/util/DictionaryMap.java
@@ -91,5 +91,4 @@ public Int2ObjectMap<String> keyToValueMap() {
   public Object2IntMap<String> valueToKeyMap() {
     return valueToKey;
   }
-
 }
diff --git a/src/test/java/com/github/lwhite1/tablesaw/api/FloatColumnTest.java b/src/test/java/com/github/lwhite1/tablesaw/api/FloatColumnTest.java
@@ -273,6 +273,7 @@ public void testCountMissing() {
     assertEquals(10, floats.countMissing());
   }
 
+
   @Test
   public void testCountUnique() {
     FloatColumn floats = new FloatColumn("floats", 10);
diff --git a/src/test/java/com/github/lwhite1/tablesaw/api/ml/association/FrequentItemsetExample.java b/src/test/java/com/github/lwhite1/tablesaw/api/ml/association/FrequentItemsetExample.java
@@ -1,5 +1,7 @@
 package com.github.lwhite1.tablesaw.api.ml.association;
 
+import com.github.lwhite1.tablesaw.api.CategoryColumn;
+import com.github.lwhite1.tablesaw.api.ShortColumn;
 import com.github.lwhite1.tablesaw.api.Table;
 import it.unimi.dsi.fastutil.ints.IntRBTreeSet;
 import it.unimi.dsi.fastutil.objects.Object2DoubleMap;
@@ -17,7 +19,18 @@ public static void main(String[] args) throws Exception {
 
     Table table = Table.createFromCsv("data/movielens.data", true, '\t');
     out(table.structure().print());
-    FrequentItemset model = new FrequentItemset(table.shortColumn("user"), table.shortColumn("movie"), .24);
+    out(table.shape());
+    ShortColumn movie = table.shortColumn("movie");
+    CategoryColumn moviecat = CategoryColumn.create("MovieCat");
+    for (int i = 0; i < movie.size(); i++) {
+      moviecat.addCell(movie.getString(i));
+    }
+    table.addColumn(moviecat);
+
+    out(table.shortColumn("user").unique().size());
+    out(table.shortColumn("movie").unique().size());
+
+    FrequentItemset model = new FrequentItemset(table.shortColumn("user"), table.categoryColumn("MovieCat"), .24);
     List<ItemSet> itemSetList = model.learn();
 
     out("Frequent Itemsets");
diff --git a/src/test/java/com/github/lwhite1/tablesaw/api/ml/classification/ConfusionMatrixTest.java b/src/test/java/com/github/lwhite1/tablesaw/api/ml/classification/ConfusionMatrixTest.java
@@ -1,5 +1,6 @@
 package com.github.lwhite1.tablesaw.api.ml.classification;
 
+import com.github.lwhite1.tablesaw.api.BooleanColumn;
 import com.github.lwhite1.tablesaw.api.Table;
 import com.github.lwhite1.tablesaw.util.DoubleArrays;
 import com.github.lwhite1.tablesaw.util.Example;
@@ -9,6 +10,8 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+import static com.github.lwhite1.tablesaw.api.QueryHelper.column;
+
 /**
  *
  */
@@ -18,6 +21,8 @@ public class ConfusionMatrixTest extends Example {
   public void testAsTable() throws Exception {
 
     Table example = Table.createFromCsv("data/KNN_Example_1.csv");
+    BooleanColumn booleanTarget = example.selectIntoColumn("bt", column("Label").isEqualTo(1));
+
     Table[] splits = example.sampleSplit(.5);
     Table train = splits[0];
     Table test = splits[1];
@@ -37,4 +42,34 @@ public void testAsTable() throws Exception {
       confusion.increment((int) test.shortColumn(2).get(row), predicted[row]);
     }
   }
+
+  @Test
+  public void testWithBooleanColumn() throws Exception {
+
+    Table example = Table.createFromCsv("data/KNN_Example_1.csv");
+    BooleanColumn booleanTarget = example.selectIntoColumn("bt", column("Label").isEqualTo(1));
+    example.addColumn(booleanTarget);
+    Table[] splits = example.sampleSplit(.5);
+    Table train = splits[0];
+    Table test = splits[1];
+
+    LogisticRegression lr = LogisticRegression.learn(
+        train.booleanColumn(3), train.nCol("X"), train.nCol("Y"));
+
+    System.out.println(lr.predictMatrix(test.booleanColumn(3), test.floatColumn(0), test.floatColumn(1)).toString());
+
+
+    int[] predicted = new int[test.rowCount()];
+    SortedSet<Object> lableSet = new TreeSet<>(train.shortColumn(2).asSet());
+    ConfusionMatrix confusion = new StandardConfusionMatrix(lableSet);
+    for (int row : test) {
+      double[] data = new double[2];
+      data[0] = test.floatColumn(0).getFloat(row);
+      data[1] = test.floatColumn(1).getFloat(row);
+      predicted[row] = lr.predict(data);
+      confusion.increment((int) test.shortColumn(2).get(row), predicted[row]);
+    }
+
+    System.out.println(confusion.toString());
+  }
 }
diff --git a/src/test/java/com/github/lwhite1/tablesaw/api/ml/clustering/KmeansExample.java b/src/test/java/com/github/lwhite1/tablesaw/api/ml/clustering/KmeansExample.java
@@ -13,7 +13,6 @@ public class KmeansExample {
 
   public static void main(String[] args) throws IOException {
     Table t = Table.createFromCsv("data/whiskey.csv");
-    //out(t.structure().printHtml());
 
     Kmeans model = new Kmeans(
         5,
@@ -38,7 +37,7 @@ public static void main(String[] args) throws IOException {
 
     //out(model.clustered(t.column(1)).printHtml());
 
-    out(model.labeledCentroids().printHtml());
+    out(model.labeledCentroids().print());
 
     int n = t.rowCount();
     double[] kValues = new double[n - 2];
diff --git a/src/test/java/com/github/lwhite1/tablesaw/examples/SfCrimeTest.java b/src/test/java/com/github/lwhite1/tablesaw/examples/SfCrimeTest.java

Original file line number	Diff line number	Diff line change
`@@ -241,6 +241,9 @@ default NumericColumn numericColumn(int columnIndex) {`
`241`	`241`	`if (c.type() == ColumnType.CATEGORY) {`
`242`	`242`	`CategoryColumn categoryColumn = (CategoryColumn) c;`
`243`	`243`	`return categoryColumn.toIntColumn();`
	`244`	`+ } else if (c.type() == ColumnType.BOOLEAN) {`
	`245`	`+ BooleanColumn booleanColumn = (BooleanColumn) c;`
	`246`	`+ return booleanColumn.toIntColumn();`
`244`	`247`	`}`
`245`	`248`	`return (NumericColumn) column(columnIndex);`
`246`	`249`	`}`
`@@ -250,6 +253,9 @@ default NumericColumn numericColumn(String columnName) {`
`250`	`253`	`if (c.type() == ColumnType.CATEGORY) {`
`251`	`254`	`CategoryColumn categoryColumn = (CategoryColumn) c;`
`252`	`255`	`return categoryColumn.toIntColumn();`
	`256`	`+ } else if (c.type() == ColumnType.BOOLEAN) {`
	`257`	`+ BooleanColumn booleanColumn = (BooleanColumn) c;`
	`258`	`+ return booleanColumn.toIntColumn();`
`253`	`259`	`}`
`254`	`260`	`return (NumericColumn) column(columnName);`
`255`	`261`	`}`
Original file line number	Diff line number	Diff line change
`@@ -91,5 +91,4 @@ public Int2ObjectMap<String> keyToValueMap() {`
`91`	`91`	`public Object2IntMap<String> valueToKeyMap() {`
`92`	`92`	`return valueToKey;`
`93`	`93`	`}`
`94`		`-`
`95`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -273,6 +273,7 @@ public void testCountMissing() {`
`273`	`273`	`assertEquals(10, floats.countMissing());`
`274`	`274`	`}`
`275`	`275`
	`276`	`+`
`276`	`277`	`@Test`
`277`	`278`	`public void testCountUnique() {`
`278`	`279`	`FloatColumn floats = new FloatColumn("floats", 10);`