twitter · jcoveney · May 21, 2014 · Apr 30, 2014 · May 1, 2014 · May 2, 2014
diff --git a/project/Build.scala b/project/Build.scala
@@ -36,6 +36,7 @@ object SummingbirdBuild extends Build {
     javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
 
     libraryDependencies ++= Seq(
+      "junit" % "junit" % "4.11" % "test",
       "org.slf4j" % "slf4j-api" % slf4jVersion,
       "org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
       // These satisify's scaldings log4j needs when in test mode
@@ -45,6 +46,8 @@ object SummingbirdBuild extends Build {
 
     libraryDependencies <+= scalaVersion(specs2Import(_)),
 
+    libraryDependencies += "com.novocode" % "junit-interface" % "0.10" % "test",
+
     resolvers ++= Seq(
       Opts.resolver.sonatypeSnapshots,
       Opts.resolver.sonatypeReleases,
@@ -121,12 +124,14 @@ object SummingbirdBuild extends Build {
     publishLocal := { }
   ).aggregate(
     summingbirdCore,
+    summingbirdCoreJava,
     summingbirdBatch,
     summingbirdBatchHadoop,
     summingbirdOnline,
     summingbirdClient,
     summingbirdStorm,
     summingbirdStormTest,
+    summingbirdStormJava,
     summingbirdScalding,
     summingbirdScaldingTest,
     summingbirdSpark,
@@ -197,6 +202,10 @@ object SummingbirdBuild extends Build {
     libraryDependencies += "com.twitter" %% "algebird-core" % algebirdVersion
   )
 
+  lazy val summingbirdCoreJava = module("core-java").dependsOn(
+    summingbirdCore % "test->test;compile->compile"
+  )
+
   lazy val summingbirdOnline = module("online").settings(
     libraryDependencies ++= Seq(
       "com.twitter" %% "algebird-core" % algebirdVersion,
@@ -249,6 +258,16 @@ object SummingbirdBuild extends Build {
     summingbirdStorm
   )
 
+  lazy val summingbirdStormJava = module("storm-java").settings(
+    libraryDependencies ++= Seq(
+      "storm" % "storm" % "0.9.0-wip15" % "provided"
+    )
+  ).dependsOn(
+    summingbirdCore % "test->test;compile->compile",
+    summingbirdCoreJava % "test->test;compile->compile",
+    summingbirdStorm % "test->test;compile->compile"
+  )
+
   lazy val summingbirdScalding = module("scalding").settings(
     libraryDependencies ++= Seq(
       "com.backtype" % "dfs-datastores" % dfsDatastoresVersion,
@@ -313,7 +332,7 @@ object SummingbirdBuild extends Build {
       "com.twitter" %% "tormenta-twitter" % tormentaVersion,
       "com.twitter" %% "storehaus-memcache" % storehausVersion
     )
-  ).dependsOn(summingbirdCore, summingbirdStorm)
+  ).dependsOn(summingbirdCore, summingbirdCoreJava, summingbirdStorm, summingbirdStormJava)
 
   lazy val sparkAssemblyMergeSettings = assemblySettings :+ {
     mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>

diff --git a/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/Buffer.java b/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/Buffer.java
@@ -0,0 +1,21 @@
+package com.twitter.summingbird.javaapi;
+
+import com.twitter.summingbird.Platform;
+
+/**
+ * a Buffer used in left joins
+ *
+ * @author Julien Le Dem
+ *
+ * @param <P> the underlying platform
+ * @param <BUFFER> Is the actual type used by the underlying Platform it is parameterized in <K,V>
+ * @param <K> key
+ * @param <V> value
+ */
+public class Buffer<P extends Platform<P>, BUFFER, K, V> extends Wrapper<BUFFER> {
+
+  public Buffer(BUFFER buffer) {
+    super(buffer);
+  }
+
+}
diff --git a/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/Function.java b/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/Function.java
@@ -0,0 +1,15 @@
+package com.twitter.summingbird.javaapi;
+
+/**
+ *
+ * A function that takes one parameter.
+ * lambdas in java 8 can be passed to methods taking an interface defining only one method.
+ *
+ * @author Julien Le Dem
+ *
+ * @param <IN> parameter
+ * @param <OUT> return type
+ */
+public interface Function<IN, OUT> {
+  OUT apply(IN p);
+}
diff --git a/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JKeyedProducer.java b/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JKeyedProducer.java
@@ -0,0 +1,108 @@
+package com.twitter.summingbird.javaapi;
+
+import scala.Option;
+import scala.Tuple2;
+
+import com.twitter.algebird.Semigroup;
+import com.twitter.summingbird.KeyedProducer;
+import com.twitter.summingbird.Platform;
+
+/**
+ * Wraps a KeyedProducer (operations on key-value tuples)
+ *
+ * @author Julien Le Dem
+ *
+ * @param <P> underlying Platform
+ * @param <K> key type
+ * @param <V> value type
+ */
+public interface JKeyedProducer<P extends Platform<P>, K, V> extends JProducer<P, Tuple2<K, V>> {
+
+  /**
+   * @return the wrapped KeyedProducer
+   */
+  KeyedProducer<P, K, V> unwrap();
+
+  // collectKeys and collectValues not useful in java as there is no PartialFunction
+
+  /**
+   * filters the producer by keeping the key-value tuples whose keys satisfy the predicate
+   * @param p predicate
+   * @return the filtered producer
+   */
+  JKeyedProducer<P, K, V> filterKeys(Predicate<K> p);
+
+  /**
+   * filters the producer by keeping the key value tuples whose values satisfy the predicate
+   * @param p predicate
+   * @return the filtered producer
+   */
+  JKeyedProducer<P, K, V> filterValues(Predicate<V> p);
+
+  /**
+   * flattens the result of f on keys into a new Producer
+   * @param f
+   * @return the resulting producer
+   */
+  <K2> JKeyedProducer<P, K2, V> flatMapKeys(Function<K, ? extends Iterable<K2>> f);
+
+  /**
+   * flattens the result of f on values into a new Producer
+   * @param f
+   * @return the resulting producer
+   */
+  <V2> JKeyedProducer<P, K, V2> flatMapValues(Function<V, ? extends Iterable<V2>> f);
+
+  /**
+   * @return a producer with only the keys
+   */
+  JProducer<P, K> keys();
+
+  /**
+   * looks up the keys in the provided service and adds the Optional result to the tuple
+   * @param service
+   * @return the resulting Producer
+   */
+  <RightV> JKeyedProducer<P, K, Tuple2<V, Option<RightV>>> leftJoin(Service<P, ?, K, RightV> service);
+
+  /**
+   * Joins the two keyed producers using the provided Buffer
+   * @param stream the right stream to join this with
+   * @param buffer the buffer used to store stream for joining with this.
+   * @return the resulting Producer
+   */
+  <RightV> JKeyedProducer<P, K, Tuple2<V, Option<RightV>>> leftJoin(JKeyedProducer<P, K, RightV> stream, Buffer<P, ?, K, RightV> buffer);
+
+  /**
+   * applies f to keys
+   * @param f
+   * @return
+   */
+  <K2> JKeyedProducer<P, K2, V> mapKeys(Function<K, K2> f);
+
+  /**
+   * applies f to values
+   * @param f
+   * @return
+   */
+  <V2> JKeyedProducer<P, K, V2> mapValues(Function<V, V2> f);
+
+  /**
+   * sums by key using the provided store to persist the result
+   * @param store
+   * @param semigroup the definition of the Sum to apply
+   * @return
+   */
+  JSummer<P, K, V> sumByKey(Store<P, ?, K, V> store, Semigroup<V> semigroup);
+
+  /**
+   * swaps key and value
+   * @return
+   */
+  JKeyedProducer<P, V, K> swap();
+
+  /**
+   * @return a producer with only the values
+   */
+  JProducer<P, V> values();
+}
diff --git a/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JProducer.java b/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JProducer.java
@@ -0,0 +1,95 @@
+package com.twitter.summingbird.javaapi;
+
+import scala.Option;
+import scala.Tuple2;
+
+import com.twitter.summingbird.Platform;
+import com.twitter.summingbird.Producer;
+
+/**
+ * Wraps a Producer
+ *
+ * @author Julien Le Dem
+ *
+ * @param <P> underlying Platform
+ * @param <T> record type
+ */
+public interface JProducer<P extends Platform<P>, T> {
+
+  /**
+   * @return the wrapped Producer
+   */
+  Producer<P, T> unwrap();
+
+  /**
+   * names this producer
+   * @param id the name of the producer
+   * @return
+   */
+  JProducer<P, T> name(String id);
+
+  // merge, lookup and write are invariant unlike the scala api because java does not allow lower bounds in type parameters
+
+  /**
+   * merges two producers of the same type
+   * @param r
+   * @return
+   */
+  JProducer<P, T> merge(JProducer<P, T> r);
+
+  // collect does not really apply in java < 8
+
+  /**
+   * filters this by applying the provided predicate
+   * @param p
+   * @return
+   */
+  JProducer<P, T> filter(Predicate<T> p);
+
+  /**
+   * looks up the values in the provided service and returns key-value tuples of the optional result
+   * @param service
+   * @return
+   */
+  <V> JKeyedProducer<P, T, Option<V>> lookup(Service<P, ?, T, V> service);
+
+  /**
+   * applies the function to the values
+   * @param f
+   * @return
+   */
+  <U> JProducer<P, U> map(Function<T, U> f);
+
+  /**
+   * applies the function to the values and flattens the result
+   * (preferred to flatMap when possible to enable optimizations)
+   * @param f
+   * @return
+   */
+  <U> JProducer<P, U> optionMap(Function<T, Option<U>> f);
+
+  /**
+   * applies the function to the values and flattens the result
+   * @param f
+   * @return
+   */
+  <U> JProducer<P, U> flatMap(Function<T, ? extends Iterable<U>> f);
+
+  /**
+   * writes the result to the provided Sink
+   * @param sink
+   * @return
+   */
+  JTailProducer<P, T> write(Sink<P, ?, T> sink);
+
+  // Either does not cross compile between scala 2.9 and 2.10 as it moved from scala to scala.util
+//  <U> JProducer<P, Either<T, U>> either(JProducer<P, U> other);
+
+  /**
+   * turns a Producer into a KeyedProducer by applying the provided function returning key-value tuples
+   * @param f
+   * @return
+   */
+  <K, V> JKeyedProducer<P, K, V> mapToKeyed(Function<T, Tuple2<K, V>> f);
+
+}
diff --git a/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JProducers.java b/summingbird-core-java/src/main/java/com/twitter/summingbird/javaapi/JProducers.java
@@ -0,0 +1,74 @@
+package com.twitter.summingbird.javaapi;
+
+import scala.Option;
+import scala.Some;
+import scala.Tuple2;
+
+import com.twitter.summingbird.Platform;
+import com.twitter.summingbird.Producer;
+import com.twitter.summingbird.javaapi.impl.JKeyedProducerImpl;
+import com.twitter.summingbird.javaapi.impl.JProducerImpl;
+
+
+/**
+ * Helpers to help interfacing with scala
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class JProducers {
+
+  /**
+   * Wraps a source to pass to get a JProducer
+   * @param source
+   * @return
+   */
+  public static <P extends Platform<P>, T> JProducer<P, T> source(Source<P, ?, T> source) {
+    return JProducerImpl.source(source);
+  }
+
+  /**
+   * Wraps a source to pass to get a JProducer
+   * @param source
+   * @return
+   */
+  public static <P extends Platform<P>, T> JProducer<P, T> source(Producer<P, T> source) {
+    return JProducerImpl.source(source);
+  }
+
+  /**
+   * converts a JProducer to a JKeyedProducer when T is actually a Tuple2<K, V>
+   * @param producer
+   * @return
+   */
+  public static <P extends Platform<P>, T, K, V> JKeyedProducer<P, K, V> toKeyed(JProducer<P, Tuple2<K, V>> producer) {
+    return JKeyedProducerImpl.toKeyed(producer);
+  }
+
+  /**
+   * @return None<T>
+   */
+  public static <T> Option<T> none() {
+    return scala.Option$.MODULE$.<T>empty();
+  }
+
+  /**
+   * @param t
+   * @return Some<T>(t)
+   */
+  public static <T> Option<T> some(T t) {
+    if (t == null) {
+      throw new NullPointerException("some(null)");
+    }
+    return new Some<T>(t);
+  }
+
+  /**
+   * @param t
+   * @return Option<T>(t)
+   */
+  public static <T> Option<T> option(T t) {
+    return scala.Option$.MODULE$.apply(t);
+  }
+
+}