twitter · ianoc · Apr 9, 2014 · Apr 1, 2014 · Apr 1, 2014 · ianoc
diff --git a/...bird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/option/FlatMapShards.scala b/...bird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/option/FlatMapShards.scala
@@ -20,4 +20,7 @@ object FlatMapShards {
   val default = FlatMapShards(0)
 }
 
+/**
+ * Setting this will force a random shuffle to the specified number of reducers.
+ */
 case class FlatMapShards(count: Int)
diff --git a/summingbird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/option/Reducers.scala b/summingbird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/option/Reducers.scala
@@ -20,4 +20,7 @@ object Reducers {
   val default = Reducers(-1)
 }
 
+/**
+ * This specifies the number of reducers to be used in the context of the map-reduce framework.
+ */
 case class Reducers(count: Int)
diff --git a/summingbird-online/src/main/scala/com/twitter/summingbird/online/option/AllOpts.scala b/summingbird-online/src/main/scala/com/twitter/summingbird/online/option/AllOpts.scala
@@ -1,4 +1,5 @@
 package com.twitter.summingbird.online.option
+
 import com.twitter.util.Duration
 
 case class OnlineSuccessHandler(handlerFn: Unit => Unit)
@@ -41,36 +42,38 @@ case class MaxWaitingFutures(get: Int)
  */
 case class MaxFutureWaitTime(get: Duration)
 
-/*
-  FlushFrequency is how often regardless of traffic a given Cache should be flushed to the network.
-*/
+/**
+ * FlushFrequency is how often, regardless of traffic, a given Cache should be flushed to the network.
+ */
 case class FlushFrequency(get: Duration)
 
-/*
- UseAsyncCache is used to enable a background asynchronous cache. These do all cache related operations in background threads.
+/**
+ * UseAsyncCache is used to enable a background asynchronous cache. These do all cache related operations in
+ * background threads.
  */
 case class UseAsyncCache(get: Boolean)
 
-/*
-  AsyncPoolSize controls the size of the fixed thread pool used to back an asynchronous cache.
-  Only will have an effect if UseAsyncCache is true
-*/
+/**
+ * AsyncPoolSize controls the size of the fixed thread pool used to back an asynchronous cache.
+ * Only will have an effect if UseAsyncCache is true
+ */
 case class AsyncPoolSize(get: Int)
 
-/*
-  MaxEmitPerExecute controls the number of elements that can at once be emitted to the underlying platform.
-  Must be careful this is >> than your fan out or more tuples could be generated than are emitted.
-*/
+/**
+ * MaxEmitPerExecute controls the number of elements that can at once be emitted to the underlying platform.
+ * Must be careful this is >> than your fan out or more tuples could be generated than are emitted.
+ */
 case class MaxEmitPerExecute(get: Int)
 
-/*
- SoftMemoryFlushPercent is the percentage of memory used in the JVM at which a flush will be triggered of the cache.
-*/
+/**
+ * SoftMemoryFlushPercent is the percentage of memory used in the JVM at which a flush will be triggered of the cache.
+ */
 case class SoftMemoryFlushPercent(get: Float) {
   require(0 < get && get <= 100.0, "must be a percentage.")
 }
 
-/*
-  ValueCombinerCacheSize is used in cache's that support it as a trigger to crush down a high locality of values without emitting
-*/
+/**
+ * ValueCombinerCacheSize is used in caches that support it as a trigger to crush down a high locality of
+ * values without emitting.
+ */
 case class ValueCombinerCacheSize(get: Int)
diff --git a/summingbird-storm/src/main/scala/com/twitter/summingbird/storm/option/FlatMapOptions.scala b/summingbird-storm/src/main/scala/com/twitter/summingbird/storm/option/FlatMapOptions.scala
@@ -33,7 +33,7 @@ case class SpoutParallelism(parHint: Int)
 case class FlatMapParallelism(parHint: Int)
 
 /**
-  * This stupidity is necessary because val parameters can't be
+  * This workaround is necessary because val parameters can't be
   * call-by-name.  We pass a function so that the metrics aren't
   * serialized. Beyond the storm IMetric not being serializable,
   * passing a value also causes problems with the instance registered
@@ -45,6 +45,10 @@ object FlatMapStormMetrics {
   def unapply(metrics: FlatMapStormMetrics) = Some(metrics.metrics)
 }
 
+/**
+ * When a bolt is prepared, these metrics will be use by being called with the TopologyContext for the storm
+ * bolt.
+ */
 class FlatMapStormMetrics(val metrics: () => TraversableOnce[StormMetric[IMetric]])
 
 
@@ -58,6 +62,17 @@ class SpoutStormMetrics(val metrics: () => TraversableOnce[StormMetric[IMetric]]
     {() => metrics().map{ x: StormMetric[IMetric] => Metric(x.name, x.metric, x.interval.inSeconds)}}
 }
 
+/**
+ * This signals that the storm bolts should use localOrShuffleGrouping, which means that if the downstream bolt
+ * has a task on the same local worker, the output will only go to those tasks. Otherwise, shuffling
+ * happens normally. This is important to understand as this can create hot spots in the topology.
+ */
 case class PreferLocalDependency(get: Boolean)
 
+/**
+ * If this is set to true, this means that a bolt will ack a tuple as soon as it is received and processing begins;
+ * otherwise, the tuple will be acked when the bolt completes. Acking signals to storm that a tuple has been fully
+ * processed, so if a tuple is acked on entry and then there is a failure it will not be replayed per storm's
+ * normal replay mechanisms.
+ */
 case class AckOnEntry(get: Boolean)
diff --git a/summingbird-storm/src/main/scala/com/twitter/summingbird/storm/option/SummerOptions.scala b/summingbird-storm/src/main/scala/com/twitter/summingbird/storm/option/SummerOptions.scala
@@ -47,5 +47,9 @@ object SummerStormMetrics {
 }
 class SummerStormMetrics(val metrics: () => TraversableOnce[StormMetric[_]])
 
-
-case class SummerBatchMultiplier(get: Int)
+/**
+ * This value is mulitplied by the summer parallelism to set the true value used to hash and shard the
+ * key/value pairs. This allows for there to be more, smaller batches sent out to a number of threads
+ * which are set by SummerParallelism.
+ */
+case class SummerBatchMultiplier(get: Int)