twitter · johnynek · Oct 22, 2015 · Oct 20, 2015 · Oct 20, 2015 · Oct 20, 2015
diff --git a/summingbird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/BatchConfig.scala b/summingbird-batch-hadoop/src/main/scala/com/twitter/summingbird/batch/BatchConfig.scala
diff --git a/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/BatchID.scala b/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/BatchID.scala
@@ -46,10 +46,9 @@ import scala.collection.Iterator.iterate
  */
 
 object BatchID {
+  import OrderedFromOrderingExt._
   implicit val equiv: Equiv[BatchID] = Equiv.by(_.id)
 
-  def apply(ts: Timestamp) = new BatchID(ts.milliSinceEpoch)
-
   // Enables BatchID(someBatchID.toString) roundtripping
   def apply(str: String) = new BatchID(str.split("\\.")(1).toLong)
 
@@ -124,20 +123,18 @@ object BatchID {
   implicit val batchID2Bytes: Injection[BatchID, Array[Byte]] =
     Injection.connect[BatchID, Long, Array[Byte]]
 
-  implicit val ord: Ordering[BatchID] = Ordering.by(_.id)
+  implicit val batchIdOrdering: Ordering[BatchID] = Ordering.by(_.id)
 }
 
-case class BatchID(id: Long) extends Ordered[BatchID] with java.io.Serializable {
+case class BatchID(id: Long) extends AnyVal {
+  import OrderedFromOrderingExt._
   def next: BatchID = new BatchID(id + 1)
   def prev: BatchID = new BatchID(id - 1)
   def +(cnt: Long) = new BatchID(id + cnt)
-  def -(cnt: Long) = new BatchID(id - cnt)
-
-  def compare(b: BatchID) = id.compareTo(b.id)
 
+  def -(cnt: Long) = new BatchID(id - cnt)
   def max(b: BatchID) = if (this >= b) this else b
   def min(b: BatchID) = if (this < b) this else b
 
-  override lazy val toString = "BatchID." + id.toString
-  override def hashCode: Int = id.hashCode
+  override def toString = "BatchID." + id.toString
 }
diff --git a/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/CombinedBatcher.scala b/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/CombinedBatcher.scala
@@ -29,7 +29,7 @@ import com.twitter.algebird.ExclusiveUpper
 class CombinedBatcher(before: Batcher,
     beforeBound: ExclusiveUpper[Timestamp],
     after: Batcher) extends Batcher {
-
+  import OrderedFromOrderingExt._
   val batchAtBound: BatchID = before.batchOf(beforeBound.upper.prev) + 1L
   val afterBatchDelta: BatchID = after.batchOf(beforeBound.upper)
 

diff --git a/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/OrderedFromOrderingExt.scala b/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/OrderedFromOrderingExt.scala
@@ -0,0 +1,29 @@
+/*
+Copyright 2013 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.twitter.summingbird.batch
+
+object OrderedFromOrderingExt {
+  // Use an implicit class to give the ordering operations on T
+  // This lets us use an Ordering similar to the convenince of an Ordered
+  implicit class HelperClass[T](val ts: T) extends AnyVal {
+    def >(other: T)(implicit ord: Ordering[T]) = ord.gt(ts, other)
+    def <(other: T)(implicit ord: Ordering[T]) = ord.lt(ts, other)
+    def <=(other: T)(implicit ord: Ordering[T]) = ord.lteq(ts, other)
+    def >=(other: T)(implicit ord: Ordering[T]) = ord.gteq(ts, other)
+    def compare(other: T)(implicit ord: Ordering[T]) = ord.compare(ts, other)
+  }
+}
diff --git a/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/TimeStamp.scala b/summingbird-batch/src/main/scala/com/twitter/summingbird/batch/TimeStamp.scala
@@ -21,7 +21,7 @@ import com.twitter.bijection.Bijection
 import java.util.Date
 import com.twitter.scalding.RichDate
 
-case class Timestamp(milliSinceEpoch: Long) extends Ordered[Timestamp] {
+case class Timestamp(milliSinceEpoch: Long) extends AnyVal {
   def compare(that: Timestamp) = milliSinceEpoch.compare(that.milliSinceEpoch)
   def prev = copy(milliSinceEpoch = milliSinceEpoch - 1)
   def next = copy(milliSinceEpoch = milliSinceEpoch + 1)
@@ -36,7 +36,6 @@ case class Timestamp(milliSinceEpoch: Long) extends Ordered[Timestamp] {
   def incrementMinutes(minutes: Long) = Timestamp(milliSinceEpoch + (minutes * 1000 * 60))
   def incrementHours(hours: Long) = Timestamp(milliSinceEpoch + (hours * 1000 * 60 * 60))
   def incrementDays(days: Long) = Timestamp(milliSinceEpoch + (days * 1000 * 60 * 60 * 24))
-  override def hashCode: Int = milliSinceEpoch.hashCode
 }
 
 object Timestamp {
@@ -74,8 +73,11 @@ object Timestamp {
     override def sumOption(ti: TraversableOnce[Timestamp]) =
       if (ti.isEmpty) None
       else {
-        var last: Timestamp = null
-        ti.foreach { last = _ }
+        val iter = ti.toIterator
+        var last: Timestamp = iter.next
+        while (iter.hasNext) {
+          last = iter.next
+        }
         Some(last)
       }
   }

diff --git a/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatchLaws.scala b/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatchLaws.scala
@@ -34,7 +34,7 @@ object BatchLaws extends Properties("BatchID") {
 
   property("BatchID should respect ordering") =
     forAll { (a: Long, b: Long) =>
-      a.compare(b) == BatchID(a).compare(BatchID(b))
+      a.compare(b) == implicitly[Ordering[BatchID]].compare(BatchID(a), BatchID(b))
     }
 
   property("BatchID should respect addition and subtraction") =

diff --git a/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatcherLaws.scala b/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatcherLaws.scala
@@ -26,19 +26,23 @@ import java.util.concurrent.TimeUnit
 
 object BatcherLaws extends Properties("Batcher") {
   import Generators._
+  import OrderedFromOrderingExt._
 
   def batchIdIdentity(batcher: Batcher) = { (b: BatchID) =>
     batcher.batchOf(batcher.earliestTimeOf(b))
   }
 
   def earliestIs_<=(batcher: Batcher) = forAll { (d: Timestamp) =>
-    (batcher.earliestTimeOf(batcher.batchOf(d)).compareTo(d) <= 0)
+    val ord = implicitly[Ordering[Timestamp]]
+    ord.compare(batcher.earliestTimeOf(batcher.batchOf(d)), d) <= 0
   }
 
   def batchesAreWeakOrderings(batcher: Batcher) = forAll { (d1: Timestamp, d2: Timestamp) =>
-    batcher.batchOf(d1).compare(batcher.batchOf(d2)) match {
+    val ord = implicitly[Ordering[BatchID]]
+    val ordT = implicitly[Ordering[Timestamp]]
+    ord.compare(batcher.batchOf(d1), batcher.batchOf(d2)) match {
       case 0 => true // can't say much
-      case x => d1.compareTo(d2) == x
+      case x => ordT.compare(d1, d2) == x
     }
   }
 

diff --git a/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatcherSpec.scala b/summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatcherSpec.scala
@@ -20,7 +20,7 @@ import org.scalatest.WordSpec
 
 class BatcherSpec extends WordSpec {
   val hourlyBatcher = Batcher.ofHours(1)
-
+  import OrderedFromOrderingExt._
   def assertRelation(other: Batcher, m: Map[Long, Iterable[Long]]) =
     m.foreach {
       case (input, expected) =>

diff --git a/summingbird-builder/src/main/scala/com/twitter/summingbird/scalding/ScaldingEnv.scala b/summingbird-builder/src/main/scala/com/twitter/summingbird/scalding/ScaldingEnv.scala
@@ -16,7 +16,7 @@ limitations under the License.
 
 package com.twitter.summingbird.scalding
 
-import com.twitter.scalding.{ Args, Hdfs, RichDate, DateParser }
+import com.twitter.scalding.{ Args, Config, Hdfs, RichDate, DateParser }
 import com.twitter.summingbird.batch.store.HDFSMetadata
 import com.twitter.summingbird.{ Env, Summer, TailProducer, AbstractJob }
 import com.twitter.summingbird.batch.{ BatchID, Batcher, Timestamp }
@@ -116,7 +116,7 @@ case class ScaldingEnv(override val jobName: String, inargs: Array[String])
     val scald = Scalding(name, opts)
       .withRegistrars(ajob.registrars ++ builder.registrar.getRegistrars.asScala)
       .withConfigUpdater { c =>
-        c.updated(ajob.transformConfig(c.toMap))
+        Config.tryFrom(ajob.transformConfig(c.toMap).toMap).get
       }
 
     def getStatePath(ss: Store[_, _]): Option[String] =

diff --git a/summingbird-builder/src/test/scala/com/twitter/summingbird/builder/OptionsTest.scala b/summingbird-builder/src/test/scala/com/twitter/summingbird/builder/OptionsTest.scala
@@ -54,10 +54,10 @@ class OptionsTest extends WordSpec {
     assert(scalding.build.platform.jobName == "com.twitter.summingbird.builder.TestJob1")
 
     val conf = new Configuration
-    scalding.build.platform.updateConfig(conf)
-    assert(conf.get("com.twitter.chill.config.configuredinstantiator") != null)
-    assert(conf.get("summingbird.options") == scalding.build.platform.options.toString)
-    assert(conf.get("cascading.aggregateby.threshold") == "100000")
+    val cfg = scalding.build.platform.buildConfig(conf)
+    assert(cfg.get("com.twitter.chill.config.configuredinstantiator") != None)
+    assert(cfg.get("summingbird.options") == Some(scalding.build.platform.options.toString))
+    assert(cfg.get("cascading.aggregateby.threshold") == Some("100000"))
 
     val opts = scalding.build.platform.options
     val dependants = Dependants(scalding.build.toRun)

diff --git a/summingbird-scalding-test/src/main/scala/com/twitter/summingbird/scalding/TestService.scala b/summingbird-scalding-test/src/main/scala/com/twitter/summingbird/scalding/TestService.scala
@@ -44,7 +44,7 @@ class TestService[K, V](service: String,
     tconv: TupleConverter[(Timestamp, (K, Option[V]))],
     tconv2: TupleConverter[(Timestamp, (K, V))])
     extends BBatchedService[K, V] {
-
+  import OrderedFromOrderingExt._
   val batcher = inBatcher
   val ordering = ord
   val reducers = None

diff --git a/summingbird-scalding-test/src/main/scala/com/twitter/summingbird/scalding/TestStore.scala b/summingbird-scalding-test/src/main/scala/com/twitter/summingbird/scalding/TestStore.scala
@@ -42,7 +42,7 @@ object TestStore {
 
 class TestStore[K, V](store: String, inBatcher: Batcher, val initBatch: BatchID, initStore: Iterable[(K, V)], lastBatch: BatchID, override val pruning: PrunedSpace[(K, V)])(implicit ord: Ordering[K], tset: TupleSetter[(K, V)], tconv: TupleConverter[(K, V)])
     extends batch.BatchedStore[K, V] {
-
+  import OrderedFromOrderingExt._
   var writtenBatches = Set[BatchID](initBatch)
   val batches: Map[BatchID, Mappable[(K, V)]] =
     BatchID.range(initBatch, lastBatch).map { b => (b, mockFor(b)) }.toMap

diff --git a/summingbird-scalding-test/src/test/scala/com/twitter/summingbird/scalding/ScaldingLaws.scala b/summingbird-scalding-test/src/test/scala/com/twitter/summingbird/scalding/ScaldingLaws.scala
@@ -593,7 +593,7 @@ class ScaldingLaws extends WordSpec {
       val summer = TestGraphs.jobWithStats[Scalding, (Long, Int), Int, Int](jobID, source, testStore)(t =>
         fn(t._2))
       val scald = Scalding("scalaCheckJob").withConfigUpdater { sbconf =>
-        sbconf.put("scalding.job.uniqueId", jobID.get)
+        sbconf.+("scalding.job.uniqueId", jobID.get)
       }
       val ws = new LoopState(intr)
       val conf: Configuration = new Configuration()

diff --git a/...ding-test/src/test/scala/com/twitter/summingbird/scalding/ScaldingSerializationSpec.scala b/...ding-test/src/test/scala/com/twitter/summingbird/scalding/ScaldingSerializationSpec.scala
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapred.RecordReader
 import org.apache.hadoop.mapred.OutputCollector
 
 import org.scalatest.WordSpec
+import com.twitter.scalding.Config
 
 class ScaldingSerializationSpecs extends WordSpec {
   implicit def tupleExtractor[T <: (Long, _)]: TimeExtractor[T] = TimeExtractor(_._1)
@@ -67,7 +68,7 @@ class ScaldingSerializationSpecs extends WordSpec {
       val intr = Interval.leftClosedRightOpen(Timestamp(0L), Timestamp(inWithTime.size.toLong))
       val scald = Scalding("scalaCheckJob")
 
-      assert((try { scald.toFlow(intr, mode, scald.plan(summer)); true }
+      assert((try { scald.toFlow(Config.default, intr, mode, scald.plan(summer)); true }
       catch { case t: Throwable => println(toTry(t)); false }) == true)
     }
   }

diff --git a/summingbird-scalding/src/main/scala/com/twitter/summingbird/scalding/Executor.scala b/summingbird-scalding/src/main/scala/com/twitter/summingbird/scalding/Executor.scala
@@ -82,7 +82,7 @@ object Executor {
 
     val scaldPlatform = Scalding(config.name, options)
       .withRegistrars(config.registrars)
-      .withConfigUpdater { c => c.updated(config.transformConfig(c.toMap)) }
+      .withConfigUpdater { c => com.twitter.scalding.Config.tryFrom(config.transformConfig(c.toMap).toMap).get }
 
     val toRun = scaldPlatform.plan(config.graph)