Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jan 20, 2022. It is now read-only.

Add fully satisfiable flag to UniqueKeyedService #489

Merged
merged 2 commits into from
Mar 28, 2014
Merged

Add fully satisfiable flag to UniqueKeyedService #489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ac1d97
Add fully satisfiable flag to UniqueKeyedService
isnotinvain Mar 27, 2014
48c32f8
cleanup ws
isnotinvain Mar 28, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions summingbird-scalding/src/main/scala/com/twitter/summingbird/scalding/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ package object scalding {
implicit def flowDefFromTuple(implicit fm: (FlowDef, Mode)): FlowDef = fm._1
implicit def toPipeFactoryOps[T](pipeF: PipeFactory[T]) = new PipeFactoryOps(pipeF)

def toTry(e: Throwable): Try[Nothing] = {
def toTry(e: Throwable, msg: String = ""): Try[Nothing] = {
val writer = new java.io.StringWriter
val printWriter = new java.io.PrintWriter(writer)
e.printStackTrace(printWriter)
printWriter.flush
Left(List(writer.toString))
Left(List(msg + writer.toString))
}

val ScaldingConfig = summingbird.batch.BatchConfig
Expand Down
32 changes: 25 additions & 7 deletions ...scalding/src/main/scala/com/twitter/summingbird/scalding/service/UniqueKeyedService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,13 @@
*/

package com.twitter.summingbird.scalding.service
import com.twitter.algebird.monad.{Reader, StateWithError}
import com.twitter.algebird.Interval

import com.twitter.bijection.Conversion.asMethod
import com.twitter.summingbird.scalding._
import com.twitter.summingbird.batch.Timestamp
import com.twitter.scalding.{Grouped => _, Source => SSource, _ }
import com.twitter.scalding.{Grouped => _, Source => SSource, _}
import com.twitter.scalding.typed.{TypedPipe => _, _ }
import cascading.flow.FlowDef
import scala.util.control.NonFatal

/** A UniqueKeyedService covers the case where Keys are globally
* unique and either are not present or have one value.
Expand Down Expand Up @@ -68,19 +66,39 @@ trait SourceUniqueKeyedService[S <: SSource, K, V] extends UniqueKeyedService[K,
}

object UniqueKeyedService extends java.io.Serializable {
def from[K:Ordering,V](fn: DateRange => Mappable[(K,V)], reducers: Option[Int] = None): UniqueKeyedService[K, V] =
fromAndThen[(K,V),K,V](fn, identity, reducers)

def from[K:Ordering, V](fn: DateRange => Mappable[(K,V)],
reducers: Option[Int] = None,
requireFullySatisfiable: Boolean = false): UniqueKeyedService[K, V] =
fromAndThen[(K,V),K,V](fn, identity, reducers, requireFullySatisfiable)

/** The Mappable is the subclass of Source that knows about the file system. */
def fromAndThen[T,K:Ordering,V](
fn: DateRange => Mappable[T],
andThen: TypedPipe[T] => TypedPipe[(K,V)],
inputReducers: Option[Int] = None): UniqueKeyedService[K, V] =
inputReducers: Option[Int] = None,
requireFullySatisfiable: Boolean = false): UniqueKeyedService[K, V] =
new SourceUniqueKeyedService[Mappable[T], K, V] {
def ordering = Ordering[K]
def source(dr: DateRange) = fn(dr)
def toPipe(mappable: Mappable[T])(implicit flow: FlowDef, mode: Mode) =
andThen(TypedPipe.from(mappable)(flow, mode)) // converter is removed in 0.9.0
def reducers: Option[Int] = inputReducers

override def satisfiable(requested: DateRange, mode: Mode): Try[DateRange] = {
if (requireFullySatisfiable) {
val s = fn(requested)
try {
s.validateTaps(mode)
Right(requested)
} catch {
case NonFatal(e) =>
val msg = "Requested range: %s for source %s was not fully satisfiable.\n".format(requested, s)
toTry(e, msg)
}
} else {
super.satisfiable(requested, mode)
}
}
}
}