Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jan 20, 2022. It is now read-only.

Fix root cause of duplicate summer execution in Memory platform #667

Merged
merged 7 commits into from
Jun 21, 2016
Merged

Fix root cause of duplicate summer execution in Memory platform #667

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
68e1ebb
d3b5808 doesn't address the root cause of duplicate sums with
pankajroark Jun 20, 2016
baf2174
Refactor toStream to make the diff more readable.
pankajroark Jun 20, 2016
5c68c57
Minor cleanup
pankajroark Jun 20, 2016
032a230
Remove unused type parameters.
pankajroark Jun 20, 2016
23d0018
Use an immutable map with reference equality.
pankajroark Jun 20, 2016
954ba40
Lazily evaluate left producer of also producer so that no execution
pankajroark Jun 20, 2016
3b8f571
Remove unused imports.
pankajroark Jun 21, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 32 additions & 3 deletions summingbird-core/src/main/scala/com/twitter/summingbird/memory/Memory.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,27 +25,55 @@ import collection.mutable.{ Map => MutableMap }
object Memory {
implicit def toSource[T](traversable: TraversableOnce[T])(implicit mf: Manifest[T]): Producer[Memory, T] =
Producer.source[Memory, T](traversable)

case class Identity[T <: AnyRef](val unwrap: T) {
override def equals(that: Any) = that match {
case i: Identity[_] => unwrap.eq(i.unwrap)
case _ => false
}
override def hashCode = System.identityHashCode(unwrap)
}

}

trait MemoryService[-K, +V] {
def get(k: K): Option[V]
}

class Memory(implicit jobID: JobId = JobId("default.memory.jobId")) extends Platform[Memory] {
import Memory.Identity

type Source[T] = TraversableOnce[T]
type Store[K, V] = MutableMap[K, V]
type Sink[-T] = (T => Unit)
type Service[-K, +V] = MemoryService[K, V]
type Plan[T] = Stream[T]

private type Prod[T] = Producer[Memory, T]
private type JamfMap = HMap[Prod, Stream]
private type IProd[T] = Identity[Producer[Memory, T]]

// Key is wrapped in Identity to get reference equality semantics
private type JamfMap = HMap[IProd, Stream]

def counter(group: Group, name: Name): Option[Long] =
MemoryStatProvider.getCountersForJob(jobID).flatMap { _.get(group.getString + "/" + name.getString).map { _.get } }

/**
* On the memory platform the notion of summingbird stream literally
* translates to scala streams. We plan the topology by creating
* streams from sources and transforming them using other components
* of the topology. Execution is then just a matter of forcing this stream.
*
* Since the topology is a DAG, some parts of the graph can be shared
* between multiple roots(TailProducers combined with AlsoProducer). To
* avoid duplicating the shared parts of the graph we keep track of
* planned portions in a map. This map uses reference equality because
* we care about the root components themselves and not their content.
* e.g. summer uses a mutable map for store. If the content of this
* mutable map changes it doesn't mean that we have a different summer.
*/
private def toStream[T](outerProducer: Prod[T], jamfs: JamfMap): (Stream[T], JamfMap) =
jamfs.get(outerProducer) match {
jamfs.get(Identity(outerProducer)) match {
case Some(s) => (s, jamfs)
case None =>
val (s, m) = outerProducer match {
Expand Down Expand Up @@ -83,6 +111,7 @@ class Memory(implicit jobID: JobId = JobId("default.memory.jobId")) extends Plat
//Plan the first one, but ignore it
val (left, leftM) = toStream(l, jamfs)
val (right, rightM) = toStream(r, leftM)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this comment is still true (note you have a cast).

// We need to force all of left to make sure any
// side effects in write happen
lazy val lforcedEmpty = left.filter(_ => false)
Expand Down Expand Up @@ -113,7 +142,7 @@ class Memory(implicit jobID: JobId = JobId("default.memory.jobId")) extends Plat
}
// scala can't infer that s is the right type through case statements above
val st = s.asInstanceOf[Stream[T]]
(st, m + (outerProducer -> st))
(st, m + (Identity(outerProducer) -> st))
}

def plan[T](prod: TailProducer[Memory, T]): Stream[T] = {
Expand Down