Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jan 20, 2022. It is now read-only.

Fixed Storm stats bug with Promise #514

Merged
merged 5 commits into from
May 22, 2014
Merged

Fixed Storm stats bug with Promise #514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
73dd250
debugging
May 22, 2014
2a5379c
debugging
May 22, 2014
4bf0dc0
debug
May 22, 2014
f8b5e2b
Storm bug fix -> Promise for StormStatProvider
May 22, 2014
346e809
Added comments & cleaned up Storm bug fix
May 22, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions summingbird-storm/src/main/scala/com/twitter/summingbird/storm/BaseBolt.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,8 @@ case class BaseBolt[I,O](jobID: JobId,
collector = oc
metrics().foreach { _.register(context) }
executor.init(context)
val metricProvider = new StormStatProvider(jobID, context, countersForBolt)
metricProvider.registerMetrics
SummingbirdRuntimeStats.addPlatformStatProvider(metricProvider)
StormStatProvider.registerMetrics(jobID, context, countersForBolt)
SummingbirdRuntimeStats.addPlatformStatProvider(StormStatProvider)
logger.debug("In Bolt prepare: added jobID stat provider for jobID {}", jobID)
}

Expand Down
53 changes: 36 additions & 17 deletions summingbird-storm/src/main/scala/com/twitter/summingbird/storm/StormStatProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import backtype.storm.metric.api.CountMetric
import backtype.storm.task.TopologyContext
import com.twitter.summingbird.{ CounterIncrementor, PlatformStatProvider }
import com.twitter.summingbird.option.JobId
import com.twitter.util.{ Promise, Await }
import java.util.concurrent.ConcurrentHashMap
import org.slf4j.LoggerFactory

// Incrementor for Storm Counters
Expand All @@ -12,27 +14,44 @@ private[summingbird] case class StormCounterIncrementor(metric: CountMetric) ext
def incrBy(by: Long): Unit = metric.incrBy(by)
}

private[summingbird] case class StormStatProvider(jobID: JobId,
context: TopologyContext,
metrics: List[(String, String)]) extends PlatformStatProvider {
@transient private val logger = LoggerFactory.getLogger(classOf[StormStatProvider])
// StormStatProvider global object that contains counter information for the Storm job(s)
private [summingbird] object StormStatProvider extends PlatformStatProvider {
@transient private val logger = LoggerFactory.getLogger(StormStatProvider.getClass)

val stormMetrics: Map[String, CountMetric] = metrics.map {
case (groupName, metricName) => (groupName + "/" + metricName, new CountMetric)
}.toMap
logger.debug("Stats for this Bolt: {}", stormMetrics.keySet mkString)
// Keep a HashMap of JobId->Promise Map[String, CountMetric] where String is the Counter name
// Each metrics map will be initialized once (via Promise) and then referred to by other Bolts
private val metricsForJob = new ConcurrentHashMap[JobId, Promise[Map[String, CountMetric]]]()

def registerMetrics(jobID: JobId,
context: TopologyContext,
metrics: List[(String, String)]) {

val metricsPromise = Promise[Map[String, CountMetric]]

if (metricsForJob.putIfAbsent(jobID, metricsPromise) == null) {
val stormMetrics = metrics.map { case (groupName, metricName) =>
(groupName + "/" + metricName, new CountMetric)
}.toMap
logger.debug("Stats for this Bolt: {}", stormMetrics.keySet.mkString)

// Register metrics with the Storm TopologyContext
stormMetrics.foreach { case (name, metric) =>
logger.info("Registered metric {} with TopologyContext", name)
context.registerMetric(name, metric, 60)
}
// fullfill Promise
metricsPromise.setValue(stormMetrics)
}
}

// returns Storm counter incrementor to the Counter object in Summingbird job
def counterIncrementor(passedJobId: JobId, group: String, name: String): Option[StormCounterIncrementor] =
if(passedJobId.get == jobID.get) {
val metric = stormMetrics.getOrElse(group + "/" + name, sys.error("It is only valid to create counter objects during submission"))
Some(StormCounterIncrementor(metric))
if(metricsForJob.containsKey(passedJobId)) {
// Get the stormMetrics once Promise was fullfilled
val stormMetrics = Await.result(metricsForJob.get(passedJobId))
val metric = stormMetrics.getOrElse(group + "/" + name, sys.error("It is only valid to create counter objects during submission"))
Some(StormCounterIncrementor(metric))
} else {
None
}

def registerMetrics: Unit =
stormMetrics.foreach { case (name, metric) =>
logger.debug("Registered metric {} with TopologyContext", name)
context.registerMetric(name, metric, 60)
}
}