Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jan 20, 2022. It is now read-only.

Fix checking delta batch in readDeltaTimestamps in batched store when merging #612

Merged
merged 6 commits into from
Mar 19, 2015
Merged

Fix checking delta batch in readDeltaTimestamps in batched store when merging #612

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da0cf7c
println
tsdeng Mar 17, 2015
d02909f
more
tsdeng Mar 18, 2015
26ac6b8
fix
tsdeng Mar 18, 2015
b30b36c
again
tsdeng Mar 18, 2015
2fb1997
prepare PR
tsdeng Mar 19, 2015
c97f459
fix log
tsdeng Mar 19, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions ...ingbird-scalding/src/main/scala/com/twitter/summingbird/scalding/batch/BatchedStore.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ trait BatchedStore[K, V] extends scalding.Store[K, V] { self =>
// get the batches read from the readTimespan
val batchIntr = batcher.batchesCoveredBy(readTimespan)

logger.info("readTimeSpan {}", readTimespan)
val batches = BatchID.toIterable(batchIntr).toList
val finalBatch = batches.last // batches won't be empty.
val filteredBatches = select(batches).sorted
Expand Down Expand Up @@ -283,6 +284,8 @@ trait BatchedStore[K, V] extends scalding.Store[K, V] { self =>
// Now get the first timestamp that we need input data for.
firstDeltaTimestamp = lastTimeWrittenToStore.next

firstDeltaBatch = lastBatch.next

// Get the requested timeSpan.
tsMode <- getState[FactoryInput]
(timeSpan, mode) = tsMode
Expand All @@ -292,19 +295,25 @@ trait BatchedStore[K, V] extends scalding.Store[K, V] { self =>

// Get the total time we want to cover. If the lower bound of the requested timeSpan
// is not the firstDeltaTimestamp, adjust it to that.
deltaTimes = setLower(InclusiveLower(firstDeltaTimestamp), timeSpan)
deltaTimes: Interval[Timestamp] = setLower(InclusiveLower(firstDeltaTimestamp), timeSpan)

// Try to read the range covering the time we want; get the time we can completely
// cover and the data from input in that range.
readTimeFlow <- fromEither(batchOps.readAvailableTimes(deltaTimes, mode, input))

(readDeltaTimestamps, readFlow) = readTimeFlow

firstDeltaBatchInterval: Interval[Timestamp] = batcher.toInterval(firstDeltaBatch)
// Make sure that the time we can read includes the time just after the last
// snapshot. We can't roll the store forward without this.
_ <- fromEither[FactoryInput](if (readDeltaTimestamps.contains(firstDeltaTimestamp)) Right(()) else
Left(List("Cannot load initial timestamp " + firstDeltaTimestamp.toString + " of deltas " +
" at " + this.toString + " only " + readDeltaTimestamps.toString)))
_ <- fromEither[FactoryInput] {
logger.info("firstBatchInterval is {}", firstDeltaBatchInterval)
if (readDeltaTimestamps.intersect(firstDeltaBatchInterval) == firstDeltaBatchInterval) //readDeltaTimestamps should include the firstDeltaBatchInterval
Right(())
else
Left(List("Cannot load initial timestamp interval " + firstDeltaBatchInterval.toString + " of deltas " +
" at " + this.toString + " only " + readDeltaTimestamps.toString))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

log should replace readDeltaTimestamps with firstDeltaBatchInterval

}

// Record the timespan we actually read.
_ <- putState((readDeltaTimestamps, mode))
Expand Down