Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3b76767

Browse files
pjfanningmigesok
andauthored
Add AsyncWriteJournal option for disabling Resequencer (#2026) (#2027) (#2043)
* Add AsyncWriteJournal option for disabling Resequencer (#2026) * Add @internalapi as requested in review * Update AsyncWriteJournal.scala --------- Co-authored-by: Mikhail Sokolov <[email protected]>
1 parent 4890cff commit 3b76767

File tree

3 files changed

+166
-7
lines changed

3 files changed

+166
-7
lines changed

persistence/src/main/resources/reference.conf

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,20 @@ pekko.persistence {
151151
# replayed event.
152152
debug = off
153153
}
154+
155+
# Controls whether the journal plugin sends back write responses in the same order
156+
# as it received requests.
157+
#
158+
# Originally Akka-Persistence implementation rearranged responses to match the request order.
159+
# But this feature wasn't guaranteed by the Akka's test suite, and nothing in Akka itself relied on it.
160+
#
161+
# As this ordering is global, slow write requests for some entities can stall writes for all,
162+
# which can cause latency issues under load.
163+
#
164+
# The old behaviour is still enabled by default ("on"). After more testing on existing applications,
165+
# the default might be switched to "off", and eventually this option might be removed altogeter, leaving
166+
# "off" the only behaviour available.
167+
write-response-global-order = on
154168
}
155169

156170
# Fallback settings for snapshot store plugin configurations

persistence/src/main/scala/org/apache/pekko/persistence/journal/AsyncWriteJournal.scala

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import scala.util.control.NonFatal
2222

2323
import org.apache.pekko
2424
import pekko.actor._
25+
import pekko.annotation.InternalApi
2526
import pekko.pattern.CircuitBreaker
2627
import pekko.pattern.pipe
2728
import pekko.persistence._
@@ -67,9 +68,22 @@ trait AsyncWriteJournal extends Actor with WriteJournalBase with AsyncRecovery {
6768
final val receiveWriteJournal: Actor.Receive = {
6869
// cannot be a val in the trait due to binary compatibility
6970
val replayDebugEnabled: Boolean = config.getBoolean("replay-filter.debug")
71+
val enableGlobalWriteResponseOrder: Boolean = config.getBoolean("write-response-global-order")
72+
7073
val eventStream = context.system.eventStream // used from Future callbacks
7174
implicit val ec: ExecutionContext = context.dispatcher
7275

76+
// should be a private method in the trait, but it needs the enableGlobalWriteResponseOrder field which can't be
77+
// moved to the trait level because adding any fields there breaks bincompat
78+
@InternalApi
79+
def sendWriteResponse(msg: Any, snr: Long, target: ActorRef, sender: ActorRef): Unit = {
80+
if (enableGlobalWriteResponseOrder) {
81+
resequencer ! Desequenced(msg, snr, target, sender)
82+
} else {
83+
target.tell(msg, sender)
84+
}
85+
}
86+
7387
{
7488
case WriteMessages(messages, persistentActor, actorInstanceId) =>
7589
val cctr = resequencerCounter
@@ -100,7 +114,7 @@ trait AsyncWriteJournal extends Actor with WriteJournalBase with AsyncRecovery {
100114

101115
writeResult.onComplete {
102116
case Success(results) =>
103-
resequencer ! Desequenced(WriteMessagesSuccessful, cctr, persistentActor, self)
117+
sendWriteResponse(WriteMessagesSuccessful, cctr, persistentActor, self)
104118

105119
val resultsIter =
106120
if (results.isEmpty) Iterator.fill(atomicWriteCount)(AsyncWriteJournal.successUnit)
@@ -111,12 +125,12 @@ trait AsyncWriteJournal extends Actor with WriteJournalBase with AsyncRecovery {
111125
resultsIter.next() match {
112126
case Success(_) =>
113127
a.payload.foreach { p =>
114-
resequencer ! Desequenced(WriteMessageSuccess(p, actorInstanceId), n, persistentActor, p.sender)
128+
sendWriteResponse(WriteMessageSuccess(p, actorInstanceId), n, persistentActor, p.sender)
115129
n += 1
116130
}
117131
case Failure(e) =>
118132
a.payload.foreach { p =>
119-
resequencer ! Desequenced(
133+
sendWriteResponse(
120134
WriteMessageRejected(p, e, actorInstanceId),
121135
n,
122136
persistentActor,
@@ -126,21 +140,21 @@ trait AsyncWriteJournal extends Actor with WriteJournalBase with AsyncRecovery {
126140
}
127141

128142
case r: NonPersistentRepr =>
129-
resequencer ! Desequenced(LoopMessageSuccess(r.payload, actorInstanceId), n, persistentActor, r.sender)
143+
sendWriteResponse(LoopMessageSuccess(r.payload, actorInstanceId), n, persistentActor, r.sender)
130144
n += 1
131145
}
132146

133147
case Failure(e) =>
134-
resequencer ! Desequenced(WriteMessagesFailed(e, atomicWriteCount), cctr, persistentActor, self)
148+
sendWriteResponse(WriteMessagesFailed(e, atomicWriteCount), cctr, persistentActor, self)
135149
var n = cctr + 1
136150
messages.foreach {
137151
case a: AtomicWrite =>
138152
a.payload.foreach { p =>
139-
resequencer ! Desequenced(WriteMessageFailure(p, e, actorInstanceId), n, persistentActor, p.sender)
153+
sendWriteResponse(WriteMessageFailure(p, e, actorInstanceId), n, persistentActor, p.sender)
140154
n += 1
141155
}
142156
case r: NonPersistentRepr =>
143-
resequencer ! Desequenced(LoopMessageSuccess(r.payload, actorInstanceId), n, persistentActor, r.sender)
157+
sendWriteResponse(LoopMessageSuccess(r.payload, actorInstanceId), n, persistentActor, r.sender)
144158
n += 1
145159
}
146160
}
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.pekko.persistence.journal
19+
20+
import org.apache.pekko.persistence.{ AtomicWrite, JournalProtocol, PersistenceSpec, PersistentRepr }
21+
import org.apache.pekko.testkit.ImplicitSender
22+
23+
import scala.collection.immutable
24+
import scala.concurrent.{ ExecutionContext, Future, Promise }
25+
import scala.util.Try
26+
27+
/**
28+
* Verifies write response ordering logic for [[AsyncWriteJournal]].
29+
*
30+
* Checkout write-response-global-order config option for more information.
31+
*/
32+
class AsyncWriteJournalResponseOrderSpec
33+
extends PersistenceSpec(
34+
PersistenceSpec.config(
35+
plugin = "", // we will provide explicit plugin IDs later
36+
test = classOf[AsyncWriteJournalResponseOrderSpec].getSimpleName,
37+
extraConfig = Some(
38+
s"""
39+
|pekko.persistence.journal.reverse-plugin {
40+
| with-global-order {
41+
| class = "${classOf[AsyncWriteJournalResponseOrderSpec.ReversePlugin].getName}"
42+
|
43+
| write-response-global-order = on
44+
| }
45+
| no-global-order {
46+
| class = "${classOf[AsyncWriteJournalResponseOrderSpec.ReversePlugin].getName}"
47+
|
48+
| write-response-global-order = off
49+
| }
50+
|}
51+
|""".stripMargin
52+
))) with ImplicitSender {
53+
54+
import AsyncWriteJournalResponseOrderSpec._
55+
56+
"AsyncWriteJournal" must {
57+
"return write responses in request order if global response order is enabled" in {
58+
val pluginRef =
59+
extension.journalFor(journalPluginId = "pekko.persistence.journal.reverse-plugin.with-global-order")
60+
61+
pluginRef ! mkWriteMessages(1)
62+
pluginRef ! mkWriteMessages(2)
63+
pluginRef ! mkWriteMessages(3)
64+
65+
pluginRef ! CompleteWriteOps
66+
67+
getMessageNumsFromResponses(receiveN(6)) shouldEqual Vector(1, 2, 3)
68+
}
69+
70+
"return write responses in completion order if global response order is disabled" in {
71+
val pluginRef =
72+
extension.journalFor(journalPluginId = "pekko.persistence.journal.reverse-plugin.no-global-order")
73+
74+
pluginRef ! mkWriteMessages(1)
75+
pluginRef ! mkWriteMessages(2)
76+
pluginRef ! mkWriteMessages(3)
77+
78+
pluginRef ! CompleteWriteOps
79+
80+
getMessageNumsFromResponses(receiveN(6)) shouldEqual Vector(3, 2, 1)
81+
}
82+
}
83+
84+
private def mkWriteMessages(num: Int): JournalProtocol.WriteMessages = JournalProtocol.WriteMessages(
85+
messages = Vector(AtomicWrite(PersistentRepr(
86+
payload = num,
87+
sequenceNr = 0L,
88+
persistenceId = num.toString
89+
))),
90+
persistentActor = self,
91+
actorInstanceId = 1
92+
)
93+
94+
private def getMessageNumsFromResponses(responses: Seq[AnyRef]): Vector[Int] = responses.collect {
95+
case successResponse: JournalProtocol.WriteMessageSuccess =>
96+
successResponse.persistent.payload.asInstanceOf[Int]
97+
}.toVector
98+
}
99+
100+
private object AsyncWriteJournalResponseOrderSpec {
101+
case object CompleteWriteOps
102+
103+
/**
104+
* Accumulates asyncWriteMessages requests and completes them in reverse receive order on [[CompleteWriteOps]] command
105+
*/
106+
class ReversePlugin extends AsyncWriteJournal {
107+
108+
private implicit val ec: ExecutionContext = context.dispatcher
109+
110+
private var pendingOps: Vector[Promise[Unit]] = Vector.empty
111+
112+
override def receivePluginInternal: Receive = {
113+
case CompleteWriteOps =>
114+
pendingOps.reverse.foreach(_.success(()))
115+
pendingOps = Vector.empty
116+
}
117+
118+
override def asyncWriteMessages(messages: immutable.Seq[AtomicWrite]): Future[immutable.Seq[Try[Unit]]] = {
119+
val responsePromise = Promise[Unit]()
120+
pendingOps = pendingOps :+ responsePromise
121+
responsePromise.future.map(_ => Vector.empty)
122+
}
123+
124+
override def asyncDeleteMessagesTo(persistenceId: String, toSequenceNr: Long): Future[Unit] = ???
125+
126+
override def asyncReplayMessages(persistenceId: String, fromSequenceNr: Long, toSequenceNr: Long, max: Long)(
127+
recoveryCallback: PersistentRepr => Unit): Future[Unit] = ???
128+
129+
override def asyncReadHighestSequenceNr(persistenceId: String, fromSequenceNr: Long): Future[Long] = ???
130+
}
131+
}

0 commit comments

Comments
 (0)