Lock wait timeout exceeded; try restarting transaction #21293
Description
Before reporting an issue
- I have searched existing issues
- I have reproduced the issue with the latest release
Area
dist/quarkus
Describe the bug
Hi Team,
After upgrading to 21.1.1 we are seeing "Lock wait timeout exceeded; try restarting transaction" issue in our Keycloack nodes due to which some of the realm creation is failing. I am attaching stack trace [error.json] for more information.
While running our automation test which comprise of multiple realm creation we have experienced this error.
Our setup is running in clustered mode on Aurora-Mysql 8.x
Version
21.1.1
Expected behavior
Realm Creation should be successful
Actual behavior
Lock wait timeout exceeded; try restarting, Uncaught server error
"threadName": "executor-thread-0",
"threadId": 18,
"mdc": {},
"ndc": "",
"hostName": "kc-13-prospectfarm1-aw-us-0",
"processName": "QuarkusEntryPoint",
"processId": 1,
"exception": {
"refId": 1,
"exceptionType": "org.keycloak.models.ModelException",
"message": "javax.persistence.PessimisticLockException: could not execute statement",
"frames": [
{
"class": "org.keycloak.connections.jpa.PersistenceExceptionConverter",
"method": "convert",
"line": 84
},
{
"class": "org.keycloak.connections.jpa.PersistenceExceptionConverter",
"method": "invoke",
"line": 62
},
{
"class": "jdk.proxy2.$Proxy71",
"method": "flush"
},
{
"class": "org.keycloak.models.jpa.RealmAdapter",
"method": "setMasterAdminClient",
"line": 1181
},
{
"class": "org.keycloak.services.managers.RealmManager",
"method": "createMasterAdminManagement",
"line": 337
},
{
"class": "org.keycloak.services.managers.RealmManager",
"method": "setupMasterAdminManagement",
"line": 310
},
{
"class": "org.keycloak.services.managers.RealmManager",
"method": "importRealm",
"line": 539
},
{
"class": "org.keycloak.services.managers.RealmManager",
"method": "importRealm",
"line": 504
},
{
"class": "org.keycloak.services.managers.RealmManagerProviderFactory",
"method": "lambda$postInit$0",
"line": 54
},
{
"class": "org.keycloak.services.DefaultKeycloakSessionFactory",
"method": "publish",
"line": 91
},
{
"class": "org.keycloak.storage.ImportRealmFromRepresentationEvent",
"method": "fire",
"line": 50
},
{
"class": "org.keycloak.storage.datastore.LegacyExportImportManager",
"method": "importRealm",
"line": 159
},
{
"class": "org.keycloak.services.resources.admin.RealmsAdminResource",
"method": "importRealm",
"line": 133
},
{
"class": "jdk.internal.reflect.NativeMethodAccessorImpl",
"method": "invoke0"
},
{
"class": "jdk.internal.reflect.NativeMethodAccessorImpl",
"method": "invoke",
"line": 77
},
{
"class": "jdk.internal.reflect.DelegatingMethodAccessorImpl",
"method": "invoke",
"line": 43
},
{
"class": "java.lang.reflect.Method",
"method": "invoke",
"line": 568
},
{
"class": "org.jboss.resteasy.core.MethodInjectorImpl",
"method": "invoke",
"line": 170
},
{
"class": "org.jboss.resteasy.core.MethodInjectorImpl",
"method": "invoke",
"line": 130
},
{
"class": "org.jboss.resteasy.core.ResourceMethodInvoker",
"method": "internalInvokeOnTarget",
"line": 660
},
{
"class": "org.jboss.resteasy.core.ResourceMethodInvoker",
"method": "invokeOnTargetAfterFilter",
"line": 524
},
{
"class": "org.jboss.resteasy.core.ResourceMethodInvoker",
"method": "lambda$invokeOnTarget$2",
"line": 474
},
{
"class": "org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext",
"method": "filter",
"line": 364
},
{
"class": "org.jboss.resteasy.core.ResourceMethodInvoker",
"method": "invokeOnTarget",
"line": 476
},
{
"class": "org.jboss.resteasy.core.ResourceMethodInvoker",
"method": "invoke",
"line": 434
},
{
"class": "org.jboss.resteasy.core.ResourceLocatorInvoker",
"method": "invokeOnTargetObject",
"line": 192
},
{
"class": "org.jboss.resteasy.core.ResourceLocatorInvoker",
"method": "invoke",
"line": 141
},
{
"class": "org.jboss.resteasy.core.ResourceLocatorInvoker",
"method": "invoke",
"line": 32
},
{
"class": "org.jboss.resteasy.core.SynchronousDispatcher",
"method": "invoke",
"line": 492
},
{
"class": "org.jboss.resteasy.core.SynchronousDispatcher",
"method": "lambda$invoke$4",
"line": 261
},
{
"class": "org.jboss.resteasy.core.SynchronousDispatcher",
"method": "lambda$preprocess$0",
"line": 161
},
{
"class": "org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext",
"method": "filter",
"line": 364
},
{
"class": "org.jboss.resteasy.core.SynchronousDispatcher",
"method": "preprocess",
"line": 164
},
{
"class": "org.jboss.resteasy.core.SynchronousDispatcher",
"method": "invoke",
"line": 247
},
{
"class": "io.quarkus.resteasy.runtime.standalone.RequestDispatcher",
"method": "service",
"line": 73
},
{
"class": "io.quarkus.resteasy.runtime.standalone.VertxRequestHandler",
"method": "dispatch",
"line": 151
},
{
"class": "io.quarkus.resteasy.runtime.standalone.VertxRequestHandler",
"method": "handle",
"line": 82
},
{
"class": "io.quarkus.resteasy.runtime.standalone.VertxRequestHandler",
"method": "handle",
"line": 42
},
{
"class": "io.vertx.ext.web.impl.RouteState",
"method": "handleContext",
"line": 1284
},
{
"class": "io.vertx.ext.web.impl.RoutingContextImplBase",
"method": "iterateNext",
"line": 173
},
{
"class": "io.vertx.ext.web.impl.RoutingContextWrapper",
"method": "next",
"line": 200
},
{
"class": "io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2",
"method": "handle",
"line": 84
},
{
"class": "io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2",
"method": "handle",
"line": 71
},
{
"class": "io.vertx.ext.web.impl.RouteState",
"method": "handleContext",
"line": 1284
},
{
"class": "io.vertx.ext.web.impl.RoutingContextImplBase",
"method": "iterateNext",
"line": 173
},
{
"class": "io.vertx.ext.web.impl.RoutingContextWrapper",
"method": "next",
"line": 200
},
{
"class": "io.quarkus.vertx.http.runtime.VertxHttpRecorder$6",
"method": "handle",
"line": 430
},
{
"class": "io.quarkus.vertx.http.runtime.VertxHttpRecorder$6",
"method": "handle",
"line": 408
},
{
"class": "io.vertx.ext.web.impl.RouteState",
"method": "handleContext",
"line": 1284
},
{
"class": "io.vertx.ext.web.impl.RoutingContextImplBase",
"method": "iterateNext",
"line": 173
},
{
"class": "io.vertx.ext.web.impl.RoutingContextWrapper",
"method": "next",
"line": 200
},
{
"class": "org.keycloak.quarkus.runtime.integration.web.QuarkusRequestFilter",
"method": "lambda$createBlockingHandler$0",
"line": 82
},
{
"class": "io.quarkus.vertx.core.runtime.VertxCoreRecorder$14",
"method": "runWith",
"line": 576
},
{
"class": "org.jboss.threads.EnhancedQueueExecutor$Task",
"method": "run",
"line": 2449
},
{
"class": "org.jboss.threads.EnhancedQueueExecutor$ThreadBody",
"method": "run",
"line": 1478
},
{
"class": "org.jboss.threads.DelegatingRunnable",
"method": "run",
"line": 29
},
{
"class": "org.jboss.threads.ThreadLocalResettingRunnable",
"method": "run",
"line": 29
},
{
"class": "io.netty.util.concurrent.FastThreadLocalRunnable",
"method": "run",
"line": 30
},
{
"class": "java.lang.Thread",
"method": "run",
"line": 833
}
],
"causedBy": {
"exception": {
"refId": 2,
"exceptionType": "javax.persistence.PessimisticLockException",
"message": "could not execute statement",
"frames": [
{
"class": "org.hibernate.internal.ExceptionConverterImpl",
"method": "wrapLockException",
"line": 273
},
{
"class": "org.hibernate.internal.ExceptionConverterImpl",
"method": "convert",
"line": 108
},
{
"class": "org.hibernate.internal.ExceptionConverterImpl",
"method": "convert",
"line": 181
},
{
"class": "org.hibernate.internal.ExceptionConverterImpl",
"method": "convert",
"line": 188
},
{
"class": "org.hibernate.internal.SessionImpl",
"method": "doFlush",
"line": 1411
},
{
"class": "org.hibernate.internal.SessionImpl",
"method": "flush",
"line": 1394
},
{
"class": "jdk.internal.reflect.NativeMethodAccessorImpl",
"method": "invoke0"
},
{
"class": "jdk.internal.reflect.NativeMethodAccessorImpl",
"method": "invoke",
"line": 77
},
{
"class": "jdk.internal.refle"
}
How to Reproduce?
Create multiple realms in loop to reproduce this. We are able to reproduce this is very regular manner
Anything else?
N/A