diff --git a/src/Symfony/Component/Security/Http/Firewall/ContextListener.php b/src/Symfony/Component/Security/Http/Firewall/ContextListener.php
index e1c9dc2bbddf9..a81b5a53b7c7b 100644
--- a/src/Symfony/Component/Security/Http/Firewall/ContextListener.php
+++ b/src/Symfony/Component/Security/Http/Firewall/ContextListener.php
@@ -21,6 +21,7 @@
 use Symfony\Component\HttpKernel\KernelEvents;
 use Symfony\Component\Security\Core\Authentication\AuthenticationTrustResolver;
 use Symfony\Component\Security\Core\Authentication\AuthenticationTrustResolverInterface;
+use Symfony\Component\Security\Core\Authentication\Token\AbstractToken;
 use Symfony\Component\Security\Core\Authentication\Token\AnonymousToken;
 use Symfony\Component\Security\Core\Authentication\Token\RememberMeToken;
 use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
@@ -237,7 +238,7 @@ protected function refreshUser(TokenInterface $token): ?TokenInterface
                 $newToken->setUser($refreshedUser, false);
 
                 // tokens can be deauthenticated if the user has been changed.
-                if ($this->hasUserChanged($user, $newToken)) {
+                if ($token instanceof AbstractToken && $this->hasUserChanged($user, $newToken)) {
                     $userDeauthenticated = true;
                     // @deprecated since Symfony 5.4
                     if (method_exists($newToken, 'setAuthenticated')) {
diff --git a/src/Symfony/Component/Security/Http/Tests/Firewall/ContextListenerTest.php b/src/Symfony/Component/Security/Http/Tests/Firewall/ContextListenerTest.php
index b59cc7d0e7e2e..b15721ca90035 100644
--- a/src/Symfony/Component/Security/Http/Tests/Firewall/ContextListenerTest.php
+++ b/src/Symfony/Component/Security/Http/Tests/Firewall/ContextListenerTest.php
@@ -29,6 +29,7 @@
 use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorage;
 use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
 use Symfony\Component\Security\Core\Authentication\Token\Storage\UsageTrackingTokenStorage;
+use Symfony\Component\Security\Core\Authentication\Token\TokenInterface;
 use Symfony\Component\Security\Core\Authentication\Token\UsernamePasswordToken;
 use Symfony\Component\Security\Core\Exception\UnsupportedUserException;
 use Symfony\Component\Security\Core\Exception\UserNotFoundException;
@@ -233,6 +234,27 @@ public function testIfTokenIsDeauthenticated()
         $this->assertNull($tokenStorage->getToken());
     }
 
+    public function testTokenIsNotDeauthenticatedOnUserChangeIfNotAnInstanceOfAbstractToken()
+    {
+        $tokenStorage = new TokenStorage();
+        $refreshedUser = new InMemoryUser('changed', 'baz');
+
+        $token = new CustomToken(new InMemoryUser('original', 'foo'), ['ROLE_FOO']);
+
+        $session = new Session(new MockArraySessionStorage());
+        $session->set('_security_context_key', serialize($token));
+
+        $request = new Request();
+        $request->setSession($session);
+        $request->cookies->set('MOCKSESSID', true);
+
+        $listener = new ContextListener($tokenStorage, [new NotSupportingUserProvider(true), new NotSupportingUserProvider(false), new SupportingUserProvider($refreshedUser)], 'context_key');
+        $listener(new RequestEvent($this->createMock(HttpKernelInterface::class), $request, HttpKernelInterface::MAIN_REQUEST));
+
+        $this->assertInstanceOf(CustomToken::class, $tokenStorage->getToken());
+        $this->assertSame($refreshedUser, $tokenStorage->getToken()->getUser());
+    }
+
     public function testIfTokenIsNotDeauthenticated()
     {
         $tokenStorage = new TokenStorage();
@@ -523,3 +545,104 @@ public function supportsClass($class): bool
         return InMemoryUser::class === $class;
     }
 }
+
+class CustomToken implements TokenInterface
+{
+    private $user;
+    private $roles;
+
+    public function __construct(UserInterface $user, array $roles)
+    {
+        $this->user = $user;
+        $this->roles = $roles;
+    }
+
+    public function __serialize(): array
+    {
+        return [$this->user, $this->roles];
+    }
+
+    public function serialize(): string
+    {
+        return serialize($this->__serialize());
+    }
+
+    public function __unserialize(array $data): void
+    {
+        [$this->user, $this->roles] = $data;
+    }
+
+    public function unserialize($serialized)
+    {
+        $this->__unserialize(\is_array($serialized) ? $serialized : unserialize($serialized));
+    }
+
+    public function __toString(): string
+    {
+        return $this->user->getUserIdentifier();
+    }
+
+    public function getRoleNames(): array
+    {
+        return $this->roles;
+    }
+
+    public function getCredentials()
+    {
+    }
+
+    public function getUser(): UserInterface
+    {
+        return $this->user;
+    }
+
+    public function setUser($user)
+    {
+        $this->user = $user;
+    }
+
+    public function getUsername(): string
+    {
+        return $this->user->getUserIdentifier();
+    }
+
+    public function getUserIdentifier(): string
+    {
+        return $this->getUserIdentifier();
+    }
+
+    public function isAuthenticated(): bool
+    {
+        return true;
+    }
+
+    public function setAuthenticated(bool $isAuthenticated)
+    {
+    }
+
+    public function eraseCredentials()
+    {
+    }
+
+    public function getAttributes(): array
+    {
+        return [];
+    }
+
+    public function setAttributes(array $attributes)
+    {
+    }
+
+    public function hasAttribute(string $name): bool
+    {
+        return false;
+    }
+
+    public function getAttribute(string $name)
+    {
+    }
+
+    public function setAttribute(string $name, $value)
+    {
+    }
+}