Fix *SslEngineTest to not throw ClassCastException and pass in all cases (#9588)

normanmaurer · web-flow · commit 8648171abc27 · 2019-09-21T14:58:36.000+02:00
Motivation:

Due some bug we did endup with ClassCastExceptions in some cases. Beside this we also did not correctly handle the case when ReferenceCountedOpenSslEngineTest did produce tasks to run in on test.

Modifications:

- Correctly unwrap the engine before to fix ClassCastExceptions
- Run delegated tasks when needed.

Result:

All tests pass with different OpenSSL implementations (OpenSSL, BoringSSL etc)
diff --git a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
@@ -36,6 +36,7 @@
 import javax.crypto.spec.SecretKeySpec;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.HandshakeStatus;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLParameters;
 import java.nio.ByteBuffer;
@@ -61,6 +62,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
@@ -1131,6 +1133,19 @@ public boolean permits(
         }
     }
 
+    private static void runTasksIfNeeded(SSLEngine engine) {
+        if (engine.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
+            for (;;) {
+                Runnable task = engine.getDelegatedTask();
+                if (task == null) {
+                    assertNotEquals(HandshakeStatus.NEED_TASK, engine.getHandshakeStatus());
+                    break;
+                }
+                task.run();
+            }
+        }
+    }
+
     @Test
     public void testExtractMasterkeyWorksCorrectly() throws Exception {
         SelfSignedCertificate cert = new SelfSignedCertificate();
@@ -1199,17 +1214,23 @@ public void testExtractMasterkeyWorksCorrectly() throws Exception {
                 cTOs.flip();
                 sTOc.flip();
 
+                runTasksIfNeeded(clientEngine);
+                runTasksIfNeeded(serverEngine);
+
                 clientEngine.unwrap(sTOc, clientIn);
                 serverEngine.unwrap(cTOs, serverIn);
 
+                runTasksIfNeeded(clientEngine);
+                runTasksIfNeeded(serverEngine);
+
                 // check when the application data has fully been consumed and sent
                 // for both the client and server
                 if ((clientOut.limit() == serverIn.position()) &&
                         (serverOut.limit() == clientIn.position())) {
-                    byte[] serverRandom = SSL.getServerRandom(((OpenSslEngine) serverEngine).sslPointer());
-                    byte[] clientRandom = SSL.getClientRandom(((OpenSslEngine) clientEngine).sslPointer());
-                    byte[] serverMasterKey = SSL.getMasterKey(((OpenSslEngine) serverEngine).sslPointer());
-                    byte[] clientMasterKey = SSL.getMasterKey(((OpenSslEngine) clientEngine).sslPointer());
+                    byte[] serverRandom = SSL.getServerRandom(unwrapEngine(serverEngine).sslPointer());
+                    byte[] clientRandom = SSL.getClientRandom(unwrapEngine(clientEngine).sslPointer());
+                    byte[] serverMasterKey = SSL.getMasterKey(unwrapEngine(serverEngine).sslPointer());
+                    byte[] clientMasterKey = SSL.getMasterKey(unwrapEngine(clientEngine).sslPointer());
 
                     asserted = true;
                     assertArrayEquals(serverMasterKey, clientMasterKey);
@@ -1318,7 +1339,9 @@ ReferenceCountedOpenSslEngine unwrapEngine(SSLEngine engine) {
 
     @Override
     protected SslContext wrapContext(SslContext context) {
-        ((OpenSslContext) context).setUseTasks(useTasks);
+        if (context instanceof OpenSslContext) {
+            ((OpenSslContext) context).setUseTasks(useTasks);
+        }
         return context;
     }
 }
diff --git a/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java
@@ -72,7 +72,9 @@ public void testNotLeakOnException() throws Exception {
 
     @Override
     protected SslContext wrapContext(SslContext context) {
-        ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);
+        if (context instanceof ReferenceCountedOpenSslContext) {
+            ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);
+        }
         return context;
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,9 @@ public void testNotLeakOnException() throws Exception {`
`72`	`72`
`73`	`73`	`@Override`
`74`	`74`	`protected SslContext wrapContext(SslContext context) {`
`75`		`- ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);`
	`75`	`+ if (context instanceof ReferenceCountedOpenSslContext) {`
	`76`	`+ ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);`
	`77`	`+ }`
`76`	`78`	`return context;`
`77`	`79`	`}`
`78`	`80`	`}`