[netty#1242] Fix infinity-loop which was triggered when a write failed and AioSocketChannel was used

Norman Maurer · Norman Maurer · commit 725156f08262 · 2013-04-10T15:39:58.000+02:00
diff --git a/transport/src/main/java/io/netty/channel/socket/aio/AioSocketChannel.java b/transport/src/main/java/io/netty/channel/socket/aio/AioSocketChannel.java
@@ -76,7 +76,12 @@ private static AsynchronousSocketChannel newSocket(AsynchronousChannelGroup grou
     private boolean readInProgress;
     private boolean inDoBeginRead;
     private boolean readAgain;
-    private boolean writeInProgress;
+
+    private static final int NO_WRITE_IN_PROGRESS = 0;
+    private static final int WRITE_IN_PROGRESS = 1;
+    private static final int WRITE_FAILED = -2;
+
+    private int writeInProgress;
     private boolean inDoFlushByteBuffer;
 
     /**
@@ -246,7 +251,7 @@ protected boolean isFlushPending() {
 
     @Override
     protected void doFlushByteBuffer(ByteBuf buf) throws Exception {
-        if (inDoFlushByteBuffer || writeInProgress) {
+        if (inDoFlushByteBuffer || writeInProgress != NO_WRITE_IN_PROGRESS) {
             return;
         }
 
@@ -263,7 +268,7 @@ protected void doFlushByteBuffer(ByteBuf buf) throws Exception {
                     // discardReadBytes() later, modifying the readerIndex and the writerIndex unexpectedly.
                     buf.discardReadBytes();
 
-                    writeInProgress = true;
+                    writeInProgress = WRITE_IN_PROGRESS;
                     if (buf.nioBufferCount() == 1) {
                         javaChannel().write(
                                 buf.nioBuffer(), config.getWriteTimeout(), TimeUnit.MILLISECONDS, this, WRITE_HANDLER);
@@ -279,7 +284,13 @@ protected void doFlushByteBuffer(ByteBuf buf) throws Exception {
                         }
                     }
 
-                    if (writeInProgress) {
+                    if (writeInProgress != NO_WRITE_IN_PROGRESS) {
+                        if (writeInProgress == WRITE_FAILED) {
+                            // failed because of an exception so reset state and break out of the loop now
+                            // See #1242
+                            writeInProgress = NO_WRITE_IN_PROGRESS;
+                            break;
+                        }
                         // JDK decided to write data (or notify handler) later.
                         buf.suspendIntermediaryDeallocations();
                         break;
@@ -368,7 +379,7 @@ private static final class WriteHandler<T extends Number> extends AioCompletionH
 
         @Override
         protected void completed0(T result, AioSocketChannel channel) {
-            channel.writeInProgress = false;
+            channel.writeInProgress = NO_WRITE_IN_PROGRESS;
 
             ByteBuf buf = channel.unsafe().directOutboundContext().outboundByteBuffer();
             if (buf.refCnt() == 0) {
@@ -407,7 +418,7 @@ protected void completed0(T result, AioSocketChannel channel) {
 
         @Override
         protected void failed0(Throwable cause, AioSocketChannel channel) {
-            channel.writeInProgress = false;
+            channel.writeInProgress = WRITE_FAILED;
             channel.flushFutureNotifier.notifyFlushFutures(cause);
 
             // Check if the exception was raised because of an InterruptedByTimeoutException which means that the
