Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d792df5

Browse files
fang-techAlexxigang
fang-tech
authored and
Alexxigang
committed
fix(tool): prevent pipe buffer deadlock in ShellCommandTool (agentscope-ai#619)
## AgentScope-Java Version [The version of AgentScope-Java you are working on, e.g. 1.0.7, check your pom.xml dependency version or run `mvn dependency:tree | grep agentscope-parent:pom`(only mac/linux)] ## Description - Fixed deadlock issue when executing commands with large output (>4-64KB) - Implemented asynchronous stream reading using CompletableFuture to prevent pipe buffer from filling up and blocking child process - Added getOutputWithTimeout() helper method for safe retrieval of async stream reader results with proper timeout and error handling - All existing tests pass without regression close agentscope-ai#617 ## Checklist Please check the following items before code is ready to be reviewed. - [x] Code has been formatted with `mvn spotless:apply` - [x] All tests are passing (`mvn test`) - [x] Javadoc comments are complete and follow project conventions - [x] Related documentation has been updated (e.g. links, examples, etc.) - [x] Code is ready for review
1 parent a3462fd commit d792df5

3 files changed

Lines changed: 1269 additions & 27 deletions

File tree

‎agentscope-core/src/main/java/io/agentscope/core/tool/coding/ShellCommandTool.java‎

Lines changed: 121 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import io.agentscope.core.tool.ToolCallParam;
2222
import java.io.BufferedReader;
2323
import java.io.IOException;
24+
import java.io.InputStream;
2425
import java.io.InputStreamReader;
2526
import java.nio.charset.StandardCharsets;
2627
import java.time.Duration;
@@ -29,9 +30,16 @@
2930
import java.util.List;
3031
import java.util.Map;
3132
import java.util.Set;
33+
import java.util.concurrent.Callable;
3234
import java.util.concurrent.ConcurrentHashMap;
35+
import java.util.concurrent.ExecutionException;
36+
import java.util.concurrent.ExecutorService;
37+
import java.util.concurrent.Executors;
38+
import java.util.concurrent.Future;
39+
import java.util.concurrent.ThreadFactory;
3340
import java.util.concurrent.TimeUnit;
3441
import java.util.concurrent.TimeoutException;
42+
import java.util.concurrent.atomic.AtomicInteger;
3543
import java.util.function.Function;
3644
import java.util.stream.Collectors;
3745
import org.slf4j.Logger;
@@ -58,6 +66,28 @@ public class ShellCommandTool implements AgentTool {
5866
private static final Logger logger = LoggerFactory.getLogger(ShellCommandTool.class);
5967
private static final int DEFAULT_TIMEOUT = 300;
6068

69+
/**
70+
* Shared thread pool for asynchronous stream reading.
71+
* Uses cached thread pool for dynamic scaling with 60s idle timeout.
72+
* Daemon threads ensure they don't prevent JVM shutdown.
73+
*/
74+
private static final ExecutorService STREAM_READER_POOL =
75+
Executors.newCachedThreadPool(
76+
new ThreadFactory() {
77+
private final AtomicInteger counter = new AtomicInteger(0);
78+
79+
@Override
80+
public Thread newThread(Runnable r) {
81+
Thread t =
82+
new Thread(
83+
r,
84+
"ShellCommand-StreamReader-"
85+
+ counter.incrementAndGet());
86+
t.setDaemon(true); // Daemon thread won't prevent JVM shutdown
87+
return t;
88+
}
89+
});
90+
6191
private final Set<String> allowedCommands;
6292
private final Function<String, Boolean> approvalCallback;
6393
private final CommandValidator commandValidator;
@@ -337,13 +367,31 @@ private ToolResultBlock executeCommand(String command, int timeoutSeconds) {
337367
}
338368

339369
Process process = null;
370+
Future<String> stdoutFuture = null;
371+
Future<String> stderrFuture = null;
372+
340373
try {
341374
long startTime = System.currentTimeMillis();
342375
logger.debug("Starting command execution: {}", command);
343376

344377
// Start the process
345378
process = processBuilder.start();
346379

380+
// CRITICAL FIX: Start asynchronous stream readers immediately to prevent pipe buffer
381+
// deadlock
382+
// When a child process writes more data than the pipe buffer can hold (typically
383+
// 4-64KB),
384+
// it will block waiting for the parent process to read the data. If the parent is
385+
// blocked
386+
// in waitFor(), this creates a deadlock. By reading streams asynchronously in
387+
// background
388+
// threads from the thread pool, we ensure the pipe buffers are continuously drained,
389+
// preventing the deadlock.
390+
stdoutFuture =
391+
STREAM_READER_POOL.submit(new StreamReader(process.getInputStream(), "stdout"));
392+
stderrFuture =
393+
STREAM_READER_POOL.submit(new StreamReader(process.getErrorStream(), "stderr"));
394+
347395
// Wait for the process to complete with timeout
348396
logger.debug("Waiting for process with timeout: {} seconds", timeoutSeconds);
349397
boolean completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS);
@@ -359,13 +407,13 @@ private ToolResultBlock executeCommand(String command, int timeoutSeconds) {
359407
timeoutSeconds,
360408
waitElapsed);
361409

362-
// Try to capture partial output before terminating
363-
String stdout = readStream(process.getInputStream());
364-
String stderr = readStream(process.getErrorStream());
365-
366410
// Terminate the process
367411
process.destroyForcibly();
368412

413+
// Get partial output from async readers with short timeout
414+
String stdout = getOutputWithTimeout(stdoutFuture, 1, TimeUnit.SECONDS);
415+
String stderr = getOutputWithTimeout(stderrFuture, 1, TimeUnit.SECONDS);
416+
369417
String timeoutMessage =
370418
String.format(
371419
"TimeoutError: The command execution exceeded the timeout of %d"
@@ -384,8 +432,11 @@ private ToolResultBlock executeCommand(String command, int timeoutSeconds) {
384432

385433
// Process completed normally
386434
int returnCode = process.exitValue();
387-
String stdout = readStream(process.getInputStream());
388-
String stderr = readStream(process.getErrorStream());
435+
436+
// Get complete output from async readers
437+
// Process has finished, so readers should complete quickly
438+
String stdout = getOutputWithTimeout(stdoutFuture, 5, TimeUnit.SECONDS);
439+
String stderr = getOutputWithTimeout(stderrFuture, 5, TimeUnit.SECONDS);
389440

390441
logger.debug("Command '{}' completed with return code: {}", command, returnCode);
391442

@@ -411,37 +462,40 @@ private ToolResultBlock executeCommand(String command, int timeoutSeconds) {
411462
// Clean up process resources
412463
if (process != null && process.isAlive()) {
413464
// Destroy the process if still alive
414-
// Note: Streams are already closed by try-with-resources in readStream()
465+
// Note: Streams are already closed by try-with-resources in StreamReader
415466
process.destroyForcibly();
416467
}
417468
}
418469
}
419470

420471
/**
421-
* Read all content from an input stream.
472+
* Get output from a Future with timeout.
422473
*
423-
* @param inputStream The input stream to read from
424-
* @return The content as a string
474+
* <p>This helper method safely retrieves the output from an asynchronous stream reader,
475+
* with proper timeout and error handling. If the future times out or fails, it will be
476+
* cancelled and an empty string will be returned.
477+
*
478+
* @param future The Future containing the output string
479+
* @param timeout The timeout value
480+
* @param unit The timeout unit
481+
* @return The output string, or empty string if timeout or error occurs
425482
*/
426-
private String readStream(java.io.InputStream inputStream) {
427-
if (inputStream == null) {
483+
private String getOutputWithTimeout(Future<String> future, long timeout, TimeUnit unit) {
484+
try {
485+
return future.get(timeout, unit);
486+
} catch (TimeoutException e) {
487+
logger.warn("Timeout waiting for stream reader to complete");
488+
future.cancel(true); // Cancel the task
489+
return "";
490+
} catch (InterruptedException e) {
491+
Thread.currentThread().interrupt();
492+
logger.warn("Interrupted while waiting for stream reader");
493+
future.cancel(true); // Cancel the task
494+
return "";
495+
} catch (ExecutionException e) {
496+
logger.error("Error in stream reader: {}", e.getCause().getMessage(), e.getCause());
428497
return "";
429498
}
430-
431-
StringBuilder output = new StringBuilder();
432-
try (BufferedReader reader =
433-
new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
434-
String line;
435-
while ((line = reader.readLine()) != null) {
436-
if (output.length() > 0) {
437-
output.append("\n");
438-
}
439-
output.append(line);
440-
}
441-
} catch (IOException e) {
442-
logger.error("Error reading stream: {}", e.getMessage(), e);
443-
}
444-
return output.toString();
445499
}
446500

447501
/**
@@ -491,4 +545,44 @@ private boolean requestUserApproval(String command) {
491545
return false;
492546
}
493547
}
548+
549+
/**
550+
* Callable task for reading process output streams asynchronously.
551+
* This prevents pipe buffer deadlock by continuously draining stdout/stderr.
552+
*/
553+
private static class StreamReader implements Callable<String> {
554+
private final InputStream inputStream;
555+
private final String streamType;
556+
557+
StreamReader(InputStream inputStream, String streamType) {
558+
this.inputStream = inputStream;
559+
this.streamType = streamType;
560+
}
561+
562+
@Override
563+
public String call() throws Exception {
564+
if (inputStream == null) {
565+
return "";
566+
}
567+
568+
logger.debug("StreamReader [{}] started", streamType);
569+
StringBuilder output = new StringBuilder();
570+
try (BufferedReader reader =
571+
new BufferedReader(
572+
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
573+
String line;
574+
while ((line = reader.readLine()) != null) {
575+
if (output.length() > 0) {
576+
output.append("\n");
577+
}
578+
output.append(line);
579+
}
580+
} catch (IOException e) {
581+
logger.error("Error reading {} stream: {}", streamType, e.getMessage(), e);
582+
throw e;
583+
}
584+
logger.debug("StreamReader [{}] completed, read {} bytes", streamType, output.length());
585+
return output.toString();
586+
}
587+
}
494588
}

‎agentscope-core/src/test/java/io/agentscope/core/tool/coding/ShellCommandToolTest.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -780,4 +780,152 @@ void testConcurrentDescriptionGeneration() throws InterruptedException {
780780
assertEquals(threadCount, successCount.get());
781781
}
782782
}
783+
784+
@Nested
785+
@DisplayName("Buffer Deadlock Fix Tests (Issue #617)")
786+
class BufferDeadlockTests {
787+
788+
@Test
789+
@DisplayName("Should handle large output without deadlock using seq command")
790+
@EnabledOnOs({OS.LINUX, OS.MAC})
791+
void reproducePipeBufferDeadlockWithSeq() {
792+
// Generate ~8KB output using seq command, which exceeds typical pipe buffer
793+
// (4-8KB)
794+
// This command completes in milliseconds with the fix (Apache Commons Exec's
795+
// PumpStreamHandler)
796+
String command = "seq 1 20000"; // Approximately 8000 bytes
797+
798+
// Set a reasonable timeout - should complete quickly now
799+
Mono<ToolResultBlock> result = tool.executeShellCommand(command, 60);
800+
801+
StepVerifier.create(result)
802+
.assertNext(
803+
block -> {
804+
String text = extractText(block);
805+
// Expected: Should complete successfully without timeout
806+
// The fix uses PumpStreamHandler to consume output in separate
807+
// threads
808+
assertFalse(
809+
text.contains("TimeoutError"),
810+
"Should not timeout with Apache Commons Exec fix, but got: "
811+
+ text);
812+
assertTrue(
813+
text.contains("<returncode>0</returncode>"),
814+
"Expected successful execution");
815+
assertTrue(
816+
text.contains("20000"),
817+
"Expected output to contain the last number");
818+
})
819+
.verifyComplete();
820+
}
821+
822+
@Test
823+
@DisplayName("Should handle large file cat without deadlock")
824+
@EnabledOnOs({OS.LINUX, OS.MAC})
825+
void reproduceLargeFileCatDeadlock() throws Exception {
826+
// Create a temporary large file
827+
java.nio.file.Path tempFile = java.nio.file.Files.createTempFile("test_large_", ".txt");
828+
try {
829+
// Write 20KB of data (far exceeds typical pipe buffer size)
830+
StringBuilder content = new StringBuilder();
831+
for (int i = 0; i < 2000; i++) {
832+
content.append("Line ")
833+
.append(i)
834+
.append(": ")
835+
.append("Some test content to fill the buffer\n");
836+
}
837+
java.nio.file.Files.writeString(tempFile, content.toString());
838+
839+
String command = "cat " + tempFile.toString();
840+
841+
// Set a reasonable timeout - should complete quickly with the fix
842+
Mono<ToolResultBlock> result = tool.executeShellCommand(command, 10);
843+
844+
StepVerifier.create(result)
845+
.assertNext(
846+
block -> {
847+
String text = extractText(block);
848+
// Expected: Should complete successfully and return the file
849+
// content
850+
assertFalse(
851+
text.contains("TimeoutError"),
852+
"Should not timeout with Apache Commons Exec fix, but"
853+
+ " got: "
854+
+ text);
855+
assertTrue(
856+
text.contains("<returncode>0</returncode>"),
857+
"Expected successful execution");
858+
assertTrue(
859+
text.contains("Line 1999"),
860+
"Expected output to contain last line");
861+
})
862+
.verifyComplete();
863+
} finally {
864+
java.nio.file.Files.deleteIfExists(tempFile);
865+
}
866+
}
867+
868+
@Test
869+
@DisplayName("Should handle pre-created large test file without deadlock")
870+
@EnabledOnOs({OS.LINUX, OS.MAC})
871+
void reproduceLargeFileCatDeadlockWithTestResource() {
872+
// Use the pre-created test resource file (20KB)
873+
String resourcePath =
874+
getClass().getClassLoader().getResource("large_output_test.txt").getPath();
875+
String command = "cat " + resourcePath;
876+
877+
// Set a reasonable timeout - should complete quickly with the fix
878+
Mono<ToolResultBlock> result = tool.executeShellCommand(command, 10);
879+
880+
StepVerifier.create(result)
881+
.assertNext(
882+
block -> {
883+
String text = extractText(block);
884+
// Expected: Should complete successfully and return the file
885+
// content
886+
assertFalse(
887+
text.contains("TimeoutError"),
888+
"Should not timeout with Apache Commons Exec fix, but got: "
889+
+ text);
890+
assertTrue(
891+
text.contains("<returncode>0</returncode>"),
892+
"Expected successful execution");
893+
assertTrue(
894+
text.contains("This is test content"),
895+
"Expected output to contain file content");
896+
})
897+
.verifyComplete();
898+
}
899+
900+
@Test
901+
@DisplayName("Should handle yes command piped to head without deadlock")
902+
@EnabledOnOs({OS.LINUX, OS.MAC})
903+
void reproducePipeBufferDeadlockWithYesCommand() {
904+
// Generate large output using yes command
905+
// This produces continuous output that will definitely fill the buffer
906+
String command = "yes 'This is a test line with some content' | head -n 1000";
907+
908+
// Set a reasonable timeout - should complete quickly with the fix
909+
Mono<ToolResultBlock> result = tool.executeShellCommand(command, 10);
910+
911+
StepVerifier.create(result)
912+
.assertNext(
913+
block -> {
914+
String text = extractText(block);
915+
// Expected: Should complete successfully without timeout
916+
// PumpStreamHandler consumes output in separate threads
917+
assertFalse(
918+
text.contains("TimeoutError"),
919+
"Should not timeout with Apache Commons Exec fix, but got: "
920+
+ text);
921+
assertTrue(
922+
text.contains("<returncode>0</returncode>"),
923+
"Expected successful execution");
924+
assertTrue(
925+
text.contains("This is a test line"),
926+
"Expected output to contain the repeated line");
927+
})
928+
.verifyComplete();
929+
}
930+
}
783931
}

0 commit comments

Comments
 (0)