From e68284378c32ab61432d23b51657855a0931e12e Mon Sep 17 00:00:00 2001 From: Andrei Cheboksarov <37665782+0xaa4eb@users.noreply.github.com> Date: Sat, 7 Dec 2024 10:02:45 +0300 Subject: [PATCH 1/3] Allow to specify several collection recorder modes --- .../java/com/ulyp/agent/RecorderContext.java | 10 +++++----- .../agent/RecordingThreadLocalContext.java | 2 +- .../com/ulyp/agent/options/AgentOptions.java | 16 +++++++++------ .../collections/CollectionRecorder.java | 15 ++++++-------- .../collections/CollectionsRecordingMode.java | 20 +++++++++++++++++++ .../recorders/collections/MapRecorder.java | 9 +++------ .../collections/MapRecorderTest.java | 10 +++------- 7 files changed, 48 insertions(+), 34 deletions(-) diff --git a/ulyp-agent-core/src/main/java/com/ulyp/agent/RecorderContext.java b/ulyp-agent-core/src/main/java/com/ulyp/agent/RecorderContext.java index 4971a846..e8a9f4f5 100644 --- a/ulyp-agent-core/src/main/java/com/ulyp/agent/RecorderContext.java +++ b/ulyp-agent-core/src/main/java/com/ulyp/agent/RecorderContext.java @@ -42,23 +42,23 @@ private void configureArrayRecorders() { private void configureCollectionRecorders() { CollectionRecorder recorder = (CollectionRecorder) ObjectRecorderRegistry.COLLECTION_RECORDER.getInstance(); - recorder.setMode(options.getCollectionsRecordingMode().get()); + recorder.setModes(options.getCollectionsRecordingMode().get()); recorder.setMaxElementsToRecord(options.getMaxItemsCollectionsRecordingOption().get()); ListRecorder listRecorder = (ListRecorder) ObjectRecorderRegistry.LIST_RECORDER.getInstance(); - listRecorder.setMode(options.getCollectionsRecordingMode().get()); + listRecorder.setModes(options.getCollectionsRecordingMode().get()); listRecorder.setMaxElementsToRecord(options.getMaxItemsCollectionsRecordingOption().get()); SetRecorder setRecorder = (SetRecorder) ObjectRecorderRegistry.SET_RECORDER.getInstance(); - setRecorder.setMode(options.getCollectionsRecordingMode().get()); + setRecorder.setModes(options.getCollectionsRecordingMode().get()); setRecorder.setMaxElementsToRecord(options.getMaxItemsCollectionsRecordingOption().get()); QueueRecorder queueRecorder = (QueueRecorder) ObjectRecorderRegistry.QUEUE_RECORDER.getInstance(); - queueRecorder.setMode(options.getCollectionsRecordingMode().get()); + queueRecorder.setModes(options.getCollectionsRecordingMode().get()); queueRecorder.setMaxElementsToRecord(options.getMaxItemsCollectionsRecordingOption().get()); MapRecorder mapRecorder = (MapRecorder) ObjectRecorderRegistry.MAP_RECORDER.getInstance(); - mapRecorder.setMode(options.getCollectionsRecordingMode().get()); + mapRecorder.setModes(options.getCollectionsRecordingMode().get()); mapRecorder.setMaxEntriesToRecord(options.getMaxItemsCollectionsRecordingOption().get()); } } diff --git a/ulyp-agent-core/src/main/java/com/ulyp/agent/RecordingThreadLocalContext.java b/ulyp-agent-core/src/main/java/com/ulyp/agent/RecordingThreadLocalContext.java index 6c8afcba..0b2c03db 100644 --- a/ulyp-agent-core/src/main/java/com/ulyp/agent/RecordingThreadLocalContext.java +++ b/ulyp-agent-core/src/main/java/com/ulyp/agent/RecordingThreadLocalContext.java @@ -34,7 +34,7 @@ public class RecordingThreadLocalContext { private RecordingEventBuffer eventBuffer; public RecordingThreadLocalContext(AgentOptions options, TypeResolver typeResolver) { - if (options.getCollectionsRecordingMode().get() == CollectionsRecordingMode.NONE && !options.getArraysRecordingOption().get()) { + if (CollectionsRecordingMode.isDisabled(options.getCollectionsRecordingMode().get()) && !options.getArraysRecordingOption().get()) { this.objectConverter = PassByRefRecordedObjectConverter.INSTANCE; } else { this.objectConverter = new ByTypeRecordedObjectConverter(typeResolver); diff --git a/ulyp-agent-core/src/main/java/com/ulyp/agent/options/AgentOptions.java b/ulyp-agent-core/src/main/java/com/ulyp/agent/options/AgentOptions.java index 2ac13747..52aae613 100644 --- a/ulyp-agent-core/src/main/java/com/ulyp/agent/options/AgentOptions.java +++ b/ulyp-agent-core/src/main/java/com/ulyp/agent/options/AgentOptions.java @@ -13,6 +13,8 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.singletonList; + /** * Agent options which define what packages to instrument, at which method recording should start, etc. * It's only possible to set settings via JMV system properties at the time. @@ -112,13 +114,15 @@ public class AgentOptions { "Value 'delay:X' allows to set delay after which recording can start. X is specified in seconds. For example, 'delay:60'. " + "Value 'api' makes the agent behaviour controllable through remote Grpc API." ); - private final AgentOption collectionsRecordingMode = new AgentOption<>( + private final AgentOption> collectionsRecordingMode = new AgentOption<>( RECORD_COLLECTIONS_PROPERTY, - CollectionsRecordingMode.NONE, - CollectionsRecordingMode::valueOf, - "Defines if collections, maps and arrays should be recorded. Defaults to 'NONE' which allows the agent to pass all objects by reference" + - " to the background thread. 'JAVA' enables recording of Java standard library collections, maps and arrays. 'ALL' " + - "will record all collections (event 3rd party library collections) which might be very unpleasant, so use with care." + singletonList(CollectionsRecordingMode.NONE), + new ListParser<>(CollectionsRecordingMode::valueOf), + "Defines if collections, maps and arrays should be recorded. " + + "Defaults to 'NONE' which allows the agent to pass all objects by reference to the background thread. " + + "'JAVA' enables recording of Java standard library collections, maps and arrays. " + + "'KT' enables recording of Kotlin standard library collections. " + + "'ALL' will record all collections (event 3rd party library collections) which may incur side effects, so use with care." ); private final AgentOption maxItemsCollectionsRecordingOption = new AgentOption<>( RECORD_COLLECTIONS_MAX_ITEMS_PROPERTY, diff --git a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionRecorder.java b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionRecorder.java index 6fa73c39..2341b8db 100644 --- a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionRecorder.java +++ b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionRecorder.java @@ -15,10 +15,7 @@ import org.jetbrains.annotations.NotNull; import javax.annotation.concurrent.ThreadSafe; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; +import java.util.*; @Slf4j @ThreadSafe @@ -30,7 +27,7 @@ public class CollectionRecorder extends ObjectRecorder { private volatile boolean active = true; @Setter private int maxElementsToRecord; - private CollectionsRecordingMode mode = CollectionsRecordingMode.NONE; + private List modes = Collections.singletonList(CollectionsRecordingMode.NONE); public CollectionRecorder(byte id) { super(id); @@ -38,7 +35,7 @@ public CollectionRecorder(byte id) { @Override public boolean supports(Class type) { - return mode.supports(type) && Collection.class.isAssignableFrom(type); + return modes.stream().anyMatch(mode -> mode.supports(type)) && Collection.class.isAssignableFrom(type); } @Override @@ -46,9 +43,9 @@ public boolean supportsAsyncRecording() { return false; } - public void setMode(CollectionsRecordingMode mode) { - this.mode = mode; - log.info("Set collection recording mode to {}", mode); + public void setModes(List modes) { + this.modes = modes; + log.info("Set collection recording mode to {}", modes); } @Override diff --git a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionsRecordingMode.java b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionsRecordingMode.java index 5dcc9aa4..8059b965 100644 --- a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionsRecordingMode.java +++ b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/CollectionsRecordingMode.java @@ -1,5 +1,7 @@ package com.ulyp.core.recorders.collections; +import java.util.List; + public enum CollectionsRecordingMode { /** @@ -18,6 +20,20 @@ public String toString() { return "Recording java.* collections"; } }, + /** + * Records all kotlin standard library collections + */ + KT { + @Override + public boolean supports(Class type) { + return type.getName().startsWith("kotlin."); + } + + @Override + public String toString() { + return "Recording java.* collections"; + } + }, /** * The most intrusive mode: try to record elements on every collection. Might break things */ @@ -48,4 +64,8 @@ public String toString() { }; public abstract boolean supports(Class type); + + public static boolean isDisabled(List modes) { + return modes.size() == 1 && modes.get(0) == NONE; + } } diff --git a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/MapRecorder.java b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/MapRecorder.java index ef580a5c..1d045505 100644 --- a/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/MapRecorder.java +++ b/ulyp-common/src/main/java/com/ulyp/core/recorders/collections/MapRecorder.java @@ -13,10 +13,7 @@ import org.jetbrains.annotations.NotNull; import javax.annotation.concurrent.ThreadSafe; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; @ThreadSafe public class MapRecorder extends ObjectRecorder { @@ -27,7 +24,7 @@ public class MapRecorder extends ObjectRecorder { @Setter private int maxEntriesToRecord; @Setter - private volatile CollectionsRecordingMode mode = CollectionsRecordingMode.NONE; + private List modes = Collections.singletonList(CollectionsRecordingMode.NONE); private volatile boolean active = true; public MapRecorder(byte id) { @@ -36,7 +33,7 @@ public MapRecorder(byte id) { @Override public boolean supports(Class type) { - return mode.supports(type) && Map.class.isAssignableFrom(type); + return modes.stream().anyMatch(mode -> mode.supports(type)) && Map.class.isAssignableFrom(type); } @Override diff --git a/ulyp-common/src/test/java/com/ulyp/core/recorders/collections/MapRecorderTest.java b/ulyp-common/src/test/java/com/ulyp/core/recorders/collections/MapRecorderTest.java index eb18c521..0d2b92f3 100644 --- a/ulyp-common/src/test/java/com/ulyp/core/recorders/collections/MapRecorderTest.java +++ b/ulyp-common/src/test/java/com/ulyp/core/recorders/collections/MapRecorderTest.java @@ -1,10 +1,6 @@ package com.ulyp.core.recorders.collections; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.agrona.concurrent.UnsafeBuffer; import org.junit.jupiter.api.Assertions; @@ -52,7 +48,7 @@ void test() throws Exception { map.put("ABC", new XYZ()); map.put("ZXC", new XYZ()); - mapRecorder.setMode(CollectionsRecordingMode.ALL); + mapRecorder.setModes(Collections.singletonList(CollectionsRecordingMode.ALL)); mapRecorder.write(map, out, typeResolver); @@ -76,7 +72,7 @@ public Set> entrySet() { map.put("ABC", new XYZ()); map.put("ZXC", new XYZ()); - mapRecorder.setMode(CollectionsRecordingMode.ALL); + mapRecorder.setModes(Collections.singletonList(CollectionsRecordingMode.ALL)); mapRecorder.write(map, out, typeResolver); IdentityObjectRecord mapRecord = (IdentityObjectRecord) mapRecorder.read(typeResolver.get(map), in, typeResolver::get); From 32653c2fea9db56702910dd7f6ad67a1ea2968bc Mon Sep 17 00:00:00 2001 From: Andrei Cheboksarov <37665782+0xaa4eb@users.noreply.github.com> Date: Sat, 7 Dec 2024 10:19:02 +0300 Subject: [PATCH 2/3] Support kotlin.EmptyList recording --- .../tests/recorders/kotlin/ListRecorderTest.java | 15 +++++++++++++++ .../com/agent/tests/util/ForkProcessBuilder.java | 15 ++++++++++----- .../tests/recorders/kotlin/CollectionsTest.kt | 4 ++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java b/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java index 54fc3fd9..36ffc610 100644 --- a/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java +++ b/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java @@ -18,6 +18,20 @@ class ListRecorderTest extends AbstractInstrumentationTest { + @Test + void shouldRecordEmptyList() { + CallRecord root = runSubprocessAndReadFile( + new ForkProcessBuilder() + .withMain(TestCase.class) + .withMethodToRecord(MethodMatcher.parse("**.CollectionsTestKt.getEmptyList")) + .withRecordCollections(CollectionsRecordingMode.JDK, CollectionsRecordingMode.KT) + ); + + CollectionRecord collection = (CollectionRecord) root.getReturnValue(); + + assertEquals(0, collection.getSize()); + } + @Test void shouldRecordImmutableListEntries() { CallRecord root = runSubprocessAndReadFile( @@ -78,6 +92,7 @@ static class TestCase { public static void main(String[] args) { System.out.println(CollectionsTestKt.getImmutableList()); + System.out.println(CollectionsTestKt.getEmptyList()); System.out.println(CollectionsTestKt.getMutableList()); } } diff --git a/ulyp-agent-tests/src/test/java/com/agent/tests/util/ForkProcessBuilder.java b/ulyp-agent-tests/src/test/java/com/agent/tests/util/ForkProcessBuilder.java index ff1d461c..b302edca 100644 --- a/ulyp-agent-tests/src/test/java/com/agent/tests/util/ForkProcessBuilder.java +++ b/ulyp-agent-tests/src/test/java/com/agent/tests/util/ForkProcessBuilder.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; public class ForkProcessBuilder { @@ -22,7 +23,7 @@ public class ForkProcessBuilder { private List instrumentedPackages = new ArrayList<>(); private String excludeClassesProperty = null; private List excludedFromInstrumentationPackages = new ArrayList<>(); - private CollectionsRecordingMode collectionsRecordingMode; + private List collectionsRecordingModes; private String printTypes = null; private String logLevel = "INFO"; private Boolean agentDisabled = null; @@ -45,8 +46,8 @@ public ForkProcessBuilder withMain(Class mainClass) { return this; } - public ForkProcessBuilder withRecordCollections(CollectionsRecordingMode mode) { - collectionsRecordingMode = mode; + public ForkProcessBuilder withRecordCollections(CollectionsRecordingMode... modes) { + collectionsRecordingModes = Arrays.asList(modes); return this; } @@ -166,8 +167,12 @@ public List toCmdJavaProps() { if (recordCollectionItems != null) { params.add("-D" + AgentOptions.RECORD_COLLECTIONS_MAX_ITEMS_PROPERTY + "=" + recordCollectionItems); } - if (collectionsRecordingMode != null) { - params.add("-D" + AgentOptions.RECORD_COLLECTIONS_PROPERTY + "=" + collectionsRecordingMode.name()); + if (collectionsRecordingModes != null) { + params.add("-D" + AgentOptions.RECORD_COLLECTIONS_PROPERTY + "=" + + collectionsRecordingModes.stream() + .map(CollectionsRecordingMode::name) + .collect(Collectors.joining(",")) + ); } params.add("-Dulyp.recording-queue.serialization-buffer-size=" + 2048); diff --git a/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt b/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt index 587c76e1..ce90e29c 100644 --- a/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt +++ b/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt @@ -12,6 +12,10 @@ fun getImmutableList(): List { return listOf("ABC", "CDE", "EFG", "FGH", "HJK") } +fun getEmptyList(): List { + return emptyList() +} + fun getMutableList(): MutableList { return mutableListOf("ABC", "CDE", "EFG", "FGH", "HJK") } From 1543d0902b31bc2a91dc582954d318c441910368 Mon Sep 17 00:00:00 2001 From: Andrei Cheboksarov <37665782+0xaa4eb@users.noreply.github.com> Date: Sat, 7 Dec 2024 10:28:49 +0300 Subject: [PATCH 3/3] Test recording array dequeue --- .../recorders/kotlin/ListRecorderTest.java | 21 +++++++++++++++++++ .../tests/recorders/kotlin/CollectionsTest.kt | 10 +++++++++ 2 files changed, 31 insertions(+) diff --git a/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java b/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java index 36ffc610..46552dcb 100644 --- a/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java +++ b/ulyp-agent-tests/src/test/java/com/agent/tests/recorders/kotlin/ListRecorderTest.java @@ -54,6 +54,26 @@ void shouldRecordImmutableListEntries() { assertThat(elements.get(2), RecordingMatchers.isString("EFG")); } + @Test + void shouldRecordArrayDequeue() { + CallRecord root = runSubprocessAndReadFile( + new ForkProcessBuilder() + .withMain(TestCase.class) + .withMethodToRecord(MethodMatcher.parse("**.CollectionsTestKt.getArrayDequeue")) + .withRecordCollections(CollectionsRecordingMode.JDK, CollectionsRecordingMode.KT) + ); + + CollectionRecord collection = (CollectionRecord) root.getReturnValue(); + + assertEquals(5, collection.getSize()); + List elements = collection.getElements(); + assertEquals(3, elements.size()); + + assertThat(elements.get(0), RecordingMatchers.isString("A")); + assertThat(elements.get(1), RecordingMatchers.isString("B")); + assertThat(elements.get(2), RecordingMatchers.isString("C")); + } + @Test void shouldRecordMutableListEntries() { CallRecord root = runSubprocessAndReadFile( @@ -94,6 +114,7 @@ public static void main(String[] args) { System.out.println(CollectionsTestKt.getImmutableList()); System.out.println(CollectionsTestKt.getEmptyList()); System.out.println(CollectionsTestKt.getMutableList()); + System.out.println(CollectionsTestKt.getArrayDequeue()); } } } diff --git a/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt b/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt index ce90e29c..3cac97d6 100644 --- a/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt +++ b/ulyp-agent-tests/src/test/kotlin/com/agent/tests/recorders/kotlin/CollectionsTest.kt @@ -16,6 +16,16 @@ fun getEmptyList(): List { return emptyList() } +fun getArrayDequeue(): ArrayDeque { + val deq = ArrayDeque(1) + deq.add("A") + deq.add("B") + deq.add("C") + deq.add("E") + deq.add("F") + return deq +} + fun getMutableList(): MutableList { return mutableListOf("ABC", "CDE", "EFG", "FGH", "HJK") }