diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 7ae0b50faafb..5c9138a92de4 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -203,6 +203,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -211,6 +217,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true @@ -308,6 +318,12 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml + + true + 1 + 256 + paranoid + @@ -329,6 +345,12 @@ Licensed under the MIT License. src/test/resources/fast-testng.xml + + true + 1 + 256 + paranoid + @@ -350,6 +372,12 @@ Licensed under the MIT License. src/test/resources/split-testng.xml + + true + 1 + 256 + paranoid + @@ -371,6 +399,12 @@ Licensed under the MIT License. src/test/resources/cfp-split-testng.xml + + true + 1 + 256 + paranoid + @@ -392,6 +426,12 @@ Licensed under the MIT License. src/test/resources/query-testng.xml + + true + 1 + 256 + paranoid + @@ -413,6 +453,12 @@ Licensed under the MIT License. src/test/resources/long-testng.xml + + true + 1 + 256 + paranoid + @@ -434,6 +480,12 @@ Licensed under the MIT License. src/test/resources/direct-testng.xml + + true + 1 + 256 + paranoid + @@ -455,6 +507,12 @@ Licensed under the MIT License. src/test/resources/multi-master-testng.xml + + true + 1 + 256 + paranoid + @@ -476,6 +534,12 @@ Licensed under the MIT License. src/test/resources/flaky-multi-master-testng.xml + + true + 1 + 256 + paranoid + @@ -497,6 +561,12 @@ Licensed under the MIT License. src/test/resources/fi-multi-master-testng.xml + + true + 1 + 256 + paranoid + @@ -519,6 +589,12 @@ Licensed under the MIT License. src/test/resources/examples-testng.xml + + true + 1 + 256 + paranoid + @@ -548,6 +624,12 @@ Licensed under the MIT License. src/test/resources/emulator-testng.xml + + true + 1 + 256 + paranoid + @@ -569,6 +651,12 @@ Licensed under the MIT License. src/test/resources/e2e-testng.xml + + true + 1 + 256 + paranoid + diff --git a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java index a3d5b79ad4ab..5584f243c419 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java +++ b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java @@ -498,4 +498,13 @@ protected String getDocumentLink(Document doc) { return doc.getSelfLink(); } } + + @Override + void shutdown() { + if (this.client != null) { + this.client.close(); + } + + super.shutdown(); + } } diff --git a/sdk/cosmos/azure-cosmos-encryption/README.md b/sdk/cosmos/azure-cosmos-encryption/README.md index 97bc4c8ef534..d460dca455f1 100644 --- a/sdk/cosmos/azure-cosmos-encryption/README.md +++ b/sdk/cosmos/azure-cosmos-encryption/README.md @@ -12,7 +12,7 @@ The Azure Cosmos Encryption Plugin is used for encrypting data with a user-provi com.azure azure-cosmos-encryption - 2.24.0 + 2.25.0-beta.1 ``` [//]: # ({x-version-update-end}) diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index a9984111c017..e4810e35139d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -49,6 +49,8 @@ Licensed under the MIT License. --add-opens com.azure.cosmos.encryption/com.azure.cosmos.encryption.util=ALL-UNNAMED --add-opens com.azure.cosmos.encryption/com.azure.cosmos.encryption.models=ALL-UNNAMED --add-opens com.azure.cosmos/com.azure.cosmos.implementation=ALL-UNNAMED + --add-modules java.management + --add-reads com.azure.cosmos.encryption=java.management - true @@ -217,6 +219,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -225,6 +233,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.encryption.CosmosNettyLeakDetectorFactory + true @@ -263,6 +275,12 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml + + true + 1 + 256 + paranoid + @@ -284,7 +302,12 @@ Licensed under the MIT License. src/test/resources/encryption-testng.xml - + + true + 1 + 256 + paranoid + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java index d396a824659b..d91fdd7c7af8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java @@ -20,6 +20,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -29,16 +30,15 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class CosmosEncryptionAsyncClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(CosmosEncryptionAsyncClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; private static final ImplementationBridgeHelpers.CosmosClientBuilderHelper.CosmosClientBuilderAccessor cosmosClientBuilderAccessor = ImplementationBridgeHelpers.CosmosClientBuilderHelper.getCosmosClientBuilderAccessor(); - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosEncryptionAsyncClientTest() { this(new CosmosClientBuilder()); @@ -52,78 +52,6 @@ public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } - @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index c2b5f179131f..759b4089e6c8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -2,23 +2,47 @@ // Licensed under the MIT License. package com.azure.cosmos.encryption; -import com.azure.cosmos.implementation.StackTraceUtil; +import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.RxDocumentClientImpl; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; +import io.netty.util.internal.PlatformDependent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.IClassListener; import org.testng.IExecutionListener; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; +import org.testng.ITestClass; +import org.testng.ITestContext; +import org.testng.ITestNGMethod; +import org.testng.ITestResult; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Fail.fail; + +public final class CosmosNettyLeakDetectorFactory + extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { -public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); private final static List identifiedLeaks = new ArrayList<>(); private final static Object staticLock = new Object(); + + private final static Map testClassInventory = new HashMap<>(); private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; + private volatile Map activeClientsAtBegin = new HashMap<>(); + public CosmosNettyLeakDetectorFactory() { } @@ -28,16 +52,148 @@ public void onExecutionStart() { } @Override - public void onExecutionFinish() { - // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. - System.gc(); - try { - Thread.sleep(1_000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public void onBeforeClass(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int alreadyInitializedInstanceCount = instanceCountSnapshot.getAndIncrement(); + if (alreadyInitializedInstanceCount == 0) { + logger.info("LEAK DETECTION INITIALIZATION for test class {}", testClassName); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE CLASS", testClassName); + } + } + } + + @Override + public void onAfterClass(ITestClass testClass) { + // Unfortunately can't use this consistently in TestNG 7.51 because execution is not symmetric + // IClassListener.onBeforeClass + // TestClassBase.@BeforeClass + // TestClass.@BeforeClass + // IClassListener.onAfterClass + // TestClass.@AfterClass + // TestClassBase.@AfterClass + // we would want the IClassListener.onAfterClass to be called last - which system property + // -Dtestng.listener.execution.symmetric=true allows, but this is only available + // in TestNG 7.7.1 - which requires Java11 + // So, this class simulates this behavior by hooking into IInvokedMethodListener + } + + @Override + public void afterInvocation(IInvokedMethod method, ITestResult result, ITestContext ctx) { + ITestClass testClass = (ITestClass)result.getTestClass(); + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + + boolean isImplementedAfterClassMethod = testClassHasAfterClassMethods + && method.isConfigurationMethod() + && method.getTestMethod().isAfterClassConfiguration(); + + ITestNGMethod[] testMethods = ctx.getAllTestMethods(); + + boolean isLastTestMethodOnTestClassWithoutAfterClassMethod = !testClassHasAfterClassMethods + && method.isTestMethod() + && method.getTestMethod().isTest() + && method.getTestMethod().getEnabled() + && testMethods.length > 0 + && method.getTestMethod() == testMethods[testMethods.length - 1]; + + if (isImplementedAfterClassMethod || isLastTestMethodOnTestClassWithoutAfterClassMethod) { + this.onAfterClassCore(testClass); + } + } + + private void onAfterClassCore(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); + if (remainingInstanceCount == 0) { + String failMessage = ""; + logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + failMessage = "COSMOS CLIENT LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + if (failMessage.length() > 0) { + failMessage += "\n\n"; + } + + failMessage += "NETTY LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; + + } + + if (failMessage.length() > 0) { + logger.error(failMessage); + fail(failMessage); + } + + this.logMemoryUsage("AFTER CLASS", testClassName); + } + } + } + + private void logMemoryUsage(String name, String className) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", className, name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); } } + // This method must be called as early as possible in the lifecycle of a process // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { @@ -50,15 +206,18 @@ public static void ingestIntoNetty() { return; } - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); System.setProperty("io.netty.leakDetection.targetRecords", "256"); + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); - logger.info("NETTY LEAK detection initialized"); + logger.info( + "NETTY LEAK detection initialized, CosmosClient leak detection enabled: {}", + Configs.isClientLeakDetectionEnabled()); isInitialized = true; } } @@ -82,7 +241,7 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { - logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + logger.warn("Disabling Leak detection:"); isLeakDetectionDisabled = true; return new DisableLeakDetectionScope(); @@ -131,8 +290,10 @@ protected void reportInstancesLeak(String resourceType) { synchronized (staticLock) { if (!isLeakDetectionDisabled) { String msg = "NETTY LEAK (instances) type=" + resourceType; + identifiedLeaks.add(msg); logger.error(msg); + } } } @@ -144,6 +305,13 @@ private static final class DisableLeakDetectionScope implements AutoCloseable { @Override public void close() { synchronized (staticLock) { + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); isLeakDetectionDisabled = false; logger.info("Leak detection enabled again."); } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index 1fbda8eb17f0..c9ee457fa750 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -509,117 +509,118 @@ public void crudOnDifferentOverload() { String databaseId = UUID.randomUUID().toString(); try { createNewDatabaseWithClientEncryptionKey(databaseId); - CosmosAsyncClient asyncClient = getClientBuilder().buildAsyncClient(); - KeyEncryptionKeyResolver keyEncryptionKeyResolver = new TestKeyEncryptionKeyResolver(); - CosmosEncryptionAsyncClient cosmosEncryptionAsyncClient = new CosmosEncryptionClientBuilder().cosmosAsyncClient(asyncClient).keyEncryptionKeyResolver( - keyEncryptionKeyResolver).keyEncryptionKeyResolverName("TEST_KEY_RESOLVER").buildAsyncClient(); - CosmosEncryptionAsyncDatabase cosmosEncryptionAsyncDatabase = - cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); - - String containerId = UUID.randomUUID().toString(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(1), 1); - createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); - CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = - cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); - - List actualProperties = new ArrayList<>(); - // Read item - EncryptionPojo properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse = encryptionAsyncContainerOriginal.createItem(properties).block(); - assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem = itemResponse.getItem(); - validateResponse(properties, responseItem); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse1 = encryptionAsyncContainerOriginal.createItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem1 = itemResponse1.getItem(); - validateResponse(properties, responseItem1); - actualProperties.add(properties); - - //Upsert Item - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse1 = encryptionAsyncContainerOriginal.upsertItem(properties).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem2 = upsertResponse1.getItem(); - validateResponse(properties, responseItem2); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse2 = encryptionAsyncContainerOriginal.upsertItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem3 = upsertResponse2.getItem(); - validateResponse(properties, responseItem3); - actualProperties.add(properties); - - //Read Item - EncryptionPojo readItem = encryptionAsyncContainerOriginal.readItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); - validateResponse(actualProperties.get(0), readItem); - - //Query Item - String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); - - CosmosPagedFlux feedResponseIterator = - encryptionAsyncContainerOriginal.queryItems(query, EncryptionPojo.class); - List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); - assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + try (CosmosAsyncClient asyncClient = getClientBuilder().buildAsyncClient()) { + KeyEncryptionKeyResolver keyEncryptionKeyResolver = new TestKeyEncryptionKeyResolver(); + CosmosEncryptionAsyncClient cosmosEncryptionAsyncClient = new CosmosEncryptionClientBuilder().cosmosAsyncClient(asyncClient).keyEncryptionKeyResolver( + keyEncryptionKeyResolver).keyEncryptionKeyResolverName("TEST_KEY_RESOLVER").buildAsyncClient(); + CosmosEncryptionAsyncDatabase cosmosEncryptionAsyncDatabase = + cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); + + String containerId = UUID.randomUUID().toString(); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(1), 1); + createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); + CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = + cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); + + List actualProperties = new ArrayList<>(); + // Read item + EncryptionPojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = encryptionAsyncContainerOriginal.createItem(properties).block(); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem = itemResponse.getItem(); + validateResponse(properties, responseItem); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse1 = encryptionAsyncContainerOriginal.createItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem1 = itemResponse1.getItem(); + validateResponse(properties, responseItem1); + actualProperties.add(properties); + + //Upsert Item + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse1 = encryptionAsyncContainerOriginal.upsertItem(properties).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem2 = upsertResponse1.getItem(); + validateResponse(properties, responseItem2); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse2 = encryptionAsyncContainerOriginal.upsertItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem3 = upsertResponse2.getItem(); + validateResponse(properties, responseItem3); + actualProperties.add(properties); + + //Read Item + EncryptionPojo readItem = encryptionAsyncContainerOriginal.readItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); + validateResponse(actualProperties.get(0), readItem); + + //Query Item + String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); + + CosmosPagedFlux feedResponseIterator = + encryptionAsyncContainerOriginal.queryItems(query, EncryptionPojo.class); + List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); + assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); + CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); - CosmosPagedFlux feedResponseIterator1 = - encryptionAsyncContainerOriginal.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); - List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); - assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse1) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + CosmosPagedFlux feedResponseIterator1 = + encryptionAsyncContainerOriginal.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); + List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); + assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse1) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); - SqlQuerySpec querySpec = new SqlQuerySpec(query); - - CosmosPagedFlux feedResponseIterator2 = - encryptionAsyncContainerOriginal.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); - List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); - assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse2) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); + SqlQuerySpec querySpec = new SqlQuerySpec(query); + + CosmosPagedFlux feedResponseIterator2 = + encryptionAsyncContainerOriginal.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); + List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); + assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse2) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - //Replace Item - CosmosItemResponse replaceResponse = - encryptionAsyncContainerOriginal.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), - new PartitionKey(actualProperties.get(2).getMypk())).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - responseItem = replaceResponse.getItem(); - validateResponse(actualProperties.get(2), responseItem); - - //Delete Item - CosmosItemResponse deleteResponse = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk())).block(); - assertThat(deleteResponse.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse1 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(1).getId(), - new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse2 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(2), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse3 = encryptionAsyncContainerOriginal.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); + //Replace Item + CosmosItemResponse replaceResponse = + encryptionAsyncContainerOriginal.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), + new PartitionKey(actualProperties.get(2).getMypk())).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + responseItem = replaceResponse.getItem(); + validateResponse(actualProperties.get(2), responseItem); + + //Delete Item + CosmosItemResponse deleteResponse = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk())).block(); + assertThat(deleteResponse.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse1 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(1).getId(), + new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse2 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(2), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse3 = encryptionAsyncContainerOriginal.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); + } } finally { try { //deleting the database created for this test diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 3457351177d4..9186c4099d70 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -82,8 +82,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends CosmosEncryptionAsyncClientTest { +public abstract class TestSuiteBase extends CosmosEncryptionAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -170,6 +169,7 @@ protected static CosmosAsyncContainer getSharedSinglePartitionCosmosContainer(Co } static { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); accountConsistency = parseConsistency(TestConfigurations.CONSISTENCY); desiredConsistencies = immutableListOrNull( ObjectUtils.defaultIfNull(parseDesiredConsistencies(TestConfigurations.DESIRED_CONSISTENCIES), @@ -256,13 +256,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { @@ -800,18 +793,6 @@ static protected void safeDeleteCollection(CosmosAsyncDatabase database, String } } - static protected void safeCloseAsync(CosmosAsyncClient client) { - if (client != null) { - new Thread(() -> { - try { - client.close(); - } catch (Exception e) { - logger.error("failed to close client", e); - } - }).start(); - } - } - static protected void safeClose(CosmosAsyncClient client) { if (client != null) { try { @@ -1482,9 +1463,4 @@ protected static void validateResponse(EncryptionPojo originalItem, EncryptionPo assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()); assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()); } - - - static { - CosmosNettyLeakDetectorFactory.ingestIntoNetty(); - } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 1693af8dc8fb..f3663f541eef 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -276,6 +276,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -284,6 +290,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true @@ -533,7 +543,12 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml - + + true + 1 + 256 + paranoid + @@ -555,7 +570,12 @@ Licensed under the MIT License. src/test/resources/kafka-emulator-testng.xml - + + true + 1 + 256 + paranoid + @@ -577,7 +597,12 @@ Licensed under the MIT License. src/test/resources/kafka-testng.xml - + + true + 1 + 256 + paranoid + @@ -599,7 +624,12 @@ Licensed under the MIT License. src/test/resources/kafka-integration-testng.xml - + + true + 1 + 256 + paranoid + diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 7391f7308117..a9c332d7af1c 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -696,6 +696,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -703,6 +709,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 948990bafe7f..7b797b4cd36f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -118,6 +118,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -125,6 +131,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index fcecfd210ea5..afaa80a9e50f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -118,6 +118,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -125,6 +131,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index 435e9fd672e5..a8ffcf3b2d5d 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -118,6 +118,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -125,6 +131,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index 72eee9af0f10..58e12cd940f0 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -747,6 +747,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -754,6 +760,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index ee090944ee95..fe5dee23f566 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -152,6 +152,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -160,6 +166,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index d02ae9502bdb..e5898d7c8f8b 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -233,6 +233,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -241,6 +247,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true @@ -314,7 +324,12 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml - + + true + 1 + 256 + paranoid + @@ -336,7 +351,12 @@ Licensed under the MIT License. src/test/resources/fast-testng.xml - + + true + 1 + 256 + paranoid + @@ -358,7 +378,12 @@ Licensed under the MIT License. src/test/resources/split-testng.xml - + + true + 1 + 256 + paranoid + @@ -380,7 +405,12 @@ Licensed under the MIT License. src/test/resources/cfp-split-testng.xml - + + true + 1 + 256 + paranoid + @@ -402,7 +432,12 @@ Licensed under the MIT License. src/test/resources/query-testng.xml - + + true + 1 + 256 + paranoid + @@ -424,7 +459,12 @@ Licensed under the MIT License. src/test/resources/long-testng.xml - + + true + 1 + 256 + paranoid + @@ -446,7 +486,12 @@ Licensed under the MIT License. src/test/resources/direct-testng.xml - + + true + 1 + 256 + paranoid + @@ -468,7 +513,12 @@ Licensed under the MIT License. src/test/resources/multi-master-testng.xml - + + true + 1 + 256 + paranoid + @@ -490,7 +540,12 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-read-all-read-many-testng.xml - + + true + 1 + 256 + paranoid + @@ -512,7 +567,12 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-misc-direct-testng.xml - + + true + 1 + 256 + paranoid + @@ -534,7 +594,12 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-misc-gateway-testng.xml - + + true + 1 + 256 + paranoid + @@ -556,7 +621,12 @@ Licensed under the MIT License. src/test/resources/flaky-multi-master-testng.xml - + + true + 1 + 256 + paranoid + @@ -578,7 +648,12 @@ Licensed under the MIT License. src/test/resources/fi-multi-master-testng.xml - + + true + 1 + 256 + paranoid + @@ -600,7 +675,12 @@ Licensed under the MIT License. src/test/resources/multi-region-testng.xml - + + true + 1 + 256 + paranoid + @@ -623,7 +703,12 @@ Licensed under the MIT License. src/test/resources/examples-testng.xml - + + true + 1 + 256 + paranoid + @@ -653,7 +738,12 @@ Licensed under the MIT License. src/test/resources/emulator-testng.xml - + + true + 1 + 256 + paranoid + @@ -675,7 +765,12 @@ Licensed under the MIT License. src/test/resources/long-emulator-testng.xml - + + true + 1 + 256 + paranoid + @@ -697,7 +792,12 @@ Licensed under the MIT License. src/test/resources/emulator-vnext-testng.xml - + + true + 1 + 256 + paranoid + @@ -719,7 +819,12 @@ Licensed under the MIT License. src/test/resources/e2e-testng.xml - + + true + 1 + 256 + paranoid + @@ -741,7 +846,9 @@ Licensed under the MIT License. src/test/resources/thinclient-testng.xml - + + true + @@ -763,6 +870,9 @@ Licensed under the MIT License. src/test/resources/fi-thinclient-multi-region-testng.xml + + true + @@ -784,6 +894,9 @@ Licensed under the MIT License. src/test/resources/fi-thinclient-multi-master-testng.xml + + true + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java index 2bcb312b0dd1..cc0082e492d9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java @@ -159,7 +159,7 @@ public void extractContinuationTokens() { @AfterClass(groups = { "emulator" }, timeOut = 3 * SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { logger.info("starting ...."); - safeCloseAsync(this.client); + safeClose(this.client); } private static class TestItem { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java index 24fe8f9b06b6..72c2f1d43a04 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -18,6 +18,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -29,6 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class CosmosAsyncClientTest implements ITest { public static final String ROUTING_GATEWAY_EMULATOR_PORT = ":8081"; @@ -36,10 +38,8 @@ public abstract class CosmosAsyncClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(CosmosAsyncClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosAsyncClientTest() { this(new CosmosClientBuilder()); @@ -49,81 +49,6 @@ public CosmosAsyncClientTest(CosmosClientBuilder clientBuilder) { this.clientBuilder = clientBuilder; } - @BeforeClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java index 53c03033ee3c..79121376b871 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java @@ -61,12 +61,11 @@ public void validateBadPreferredRegions1() { @Test(groups = "unit") public void validateBadPreferredRegions2() { - try { - CosmosAsyncClient client = new CosmosClientBuilder() + try (CosmosAsyncClient client = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(hostName) .preferredRegions(Arrays.asList(" ")) - .buildAsyncClient(); + .buildAsyncClient()) { client.close(); } catch (Exception e) { assertThat(e).isInstanceOf(RuntimeException.class); @@ -92,9 +91,11 @@ public void validateApiTypePresent() { ImplementationBridgeHelpers.CosmosClientBuilderHelper.getCosmosClientBuilderAccessor(); accessor.setCosmosClientApiType(cosmosClientBuilder, apiType); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(cosmosClientBuilder.buildAsyncClient()); - assertThat(ReflectionUtils.getApiType(documentClient)).isEqualTo(apiType); + try (CosmosAsyncClient cosmosClient = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(cosmosClient); + assertThat(ReflectionUtils.getApiType(documentClient)).isEqualTo(apiType); + } } @Test(groups = "emulator", dataProvider = "regionScopedSessionContainerConfigs") @@ -111,23 +112,24 @@ public void validateSessionTokenCapturingForAccountDefaultConsistency(boolean sh .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("custom-direct-client"); - CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient(); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); + try (CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); - if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { - throw new SkipException("This test is only applicable when default account-level consistency is Session."); - } + if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { + throw new SkipException("This test is only applicable when default account-level consistency is Session."); + } - ISessionContainer sessionContainer = documentClient.getSession(); + ISessionContainer sessionContainer = documentClient.getSession(); - if (System.getProperty("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getProperty("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { - assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); - } else { - assertThat(sessionContainer instanceof SessionContainer).isTrue(); - } + if (System.getProperty("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getProperty("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { + assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); + } else { + assertThat(sessionContainer instanceof SessionContainer).isTrue(); + } - assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + } } finally { System.clearProperty("COSMOS.SESSION_CAPTURING_TYPE"); } @@ -144,23 +146,24 @@ public void validateSessionTokenCapturingForAccountDefaultConsistencyWithEnvVari .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("custom-direct-client"); - CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient(); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); + try (CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); - if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { - throw new SkipException("This test is only applicable when default account-level consistency is Session."); - } + if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { + throw new SkipException("This test is only applicable when default account-level consistency is Session."); + } - ISessionContainer sessionContainer = documentClient.getSession(); + ISessionContainer sessionContainer = documentClient.getSession(); - if (System.getenv("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getenv("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { - assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); - } else { - assertThat(sessionContainer instanceof SessionContainer).isTrue(); - } + if (System.getenv("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getenv("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { + assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); + } else { + assertThat(sessionContainer instanceof SessionContainer).isTrue(); + } - assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + } } finally { System.clearProperty("COSMOS.SESSION_CAPTURING_TYPE"); } @@ -168,102 +171,108 @@ public void validateSessionTokenCapturingForAccountDefaultConsistencyWithEnvVari @Test(groups = "emulator") public void validateContainerCreationInterceptor() { - CosmosClient clientWithoutInterceptor = new CosmosClientBuilder() + try (CosmosClient clientWithoutInterceptor = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("noInterceptor") - .buildClient(); - - ConcurrentMap> queryCache = new ConcurrentHashMap<>(); - - CosmosClient clientWithInterceptor = new CosmosClientBuilder() - .endpoint(TestConfigurations.HOST) - .key(TestConfigurations.MASTER_KEY) - .userAgentSuffix("withInterceptor") - .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) - .buildClient(); - - CosmosAsyncClient asyncClientWithInterceptor = new CosmosClientBuilder() - .endpoint(TestConfigurations.HOST) - .key(TestConfigurations.MASTER_KEY) - .userAgentSuffix("withInterceptor") - .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) - .buildAsyncClient(); - - CosmosContainer normalContainer = clientWithoutInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(normalContainer).isNotNull(); - assertThat(normalContainer.getClass()).isEqualTo(CosmosContainer.class); - assertThat(normalContainer.asyncContainer.getClass()).isEqualTo(CosmosAsyncContainer.class); - - CosmosContainer customSyncContainer = clientWithInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(customSyncContainer).isNotNull(); - assertThat(customSyncContainer.getClass()).isEqualTo(CosmosContainer.class); - assertThat(customSyncContainer.asyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); - - CosmosAsyncContainer customAsyncContainer = asyncClientWithInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(customAsyncContainer).isNotNull(); - assertThat(customAsyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); - - try { - customSyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - try { - customAsyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - try { - customAsyncContainer.queryItems("SELECT * from c", ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - SqlQuerySpec querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); - assertThat(queryCache).size().isEqualTo(0); - - try { - List items = customSyncContainer - .queryItems(querySpec, null, ObjectNode.class) - .stream().collect(Collectors.toList()); - fail("Not yet cached - the query above should always throw"); - } catch (CosmosException cosmosException) { - // Container does not exist - when not cached should fail - assertThat(cosmosException.getStatusCode()).isEqualTo(404); - assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); - } - - queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); - assertThat(queryCache).size().isEqualTo(1); - - // Validate that CacheKey equality check works - queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); - assertThat(queryCache).size().isEqualTo(1); - - // Validate that form cache the results can be served - List items = customSyncContainer - .queryItems(querySpec, null, ObjectNode.class) - .stream().collect(Collectors.toList()); + .buildClient()) { - querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); - CosmosPagedFlux cachedPagedFlux = customAsyncContainer - .queryItems(querySpec, null, ObjectNode.class); - assertThat(cachedPagedFlux.getClass().getName()).startsWith("com.azure.cosmos.util.CosmosPagedFluxStaticListImpl"); + ConcurrentMap> queryCache = new ConcurrentHashMap<>(); - // Validate that uncached query form async Container also fails with 404 due to non-existing Container - querySpec = new SqlQuerySpec().setQueryText("SELECT * from r"); - try { - CosmosPagedFlux uncachedPagedFlux = customAsyncContainer - .queryItems(querySpec, null, ObjectNode.class); - } catch (CosmosException cosmosException) { - assertThat(cosmosException.getStatusCode()).isEqualTo(404); - assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + try (CosmosClient clientWithInterceptor = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .userAgentSuffix("withInterceptor") + .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) + .buildClient()) { + + try (CosmosAsyncClient asyncClientWithInterceptor = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .userAgentSuffix("withInterceptor") + .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) + .buildAsyncClient()) { + + CosmosContainer normalContainer = clientWithoutInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(normalContainer).isNotNull(); + assertThat(normalContainer.getClass()).isEqualTo(CosmosContainer.class); + assertThat(normalContainer.asyncContainer.getClass()).isEqualTo(CosmosAsyncContainer.class); + + CosmosContainer customSyncContainer = clientWithInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(customSyncContainer).isNotNull(); + assertThat(customSyncContainer.getClass()).isEqualTo(CosmosContainer.class); + assertThat(customSyncContainer.asyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); + + CosmosAsyncContainer customAsyncContainer = asyncClientWithInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(customAsyncContainer).isNotNull(); + assertThat(customAsyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); + + try { + customSyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + try { + customAsyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + try { + customAsyncContainer.queryItems("SELECT * from c", ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + SqlQuerySpec querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); + assertThat(queryCache).size().isEqualTo(0); + + try { + List items = customSyncContainer + .queryItems(querySpec, null, ObjectNode.class) + .stream().collect(Collectors.toList()); + fail("Not yet cached - the query above should always throw"); + } catch (CosmosException cosmosException) { + // Container does not exist - when not cached should fail + assertThat(cosmosException.getStatusCode()).isEqualTo(404); + assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + } + + queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); + assertThat(queryCache).size().isEqualTo(1); + + // Validate that CacheKey equality check works + queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); + assertThat(queryCache).size().isEqualTo(1); + + // Validate that form cache the results can be served + List items = customSyncContainer + .queryItems(querySpec, null, ObjectNode.class) + .stream().collect(Collectors.toList()); + + querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); + CosmosPagedFlux cachedPagedFlux = customAsyncContainer + .queryItems(querySpec, null, ObjectNode.class); + assertThat(cachedPagedFlux.getClass().getName()).startsWith("com.azure.cosmos.util.CosmosPagedFluxStaticListImpl"); + + // Validate that uncached query form async Container also fails with 404 due to non-existing Container + querySpec = new SqlQuerySpec().setQueryText("SELECT * from r"); + try { + CosmosPagedFlux uncachedPagedFlux = customAsyncContainer + .queryItems(querySpec, null, ObjectNode.class); + } catch (CosmosException cosmosException) { + assertThat(cosmosException.getStatusCode()).isEqualTo(404); + assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + } + } + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index 5eb5b59867db..ab5d26f0d4c4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -328,41 +328,42 @@ public void queryChangeFeedIncrementalGatewayMode() throws Exception { @Test(groups = {"fast"}, timeOut = TIMEOUT) public void gatewayDiagnostics() throws Exception { - CosmosClient testClient = new CosmosClientBuilder() + try (CosmosClient testClient = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .contentResponseOnWriteEnabled(true) .userAgentSuffix(USER_AGENT_SUFFIX_GATEWAY_CLIENT) .gatewayMode() - .buildClient(); + .buildClient()) { - CosmosContainer testContainer = - testClient - .getDatabase(cosmosAsyncContainer.getDatabase().getId()) - .getContainer(cosmosAsyncContainer.getId()); - // Adding a delay to allow async VM instance metadata initialization to complete - Thread.sleep(2000); - InternalObjectNode internalObjectNode = getInternalObjectNode(); - CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); - String diagnostics = createResponse.getDiagnostics().toString(); - logger.info("DIAGNOSTICS: {}", diagnostics); - assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); - assertThat(diagnostics).contains("gatewayStatisticsList"); - assertThat(diagnostics).contains("\"operationType\":\"Create\""); - assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); - assertThat(diagnostics).contains("\"serializationType\":\"PARTITION_KEY_FETCH_SERIALIZATION\""); - assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); - assertThat(diagnostics).containsAnyOf( - "\"machineId\":\"" + tempMachineId + "\"", // logged machineId should be static uuid or - "\"machineId\":\"" + ClientTelemetry.getMachineId(null) + "\"" // the vmId from Azure - ); - assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); - assertThat(createResponse.getDiagnostics().getDuration()).isNotNull(); - assertThat(createResponse.getDiagnostics().getContactedRegionNames()).isNotNull(); - assertThat(createResponse.getDiagnostics().getRegionsContacted()).isNotEmpty(); - validateTransportRequestTimelineGateway(diagnostics); - validateRegionContacted(createResponse.getDiagnostics(), gatewayClient.asyncClient()); - isValidJSON(diagnostics); + CosmosContainer testContainer = + testClient + .getDatabase(cosmosAsyncContainer.getDatabase().getId()) + .getContainer(cosmosAsyncContainer.getId()); + // Adding a delay to allow async VM instance metadata initialization to complete + Thread.sleep(2000); + InternalObjectNode internalObjectNode = getInternalObjectNode(); + CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); + String diagnostics = createResponse.getDiagnostics().toString(); + logger.info("DIAGNOSTICS: {}", diagnostics); + assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); + assertThat(diagnostics).contains("gatewayStatisticsList"); + assertThat(diagnostics).contains("\"operationType\":\"Create\""); + assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); + assertThat(diagnostics).contains("\"serializationType\":\"PARTITION_KEY_FETCH_SERIALIZATION\""); + assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); + assertThat(diagnostics).containsAnyOf( + "\"machineId\":\"" + tempMachineId + "\"", // logged machineId should be static uuid or + "\"machineId\":\"" + ClientTelemetry.getMachineId(null) + "\"" // the vmId from Azure + ); + assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); + assertThat(createResponse.getDiagnostics().getDuration()).isNotNull(); + assertThat(createResponse.getDiagnostics().getContactedRegionNames()).isNotNull(); + assertThat(createResponse.getDiagnostics().getRegionsContacted()).isNotEmpty(); + validateTransportRequestTimelineGateway(diagnostics); + validateRegionContacted(createResponse.getDiagnostics(), gatewayClient.asyncClient()); + isValidJSON(diagnostics); + } } @Test(groups = {"fast"}, timeOut = TIMEOUT) @@ -474,31 +475,32 @@ public void systemDiagnosticsForSystemStateInformation() { @Test(groups = {"fast"}, timeOut = TIMEOUT) public void directDiagnostics() throws Exception { - CosmosClient testClient = new CosmosClientBuilder() + try (CosmosClient testClient = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .contentResponseOnWriteEnabled(true) .userAgentSuffix(USER_AGENT_SUFFIX_DIRECT_CLIENT) .directMode() - .buildClient(); + .buildClient()) { - CosmosContainer testContainer = - testClient - .getDatabase(cosmosAsyncContainer.getDatabase().getId()) - .getContainer(cosmosAsyncContainer.getId()); + CosmosContainer testContainer = + testClient + .getDatabase(cosmosAsyncContainer.getDatabase().getId()) + .getContainer(cosmosAsyncContainer.getId()); - InternalObjectNode internalObjectNode = getInternalObjectNode(); - CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); - validateDirectModeDiagnosticsOnSuccess(createResponse.getDiagnostics(), directClient, this.directClientUserAgent); - validateChannelAcquisitionContext(createResponse.getDiagnostics(), false); + InternalObjectNode internalObjectNode = getInternalObjectNode(); + CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); + validateDirectModeDiagnosticsOnSuccess(createResponse.getDiagnostics(), directClient, this.directClientUserAgent); + validateChannelAcquisitionContext(createResponse.getDiagnostics(), false); - // validate that on failed operation request timeline is populated - try { - testContainer.createItem(internalObjectNode); - fail("expected 409"); - } catch (CosmosException e) { - validateDirectModeDiagnosticsOnException(e, this.directClientUserAgent); - validateChannelAcquisitionContext(e.getDiagnostics(), false); + // validate that on failed operation request timeline is populated + try { + testContainer.createItem(internalObjectNode); + fail("expected 409"); + } catch (CosmosException e) { + validateDirectModeDiagnosticsOnException(e, this.directClientUserAgent); + validateChannelAcquisitionContext(e.getDiagnostics(), false); + } } } @@ -796,7 +798,7 @@ public void queryDiagnosticsOnOrderBy() { deleteCollection(testcontainer); } - @Test(groups = {"fast"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT) + @Test(groups = {"fast"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void directDiagnosticsOnCancelledOperation(OperationType operationType) { CosmosAsyncClient client = null; @@ -1808,12 +1810,15 @@ public void expireRecordWhenRecordAlreadyCompleteExceptionally() throws URISynta RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Read, ResourceType.Document), new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - record.completeExceptionally(exception); - // validate record.toString() will work correctly - String recordString = record.toString(); - assertThat(recordString.contains("NotFoundException")).isTrue(); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + record.completeExceptionally(exception); + // validate record.toString() will work correctly + String recordString = record.toString(); + assertThat(recordString.contains("NotFoundException")).isTrue(); + } } finally { safeClose(client); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index c94a542edfff..58d80e48e54e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -2,23 +2,47 @@ // Licensed under the MIT License. package com.azure.cosmos; -import com.azure.cosmos.implementation.StackTraceUtil; +import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.RxDocumentClientImpl; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; +import io.netty.util.internal.PlatformDependent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.IClassListener; import org.testng.IExecutionListener; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; +import org.testng.ITestClass; +import org.testng.ITestContext; +import org.testng.ITestNGMethod; +import org.testng.ITestResult; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Fail.fail; + +public final class CosmosNettyLeakDetectorFactory + extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { -public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); private final static List identifiedLeaks = new ArrayList<>(); private final static Object staticLock = new Object(); + + private final static Map testClassInventory = new HashMap<>(); private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; + private volatile Map activeClientsAtBegin = new HashMap<>(); + public CosmosNettyLeakDetectorFactory() { } @@ -28,16 +52,148 @@ public void onExecutionStart() { } @Override - public void onExecutionFinish() { - // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. - System.gc(); - try { - Thread.sleep(1_000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public void onBeforeClass(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int alreadyInitializedInstanceCount = instanceCountSnapshot.getAndIncrement(); + if (alreadyInitializedInstanceCount == 0) { + logger.info("LEAK DETECTION INITIALIZATION for test class {}", testClassName); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE CLASS", testClassName); + } + } + } + + @Override + public void onAfterClass(ITestClass testClass) { + // Unfortunately can't use this consistently in TestNG 7.51 because execution is not symmetric + // IClassListener.onBeforeClass + // TestClassBase.@BeforeClass + // TestClass.@BeforeClass + // IClassListener.onAfterClass + // TestClass.@AfterClass + // TestClassBase.@AfterClass + // we would want the IClassListener.onAfterClass to be called last - which system property + // -Dtestng.listener.execution.symmetric=true allows, but this is only available + // in TestNG 7.7.1 - which requires Java11 + // So, this class simulates this behavior by hooking into IInvokedMethodListener + } + + @Override + public void afterInvocation(IInvokedMethod method, ITestResult result, ITestContext ctx) { + ITestClass testClass = (ITestClass)result.getTestClass(); + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + + boolean isImplementedAfterClassMethod = testClassHasAfterClassMethods + && method.isConfigurationMethod() + && method.getTestMethod().isAfterClassConfiguration(); + + ITestNGMethod[] testMethods = ctx.getAllTestMethods(); + + boolean isLastTestMethodOnTestClassWithoutAfterClassMethod = !testClassHasAfterClassMethods + && method.isTestMethod() + && method.getTestMethod().isTest() + && method.getTestMethod().getEnabled() + && testMethods.length > 0 + && method.getTestMethod() == testMethods[testMethods.length - 1]; + + if (isImplementedAfterClassMethod || isLastTestMethodOnTestClassWithoutAfterClassMethod) { + this.onAfterClassCore(testClass); + } + } + + private void onAfterClassCore(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); + if (remainingInstanceCount == 0) { + String failMessage = ""; + logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + failMessage = "COSMOS CLIENT LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + if (failMessage.length() > 0) { + failMessage += "\n\n"; + } + + failMessage += "NETTY LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; + + } + + if (failMessage.length() > 0) { + logger.error(failMessage); + fail(failMessage); + } + + this.logMemoryUsage("AFTER CLASS", testClassName); + } + } + } + + private void logMemoryUsage(String name, String className) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", className, name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); } } + // This method must be called as early as possible in the lifecycle of a process // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { @@ -50,15 +206,18 @@ public static void ingestIntoNetty() { return; } - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); System.setProperty("io.netty.leakDetection.targetRecords", "256"); + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); - logger.info("NETTY LEAK detection initialized"); + logger.info( + "NETTY LEAK detection initialized, CosmosClient leak detection enabled: {}", + Configs.isClientLeakDetectionEnabled()); isInitialized = true; } } @@ -82,7 +241,7 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { - logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + logger.warn("Disabling Leak detection:"); isLeakDetectionDisabled = true; return new DisableLeakDetectionScope(); @@ -146,6 +305,13 @@ private static final class DisableLeakDetectionScope implements AutoCloseable { @Override public void close() { synchronized (staticLock) { + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); isLeakDetectionDisabled = false; logger.info("Leak detection enabled again."); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java index b88bd63db110..156b116dc8b0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java @@ -44,13 +44,18 @@ public void createDocument() throws Exception { CosmosClientMetadataCachesSnapshot state = new CosmosClientMetadataCachesSnapshot(); RxClientCollectionCache.serialize(state, cache); - CosmosAsyncClient newClient = new CosmosClientBuilder().endpoint(TestConfigurations.HOST) + try (CosmosAsyncClient newClient = new CosmosClientBuilder().endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .metadataCaches(state) - .buildAsyncClient(); + .buildAsyncClient()) { - // TODO: moderakh we should somehow verify that to collection fetch request is made and the existing collection cache is used. - newClient.getDatabase(container.getDatabase().getId()).getContainer(container.getId()).readItem(id, new PartitionKey(id), ObjectNode.class).block(); + // TODO: moderakh we should somehow verify that to collection fetch request is made and the existing collection cache is used. + newClient + .getDatabase(container.getDatabase().getId()) + .getContainer(container.getId()) + .readItem(id, new PartitionKey(id), ObjectNode.class) + .block(); + } } @BeforeClass(groups = { "emulator" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java index 000bfaffc0f0..44b609aa5a55 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java @@ -15,6 +15,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -24,13 +25,12 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class DocumentClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(DocumentClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final AsyncDocumentClient.Builder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public DocumentClientTest() { this(new AsyncDocumentClient.Builder()); @@ -44,80 +44,6 @@ public final AsyncDocumentClient.Builder clientBuilder() { return this.clientBuilder; } - @BeforeClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java index 1277992b9b8d..f1cc3e82a30c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java @@ -25,6 +25,7 @@ import com.azure.cosmos.test.faultinjection.IFaultInjectionResult; import com.azure.cosmos.test.implementation.faultinjection.FaultInjectorProvider; import com.azure.cosmos.util.CosmosPagedFlux; +import net.bytebuddy.implementation.bytecode.Throw; import org.testng.SkipException; import org.testng.annotations.BeforeClass; import org.testng.annotations.Factory; @@ -55,20 +56,24 @@ public EndToEndTimeOutValidationTests(CosmosClientBuilder clientBuilder) { .build(); } - @BeforeClass(groups = {"fast"}, timeOut = SETUP_TIMEOUT * 100) - public void beforeClass() throws Exception { - initializeClient(null); - } - - public void initializeClient(CosmosEndToEndOperationLatencyPolicyConfig e2eDefaultConfig) { + public CosmosAsyncClient initializeClient(CosmosEndToEndOperationLatencyPolicyConfig e2eDefaultConfig) { CosmosAsyncClient client = this .getClientBuilder() .endToEndOperationLatencyPolicyConfig(e2eDefaultConfig) .buildAsyncClient(); - createdContainer = getSharedMultiPartitionCosmosContainer(client); - truncateCollection(createdContainer); - createdDocuments.addAll(this.insertDocuments(DEFAULT_NUM_DOCUMENTS, null, createdContainer)); + try { + createdContainer = getSharedMultiPartitionCosmosContainer(client); + truncateCollection(createdContainer); + + createdDocuments.addAll(this.insertDocuments(DEFAULT_NUM_DOCUMENTS, null, createdContainer)); + + return client; + } catch (Throwable t) { + safeClose(client); + + throw t; + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -77,17 +82,24 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutWithClientCon throw new SkipException("Failure injection only supported for DIRECT mode"); } - initializeClient(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - TestObject itemToRead = createdDocuments.get(random.nextInt(createdDocuments.size())); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.READ_ITEM, null); + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + TestObject itemToRead = createdDocuments.get(random.nextInt(createdDocuments.size())); + rule = injectFailure(createdContainer, FaultInjectionOperationType.READ_ITEM, null); - Mono> cosmosItemResponseMono = - createdContainer.readItem(itemToRead.id, new PartitionKey(itemToRead.mypk), options, TestObject.class); + Mono> cosmosItemResponseMono = + createdContainer.readItem(itemToRead.id, new PartitionKey(itemToRead.mypk), options, TestObject.class); - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -100,7 +112,7 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutEvenWhenDisab Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED, "true"); - initializeClient(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); FaultInjectionRule rule = null; try { @@ -119,6 +131,7 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutEvenWhenDisab } System.clearProperty(Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED); + safeClose(cosmosClient); } } @@ -128,16 +141,24 @@ public void createItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.CREATE_ITEM, null); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - Mono> cosmosItemResponseMono = - createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.CREATE_ITEM, null); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + Mono> cosmosItemResponseMono = + createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options); - verifyExpectError(cosmosItemResponseMono); - faultInjectionRule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -146,18 +167,26 @@ public void replaceItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options).block(); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.REPLACE_ITEM, null); - inputObject.setName("replaceName"); - Mono> cosmosItemResponseMono = - createdContainer.replaceItem(inputObject, inputObject.id, new PartitionKey(inputObject.mypk), options); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options).block(); + rule = injectFailure(createdContainer, FaultInjectionOperationType.REPLACE_ITEM, null); + inputObject.setName("replaceName"); + Mono> cosmosItemResponseMono = + createdContainer.replaceItem(inputObject, inputObject.id, new PartitionKey(inputObject.mypk), options); - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -166,16 +195,25 @@ public void upsertItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + + rule = injectFailure(createdContainer, FaultInjectionOperationType.UPSERT_ITEM, null); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + Mono> cosmosItemResponseMono = + createdContainer.upsertItem(inputObject, new PartitionKey(inputObject.mypk), options); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.UPSERT_ITEM, null); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - Mono> cosmosItemResponseMono = - createdContainer.upsertItem(inputObject, new PartitionKey(inputObject.mypk), options); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + safeClose(cosmosClient); + } } static void verifyExpectError(Mono> cosmosItemResponseMono) { @@ -190,27 +228,36 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosEndToEndOperationLatencyPolicyConfig endToEndOperationLatencyPolicyConfig = - new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) - .build(); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosEndToEndOperationLatencyPolicyConfig endToEndOperationLatencyPolicyConfig = + new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) + .build(); - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - createdDocuments.get(random.nextInt(createdDocuments.size())); + createdDocuments.get(random.nextInt(createdDocuments.size())); - String queryText = "select top 1 * from c"; - SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); + String queryText = "select top 1 * from c"; + SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); - CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); + CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); - StepVerifier.create(queryPagedFlux) - .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException - && ((OperationCancelledException) throwable).getSubStatusCode() - == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) - .verify(); - faultInjectionRule.disable(); + StepVerifier.create(queryPagedFlux) + .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException + && ((OperationCancelledException) throwable).getSubStatusCode() + == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) + .verify(); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -223,24 +270,30 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutWithClientCo new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) .build(); - initializeClient(endToEndOperationLatencyPolicyConfig); - - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - createdDocuments.get(random.nextInt(createdDocuments.size())); + createdDocuments.get(random.nextInt(createdDocuments.size())); - String queryText = "select top 1 * from c"; - SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); + String queryText = "select top 1 * from c"; + SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); - CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); + CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); - StepVerifier.create(queryPagedFlux) - .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException - && ((OperationCancelledException) throwable).getSubStatusCode() - == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) - .verify(); - faultInjectionRule.disable(); + StepVerifier.create(queryPagedFlux) + .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException + && ((OperationCancelledException) throwable).getSubStatusCode() + == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) + .verify(); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -257,7 +310,7 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldNotTimeoutWhenSuppr "isDefaultE2ETimeoutDisabledForNonPointOperations() after setting system property {}", Configs.isDefaultE2ETimeoutDisabledForNonPointOperations()); - initializeClient( + CosmosAsyncClient cosmosClient = initializeClient( new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) .build() ); @@ -283,6 +336,7 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldNotTimeoutWhenSuppr } System.clearProperty(Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED); + safeClose(cosmosClient); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index 7d6491c1fee2..b3a09b36d3f7 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -64,6 +64,8 @@ import io.reactivex.subscribers.TestSubscriber; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import reactor.core.publisher.Mono; @@ -94,6 +96,19 @@ public class RetryContextOnDiagnosticTest extends TestSuiteBase { private IRetryPolicy retryPolicy; private RxDocumentServiceRequest serviceRequest; private AddressSelector addressSelector; + private volatile AutoCloseable disableNettyLeakDetectionScope; + + @BeforeClass(groups = {"unit", "long-emulator"}) + public void beforeClass_DisableNettyLeakDetection() { + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); + } + + @AfterClass(groups = {"unit", "long-emulator"}, alwaysRun = true) + public void afterClass_ReactivateNettyLeakDetection() throws Exception { + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } + } @Test(groups = {"unit"}, timeOut = TIMEOUT * 2) public void backoffRetryUtilityExecuteRetry() throws Exception { @@ -105,15 +120,12 @@ public void backoffRetryUtilityExecuteRetry() throws Exception { addressSelector = Mockito.mock(AddressSelector.class); CosmosException exception = new CosmosException(410, exceptionText); String rawJson = "{\"id\":\"" + responseText + "\"}"; - ByteBuf buffer = getUTF8BytesOrNull(rawJson); Mockito.when(callbackMethod.call()).thenThrow(exception, exception, exception, exception, exception) - .thenReturn(Mono.just(new StoreResponse( - null, - 200, - new HashMap<>(), - new ByteBufInputStream(buffer, true), - buffer.readableBytes()))); + .thenReturn(Mono.fromCallable(() -> StoreResponseBuilder.create() + .withContent(rawJson) + .withStatus(200) + .build())); Mono monoResponse = BackoffRetryUtility.executeRetry(callbackMethod, retryPolicy); StoreResponse response = validateSuccess(monoResponse); @@ -156,14 +168,11 @@ public void backoffRetryUtilityExecuteAsync() { CosmosException exception = new CosmosException(410, exceptionText); Mono exceptionMono = Mono.error(exception); String rawJson = "{\"id\":\"" + responseText + "\"}"; - ByteBuf buffer = getUTF8BytesOrNull(rawJson); Mockito.when(parameterizedCallbackMethod.apply(ArgumentMatchers.any())).thenReturn(exceptionMono, exceptionMono, exceptionMono, exceptionMono, exceptionMono) - .thenReturn(Mono.just(new StoreResponse( - null, - 200, - new HashMap<>(), - new ByteBufInputStream(buffer, true), - buffer.readableBytes()))); + .thenReturn(Mono.fromCallable(() -> StoreResponseBuilder.create() + .withContent(rawJson) + .withStatus(200) + .build())); Mono monoResponse = BackoffRetryUtility.executeAsync( parameterizedCallbackMethod, retryPolicy, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java index d91b97327f80..116eb5b72e0e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java @@ -9,6 +9,7 @@ import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.models.CosmosContainerResponse; import com.azure.cosmos.rx.TestSuiteBase; +import org.testng.ITestContext; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Factory; @@ -42,70 +43,74 @@ public void afterClass() { @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithoutSpecialCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("TestUserAgent") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithSpecialCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("TéstUserAgent's") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent's"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent's"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithUnicodeCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("UnicodeChar鱀InUserAgent") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UnicodeChar_InUserAgent"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UnicodeChar_InUserAgent"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithWhitespaceAndAsciiSpecialChars() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("UserAgent with space$%_^()*&") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UserAgent with space$%_^()*&"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UserAgent with space$%_^()*&"); + } } private void validateUserAgentSuffix(String actualUserAgent, String expectedUserAgentSuffix) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java index 79bc5818cc92..3b20cd2ec3a5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java @@ -2422,13 +2422,13 @@ private void execute(MutationTestConfig mutationTestConfig, boolean shouldInject } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @AfterClass(groups = {"multi-master"}) public void afterClass() { - safeCloseAsync(this.cosmosAsyncClient); + safeClose(this.cosmosAsyncClient); } private static List buildFaultInjectionRules( diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java index 261ae9ee0a27..c619b5c1d83a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java @@ -279,7 +279,7 @@ public void nonWriteOperation_WithReadSessionUnavailable_test( } } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @@ -354,13 +354,13 @@ public void writeOperation_withReadSessionUnavailable_test( } } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @AfterClass(groups = {"multi-master"}, timeOut = SHUTDOWN_TIMEOUT) public void afterClass() { - safeCloseAsync(cosmosAsyncClient); + safeClose(cosmosAsyncClient); } private Map getRegionMap(DatabaseAccount databaseAccount, boolean writeOnly) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java index 22cba6842472..de8120a84091 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java @@ -28,34 +28,42 @@ public void validateStrongConsistencyOnSyncReplication() throws Exception { throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - User userDefinition = getUserDefinition(); - userDefinition.setId(userDefinition.getId() + "validateStrongConsistencyOnSyncReplication"); - User user = safeCreateUser(this.initClient, createdDatabase.getId(), userDefinition); - validateStrongConsistency(user); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + User userDefinition = getUserDefinition(); + userDefinition.setId(userDefinition.getId() + "validateStrongConsistencyOnSyncReplication"); + User user = safeCreateUser(this.initClient, createdDatabase.getId(), userDefinition); + validateStrongConsistency(user, readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @@ -63,85 +71,109 @@ public void validateStrongConsistencyOnSyncReplication() throws Exception { public void validateConsistentLSNForDirectTCPClient() { //TODO Need to test with TCP protocol // https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355057 - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateConsistentLSNForDirectHttpsClient() { ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) public void validateConsistentLSNAndQuorumAckedLSNForDirectTCPClient() { //TODO Need to test with TCP protocol //https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355057 - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSNAndQuorumAckedLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSNAndQuorumAckedLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) @@ -164,29 +196,37 @@ public void validateBoundedStalenessDynamicQuorumSyncReplication() { @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateConsistentLSNAndQuorumAckedLSNForDirectHttpsClient() { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSNAndQuorumAckedLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSNAndQuorumAckedLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } // TODO (DANOBLE) test is flaky @@ -218,33 +258,41 @@ public void validateConsistentPrefixOnSyncReplication() throws InterruptedExcept throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - User user = safeCreateUser(this.initClient, createdDatabase.getId(), getUserDefinition()); - boolean readLagging = validateConsistentPrefix(user); - assertThat(readLagging).isFalse(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + User user = safeCreateUser(this.initClient, createdDatabase.getId(), getUserDefinition()); + boolean readLagging = validateConsistentPrefix(user, readClient, writeClient); + assertThat(readLagging).isFalse(); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) @@ -253,34 +301,42 @@ public void validateConsistentPrefixOnAsyncReplication() throws InterruptedExcep throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - Document documentDefinition = getDocumentDefinition(); - Document document = createDocument(this.initClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - boolean readLagging = validateConsistentPrefix(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + Document documentDefinition = getDocumentDefinition(); + Document document = createDocument(this.initClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + boolean readLagging = validateConsistentPrefix(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java index e13377e64999..4bc97ff928dc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java @@ -42,75 +42,91 @@ public Object[] regionScopedSessionContainerConfigs() { @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateReadSessionOnAsyncReplication(boolean shouldRegionScopedSessionContainerEnabled) throws InterruptedException { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - - Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), - null, false).block().getResource(); - Thread.sleep(5000);//WaitForServerReplication - boolean readLagging = this.validateReadSession(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), + null, false).block().getResource(); + Thread.sleep(5000);//WaitForServerReplication + boolean readLagging = this.validateReadSession(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateWriteSessionOnAsyncReplication(boolean shouldRegionScopedSessionContainerEnabled) throws InterruptedException { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - - Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), - null, false).block().getResource(); - Thread.sleep(5000);//WaitForServerReplication - boolean readLagging = this.validateWriteSession(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), + null, false).block().getResource(); + Thread.sleep(5000);//WaitForServerReplication + boolean readLagging = this.validateWriteSession(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) @@ -233,6 +249,8 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { new CosmosClientTelemetryConfig() .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) .build(); + + QueryFeedOperationState dummyState = null; try { // CREATE collection DocumentCollection parentResource = writeClient.createCollection(createdDatabase.getSelfLink(), @@ -250,7 +268,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { cosmosQueryRequestOptions.setPartitionKey(new PartitionKey(PartitionKeyInternal.Empty.toJson())); cosmosQueryRequestOptions.setSessionToken(token); - QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.ReadFeed, cosmosQueryRequestOptions, @@ -262,6 +280,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { parentResource.getSelfLink(), dummyState, Document.class); validateQueryFailure(feedObservable, validator); } finally { + safeClose(dummyState); safeClose(writeClient); safeClose(readSecondaryClient); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java index 3b5244997d2e..75b5c04bf6ea 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java @@ -36,8 +36,6 @@ public class ConsistencyTestsBase extends TestSuiteBase { static final int CONSISTENCY_TEST_TIMEOUT = 120000; static final String USER_NAME = "TestUser"; - RxDocumentClientImpl writeClient; - RxDocumentClientImpl readClient; AsyncDocumentClient initClient; Database createdDatabase; DocumentCollection createdCollection; @@ -49,7 +47,11 @@ public void before_ConsistencyTestsBase() throws Exception { createdCollection = SHARED_MULTI_PARTITION_COLLECTION; } - void validateStrongConsistency(Resource resourceToWorkWith) throws Exception { + void validateStrongConsistency( + Resource resourceToWorkWith, + RxDocumentClientImpl readClient, + RxDocumentClientImpl writeClient) throws Exception { + int numberOfTestIteration = 5; Resource writeResource = resourceToWorkWith; while (numberOfTestIteration-- > 0) //Write from a client and do point read through second client and ensure TS matches. @@ -58,47 +60,47 @@ void validateStrongConsistency(Resource resourceToWorkWith) throws Exception { Thread.sleep(1000); //Timestamp is in granularity of seconds. Resource updatedResource = null; if (resourceToWorkWith instanceof User) { - updatedResource = this.writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null).block().getResource(); + updatedResource = writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null).block().getResource(); } else if (resourceToWorkWith instanceof Document) { RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, options, false).block().getResource(); + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, options, false).block().getResource(); } assertThat(updatedResource.getTimestamp().isAfter(sourceTimestamp)).isTrue(); - User readResource = this.readClient.readUser(resourceToWorkWith.getSelfLink(), null).block().getResource(); + User readResource = readClient.readUser(resourceToWorkWith.getSelfLink(), null).block().getResource(); assertThat(updatedResource.getTimestamp().equals(readResource.getTimestamp())); } } - void validateConsistentLSN() { + void validateConsistentLSN(RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) { Document documentDefinition = getDocumentDefinition(); RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentDefinition.get("mypk"))); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - ResourceResponse response = this.writeClient.deleteDocument(document.getSelfLink(), options).block(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + ResourceResponse response = writeClient.deleteDocument(document.getSelfLink(), options).block(); assertThat(response.getStatusCode()).isEqualTo(204); long quorumAckedLSN = Long.parseLong(response.getResponseHeaders().get(WFConstants.BackendHeaders.QUORUM_ACKED_LSN)); assertThat(quorumAckedLSN > 0).isTrue(); FailureValidator validator = new FailureValidator.Builder().statusCode(404).lsnGreaterThan(quorumAckedLSN).build(); - Mono> readObservable = this.readClient.readDocument(document.getSelfLink(), options); + Mono> readObservable = readClient.readDocument(document.getSelfLink(), options); validateFailure(readObservable, validator); } - void validateConsistentLSNAndQuorumAckedLSN() { + void validateConsistentLSNAndQuorumAckedLSN(RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) { Document documentDefinition = getDocumentDefinition(); RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentDefinition.get("mypk"))); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - ResourceResponse response = this.writeClient.deleteDocument(document.getSelfLink(), options).block(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + ResourceResponse response = writeClient.deleteDocument(document.getSelfLink(), options).block(); assertThat(response.getStatusCode()).isEqualTo(204); long quorumAckedLSN = Long.parseLong(response.getResponseHeaders().get(WFConstants.BackendHeaders.QUORUM_ACKED_LSN)); assertThat(quorumAckedLSN > 0).isTrue(); FailureValidator validator = new FailureValidator.Builder().statusCode(404).lsnGreaterThanEqualsTo(quorumAckedLSN).exceptionQuorumAckedLSNInNotNull().build(); - Mono> readObservable = this.readClient.deleteDocument(document.getSelfLink(), options); + Mono> readObservable = readClient.deleteDocument(document.getSelfLink(), options); validateFailure(readObservable, validator); } @@ -119,36 +121,48 @@ void validateStrongConsistencyOnAsyncReplication(boolean useGateway) throws Inte connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); } - this.writeClient = + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; + + try { + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); - this.readClient = + readClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); - Document documentDefinition = getDocumentDefinition(); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - validateStrongConsistency(document, TestUtils.getCollectionNameLink(createdDatabase.getId(), createdCollection.getId())); + Document documentDefinition = getDocumentDefinition(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + validateStrongConsistency( + document, + TestUtils.getCollectionNameLink(createdDatabase.getId(), createdCollection.getId()), + readClient, + writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } - void validateStrongConsistency(Document documentToWorkWith, String collectionLink) throws InterruptedException { + void validateStrongConsistency(Document documentToWorkWith, String collectionLink, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Document writeDocument = documentToWorkWith; while (numberOfTestIteration-- > 0) { @@ -156,10 +170,10 @@ void validateStrongConsistency(Document documentToWorkWith, String collectionLin Thread.sleep(1000);//Timestamp is in granularity of seconds. RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentToWorkWith.get("mypk"))); - Document updatedDocument = this.writeClient.replaceDocument(writeDocument, options).block().getResource(); + Document updatedDocument = writeClient.replaceDocument(writeDocument, options).block().getResource(); assertThat(updatedDocument.getTimestamp().isAfter(sourceTimestamp)).isTrue(); - Document readDocument = this.readClient.readDocument(documentToWorkWith.getSelfLink(), options).block().getResource(); + Document readDocument = readClient.readDocument(documentToWorkWith.getSelfLink(), options).block().getResource(); assertThat(updatedDocument.getTimestamp().equals(readDocument.getTimestamp())); } } @@ -251,7 +265,7 @@ void validateSessionContainerAfterCollectionCreateReplace(boolean useGateway) { } } - boolean validateConsistentPrefix(Resource resourceToWorkWith) throws InterruptedException { + boolean validateConsistentPrefix(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = resourceToWorkWith.getTimestamp(); boolean readLagging = false; @@ -262,12 +276,12 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted Thread.sleep(1000); //Timestamp is in granularity of seconds. Resource updatedResource = null; if (resourceToWorkWith instanceof User) { - updatedResource = this.writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, + updatedResource = writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null) .block() .getResource(); } else if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, null, false) .block() .getResource(); @@ -277,13 +291,13 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted Resource readResource = null; if (resourceToWorkWith instanceof User) { - readResource = this.readClient.readUser(resourceToWorkWith.getSelfLink(), null) + readResource = readClient.readUser(resourceToWorkWith.getSelfLink(), null) .block() .getResource(); } else if (resourceToWorkWith instanceof Document) { RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); - readResource = this.readClient.readDocument(resourceToWorkWith.getSelfLink(), options) + readResource = readClient.readDocument(resourceToWorkWith.getSelfLink(), options) .block() .getResource(); } @@ -296,7 +310,7 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted return readLagging; } - boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedException { + boolean validateReadSession(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = Instant.MIN; boolean readLagging = false; @@ -307,7 +321,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep Thread.sleep(1000); Resource updatedResource = null; if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false) .block() .getResource(); @@ -319,7 +333,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep RequestOptions requestOptions = new RequestOptions(); requestOptions.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); if (resourceToWorkWith instanceof Document) { - readResource = this.readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions).block().getResource(); + readResource = readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions).block().getResource(); } assertThat(readResource.getTimestamp().compareTo(lastReadDateTime) >= 0).isTrue(); lastReadDateTime = readResource.getTimestamp(); @@ -331,7 +345,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep return readLagging; } - boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedException { + boolean validateWriteSession(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = Instant.MIN; boolean readLagging = false; @@ -342,7 +356,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce Thread.sleep(1000); Resource updatedResource = null; if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false).block().getResource(); + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false).block().getResource(); } assertThat(updatedResource.getTimestamp().isAfter(sourceTimestamp)).isTrue(); writeResource = updatedResource; @@ -352,7 +366,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce requestOptions.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); if (resourceToWorkWith instanceof Document) { readResource = - this.readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions) + readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions) .block() .getResource(); } @@ -366,7 +380,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce //Now perform write on session and update our session token and lastReadTS Thread.sleep(1000); if (resourceToWorkWith instanceof Document) { - readResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), readResource, + readResource = writeClient.upsertDocument(createdCollection.getSelfLink(), readResource, requestOptions, false) .block() .getResource(); @@ -374,7 +388,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce } assertThat(readResource.getTimestamp().isAfter(lastReadDateTime)); - this.readClient.setSession(this.writeClient.getSession()); + readClient.setSession(writeClient.getSession()); } return readLagging; } @@ -873,8 +887,6 @@ private static String getGlobalSessionToken(RxDocumentClientImpl client, Documen @AfterClass(groups = {"direct"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { safeClose(this.initClient); - safeClose(this.writeClient); - safeClose(this.readClient); } private String getDifferentLSNToken(String token, long lsnDifferent) throws Exception { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java index 23e7d569a4b1..22b3be737817 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java @@ -95,22 +95,27 @@ public void queryWithContinuationTokenLimit(CosmosQueryRequestOptions options, S client.clearCapturedRequests(); - Flux> queryObservable = client + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + try { + Flux> queryObservable = client .queryDocuments( collectionLink, query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), + dummyState, Document.class); - List results = queryObservable.flatMap(p -> Flux.fromIterable(p.getResults())) - .collectList().block(); + List results = queryObservable.flatMap(p -> Flux.fromIterable(p.getResults())) + .collectList().block(); - assertThat(results.size()).describedAs("total results").isGreaterThanOrEqualTo(1); + assertThat(results.size()).describedAs("total results").isGreaterThanOrEqualTo(1); - List requests = client.getCapturedRequests(); + List requests = client.getCapturedRequests(); - for(HttpRequest req: requests) { - validateRequestHasContinuationTokenLimit(req, options.getResponseContinuationTokenLimitInKb()); + for (HttpRequest req : requests) { + validateRequestHasContinuationTokenLimit(req, options.getResponseContinuationTokenLimitInKb()); + } + } finally { + safeClose(dummyState); } } @@ -142,6 +147,11 @@ public Document createDocument(AsyncDocumentClient client, String collectionLink @BeforeClass(groups = { "fast" }, timeOut = SETUP_TIMEOUT) public void before_DocumentQuerySpyWireContentTest() throws Exception { + SpyClientUnderTestFactory.ClientUnderTest oldSnapshot = client; + if (oldSnapshot != null) { + oldSnapshot.close(); + } + client = new SpyClientBuilder(this.clientBuilder()).build(); createdDatabase = SHARED_DATABASE; @@ -172,13 +182,17 @@ public void before_DocumentQuerySpyWireContentTest() throws Exception { client ); - // do the query once to ensure the collection is cached. - client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + try { + // do the query once to ensure the collection is cached. + client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); - // do the query once to ensure the collection is cached. - client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + // do the query once to ensure the collection is cached. + client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); + } finally { + safeClose(state); + } } @AfterClass(groups = { "fast" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java index 435cf569c90b..9f5e783d1f67 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java @@ -137,15 +137,20 @@ public void queryWithMaxIntegratedCacheStaleness(CosmosQueryRequestOptions optio client.clearCapturedRequests(); - client.queryDocuments( - collectionLink, - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), - Document.class).blockLast(); - - List requests = client.getCapturedRequests(); - for (HttpRequest httpRequest : requests) { - validateRequestHasDedicatedGatewayHeaders(httpRequest, options.getDedicatedGatewayRequestOptions()); + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + try { + client.queryDocuments( + collectionLink, + query, + dummyState, + Document.class).blockLast(); + + List requests = client.getCapturedRequests(); + for (HttpRequest httpRequest : requests) { + validateRequestHasDedicatedGatewayHeaders(httpRequest, options.getDedicatedGatewayRequestOptions()); + } + } finally { + safeClose(dummyState); } } @@ -168,11 +173,15 @@ public void queryWithMaxIntegratedCacheStalenessInNanoseconds() { client ); - assertThatThrownBy(() -> client - .queryDocuments(collectionLink, query, state, Document.class) - .blockLast()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("MaxIntegratedCacheStaleness granularity is milliseconds"); + try { + assertThatThrownBy(() -> client + .queryDocuments(collectionLink, query, state, Document.class) + .blockLast()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("MaxIntegratedCacheStaleness granularity is milliseconds"); + } finally { + safeClose(state); + } } @Test(groups = { "fast" }, timeOut = TIMEOUT) @@ -192,13 +201,17 @@ public void queryWithMaxIntegratedCacheStalenessInNegative() { client ); - client.clearCapturedRequests(); + try { + client.clearCapturedRequests(); - assertThatThrownBy(() -> client - .queryDocuments(collectionLink, query, state, Document.class) - .blockLast()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("MaxIntegratedCacheStaleness duration cannot be negative"); + assertThatThrownBy(() -> client + .queryDocuments(collectionLink, query, state, Document.class) + .blockLast()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("MaxIntegratedCacheStaleness duration cannot be negative"); + } finally { + safeClose(state); + } } @Test(dataProvider = "maxIntegratedCacheStalenessDurationProviderItemOptions", groups = { "fast" }, timeOut = diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java index 888ed2687825..e672381f247a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java @@ -78,21 +78,26 @@ public void before_SessionTest() { RequestOptions requestOptions = new RequestOptions(); requestOptions.setOfferThroughput(20000); //Making sure we have 4 physical partitions - createdCollection = createCollection(createGatewayHouseKeepingDocumentClient().build(), createdDatabase.getId(), + AsyncDocumentClient asynClient = createGatewayHouseKeepingDocumentClient().build(); + try { + createdCollection = createCollection(asynClient, createdDatabase.getId(), collection, requestOptions); - houseKeepingClient = clientBuilder().build(); - connectionMode = houseKeepingClient.getConnectionPolicy().getConnectionMode(); - - if (connectionMode == ConnectionMode.DIRECT) { - spyClient = SpyClientUnderTestFactory.createDirectHttpsClientUnderTest(clientBuilder()); - } else { - // Gateway builder has multipleWriteRegionsEnabled false by default, enabling it for multi master test - ConnectionPolicy connectionPolicy = clientBuilder().connectionPolicy; - connectionPolicy.setMultipleWriteRegionsEnabled(true); - spyClient = SpyClientUnderTestFactory.createClientUnderTest(clientBuilder().withConnectionPolicy(connectionPolicy)); + houseKeepingClient = clientBuilder().build(); + connectionMode = houseKeepingClient.getConnectionPolicy().getConnectionMode(); + + if (connectionMode == ConnectionMode.DIRECT) { + spyClient = SpyClientUnderTestFactory.createDirectHttpsClientUnderTest(clientBuilder()); + } else { + // Gateway builder has multipleWriteRegionsEnabled false by default, enabling it for multi master test + ConnectionPolicy connectionPolicy = clientBuilder().connectionPolicy; + connectionPolicy.setMultipleWriteRegionsEnabled(true); + spyClient = SpyClientUnderTestFactory.createClientUnderTest(clientBuilder().withConnectionPolicy(connectionPolicy)); + } + options = new RequestOptions(); + options.setPartitionKey(PartitionKey.NONE); + } finally { + asynClient.close(); } - options = new RequestOptions(); - options.setPartitionKey(PartitionKey.NONE); } @AfterClass(groups = { "fast", "multi-master" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) @@ -211,6 +216,8 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce // Session token validation for cross partition query spyClient.clearCapturedRequests(); queryRequestOptions = new CosmosQueryRequestOptions(); + + safeClose(dummyState); dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.Query, @@ -227,6 +234,7 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce List feedRanges = spyClient.getFeedRanges(getCollectionLink(isNameBased), true).block(); queryRequestOptions = new CosmosQueryRequestOptions(); queryRequestOptions.setFeedRange(feedRanges.get(0)); + safeClose(dummyState); dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.Query, @@ -241,6 +249,7 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce // Session token validation for readAll with partition query spyClient.clearCapturedRequests(); queryRequestOptions = new CosmosQueryRequestOptions(); + safeClose(dummyState); dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.ReadFeed, @@ -260,6 +269,7 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce spyClient.clearCapturedRequests(); queryRequestOptions = new CosmosQueryRequestOptions(); + safeClose(dummyState); dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.ReadFeed, @@ -278,10 +288,12 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce CosmosItemIdentity cosmosItemIdentity = new CosmosItemIdentity(new PartitionKey(documentCreated.getId()), documentCreated.getId()); List cosmosItemIdentities = new ArrayList<>(); cosmosItemIdentities.add(cosmosItemIdentity); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient); spyClient.readMany( cosmosItemIdentities, getCollectionLink(isNameBased), - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient), + dummyState, InternalObjectNode.class).block(); assertThat(getSessionTokensInRequests().size()).isEqualTo(1); assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); @@ -317,6 +329,8 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token } + + safeClose(dummyState); } @Test(groups = { "fast" }, timeOut = TIMEOUT, dataProvider = "sessionTestArgProvider") diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 3da097f3213c..ce0b29a47de5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -55,8 +55,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doAnswer; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends DocumentClientTest { +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) +public abstract class TestSuiteBase extends DocumentClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); protected static final int TIMEOUT = 40000; @@ -126,7 +126,9 @@ public Flux> queryDatabases(SqlQuerySpec query) { OperationType.Query, new CosmosQueryRequestOptions(), client); - return client.queryDatabases(query, state); + return client + .queryDatabases(query, state) + .doFinally(signal -> safeClose(state)); } @Override @@ -158,13 +160,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { @@ -184,128 +179,128 @@ protected static void truncateCollection(DocumentCollection collection) { try { List paths = collection.getPartitionKey().getPaths(); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - options.setMaxDegreeOfParallelism(-1); - QueryFeedOperationState state = new QueryFeedOperationState( - cosmosClient, - "truncateCollection", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); - - logger.info("Truncating DocumentCollection {} documents ...", collection.getId()); - - houseKeepingClient.queryDocuments(collection.getSelfLink(), "SELECT * FROM root", state, Document.class) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(doc -> { - RequestOptions requestOptions = new RequestOptions(); - - if (paths != null && !paths.isEmpty()) { - List pkPath = PathParser.getPathParts(paths.get(0)); - Object propertyValue = doc.getObjectByPath(pkPath); - if (propertyValue == null) { - propertyValue = Undefined.value(); - } - - requestOptions.setPartitionKey(new PartitionKey(propertyValue)); - } - - return houseKeepingClient.deleteDocument(doc.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} triggers ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateTriggers", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryTriggers(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(trigger -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = trigger.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteTrigger(trigger.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} storedProcedures ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateStoredProcs", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryStoredProcedures(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(storedProcedure -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = storedProcedure.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteStoredProcedure(storedProcedure.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} udfs ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateUserDefinedFunctions", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryUserDefinedFunctions(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(udf -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = udf.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteUserDefinedFunction(udf.getSelfLink(), requestOptions); - }).then().block(); - + .buildAsyncClient()) { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setMaxDegreeOfParallelism(-1); + QueryFeedOperationState state = new QueryFeedOperationState( + cosmosClient, + "truncateCollection", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); + + logger.info("Truncating DocumentCollection {} documents ...", collection.getId()); + + houseKeepingClient.queryDocuments(collection.getSelfLink(), "SELECT * FROM root", state, Document.class) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(doc -> { + RequestOptions requestOptions = new RequestOptions(); + + if (paths != null && !paths.isEmpty()) { + List pkPath = PathParser.getPathParts(paths.get(0)); + Object propertyValue = doc.getObjectByPath(pkPath); + if (propertyValue == null) { + propertyValue = Undefined.value(); + } + + requestOptions.setPartitionKey(new PartitionKey(propertyValue)); + } + + return houseKeepingClient.deleteDocument(doc.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} triggers ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateTriggers", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryTriggers(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(trigger -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = trigger.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteTrigger(trigger.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} storedProcedures ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateStoredProcs", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryStoredProcedures(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(storedProcedure -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = storedProcedure.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteStoredProcedure(storedProcedure.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} udfs ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateUserDefinedFunctions", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryUserDefinedFunctions(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(udf -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = udf.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteUserDefinedFunction(udf.getSelfLink(), requestOptions); + }).then().block(); + } } finally { houseKeepingClient.close(); } @@ -554,11 +549,15 @@ public static void deleteCollectionIfExists(AsyncDocumentClient client, String d new CosmosQueryRequestOptions(), client ); - List res = client.queryCollections("dbs/" + databaseId, - String.format("SELECT * FROM root r where r.id = '%s'", collectionId), state).single().block() - .getResults(); - if (!res.isEmpty()) { - deleteCollection(client, TestUtils.getCollectionNameLink(databaseId, collectionId)); + try { + List res = client.queryCollections("dbs/" + databaseId, + String.format("SELECT * FROM root r where r.id = '%s'", collectionId), state).single().block() + .getResults(); + if (!res.isEmpty()) { + deleteCollection(client, TestUtils.getCollectionNameLink(databaseId, collectionId)); + } + } finally { + safeClose(state); } } @@ -576,15 +575,19 @@ public static void deleteDocumentIfExists(AsyncDocumentClient client, String dat new CosmosQueryRequestOptions(), client ); - List res = client + try { + List res = client .queryDocuments( TestUtils.getCollectionNameLink(databaseId, collectionId), String.format("SELECT * FROM root r where r.id = '%s'", docId), state, Document.class) .single().block().getResults(); - if (!res.isEmpty()) { - deleteDocument(client, TestUtils.getDocumentNameLink(databaseId, collectionId, docId), pk, TestUtils.getCollectionNameLink(databaseId, collectionId)); + if (!res.isEmpty()) { + deleteDocument(client, TestUtils.getDocumentNameLink(databaseId, collectionId, docId), pk, TestUtils.getCollectionNameLink(databaseId, collectionId)); + } + } finally { + safeClose(state); } } @@ -601,11 +604,16 @@ public static void deleteUserIfExists(AsyncDocumentClient client, String databas new CosmosQueryRequestOptions(), client ); - List res = client + + try { + List res = client .queryUsers("dbs/" + databaseId, String.format("SELECT * FROM root r where r.id = '%s'", userId), state) .single().block().getResults(); - if (!res.isEmpty()) { - deleteUser(client, TestUtils.getUserNameLink(databaseId, userId)); + if (!res.isEmpty()) { + deleteUser(client, TestUtils.getUserNameLink(databaseId, userId)); + } + } finally { + safeClose(state); } } @@ -640,7 +648,8 @@ static protected Database createDatabaseIfNotExists(AsyncDocumentClient client, new CosmosQueryRequestOptions(), client ); - return client.queryDatabases(String.format("SELECT * FROM r where r.id = '%s'", databaseId), state).flatMap(p -> Flux.fromIterable(p.getResults())).switchIfEmpty( + try { + return client.queryDatabases(String.format("SELECT * FROM r where r.id = '%s'", databaseId), state).flatMap(p -> Flux.fromIterable(p.getResults())).switchIfEmpty( Flux.defer(() -> { Database databaseDefinition = new Database(); @@ -648,7 +657,10 @@ static protected Database createDatabaseIfNotExists(AsyncDocumentClient client, return client.createDatabase(databaseDefinition, null).map(ResourceResponse::getResource); }) - ).single().block(); + ).single().block(); + } finally { + safeClose(state); + } } static protected void safeDeleteDatabase(AsyncDocumentClient client, Database database) { @@ -674,14 +686,19 @@ static protected void safeDeleteAllCollections(AsyncDocumentClient client, Datab new CosmosQueryRequestOptions(), client ); - List collections = client.readCollections(database.getSelfLink(), state) - .flatMap(p -> Flux.fromIterable(p.getResults())) - .collectList() - .single() - .block(); - - for (DocumentCollection collection : collections) { - client.deleteCollection(collection.getSelfLink(), null).block().getResource(); + + try { + List collections = client.readCollections(database.getSelfLink(), state) + .flatMap(p -> Flux.fromIterable(p.getResults())) + .collectList() + .single() + .block(); + + for (DocumentCollection collection : collections) { + client.deleteCollection(collection.getSelfLink(), null).block().getResource(); + } + } finally { + safeClose(state); } } } @@ -716,6 +733,22 @@ static protected void safeCloseAsync(AsyncDocumentClient client) { } } + static protected void safeClose(QueryFeedOperationState state) { + if (state != null) { + safeClose(state.getClient()); + } + } + + static protected void safeClose(CosmosAsyncClient client) { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + static protected void safeClose(AsyncDocumentClient client) { if (client != null) { try { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java index 46b5e0374438..ceafb8c65778 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java @@ -15,7 +15,7 @@ public class CpuLoadMonitorTest { @Test(groups = "unit") public void noInstance() throws Exception { - assertThat(ReflectionUtils.getListeners()).hasSize(0); + assertEventualListenerCount(0); assertThat(ReflectionUtils.getFuture()).isNull(); } @@ -31,7 +31,7 @@ public void multipleInstances() throws Exception { Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); assertThat(workFuture.isCancelled()).isFalse(); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Thread.sleep(10); } @@ -41,7 +41,7 @@ public void multipleInstances() throws Exception { CpuMemoryMonitor.unregister(cpuMemoryListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); @@ -54,7 +54,7 @@ public void multipleInstances() throws Exception { CpuMemoryMonitor.register(newListener); CpuMemoryMonitor.unregister(newListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); assertThat(workFuture.isCancelled()).isFalse(); @@ -62,7 +62,7 @@ public void multipleInstances() throws Exception { CpuMemoryListener cpuMemoryListener = cpuMonitorList.remove(0); CpuMemoryMonitor.unregister(cpuMemoryListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNull(); @@ -74,12 +74,20 @@ public void handleLeak() throws Throwable { CpuMemoryMonitor.register(listener); listener = null; System.gc(); - Thread.sleep(10000); - - assertThat(ReflectionUtils.getListeners()).hasSize(0); + int secondsWaited = 0; + assertEventualListenerCount(0); assertThat(ReflectionUtils.getFuture()).isNull(); } + private static void assertEventualListenerCount(int expectedListenerCount) throws Exception { + int secondsWaited = 0; + while (secondsWaited < 30 && ReflectionUtils.getListeners().size() > expectedListenerCount) { + Thread.sleep(1_000); + } + + assertThat(ReflectionUtils.getListeners()).hasSize(expectedListenerCount); + } + class TestMemoryListener implements CpuMemoryListener { } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java index 97f86bb795d3..01e7c1f86de4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java @@ -4,6 +4,7 @@ package com.azure.cosmos.implementation.directconnectivity; import com.azure.cosmos.ConnectionMode; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ConnectionPolicy; @@ -29,34 +30,43 @@ import com.azure.cosmos.implementation.routing.RegionalRoutingContext; import io.netty.handler.ssl.SslContext; import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.Random; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; import static org.assertj.core.api.Assertions.assertThat; public class ConnectionStateListenerTest { - private static final Logger logger = LoggerFactory.getLogger(ConnectionStateListenerTest.class); private static final AtomicInteger randomPort = new AtomicInteger(1000); - private static int port = 8082; - private static String serverAddressPrefix = "rntbd://localhost:"; - private static Random random = new Random(); + private static final int port = 8082; + private static final String serverAddressPrefix = "rntbd://localhost:"; + + private volatile AutoCloseable disableNettyLeakDetectionScope; + + @BeforeClass(groups = "unit") + public void beforeClass_DisableNettyLeakDetection() { + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); + } + + @AfterClass(groups = "unit", alwaysRun = true) + public void afterClass_ReactivateNettyLeakDetection() throws Exception { + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } + } @DataProvider(name = "connectionStateListenerConfigProvider") public Object[][] connectionStateListenerConfigProvider() { @@ -88,82 +98,84 @@ public void connectionStateListener_OnConnectionEvent( boolean isTcpConnectionEndpointRediscoveryEnabled, RequestResponseType responseType, boolean markUnhealthy, - boolean markUnhealthyWhenServerShutdown) throws ExecutionException, InterruptedException, URISyntaxException { - - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - connectionPolicy.setTcpConnectionEndpointRediscoveryEnabled(isTcpConnectionEndpointRediscoveryEnabled); - - GlobalAddressResolver addressResolver = Mockito.mock(GlobalAddressResolver.class); - - SslContext sslContext = SslContextUtils.CreateSslContext("client.jks", false); - - Configs config = Mockito.mock(Configs.class); - Mockito.doReturn(sslContext).when(config).getSslContext(false, false); - - ClientTelemetry clientTelemetry = Mockito.mock(ClientTelemetry.class); - ClientTelemetryInfo clientTelemetryInfo = new ClientTelemetryInfo( - "testMachine", - "testClient", - "testProcess", - "testApp", - ConnectionMode.DIRECT, - "test-cdb-account", - "Test Region 1", - "Linux", - false, - Arrays.asList("Test Region 1", "Test Region 2")); - - Mockito.when(clientTelemetry.getClientTelemetryInfo()).thenReturn(clientTelemetryInfo); - - RntbdTransportClient client = new RntbdTransportClient( - config, - connectionPolicy, - new UserAgentContainer(), - addressResolver, - clientTelemetry, - null); - - RxDocumentServiceRequest req = - RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Document, - "dbs/fakedb/colls/fakeColls", - getDocumentDefinition(), new HashMap<>()); - req.requestContext.regionalRoutingContextToRoute = new RegionalRoutingContext(new URI("https://localhost:8080")); - - req.setPartitionKeyRangeIdentity(new PartitionKeyRangeIdentity("fakeCollectionId","fakePartitionKeyRangeId")); - - // Validate connectionStateListener will always track the latest Uri - List targetUris = new ArrayList<>(); - int serverPort = port + randomPort.getAndIncrement(); - String serverAddress = serverAddressPrefix + serverPort; - - targetUris.add(new Uri(serverAddress)); - targetUris.add(new Uri(serverAddress)); - - for (Uri uri : targetUris) { - // using a random generated server port - TcpServer server = TcpServerFactory.startNewRntbdServer(serverPort); - // Inject fake response - server.injectServerResponse(responseType); - - try { - client.invokeResourceOperationAsync(uri, req).block(); - } catch (Exception e) { - // no op here - } - - if (markUnhealthy) { - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); - TcpServerFactory.shutdownRntbdServer(server); - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); - - } else { - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + boolean markUnhealthyWhenServerShutdown) throws Exception { + + try (AutoCloseable disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope()) { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + connectionPolicy.setTcpConnectionEndpointRediscoveryEnabled(isTcpConnectionEndpointRediscoveryEnabled); + + GlobalAddressResolver addressResolver = Mockito.mock(GlobalAddressResolver.class); + + SslContext sslContext = SslContextUtils.CreateSslContext("client.jks", false); + + Configs config = Mockito.mock(Configs.class); + Mockito.doReturn(sslContext).when(config).getSslContext(false, false); + + ClientTelemetry clientTelemetry = Mockito.mock(ClientTelemetry.class); + ClientTelemetryInfo clientTelemetryInfo = new ClientTelemetryInfo( + "testMachine", + "testClient", + "testProcess", + "testApp", + ConnectionMode.DIRECT, + "test-cdb-account", + "Test Region 1", + "Linux", + false, + Arrays.asList("Test Region 1", "Test Region 2")); + + Mockito.when(clientTelemetry.getClientTelemetryInfo()).thenReturn(clientTelemetryInfo); + + RntbdTransportClient client = new RntbdTransportClient( + config, + connectionPolicy, + new UserAgentContainer(), + addressResolver, + clientTelemetry, + null); + + RxDocumentServiceRequest req = + RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Document, + "dbs/fakedb/colls/fakeColls", + getDocumentDefinition(), new HashMap<>()); + req.requestContext.regionalRoutingContextToRoute = new RegionalRoutingContext(new URI("https://localhost:8080")); + + req.setPartitionKeyRangeIdentity(new PartitionKeyRangeIdentity("fakeCollectionId", "fakePartitionKeyRangeId")); + + // Validate connectionStateListener will always track the latest Uri + List targetUris = new ArrayList<>(); + int serverPort = port + randomPort.getAndIncrement(); + String serverAddress = serverAddressPrefix + serverPort; + + targetUris.add(new Uri(serverAddress)); + targetUris.add(new Uri(serverAddress)); + + for (Uri uri : targetUris) { + // using a random generated server port + TcpServer server = TcpServerFactory.startNewRntbdServer(serverPort); + // Inject fake response + server.injectServerResponse(responseType); + + try { + client.invokeResourceOperationAsync(uri, req).block(); + } catch (Exception e) { + // no op here + } - TcpServerFactory.shutdownRntbdServer(server); - if (markUnhealthyWhenServerShutdown) { + if (markUnhealthy) { assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + TcpServerFactory.shutdownRntbdServer(server); + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + } else { assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + + TcpServerFactory.shutdownRntbdServer(server); + if (markUnhealthyWhenServerShutdown) { + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + } else { + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + } } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java index 77d68acb18bd..7e7410836eb5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java @@ -5,6 +5,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.implementation.AsyncDocumentClient.Builder; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.TestUtils; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; import com.azure.cosmos.models.CosmosClientTelemetryConfig; @@ -228,20 +229,27 @@ public void crossPartitionQuery() { options.setMaxDegreeOfParallelism(-1); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); - Flux> results = client.queryDocuments( - getCollectionLink(), - "SELECT * FROM r", - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), - Document.class); + QueryFeedOperationState dummyState = + TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + try { + Flux> results = client.queryDocuments( + getCollectionLink(), + "SELECT * FROM r", + dummyState, + Document.class); - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(documentList.size()) .exactlyContainsInAnyOrder(documentList.stream().map(Document::getResourceId).collect(Collectors.toList())).build(); - validateQuerySuccess(results, validator, QUERY_TIMEOUT); - validateNoDocumentQueryOperationThroughGateway(); - // validates only the first query for fetching query plan goes to gateway. - assertThat(client.getCapturedRequests().stream().filter(r -> r.getResourceType() == ResourceType.Document)).hasSize(1); + validateQuerySuccess(results, validator, QUERY_TIMEOUT); + validateNoDocumentQueryOperationThroughGateway(); + + // validates only the first query for fetching query plan goes to gateway. + assertThat(client.getCapturedRequests().stream().filter(r -> r.getResourceType() == ResourceType.Document)).hasSize(1); + } finally { + safeClose(dummyState); + } } private void validateNoStoredProcExecutionOperationThroughGateway() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java index c92781137a97..8285ea915603 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java @@ -168,7 +168,7 @@ public void getServerAddressesViaGateway(List partitionKeyRangeIds, } } - @Test(groups = { "direct" }, dataProvider = "protocolProvider", timeOut = TIMEOUT) + @Test(groups = { "direct" }, dataProvider = "protocolProvider", timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) @SuppressWarnings({"unchecked", "rawtypes"}) public void getMasterAddressesViaGatewayAsync(Protocol protocol) throws Exception { Configs configs = ConfigsBuilder.instance().withProtocol(protocol).build(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java index cf126f74b3fe..cf1110aec87f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java @@ -317,6 +317,15 @@ public static void setTransportClient(StoreReader storeReader, TransportClient t set(storeReader, transportClient, "transportClient"); } + public static void setTransportClient(CosmosClient client, TransportClient transportClient) { + StoreClient storeClient = getStoreClient((RxDocumentClientImpl) CosmosBridgeInternal.getAsyncDocumentClient(client)); + set(storeClient, transportClient, "transportClient"); + ReplicatedResourceClient replicatedResClient = getReplicatedResourceClient(storeClient); + ConsistencyWriter writer = getConsistencyWriter(replicatedResClient); + set(replicatedResClient, transportClient, "transportClient"); + set(writer, transportClient, "transportClient"); + } + public static TransportClient getTransportClient(ReplicatedResourceClient replicatedResourceClient) { return get(TransportClient.class, replicatedResourceClient, "transportClient"); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java index 4dd92d213297..a9ae3654115a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java @@ -7,6 +7,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ReferenceCountUtil; import java.util.List; @@ -39,6 +40,7 @@ public void channelRead(final ChannelHandlerContext context, final Object messag return; } } + context.fireChannelRead(message); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java index d9fdaec12fdf..948a09f88a21 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ReferenceCountUtil; import java.util.List; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java index 5afe521bcc51..a6f11a5f7c82 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java @@ -46,21 +46,24 @@ public void expireRecord(OperationType operationType, boolean requestSent, Class new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - if (requestSent) { - record.setSendingRequestHasStarted(); - } - record.expire(); - - try{ - record.get(); - fail("RntbdRequestRecord should complete with exception"); - } catch (ExecutionException e) { - Throwable innerException = e.getCause(); - assertThat(innerException).isInstanceOf(exceptionType); - } catch (Exception e) { - fail("Wrong exception"); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + if (requestSent) { + record.setSendingRequestHasStarted(); + } + record.expire(); + + try { + record.get(); + fail("RntbdRequestRecord should complete with exception"); + } catch (ExecutionException e) { + Throwable innerException = e.getCause(); + assertThat(innerException).isInstanceOf(exceptionType); + } catch (Exception e) { + fail("Wrong exception"); + } } } @@ -72,22 +75,25 @@ public void cancelRecord() throws URISyntaxException, InterruptedException, Json new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - Mono result = Mono.fromFuture(record) - .doOnNext(storeResponse -> fail("Record got cancelled should not reach here")) - .doOnError(throwable -> fail("Record got cancelled should not reach here")); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + Mono result = Mono.fromFuture(record) + .doOnNext(storeResponse -> fail("Record got cancelled should not reach here")) + .doOnError(throwable -> fail("Record got cancelled should not reach here")); - result.cancelOn(Schedulers.boundedElastic()).subscribe().dispose(); + result.cancelOn(Schedulers.boundedElastic()).subscribe().dispose(); - Thread.sleep(100); - assertThat(record.isCancelled()).isTrue(); + Thread.sleep(100); + assertThat(record.isCancelled()).isTrue(); - String jsonString = record.toString(); - String statusString = "{\"done\":true,\"cancelled\":true,\"completedExceptionally\":true,\"error\":{\"type\":\"java.util.concurrent.CancellationException\"}}"; - JsonNode jsonNode = Utils.getSimpleObjectMapper().readTree(jsonString); - JsonNode errorStatus = jsonNode.get("RntbdRequestRecord").get("status"); - assertThat(errorStatus).isNotNull(); - assertThat(errorStatus.toString()).isEqualTo(statusString); + String jsonString = record.toString(); + String statusString = "{\"done\":true,\"cancelled\":true,\"completedExceptionally\":true,\"error\":{\"type\":\"java.util.concurrent.CancellationException\"}}"; + JsonNode jsonNode = Utils.getSimpleObjectMapper().readTree(jsonString); + JsonNode errorStatus = jsonNode.get("RntbdRequestRecord").get("status"); + assertThat(errorStatus).isNotNull(); + assertThat(errorStatus.toString()).isEqualTo(statusString); + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java index 6de130598613..f417bec3061b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java @@ -15,50 +15,63 @@ public class RntbdTokenTests { private static final Random rnd = new Random(); @Test(groups = { "unit" }) public void getValueIsIdempotent() { - RntbdToken token = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); - byte[] blob = new byte[10]; - rnd.nextBytes(blob); - token.setValue(blob); + ByteBuf buffer = null; + ByteBuf byteBufValue1 = null; - String expectedJson = "{\"id\":90,\"name\":\"EffectivePartitionKey\",\"present\":true," - + "\"required\":false,\"value\":\"" - + Base64.getEncoder().encodeToString(blob) - + "\",\"tokenType\":\"Bytes\"}"; - assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + try { + RntbdToken token = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); + byte[] blob = new byte[10]; + rnd.nextBytes(blob); + token.setValue(blob); - Object value1 = token.getValue(); - Object value2 = token.getValue(); - assertThat(value1).isSameAs(value2); - assertThat(value1).isSameAs(blob); + String expectedJson = "{\"id\":90,\"name\":\"EffectivePartitionKey\",\"present\":true," + + "\"required\":false,\"value\":\"" + + Base64.getEncoder().encodeToString(blob) + + "\",\"tokenType\":\"Bytes\"}"; + assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); - assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + Object value1 = token.getValue(); + Object value2 = token.getValue(); + assertThat(value1).isSameAs(value2); + assertThat(value1).isSameAs(blob); - ByteBuf buffer = Unpooled.buffer(1024); - token.encode(buffer); + assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + buffer = Unpooled.buffer(1024); + token.encode(buffer); - RntbdToken decodedToken = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); - // skipping 3 bytes (2 bytes for header id + 1 byte for token type) - buffer.readerIndex(3); - // when decoding the RntbdToken.value is a ByteBuffer - not a byte[] - testing this path for idempotency as well - decodedToken.decode(buffer); - assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + RntbdToken decodedToken = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); + // skipping 3 bytes (2 bytes for header id + 1 byte for token type) + buffer.readerIndex(3); + // when decoding the RntbdToken.value is a ByteBuffer - not a byte[] - testing this path for idempotency as well + decodedToken.decode(buffer); + assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); - value1 = decodedToken.getValue(); - assertThat(value1).isInstanceOf(ByteBuf.class); - ByteBuf byteBufValue1 = (ByteBuf)value1; - assertThat(byteBufValue1.readableBytes()).isEqualTo(10); - byte[] byteArray1 = new byte[10]; - byteBufValue1.getBytes(byteBufValue1.readerIndex(), byteArray1); - assertThat(byteArray1).isEqualTo(blob); + value1 = decodedToken.getValue(); + assertThat(value1).isInstanceOf(ByteBuf.class); - value2 = decodedToken.getValue(); - assertThat(value1).isSameAs(value2); - ByteBuf byteBufValue2 = (ByteBuf)value2; - assertThat(byteBufValue2.readableBytes()).isEqualTo(10); - byte[] byteArray2 = new byte[10]; - byteBufValue2.getBytes(byteBufValue2.readerIndex(), byteArray2); - assertThat(byteArray2).isEqualTo(blob); + byteBufValue1 = (ByteBuf) value1; + assertThat(byteBufValue1.readableBytes()).isEqualTo(10); + byte[] byteArray1 = new byte[10]; + byteBufValue1.getBytes(byteBufValue1.readerIndex(), byteArray1); + assertThat(byteArray1).isEqualTo(blob); - assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + value2 = decodedToken.getValue(); + assertThat(value1).isSameAs(value2); + ByteBuf byteBufValue2 = (ByteBuf) value2; + assertThat(byteBufValue2.readableBytes()).isEqualTo(10); + byte[] byteArray2 = new byte[10]; + byteBufValue2.getBytes(byteBufValue2.readerIndex(), byteArray2); + assertThat(byteArray2).isEqualTo(blob); + + assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + } finally { + if (buffer != null) { + buffer.release(); + } + + if (byteBufValue1 != null) { + byteBufValue1.release(); + } + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java index 74a00185967a..88b85abbdf68 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java @@ -4,6 +4,8 @@ package com.azure.cosmos.implementation.http; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; +import com.azure.cosmos.TestNGLogListener; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.LifeCycleUtils; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; @@ -12,6 +14,7 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Listeners; import org.testng.annotations.Test; import java.time.Duration; @@ -26,15 +29,21 @@ public class ReactorNettyHttpClientTest { private static final Logger logger = LoggerFactory.getLogger(ReactorNettyHttpClientTest.class); private HttpClient reactorNettyHttpClient; + private volatile AutoCloseable disableNettyLeakDetectionScope; @BeforeClass(groups = "unit") public void before_ReactorNettyHttpClientTest() { this.reactorNettyHttpClient = HttpClient.createFixed(new HttpClientConfig(new Configs())); + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); } - @AfterClass(groups = "unit") - public void after_ReactorNettyHttpClientTest() { + @AfterClass(groups = "unit", alwaysRun = true) + public void after_ReactorNettyHttpClientTest() throws Exception { + LifeCycleUtils.closeQuietly(reactorNettyHttpClient); + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } } @Test(groups = "unit") diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java index abee1236ba68..f44802a44e4a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java @@ -269,25 +269,29 @@ public void before_BackPressureTest() throws Exception { rxClient ); - // increase throughput to max for a single partition collection to avoid throttling - // for bulk insert and later queries. - Offer offer = rxClient.queryOffers( + try { + // increase throughput to max for a single partition collection to avoid throttling + // for bulk insert and later queries. + Offer offer = rxClient.queryOffers( String.format("SELECT * FROM r WHERE r.offerResourceId = '%s'", createdCollection.read().block().getProperties().getResourceId()) - , state).take(1).map(FeedResponse::getResults).single().block().get(0); - offer.setThroughput(6000); - offer = rxClient.replaceOffer(offer).block().getResource(); - assertThat(offer.getThroughput()).isEqualTo(6000); - - ArrayList docDefList = new ArrayList<>(); - for(int i = 0; i < 1000; i++) { - docDefList.add(getDocumentDefinition(i)); - } + , state).take(1).map(FeedResponse::getResults).single().block().get(0); + offer.setThroughput(6000); + offer = rxClient.replaceOffer(offer).block().getResource(); + assertThat(offer.getThroughput()).isEqualTo(6000); + + ArrayList docDefList = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + docDefList.add(getDocumentDefinition(i)); + } - createdDocuments = bulkInsertBlocking(createdCollection, docDefList); + createdDocuments = bulkInsertBlocking(createdCollection, docDefList); - waitIfNeededForReplicasToCatchUp(getClientBuilder()); - warmUp(); + waitIfNeededForReplicasToCatchUp(getClientBuilder()); + warmUp(); + } finally { + safeClose(state); + } } private void warmUp() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java index 8d4e7fe2570b..446ee82008b8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java @@ -57,7 +57,6 @@ import reactor.core.publisher.Mono; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -68,7 +67,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -154,46 +152,49 @@ public ClientRetryPolicyE2ETests(CosmosClientBuilder clientBuilder) { @BeforeClass(groups = {"multi-master", "fast", "fi-multi-master", "multi-region"}, timeOut = TIMEOUT) public void beforeClass() { - CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient(); - AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); - GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + try(CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); - DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); - AccountLevelLocationContext accountLevelReadableLocationContext - = getAccountLevelLocationContext(databaseAccount, false); + AccountLevelLocationContext accountLevelReadableLocationContext + = getAccountLevelLocationContext(databaseAccount, false); - AccountLevelLocationContext accountLevelWriteableLocationContext - = getAccountLevelLocationContext(databaseAccount, true); + AccountLevelLocationContext accountLevelWriteableLocationContext + = getAccountLevelLocationContext(databaseAccount, true); - validate(accountLevelReadableLocationContext, false); - validate(accountLevelWriteableLocationContext, true); + validate(accountLevelReadableLocationContext, false); + validate(accountLevelWriteableLocationContext, true); - this.preferredRegions = accountLevelReadableLocationContext.serviceOrderedReadableRegions + this.preferredRegions = accountLevelReadableLocationContext.serviceOrderedReadableRegions .stream() .map(regionName -> regionName.toLowerCase(Locale.ROOT)) .collect(Collectors.toList()); - this.serviceOrderedReadableRegions = this.preferredRegions; + this.serviceOrderedReadableRegions = this.preferredRegions; - this.serviceOrderedWriteableRegions = accountLevelWriteableLocationContext.serviceOrderedWriteableRegions - .stream() - .map(regionName -> regionName.toLowerCase(Locale.ROOT)) - .collect(Collectors.toList()); + this.serviceOrderedWriteableRegions = accountLevelWriteableLocationContext.serviceOrderedWriteableRegions + .stream() + .map(regionName -> regionName.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); - this.clientWithPreferredRegions = getClientBuilder() - .preferredRegions(this.preferredRegions) - .endpointDiscoveryEnabled(true) - .multipleWriteRegionsEnabled(true) - .buildAsyncClient(); + this.clientWithPreferredRegions = getClientBuilder() + .preferredRegions(this.preferredRegions) + .endpointDiscoveryEnabled(true) + .multipleWriteRegionsEnabled(true) + .buildAsyncClient(); - this.clientWithoutPreferredRegions = getClientBuilder() - .endpointDiscoveryEnabled(true) - .multipleWriteRegionsEnabled(true) - .buildAsyncClient(); + this.clientWithoutPreferredRegions = getClientBuilder() + .endpointDiscoveryEnabled(true) + .multipleWriteRegionsEnabled(true) + .buildAsyncClient(); + } - this.cosmosAsyncContainerFromClientWithPreferredRegions = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithPreferredRegions); - this.cosmosAsyncContainerFromClientWithoutPreferredRegions = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithoutPreferredRegions); + this.cosmosAsyncContainerFromClientWithPreferredRegions = + getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithPreferredRegions); + this.cosmosAsyncContainerFromClientWithoutPreferredRegions = + getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithoutPreferredRegions); } @AfterClass(groups = {"multi-master", "fast", "fi-multi-master", "multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) @@ -550,52 +551,49 @@ public void dataPlaneRequestHitsLeaseNotFoundInFirstPreferredRegion( .hitLimit(hitLimit) .build(); - CosmosAsyncClient testClient = getClientBuilder() + try (CosmosAsyncClient testClient = getClientBuilder() .preferredRegions(shouldUsePreferredRegionsOnClient ? this.preferredRegions : Collections.emptyList()) .directMode() // required to force a quorum read irrespective of account consistency level .readConsistencyStrategy(ReadConsistencyStrategy.LATEST_COMMITTED) - .buildAsyncClient(); + .buildAsyncClient()) { - CosmosAsyncContainer testContainer = getSharedSinglePartitionCosmosContainer(testClient); + CosmosAsyncContainer testContainer = getSharedSinglePartitionCosmosContainer(testClient); - try { + try { - testContainer.createItem(createdItem).block(); + testContainer.createItem(createdItem).block(); - CosmosFaultInjectionHelper.configureFaultInjectionRules(testContainer, Arrays.asList(leaseNotFoundFaultRule)).block(); + CosmosFaultInjectionHelper.configureFaultInjectionRules(testContainer, Arrays.asList(leaseNotFoundFaultRule)).block(); - CosmosDiagnostics cosmosDiagnostics - = this.performDocumentOperation(testContainer, operationType, createdItem, testItem -> new PartitionKey(testItem.getMypk()), isReadMany) - .block(); + CosmosDiagnostics cosmosDiagnostics + = this.performDocumentOperation(testContainer, operationType, createdItem, testItem -> new PartitionKey(testItem.getMypk()), isReadMany) + .block(); - if (shouldRetryCrossRegion) { - assertThat(cosmosDiagnostics).isNotNull(); - assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); + if (shouldRetryCrossRegion) { + assertThat(cosmosDiagnostics).isNotNull(); + assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); - CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); + CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); - assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(2); - assertThat(diagnosticsContext.getStatusCode()).isLessThan(HttpConstants.StatusCodes.BADREQUEST); - assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); - } else { - assertThat(cosmosDiagnostics).isNotNull(); - assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); + assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(2); + assertThat(diagnosticsContext.getStatusCode()).isLessThan(HttpConstants.StatusCodes.BADREQUEST); + assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); + } else { + assertThat(cosmosDiagnostics).isNotNull(); + assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); - CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); + CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); - assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(1); - assertThat(diagnosticsContext.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.SERVICE_UNAVAILABLE); - assertThat(diagnosticsContext.getSubStatusCode()).isEqualTo(HttpConstants.SubStatusCodes.LEASE_NOT_FOUND); - assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); - } - - } finally { - leaseNotFoundFaultRule.disable(); + assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(1); + assertThat(diagnosticsContext.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.SERVICE_UNAVAILABLE); + assertThat(diagnosticsContext.getSubStatusCode()).isEqualTo(HttpConstants.SubStatusCodes.LEASE_NOT_FOUND); + assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); + } - if (testClient != null) { + } finally { + leaseNotFoundFaultRule.disable(); cleanUpContainer(testContainer); - testClient.close(); } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java index d0e79d6d3f28..a8743a1f6b9c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java @@ -62,29 +62,40 @@ public void queryOffersWithFilter() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - Flux> queryObservable = client.queryOffers( - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client)); - List allOffers = client - .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client)) - .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); - List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); + QueryFeedOperationState queryDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + QueryFeedOperationState offerDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client); - assertThat(expectedOffers).isNotEmpty(); + try { + Flux> queryObservable = client.queryOffers( + query, + queryDummyState); - Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; + List allOffers = client + .readOffers(offerDummyState) + .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); + List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + assertThat(expectedOffers).isNotEmpty(); + + Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(expectedOffers.size()) .exactlyContainsInAnyOrder(expectedOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(queryObservable, validator, 10000); + validateQuerySuccess(queryObservable, validator, 10000); + } finally { + safeClose(queryDummyState); + safeClose(offerDummyState); + } } @Test(groups = { "query" }, timeOut = TIMEOUT * 10) @@ -97,32 +108,44 @@ public void queryOffersFilterMorePages() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 1); - Flux> queryObservable = client.queryOffers( - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client)); + QueryFeedOperationState queryDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + + QueryFeedOperationState offerDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client); + + try { - List expectedOffers = client - .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client)) + Flux> queryObservable = client.queryOffers( + query, + queryDummyState); + + List expectedOffers = client + .readOffers(offerDummyState) .flatMap(f -> Flux.fromIterable(f.getResults())) .collectList() .single().block() .stream().filter(o -> collectionResourceIds.contains(o.getOfferResourceId())) .collect(Collectors.toList()); - assertThat(expectedOffers).hasSize(createdCollections.size()); + assertThat(expectedOffers).hasSize(createdCollections.size()); - Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (expectedOffers.size() + maxItemCount- 1) / maxItemCount; + Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(expectedOffers.size()) .exactlyContainsInAnyOrder(expectedOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(queryObservable, validator, 10000); + validateQuerySuccess(queryObservable, validator, 10000); + } finally { + safeClose(queryDummyState); + safeClose(offerDummyState); + } } @Test(groups = { "query" }, timeOut = TIMEOUT) @@ -130,30 +153,37 @@ public void queryCollections_NoResults() throws Exception { String query = "SELECT * from root r where r.id = '2'"; CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .containsExactly(new ArrayList<>()) - .numberOfPages(1) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient()) { + + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + try { + Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .containsExactly(new ArrayList<>()) + .numberOfPages(1) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(queryObservable, validator); + .build(); + validateQuerySuccess(queryObservable, validator); + } finally { + safeClose(dummyState); + } + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java index 6d6064d48b0a..ed4f1a523e71 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java @@ -3,6 +3,7 @@ package com.azure.cosmos.rx; import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.TestUtils; import com.azure.cosmos.models.CosmosQueryRequestOptions; @@ -42,48 +43,53 @@ public OfferReadReplaceTest(AsyncDocumentClient.Builder clientBuilder) { @Test(groups = { "emulator" }, timeOut = TIMEOUT) public void readAndReplaceOffer() { - List offers = client - .readOffers( - TestUtils.createDummyQueryFeedOperationState( - ResourceType.Offer, - OperationType.ReadFeed, - new CosmosQueryRequestOptions(), - client)) - .map(FeedResponse::getResults) - .flatMap(list -> Flux.fromIterable(list)).collectList().block(); - - int i; - for (i = 0; i < offers.size(); i++) { - if (offers.get(i).getOfferResourceId().equals(createdCollection.getResourceId())) { - break; + QueryFeedOperationState offerDummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Offer, + OperationType.ReadFeed, + new CosmosQueryRequestOptions(), + client); + + try { + List offers = client + .readOffers(offerDummyState) + .map(FeedResponse::getResults) + .flatMap(list -> Flux.fromIterable(list)).collectList().block(); + + int i; + for (i = 0; i < offers.size(); i++) { + if (offers.get(i).getOfferResourceId().equals(createdCollection.getResourceId())) { + break; + } } - } - Offer rOffer = client.readOffer(offers.get(i).getSelfLink()).single().block().getResource(); - int oldThroughput = rOffer.getThroughput(); + Offer rOffer = client.readOffer(offers.get(i).getSelfLink()).single().block().getResource(); + int oldThroughput = rOffer.getThroughput(); - Mono> readObservable = client.readOffer(offers.get(i).getSelfLink()); + Mono> readObservable = client.readOffer(offers.get(i).getSelfLink()); - // validate offer read - ResourceResponseValidator validatorForRead = new ResourceResponseValidator.Builder() - .withOfferThroughput(oldThroughput) - .notNullEtag() - .build(); + // validate offer read + ResourceResponseValidator validatorForRead = new ResourceResponseValidator.Builder() + .withOfferThroughput(oldThroughput) + .notNullEtag() + .build(); - validateSuccess(readObservable, validatorForRead); + validateSuccess(readObservable, validatorForRead); - // update offer - int newThroughput = oldThroughput + 100; - offers.get(i).setThroughput(newThroughput); - Mono> replaceObservable = client.replaceOffer(offers.get(i)); + // update offer + int newThroughput = oldThroughput + 100; + offers.get(i).setThroughput(newThroughput); + Mono> replaceObservable = client.replaceOffer(offers.get(i)); - // validate offer replace - ResourceResponseValidator validatorForReplace = new ResourceResponseValidator.Builder() - .withOfferThroughput(newThroughput) - .notNullEtag() - .build(); + // validate offer replace + ResourceResponseValidator validatorForReplace = new ResourceResponseValidator.Builder() + .withOfferThroughput(newThroughput) + .notNullEtag() + .build(); - validateSuccess(replaceObservable, validatorForReplace); + validateSuccess(replaceObservable, validatorForReplace); + } finally { + safeClose(offerDummyState); + } } @BeforeClass(groups = { "emulator" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index aff4b8d6d536..69e4bb9f4a1a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -4,6 +4,7 @@ import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.implementation.CosmosPagedFluxOptions; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.QueryFeedOperationState; @@ -53,41 +54,42 @@ public ReadFeedOffersTest(AsyncDocumentClient.Builder clientBuilder) { super(clientBuilder); } - @Test(groups = { "query" }, timeOut = FEED_TIMEOUT) + @Test(groups = { "query" }, timeOut = FEED_TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void readOffers() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - Flux> feedObservable = client.readOffers(dummyState); - - int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .buildAsyncClient()) { + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + Flux> feedObservable = client.readOffers(dummyState); + + int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(allOffers.size()) .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) @@ -99,19 +101,23 @@ public void before_ReadFeedOffersTest() { createCollections(client); } - allOffers = client.readOffers( - TestUtils.createDummyQueryFeedOperationState( - ResourceType.Offer, - OperationType.ReadFeed, - new CosmosQueryRequestOptions(), - client - ) - ) - .map(FeedResponse::getResults) - .collectList() - .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) - .single() - .block(); + QueryFeedOperationState offerDummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Offer, + OperationType.ReadFeed, + new CosmosQueryRequestOptions(), + client + ); + + try { + allOffers = client.readOffers(offerDummyState) + .map(FeedResponse::getResults) + .collectList() + .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) + .single() + .block(); + } finally { + safeClose(offerDummyState); + } } @AfterClass(groups = { "query" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java index 16764539b983..680b55050123 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos.rx; +import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosAsyncContainer; import com.azure.cosmos.CosmosAsyncDatabase; import com.azure.cosmos.CosmosBridgeInternal; @@ -49,8 +50,9 @@ public void readPartitionKeyRanges() throws Exception { @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) public void before_ReadFeedPkrTests() { - client = CosmosBridgeInternal.getAsyncDocumentClient(getClientBuilder().buildAsyncClient()); - createdDatabase = getSharedCosmosDatabase(getClientBuilder().buildAsyncClient()); + CosmosAsyncClient cosmosAsyncClient = getClientBuilder().buildAsyncClient(); + client = CosmosBridgeInternal.getAsyncDocumentClient(cosmosAsyncClient); + createdDatabase = getSharedCosmosDatabase(cosmosAsyncClient); createdCollection = createCollection(createdDatabase, getCollectionDefinition(), new CosmosContainerRequestOptions()); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java index a94dfea5b85c..ead3cdc9f785 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java @@ -6,6 +6,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; @@ -463,6 +464,7 @@ public void readDocumentFromCollPermissionWithDiffPartitionKey_WithException() t public void queryItemFromResourceToken(DocumentCollection documentCollection, Permission permission, PartitionKey partitionKey) throws Exception { AsyncDocumentClient asyncClientResourceToken = null; + QueryFeedOperationState dummyState = null; try { List permissionFeed = new ArrayList<>(); permissionFeed.add(permission); @@ -479,11 +481,15 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe CosmosQueryRequestOptions queryRequestOptions = new CosmosQueryRequestOptions(); queryRequestOptions.setPartitionKey(partitionKey); + + dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, asyncClientResourceToken); + Flux> queryObservable = asyncClientResourceToken.queryDocuments( documentCollection.getAltLink(), "select * from c", - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, asyncClientResourceToken), + dummyState, Document.class); FeedResponseListValidator validator = new FeedResponseListValidator.Builder() @@ -494,6 +500,7 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe validateQuerySuccess(queryObservable, validator); } finally { safeClose(asyncClientResourceToken); + safeClose(dummyState); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index d17e872db25e..e17c95012704 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -32,6 +32,7 @@ import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.Resource; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; @@ -95,8 +96,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends CosmosAsyncClientTest { +public abstract class TestSuiteBase extends CosmosAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -221,13 +221,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) @@ -924,15 +917,9 @@ static protected void safeDeleteCollection(CosmosAsyncDatabase database, String } } - static protected void safeCloseAsync(CosmosAsyncClient client) { - if (client != null) { - new Thread(() -> { - try { - client.close(); - } catch (Exception e) { - logger.error("failed to close client", e); - } - }).start(); + static protected void safeClose(QueryFeedOperationState state) { + if (state != null) { + safeClose(state.getClient()); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java index 79bfbc76336e..6114d930c1ec 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java @@ -10,14 +10,22 @@ import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.util.ReferenceCountUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; + /** * Handle data from client. * */ public class HttpProxyClientHandler extends ChannelInboundHandlerAdapter { + + private static final ByteBuf TUNNEL_OK = + Unpooled.unreleasableBuffer( + Unpooled.copiedBuffer("HTTP/1.1 200 Connection Established\r\n\r\n", StandardCharsets.US_ASCII)); + private final Logger logger = LoggerFactory.getLogger(HttpProxyClientHandler.class); private final String id; private Channel clientChannel; @@ -36,15 +44,27 @@ public void channelActive(ChannelHandlerContext ctx) { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { if (header.isComplete()) { - remoteChannel.writeAndFlush(msg); // just forward + if (remoteChannel != null && remoteChannel.isActive()) { + remoteChannel.writeAndFlush(msg); // just forward + } else { + ReferenceCountUtil.safeRelease(msg); + flushAndClose(clientChannel); + } return; } ByteBuf in = (ByteBuf) msg; - header.digest(in); + + try { + header.digest(in); + } catch (Throwable t) { + ReferenceCountUtil.safeRelease(in); + flushAndClose(clientChannel); + throw t; + } if (!header.isComplete()) { - in.release(); + ReferenceCountUtil.safeRelease(in); return; } @@ -52,7 +72,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { clientChannel.config().setAutoRead(false); // disable AutoRead until remote connection is ready if (header.isHttps()) { // if https, respond 200 to create tunnel - clientChannel.writeAndFlush(Unpooled.wrappedBuffer("HTTP/1.1 200 Connection Established\r\n\r\n".getBytes())); + clientChannel.writeAndFlush(TUNNEL_OK.duplicate()); } Bootstrap b = new Bootstrap(); @@ -71,20 +91,27 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { remoteChannel.writeAndFlush(in); } else { - in.release(); - clientChannel.close(); + ReferenceCountUtil.safeRelease(in); + // release header buffer if retained/allocated + ReferenceCountUtil.safeRelease(header.getByteBuf()); + flushAndClose(remoteChannel); + flushAndClose(clientChannel); } }); } @Override public void channelInactive(ChannelHandlerContext ctx) { + flushAndClose(remoteChannel); + remoteChannel = null; + clientChannel = null; } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) { - logger.error(id + " error occured", e); + logger.error(id + " error occurred", e); + flushAndClose(remoteChannel); flushAndClose(clientChannel); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java index d300cbaa5337..4a64410fb187 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java @@ -4,12 +4,20 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.util.ResourceLeakDetector; /** * The http header of client. * */ public class HttpProxyClientHeader { + private static final int MAX_HEADER_BYTES = 64 * 1024; + + private static final boolean LEAK_DEBUG = + ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal(); + + private volatile boolean touchedAlloc; + private String method; private String host; private int port; @@ -43,10 +51,6 @@ public int getPort() { return port; } - public void setPort(int port) { - this.port = port; - } - public boolean isHttps() { return https; } @@ -56,58 +60,87 @@ public void setHttps(boolean https) { } public ByteBuf getByteBuf() { - return byteBuf; - } - - public void setByteBuf(ByteBuf byteBuf) { - this.byteBuf = byteBuf; - } - - public StringBuilder getLineBuf() { - return lineBuf; + if (!complete) throw new IllegalStateException("header not complete"); + if (LEAK_DEBUG) { + this.byteBuf.touch("getByteBuf handoff"); + } + ByteBuf out = this.byteBuf; + this.byteBuf = Unpooled.EMPTY_BUFFER; // we no longer own anything + return out; // caller must write/release this } public void setComplete(boolean complete) { + if (complete) { + if (LEAK_DEBUG) { + this.byteBuf.touch("end-of-headers; https=" + https + " host=" + host + " port=" + port); + } + } this.complete = complete; } + /** Release any internal buffer we still own (idempotent). */ + public void releaseQuietly() { + if (LEAK_DEBUG) { + this.byteBuf.touch("releaseQuietly"); + } + io.netty.util.ReferenceCountUtil.safeRelease(this.byteBuf); + this.byteBuf = Unpooled.EMPTY_BUFFER; + } + public void digest(ByteBuf in) { + if (LEAK_DEBUG && !touchedAlloc) { + touchedAlloc = true; + this.byteBuf.touch("allocated header buffer"); + } + while (in.isReadable()) { - if (complete) { - throw new IllegalStateException("already complete"); - } + if (complete) throw new IllegalStateException("already complete"); String line = readLine(in); - if (line == null) { - return; - } + if (line == null) return; if (method == null) { - method = line.split(" ")[0]; // the first word is http method name - https = method.equalsIgnoreCase("CONNECT"); // method CONNECT means https + method = line.split(" ", 2)[0]; + https = "CONNECT".equalsIgnoreCase(method); } - if (line.startsWith("Host: ") || line.startsWith("host: ")) { - String[] arr = line.split(":"); - host = arr[1].trim(); - if (arr.length == 3) { - port = Integer.parseInt(arr[2]); - } else if (https) { - port = 443; // https + if (line.regionMatches(true, 0, "Host:", 0, 5)) { + // be tolerant to extra spaces and IPv6 + String value = line.substring(5).trim(); + int idx = value.lastIndexOf(':'); // last colon to allow IPv6 literals + if (idx > 0 && value.indexOf(']') < idx) { + host = value.substring(0, idx).trim(); + port = Integer.parseInt(value.substring(idx + 1).trim()); } else { - port = 80; // http + host = value; + port = https ? 443 : 80; } } if (line.isEmpty()) { if (host == null || port == 0) { - throw new IllegalStateException("cannot find header \'Host\'"); + releaseQuietly(); + throw new IllegalStateException("cannot find header 'Host'"); } - - byteBuf = byteBuf.asReadOnly(); - complete = true; + // If HTTPS, we don’t forward the CONNECT request → release now. + if (https) { + releaseQuietly(); + } else { + // non-HTTPS: make read-only for safety; caller will drain & own it + byteBuf = byteBuf.asReadOnly(); + } + setComplete(true); break; } + + // size guard to avoid OOM from giant headers + if (byteBuf.writerIndex() >= MAX_HEADER_BYTES) { + releaseQuietly(); + if (LEAK_DEBUG) { + this.byteBuf.touch("header too large at writerIndex=" + byteBuf.writerIndex()); + } + throw new IllegalStateException("header too large"); + } } } @@ -117,12 +150,14 @@ private String readLine(ByteBuf in) { byteBuf.writeByte(b); lineBuf.append((char) b); int len = lineBuf.length(); - if (len >= 2 && lineBuf.substring(len - 2).equals("\r\n")) { + if (len >= 2 && lineBuf.charAt(len - 2) == '\r' && lineBuf.charAt(len - 1) == '\n') { String line = lineBuf.substring(0, len - 2); - lineBuf.delete(0, len); + lineBuf.setLength(0); + if (LEAK_DEBUG) { + this.byteBuf.touch("CRLF reached, writerIndex=" + byteBuf.writerIndex()); + } return line; } - } return null; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java index 487aa265d104..ee9228aeeeb9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java @@ -32,18 +32,41 @@ public void channelActive(ChannelHandlerContext ctx) { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - clientChannel.writeAndFlush(msg); // just forward + if (clientChannel != null && clientChannel.isActive()) { + clientChannel.writeAndFlush(msg).addListener(f -> { + if (!f.isSuccess()) { + // transport is broken; close both sides + flushAndClose(clientChannel); + flushAndClose(remoteChannel); + } + }); + } else { + io.netty.util.ReferenceCountUtil.safeRelease(msg); // prevent leak + flushAndClose(remoteChannel); + } } @Override public void channelInactive(ChannelHandlerContext ctx) { flushAndClose(clientChannel); + remoteChannel = null; + clientChannel = null; } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) { - logger.error(id + " error occured", e); + logger.error(id + " error occurred", e); flushAndClose(remoteChannel); + flushAndClose(clientChannel); + } + + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + if (remoteChannel != null) { + boolean writable = clientChannel != null && clientChannel.isWritable(); + remoteChannel.config().setAutoRead(writable); + } + ctx.fireChannelWritabilityChanged(); } private void flushAndClose(Channel ch) { diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index b474bbe5f73b..c11f2a13b1e0 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -146,6 +146,12 @@ Licensed under the MIT License. 3.5.3 unit + + true + 1 + 256 + paranoid + %regex[.*] @@ -154,6 +160,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java index f19ccb503027..9baeb17dbde8 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java @@ -329,7 +329,14 @@ public AsyncDocumentClient build() { operationPolicies, isPerPartitionAutomaticFailoverEnabled); - client.init(state, null); + try { + client.init(state, null); + } catch (Throwable t) { + client.close(); + // TODO - should we map this to a CosmosException? + throw t; + } + return client; } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java index 03922e5b1cec..900d6b2d6768 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java @@ -155,6 +155,10 @@ public String getSpanName() { return ctxAccessor.getSpanName(this.ctxHolder.get()); } + public CosmosAsyncClient getClient() { + return this.cosmosAsyncClient; + } + public CosmosDiagnosticsContext getDiagnosticsContextSnapshot() { return this.ctxHolder.get(); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 25f02086acc3..37b4f56dbd3c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1380,6 +1380,9 @@ private Flux> createQueryInternal( private void addToActiveClients() { if (Configs.isClientLeakDetectionEnabled()) { + logger.warn( + "Cosmos Client leak detection is enabled - " + + "this will impact performance negatively - disable it in production scenarios."); synchronized (staticLock) { activeClients.put(this.clientId, StackTraceUtil.currentCallStack()); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index 680b591d38c9..f141de34aa9a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -211,13 +211,20 @@ public StoreResponse unwrapToStoreResponse( retainedContent, "Argument 'retainedContent' must not be null - use empty ByteBuf when theres is no payload."); + if (leakDetectionDebuggingEnabled) { + retainedContent.touch( + "RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: " + retainedContent.refCnt()); + logger.info("RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: {}", retainedContent.refCnt()); + } + // If there is any error in the header response this throws exception validateOrThrow(request, HttpResponseStatus.valueOf(statusCode), headers, retainedContent); int size; if ((size = retainedContent.readableBytes()) > 0) { if (leakDetectionDebuggingEnabled) { - retainedContent.touch(this); + retainedContent.touch("RxGatewayStoreModel before creating StoreResponse - refCnt: " + retainedContent.refCnt()); + logger.info("RxGatewayStoreModel before creating StoreResponse - refCnt: {}", retainedContent.refCnt()); } return new StoreResponse( @@ -406,210 +413,251 @@ private Mono toDocumentServiceResponse(Mono { - - // header key/value pairs - HttpHeaders httpResponseHeaders = httpResponse.headers(); - int httpResponseStatus = httpResponse.statusCode(); - - Mono contentObservable = httpResponse - .body() - .switchIfEmpty(Mono.just(Unpooled.EMPTY_BUFFER)) - .map(bodyByteBuf -> leakDetectionDebuggingEnabled - ? bodyByteBuf.retain().touch(this) - : bodyByteBuf.retain()) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) - .doOnDiscard(ByteBuf.class, buf -> { - if (buf.refCnt() > 0) { - // there could be a race with the catch in the .map operator below - // so, use safeRelease - ReferenceCountUtil.safeRelease(buf); - } - }); + return httpResponseMono + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) + .flatMap(httpResponse -> { + + // header key/value pairs + HttpHeaders httpResponseHeaders = httpResponse.headers(); + int httpResponseStatus = httpResponse.statusCode(); + + Mono contentObservable = httpResponse + .body() + .switchIfEmpty(Mono.just(Unpooled.EMPTY_BUFFER)) + .map(bodyByteBuf -> { + if (leakDetectionDebuggingEnabled) { + bodyByteBuf.touch("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: " + bodyByteBuf.refCnt()); + logger.info("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: {}", bodyByteBuf.refCnt()); + } - return contentObservable - .map(content -> { - if (leakDetectionDebuggingEnabled) { - content.touch(this); - } + if (bodyByteBuf != Unpooled.EMPTY_BUFFER) { + bodyByteBuf.retain(); // +1 for our downstream work + } + + if (leakDetectionDebuggingEnabled) { + bodyByteBuf.touch("RxGatewayStoreModel - touch retained buffer - refCnt: " + bodyByteBuf.refCnt()); + logger.info("RxGatewayStoreModel - touch retained buffer - refCnt: {]", bodyByteBuf.refCnt()); + } - try { - // Capture transport client request timeline - ReactorNettyRequestRecord reactorNettyRequestRecord = httpResponse.request().reactorNettyRequestRecord(); - if (reactorNettyRequestRecord != null) { - reactorNettyRequestRecord.setTimeCompleted(Instant.now()); + return bodyByteBuf; + }) + .doOnDiscard(ByteBuf.class, buf -> { + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("RxGatewayStoreModel - doOnDiscard - begin - refCnt: " + buf.refCnt()); + logger.info("RxGatewayStoreModel - doOnDiscard - begin - refCnt: {}", buf.refCnt()); + } + + // there could be a race with the catch in the .map operator below + // so, use safeRelease + ReferenceCountUtil.safeRelease(buf); + } + }) + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC); + + return contentObservable + .map(content -> { + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel - before capturing transport timeline - refCnt: " + content.refCnt()); + logger.info("RxGatewayStoreModel - before capturing transport timeline - refCnt: {}", content.refCnt()); } - StoreResponse rsp = request - .getEffectiveHttpTransportSerializer(this) - .unwrapToStoreResponse(httpRequest.uri().toString(), request, httpResponseStatus, httpResponseHeaders, content); + try { + // Capture transport client request timeline + ReactorNettyRequestRecord reactorNettyRequestRecord = httpResponse.request().reactorNettyRequestRecord(); + if (reactorNettyRequestRecord != null) { + reactorNettyRequestRecord.setTimeCompleted(Instant.now()); + } - if (reactorNettyRequestRecord != null) { - rsp.setRequestTimeline(reactorNettyRequestRecord.takeTimelineSnapshot()); + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel - before creating StoreResponse - refCnt: " + content.refCnt()); + logger.info("RxGatewayStoreModel - before creating StoreResponse - refCnt: {}", content.refCnt()); + } + StoreResponse rsp = request + .getEffectiveHttpTransportSerializer(this) + .unwrapToStoreResponse(httpRequest.uri().toString(), request, httpResponseStatus, httpResponseHeaders, content); + + if (reactorNettyRequestRecord != null) { + rsp.setRequestTimeline(reactorNettyRequestRecord.takeTimelineSnapshot()); + + if (this.gatewayServerErrorInjector != null) { + // only configure when fault injection is used + rsp.setFaultInjectionRuleId( + request + .faultInjectionRequestContext + .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); + + rsp.setFaultInjectionRuleEvaluationResults( + request + .faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + } + } - if (this.gatewayServerErrorInjector != null) { - // only configure when fault injection is used - rsp.setFaultInjectionRuleId( - request - .faultInjectionRequestContext - .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); + if (request.requestContext.cosmosDiagnostics != null) { + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, rsp, globalEndpointManager); + } - rsp.setFaultInjectionRuleEvaluationResults( - request - .faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + return rsp; + } catch (Throwable t) { + if (content.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel -exception creating StoreResponse - refCnt: " + content.refCnt()); + logger.info("RxGatewayStoreModel -exception creating StoreResponse - refCnt: {}", content.refCnt()); + } + // Unwrap failed before StoreResponse took ownership -> release our retain + // there could be a race with the doOnDiscard above - so, use safeRelease + ReferenceCountUtil.safeRelease(content); } - } - if (request.requestContext.cosmosDiagnostics != null) { - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, rsp, globalEndpointManager); + throw t; } - - return rsp; - } catch (Throwable t) { - if (content.refCnt() > 0) { - // Unwrap failed before StoreResponse took ownership -> release our retain - // there could be a race with the doOnDiscard above - so, use safeRelease - ReferenceCountUtil.safeRelease(content); + }) + .doOnDiscard(ByteBuf.class, buf -> { + // This handles the case where the retained buffer is discarded after the map operation + // but before unwrapToStoreResponse takes ownership (e.g., during cancellation) + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("RxGatewayStoreModel - doOnDiscard after map - refCnt: " + buf.refCnt()); + logger.info("RxGatewayStoreModel - doOnDiscard after map - refCnt: {}", buf.refCnt()); + } + ReferenceCountUtil.safeRelease(buf); } + }) + .single(); - throw t; - } - }) - .single(); + }).map(rsp -> { + RxDocumentServiceResponse rxDocumentServiceResponse; + if (httpRequest.reactorNettyRequestRecord() != null) { + rxDocumentServiceResponse = + new RxDocumentServiceResponse(this.clientContext, rsp, + httpRequest.reactorNettyRequestRecord().takeTimelineSnapshot()); - }).map(rsp -> { - RxDocumentServiceResponse rxDocumentServiceResponse; - if (httpRequest.reactorNettyRequestRecord() != null) { - rxDocumentServiceResponse = - new RxDocumentServiceResponse(this.clientContext, rsp, - httpRequest.reactorNettyRequestRecord().takeTimelineSnapshot()); + } else { + rxDocumentServiceResponse = + new RxDocumentServiceResponse(this.clientContext, rsp); + } + rxDocumentServiceResponse.setCosmosDiagnostics(request.requestContext.cosmosDiagnostics); + return rxDocumentServiceResponse; + }).onErrorResume(throwable -> { + Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); + if (!(unwrappedException instanceof Exception)) { + // fatal error + logger.error("Unexpected failure " + unwrappedException.getMessage(), unwrappedException); + return Mono.error(unwrappedException); + } - } else { - rxDocumentServiceResponse = - new RxDocumentServiceResponse(this.clientContext, rsp); - } - rxDocumentServiceResponse.setCosmosDiagnostics(request.requestContext.cosmosDiagnostics); - return rxDocumentServiceResponse; - }).onErrorResume(throwable -> { - Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); - if (!(unwrappedException instanceof Exception)) { - // fatal error - logger.error("Unexpected failure " + unwrappedException.getMessage(), unwrappedException); - return Mono.error(unwrappedException); - } + Exception exception = (Exception) unwrappedException; + CosmosException dce; + if (!(exception instanceof CosmosException)) { + int statusCode = 0; + if (WebExceptionUtility.isNetworkFailure(exception)) { - Exception exception = (Exception) unwrappedException; - CosmosException dce; - if (!(exception instanceof CosmosException)) { - int statusCode = 0; - if (WebExceptionUtility.isNetworkFailure(exception)) { + // wrap in CosmosException + logger.error("Network failure", exception); - // wrap in CosmosException - logger.error("Network failure", exception); + if (WebExceptionUtility.isReadTimeoutException(exception)) { + statusCode = HttpConstants.StatusCodes.REQUEST_TIMEOUT; + } else { + statusCode = HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; + } + } - if (WebExceptionUtility.isReadTimeoutException(exception)) { - statusCode = HttpConstants.StatusCodes.REQUEST_TIMEOUT; + dce = BridgeInternal.createCosmosException(request.requestContext.resourcePhysicalAddress, statusCode, exception); + BridgeInternal.setRequestHeaders(dce, request.getHeaders()); + } else { + logger.error("Non-network failure", exception); + dce = (CosmosException) exception; + } + + if (WebExceptionUtility.isNetworkFailure(dce)) { + if (WebExceptionUtility.isReadTimeoutException(dce)) { + BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT); } else { - statusCode = HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; + BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_UNAVAILABLE); } } - dce = BridgeInternal.createCosmosException(request.requestContext.resourcePhysicalAddress, statusCode, exception); - BridgeInternal.setRequestHeaders(dce, request.getHeaders()); - } else { - logger.error("Non-network failure", exception); - dce = (CosmosException) exception; - } + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setRequestUri(dce, Uri.create(httpRequest.uri().toString())); - if (WebExceptionUtility.isNetworkFailure(dce)) { - if (WebExceptionUtility.isReadTimeoutException(dce)) { - BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT); - } else { - BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_UNAVAILABLE); - } - } + if (request.requestContext.cosmosDiagnostics != null) { + if (httpRequest.reactorNettyRequestRecord() != null) { + ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); + BridgeInternal.setRequestTimeline(dce, reactorNettyRequestRecord.takeTimelineSnapshot()); - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setRequestUri(dce, Uri.create(httpRequest.uri().toString())); + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionRuleId( + dce, + request.faultInjectionRequestContext + .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); - if (request.requestContext.cosmosDiagnostics != null) { - if (httpRequest.reactorNettyRequestRecord() != null) { - ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); - BridgeInternal.setRequestTimeline(dce, reactorNettyRequestRecord.takeTimelineSnapshot()); - - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionRuleId( - dce, - request.faultInjectionRequestContext - .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); - - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionEvaluationResults( - dce, - request.faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); - } + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionEvaluationResults( + dce, + request.faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + } - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, dce, globalEndpointManager); - } + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, dce, globalEndpointManager); + } - return Mono.error(dce); - }).doFinally(signalType -> { + return Mono.error(dce); + }).doFinally(signalType -> { - if (signalType != SignalType.CANCEL) { - return; - } + if (signalType != SignalType.CANCEL) { + return; + } - if (httpRequest.reactorNettyRequestRecord() != null) { + if (httpRequest.reactorNettyRequestRecord() != null) { - OperationCancelledException oce = new OperationCancelledException("", httpRequest.uri()); + OperationCancelledException oce = new OperationCancelledException("", httpRequest.uri()); - ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); + ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); - RequestTimeline requestTimeline = reactorNettyRequestRecord.takeTimelineSnapshot(); - long transportRequestId = reactorNettyRequestRecord.getTransportRequestId(); + RequestTimeline requestTimeline = reactorNettyRequestRecord.takeTimelineSnapshot(); + long transportRequestId = reactorNettyRequestRecord.getTransportRequestId(); - GatewayRequestTimelineContext gatewayRequestTimelineContext = new GatewayRequestTimelineContext(requestTimeline, transportRequestId); + GatewayRequestTimelineContext gatewayRequestTimelineContext = new GatewayRequestTimelineContext(requestTimeline, transportRequestId); - request.requestContext.cancelledGatewayRequestTimelineContexts.add(gatewayRequestTimelineContext); + request.requestContext.cancelledGatewayRequestTimelineContexts.add(gatewayRequestTimelineContext); - if (request.requestContext.getCrossRegionAvailabilityContext() != null) { + if (request.requestContext.getCrossRegionAvailabilityContext() != null) { - CrossRegionAvailabilityContextForRxDocumentServiceRequest availabilityStrategyContextForReq = - request.requestContext.getCrossRegionAvailabilityContext(); + CrossRegionAvailabilityContextForRxDocumentServiceRequest availabilityStrategyContextForReq = + request.requestContext.getCrossRegionAvailabilityContext(); - if (availabilityStrategyContextForReq.getAvailabilityStrategyContext().isAvailabilityStrategyEnabled() && !availabilityStrategyContextForReq.getAvailabilityStrategyContext().isHedgedRequest()) { + if (availabilityStrategyContextForReq.getAvailabilityStrategyContext().isAvailabilityStrategyEnabled() && !availabilityStrategyContextForReq.getAvailabilityStrategyContext().isHedgedRequest()) { - BridgeInternal.setRequestTimeline(oce, reactorNettyRequestRecord.takeTimelineSnapshot()); + BridgeInternal.setRequestTimeline(oce, reactorNettyRequestRecord.takeTimelineSnapshot()); - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionRuleId( - oce, - request.faultInjectionRequestContext - .getFaultInjectionRuleId(transportRequestId)); + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionRuleId( + oce, + request.faultInjectionRequestContext + .getFaultInjectionRuleId(transportRequestId)); - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionEvaluationResults( - oce, - request.faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(transportRequestId)); + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionEvaluationResults( + oce, + request.faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(transportRequestId)); - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, oce, globalEndpointManager); + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, oce, globalEndpointManager); + } } } - } - }); + }); } private void validateOrThrow(RxDocumentServiceRequest request, @@ -624,11 +672,11 @@ private void validateOrThrow(RxDocumentServiceRequest request, ? status.reasonPhrase().replace(" ", "") : ""; - String body = retainedBodyAsByteBuf != null + String body = retainedBodyAsByteBuf.readableBytes() > 0 ? retainedBodyAsByteBuf.toString(StandardCharsets.UTF_8) : null; - retainedBodyAsByteBuf.release(); + ReferenceCountUtil.safeRelease(retainedBodyAsByteBuf); CosmosError cosmosError; cosmosError = (StringUtils.isNotEmpty(body)) ? new CosmosError(body) : new CosmosError(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java index 74b314881bba..d32e5d901f18 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java @@ -111,7 +111,7 @@ public StoreResponse unwrapToStoreResponse( } if (leakDetectionDebuggingEnabled) { - content.touch(this); + content.touch("ThinClientStoreModel.unwrapToStoreResponse - refCnt: " + content.refCnt()); } try { @@ -123,7 +123,7 @@ public StoreResponse unwrapToStoreResponse( ByteBuf payloadBuf = response.getContent(); if (payloadBuf != Unpooled.EMPTY_BUFFER && leakDetectionDebuggingEnabled) { - payloadBuf.touch(this); + payloadBuf.touch("ThinClientStoreModel.after RNTBD decoding - refCnt: " + payloadBuf.refCnt()); } try { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index c8539f09cba5..e092886a8d08 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -11,6 +11,7 @@ import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdEndpointStatistics; import com.fasterxml.jackson.databind.JsonNode; import io.netty.buffer.ByteBufInputStream; +import io.netty.util.IllegalReferenceCountException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,12 +72,14 @@ public StoreResponse( if (contentStream != null) { try { this.responsePayload = new JsonNodeStorePayload(contentStream, responsePayloadLength, headerMap); - } - finally { + } finally { try { contentStream.close(); - } catch (IOException e) { - logger.debug("Could not successfully close content stream.", e); + } catch (Throwable e) { + if (!(e instanceof IllegalReferenceCountException)) { + // Log as warning instead of debug to make ByteBuf leak issues more visible + logger.warn("Failed to close content stream. This may cause a Netty ByteBuf leak.", e); + } } } } else { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java index cf8724244535..13e953e47734 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +15,8 @@ class RntbdContextDecoder extends ByteToMessageDecoder { private static final Logger logger = LoggerFactory.getLogger(RntbdContextDecoder.class); + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); /** * Deserialize from an input {@link ByteBuf} to an {@link RntbdContext} instance @@ -27,6 +30,10 @@ class RntbdContextDecoder extends ByteToMessageDecoder { @Override protected void decode(final ChannelHandlerContext context, final ByteBuf in, final List out) { + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextDecoder.decode: entry"); + } + if (RntbdFramer.canDecodeHead(in)) { Object result; @@ -35,14 +42,35 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin final RntbdContext rntbdContext = RntbdContext.decode(in); context.fireUserEventTriggered(rntbdContext); result = rntbdContext; + + if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: decoded RntbdContext successfully", context.channel()); + } } catch (RntbdContextException error) { context.fireUserEventTriggered(error); result = error; + + if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: caught RntbdContextException", context.channel(), error); + } } finally { + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextDecoder.decode: before discardReadBytes in finally block"); + } in.discardReadBytes(); } logger.debug("{} DECODE COMPLETE: {}", context.channel(), result); + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: cannot decode head yet, readableBytes={}", + context.channel(), in.readableBytes()); } } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdContextDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java index c256fce6de65..ceec8d90ad46 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java @@ -6,11 +6,19 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public class RntbdContextRequestDecoder extends ByteToMessageDecoder { + private static final Logger logger = LoggerFactory.getLogger(RntbdContextRequestDecoder.class); + + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + public RntbdContextRequestDecoder() { this.setSingleDecode(true); } @@ -28,14 +36,33 @@ public void channelRead(final ChannelHandlerContext context, final Object messag if (message instanceof ByteBuf) { final ByteBuf in = (ByteBuf)message; + // BREADCRUMB: Track buffer before reading operation type + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: before reading resourceOperationType"); + } + final int resourceOperationType = in.getInt(in.readerIndex() + Integer.BYTES); if (resourceOperationType == 0) { assert this.isSingleDecode(); + // BREADCRUMB: Going through normal decode path + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: passing to super.channelRead (resourceOperationType == 0)"); + } super.channelRead(context, message); return; } + + // BREADCRUMB: Bypassing decoder - this is a potential leak point if downstream doesn't release + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: bypassing decoder (resourceOperationType != 0)"); + } } + // BREADCRUMB: Message forwarded downstream - downstream MUST release it + if (leakDetectionDebuggingEnabled && message instanceof ByteBuf) { + ((ByteBuf) message).touch("RntbdContextRequestDecoder.channelRead: forwarding to next handler"); + } + context.fireChannelRead(message); } @@ -66,4 +93,11 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin in.discardReadBytes(); out.add(request); } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdContextRequestDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java index b924f2e97169..bab67b3dd231 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java @@ -6,10 +6,19 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public final class RntbdRequestDecoder extends ByteToMessageDecoder { + + private static final Logger logger = LoggerFactory.getLogger(RntbdContextRequestDecoder.class); + + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + /** * Prepare for decoding an @{link RntbdRequest} or fire a channel readTree event to pass the input message along. * @@ -23,14 +32,32 @@ public void channelRead(final ChannelHandlerContext context, final Object messag if (message instanceof ByteBuf) { final ByteBuf in = (ByteBuf) message; + // BREADCRUMB: Track buffer before reading operation type + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: before reading resourceOperationType"); + } + final int resourceOperationType = in.getInt(in.readerIndex() + Integer.BYTES); if (resourceOperationType != 0) { + // BREADCRUMB: Going through normal decode path + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: passing to super.channelRead (resourceOperationType != 0)"); + } super.channelRead(context, message); return; } + + // BREADCRUMB: Bypassing decoder - this is a potential leak point if downstream doesn't release + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: bypassing decoder (resourceOperationType == 0)"); + } } + // BREADCRUMB: Message forwarded downstream - downstream MUST release it + if (leakDetectionDebuggingEnabled && message instanceof ByteBuf) { + ((ByteBuf) message).touch("RntbdRequestDecoder.channelRead: forwarding to next handler"); + } context.fireChannelRead(message); } @@ -65,4 +92,11 @@ protected void decode( in.discardReadBytes(); out.add(request); } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdRequestDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java index 57c1c9a1a13a..8ae5da0aa315 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,6 +19,9 @@ public final class RntbdResponseDecoder extends ByteToMessageDecoder { private static final Logger logger = LoggerFactory.getLogger(RntbdResponseDecoder.class); private static final AtomicReference decodeStartTime = new AtomicReference<>(); + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + /** * Deserialize from an input {@link ByteBuf} to an {@link RntbdResponse} instance. *

@@ -32,6 +36,11 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin decodeStartTime.compareAndSet(null, Instant.now()); + // BREADCRUMB: Track buffer at decode entry + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdResponseDecoder.decode: entry"); + } + if (RntbdFramer.canDecodeHead(in)) { final RntbdResponse response = RntbdResponse.decode(in); @@ -41,9 +50,34 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin response.setDecodeStartTime(decodeStartTime.getAndSet(null)); logger.debug("{} DECODE COMPLETE: {}", context.channel(), response); + + // BREADCRUMB: Track buffer before discard + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdResponseDecoder.decode: before discardReadBytes"); + } + in.discardReadBytes(); + + // BREADCRUMB: Track response before adding to output + if (leakDetectionDebuggingEnabled) { + response.touch("RntbdResponseDecoder.decode: before retain and adding to output"); + } + out.add(response.retain()); + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdResponseDecoder: response is null, not enough data to decode yet", + context.channel()); } + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdResponseDecoder: cannot decode head yet, readableBytes={}", + context.channel(), in.readableBytes()); } } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdResponseDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 488596e825d5..5c5bc5355b32 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -11,6 +11,8 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.logging.LogLevel; import io.netty.resolver.DefaultAddressResolverGroup; +import io.netty.util.ReferenceCountUtil; +import io.netty.util.ResourceLeakDetector; import org.reactivestreams.Publisher; import org.reactivestreams.Subscription; import org.slf4j.Logger; @@ -39,7 +41,8 @@ * HttpClient that is implemented using reactor-netty. */ public class ReactorNettyClient implements HttpClient { - + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); private static final String REACTOR_NETTY_REQUEST_RECORD_KEY = "reactorNettyRequestRecordKey"; private static final Logger logger = LoggerFactory.getLogger(ReactorNettyClient.class.getSimpleName()); @@ -348,7 +351,17 @@ public HttpHeaders headers() { public Mono body() { return ByteBufFlux .fromInbound( - bodyIntern().doOnDiscard(ByteBuf.class, io.netty.util.ReferenceCountUtil::safeRelease) + bodyIntern().doOnDiscard( + ByteBuf.class, + buf -> { + if (leakDetectionDebuggingEnabled) { + buf.touch("ReactorNettyHttpResponse.body - onDiscard - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); + } + if (buf.refCnt() > 0) { + ReferenceCountUtil.safeRelease(buf); + } + }) ) .aggregate() .doOnSubscribe(this::updateSubscriptionState); @@ -400,8 +413,21 @@ private void releaseOnNotSubscribedResponse(ReactorNettyResponseState reactorNet if (logger.isDebugEnabled()) { logger.debug("Releasing body, not yet subscribed"); } - this.bodyIntern() - .doOnNext(io.netty.util.ReferenceCountUtil::safeRelease) + + if (leakDetectionDebuggingEnabled) { + logger.info("Releasing body, not yet subscribed"); + } + + body() + .doOnNext(buf -> { + if (leakDetectionDebuggingEnabled) { + buf.touch("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); + } + if (buf.refCnt() > 0) { + ReferenceCountUtil.safeRelease(buf); + } + }) .subscribe(v -> {}, ex -> {}, () -> {}); } } diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index 2941fa16041d..75035c197bc1 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -506,6 +506,12 @@ maven-surefire-plugin 3.5.3 + + true + 1 + 256 + paranoid + **/*.* **/*Test.* @@ -513,6 +519,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 7f3e3af80997..613390ddd671 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -14,7 +14,7 @@ "-Pcircuit-breaker-read-all-read-many": "CircuitBreakerReadAllAndReadMany", "-Pmulti-region": "MultiRegion", "-Plong": "Long", - "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", "Session": "", "ubuntu": "", "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }": "", @@ -38,7 +38,7 @@ } }, "AdditionalArgs": [ - "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"" + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"" ], "ProfileFlag": "-Pe2e", "Agent": { @@ -52,7 +52,7 @@ "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pfast", "-Pdirect" ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }", "AdditionalArgs": [ - "-DargLine=\"-Dio.netty.handler.ssl.noOpenSsl=true\"" + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dio.netty.handler.ssl.noOpenSsl=true\"" ], "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } @@ -71,7 +71,7 @@ { "DESIRED_CONSISTENCY": "BoundedStaleness", "ACCOUNT_CONSISTENCY": "Strong", - "AdditionalArgs": "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"", + "AdditionalArgs": "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"", "ProfileFlag": "-Pe2e", "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "Agent": { @@ -109,7 +109,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -125,7 +125,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -141,7 +141,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -157,7 +157,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 17af3836ec2d..69d782fcc9a0 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -33,7 +33,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -65,7 +65,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -97,7 +97,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -129,7 +129,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thin-client-canary-multi-region-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-region-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thin-client-canary-multi-region-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-region-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -161,7 +161,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thin-client-canary-multi-writer-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-writer-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thin-client-canary-multi-writer-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-writer-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -183,10 +183,10 @@ extends: safeName: azurespringdatacosmos TimeoutInMinutes: 90 TestGoals: 'verify' - TestOptions: '$(ProfileFlag) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' + TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=false' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -212,5 +212,5 @@ extends: TestOptions: '$(ProfileFlag) $(AdditionalArgs)' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true'