From 1b14f78e5b5b27d9bf899c8e7480b957d868a04f Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 13 Nov 2025 17:12:33 +0200 Subject: [PATCH] Optimize AsyncSearchErrorTraceIT to avoid failures (#137716) --- .../xpack/search/AsyncSearchErrorTraceIT.java | 126 ++++++++++++------ 1 file changed, 88 insertions(+), 38 deletions(-) diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java index 83e7cf9e12096..182f764d9e624 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.search; +import org.apache.http.HttpEntity; +import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.elasticsearch.client.Request; @@ -16,7 +18,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.ErrorTraceHelper; import org.elasticsearch.search.SearchService; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockLog; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; @@ -31,7 +32,7 @@ reason = "testing debug log output to identify race condition", value = "org.elasticsearch.xpack.search.MutableSearchResponse:DEBUG,org.elasticsearch.xpack.search.AsyncSearchTask:DEBUG" ) -public class AsyncSearchErrorTraceIT extends ESIntegTestCase { +public class AsyncSearchErrorTraceIT extends AsyncSearchIntegTestCase { @Override protected boolean addMockHttpTransport() { @@ -76,10 +77,15 @@ public void testAsyncSearchFailingQueryErrorTraceDefault() throws Exception { createAsyncRequest.addParameter("keep_on_completion", "true"); createAsyncRequest.addParameter("wait_for_completion_timeout", "0ms"); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - awaitAsyncRequestDoneRunning(getAsyncRequest); + + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); } // check that the stack trace was not sent from the data node to the coordinating node ErrorTraceHelper.assertStackTraceCleared(internalCluster()); @@ -103,11 +109,16 @@ public void testAsyncSearchFailingQueryErrorTraceTrue() throws Exception { createAsyncRequest.addParameter("keep_on_completion", "true"); createAsyncRequest.addParameter("wait_for_completion_timeout", "0ms"); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - getAsyncRequest.addParameter("error_trace", "true"); - awaitAsyncRequestDoneRunning(getAsyncRequest); + + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + getAsyncRequest.addParameter("error_trace", "true"); + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); } // check that the stack trace was sent from the data node to the coordinating node ErrorTraceHelper.assertStackTraceObserved(internalCluster()); @@ -131,11 +142,16 @@ public void testAsyncSearchFailingQueryErrorTraceFalse() throws Exception { createAsyncRequest.addParameter("keep_on_completion", "true"); createAsyncRequest.addParameter("wait_for_completion_timeout", "0ms"); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - getAsyncRequest.addParameter("error_trace", "false"); - awaitAsyncRequestDoneRunning(getAsyncRequest); + + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + getAsyncRequest.addParameter("error_trace", "false"); + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); } // check that the stack trace was not sent from the data node to the coordinating node ErrorTraceHelper.assertStackTraceCleared(internalCluster()); @@ -172,19 +188,24 @@ public void testDataNodeLogsStackTrace() throws Exception { try (var mockLog = MockLog.capture(SearchService.class)) { ErrorTraceHelper.addSeenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - // Use the same value of error_trace as the search request - if (errorTraceValue == 0) { - getAsyncRequest.addParameter("error_trace", "true"); - } else if (errorTraceValue == 1) { - getAsyncRequest.addParameter("error_trace", "false"); - } // else empty - awaitAsyncRequestDoneRunning(getAsyncRequest); - } - mockLog.assertAllExpectationsMatched(); + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + // Use the same value of error_trace as the search request + if (errorTraceValue == 0) { + getAsyncRequest.addParameter("error_trace", "true"); + } else if (errorTraceValue == 1) { + getAsyncRequest.addParameter("error_trace", "false"); + } // else empty + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + + mockLog.assertAllExpectationsMatched(); + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); + } } } @@ -206,11 +227,16 @@ public void testAsyncSearchFailingQueryErrorTraceFalseOnSubmitAndTrueOnGet() thr createAsyncSearchRequest.addParameter("keep_on_completion", "true"); createAsyncSearchRequest.addParameter("wait_for_completion_timeout", "0ms"); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncSearchRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - getAsyncRequest.addParameter("error_trace", "true"); - awaitAsyncRequestDoneRunning(getAsyncRequest); + + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + getAsyncRequest.addParameter("error_trace", "true"); + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); } // check that the stack trace was not sent from the data node to the coordinating node ErrorTraceHelper.assertStackTraceCleared(internalCluster()); @@ -234,11 +260,15 @@ public void testAsyncSearchFailingQueryErrorTraceTrueOnSubmitAndFalseOnGet() thr createAsyncSearchRequest.addParameter("keep_on_completion", "true"); createAsyncSearchRequest.addParameter("wait_for_completion_timeout", "0ms"); Map createAsyncResponseEntity = performRequestAndGetResponseEntity(createAsyncSearchRequest); - if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { - String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); - Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); - getAsyncRequest.addParameter("error_trace", "false"); - awaitAsyncRequestDoneRunning(getAsyncRequest); + try { + if (Boolean.TRUE.equals(createAsyncResponseEntity.get("is_running"))) { + String asyncExecutionId = (String) createAsyncResponseEntity.get("id"); + Request getAsyncRequest = new Request("GET", "/_async_search/" + asyncExecutionId); + getAsyncRequest.addParameter("error_trace", "false"); + awaitAsyncRequestDoneRunning(getAsyncRequest); + } + } finally { + deleteAsyncSearchIfPresent(createAsyncResponseEntity); } // check that the stack trace was sent from the data node to the coordinating node ErrorTraceHelper.assertStackTraceObserved(internalCluster()); @@ -250,6 +280,26 @@ private Map performRequestAndGetResponseEntity(Request r) throws return XContentHelper.convertToMap(entityContentType.xContent(), response.getEntity().getContent(), false); } + private void deleteAsyncSearchIfPresent(Map map) throws IOException { + String id = (String) map.get("id"); + if (id == null) { + return; + } + + // Make sure the .async-search system index is green before deleting it + try { + ensureGreen(".async-search"); + } catch (Exception ignore) { + // the index may not exist + } + + Response response = getRestClient().performRequest(new Request("DELETE", "/_async_search/" + id)); + HttpEntity entity = response.getEntity(); + if (entity != null) { + EntityUtils.consumeQuietly(entity); + } + } + private void awaitAsyncRequestDoneRunning(Request getAsyncRequest) throws Exception { assertBusy(() -> { Map getAsyncResponseEntity = performRequestAndGetResponseEntity(getAsyncRequest);