From 1c627ba0e88ddd1adb9076fc44a9c5b2e998ed6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:28:23 +0400 Subject: [PATCH 01/32] Bump pnpm/action-setup from 2.2.3 to 2.2.4 (#2757) Bumps [pnpm/action-setup](https://github.com/pnpm/action-setup) from 2.2.3 to 2.2.4. - [Release notes](https://github.com/pnpm/action-setup/releases) - [Commits](https://github.com/pnpm/action-setup/compare/v2.2.3...v2.2.4) --- updated-dependencies: - dependency-name: pnpm/action-setup dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 7db1337a0f..866ee6f003 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -20,7 +20,7 @@ jobs: # Disabling shallow clone is recommended for improving relevancy of reporting fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - uses: pnpm/action-setup@v2.2.3 + - uses: pnpm/action-setup@v2.2.4 with: version: 7.4.0 - name: Install node From 3e72a7ac4ebac6af3e805e0500cab3d50767d635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:29:36 +0400 Subject: [PATCH 02/32] Bump actions/setup-node from 3.4.1 to 3.5.1 (#2747) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.4.1 to 3.5.1. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v3.4.1...v3.5.1) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 866ee6f003..baa2551d1c 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -24,7 +24,7 @@ jobs: with: version: 7.4.0 - name: Install node - uses: actions/setup-node@v3.4.1 + uses: actions/setup-node@v3.5.1 with: node-version: "16.15.0" cache: "pnpm" From daba2a1f51686aa1260c9c9350883b1af79df04e Mon Sep 17 00:00:00 2001 From: Narekmat <47845266+Narekmat@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:35:08 +0400 Subject: [PATCH 03/32] Fix helm charts validation (#2750) * add workflows * fix * add flow * fix * change names * add comments * fix helm.yaml file * fix helm.yaml * fix helm.yaml * fix realse-helm.yaml * fix helm.yaml Co-authored-by: Roman Zabaluev --- .github/workflows/helm.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index 255f92260f..d8c2875aed 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -15,6 +15,18 @@ jobs: uses: Azure/setup-helm@v1 - name: Setup Kubeval uses: lra/setup-kubeval@v1.0.1 + #check, was helm version increased in Chart.yaml? + - name: Check version + shell: bash + run: | + git fetch + git checkout master + helm_version_old=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') + git checkout $GITHUB_HEAD_REF + helm_version_new=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') + echo $helm_version_old + echo $helm_version_new + if [[ "$helm_version_new" > "$helm_version_old" ]]; then exit 0 ; else exit 1 ; fi - name: Run kubeval shell: bash run: | From f5e1dcbd82b5d16210736604eda572d994ebf113 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:42:24 +0400 Subject: [PATCH 04/32] Bump actions/stale from 5 to 6 (#2639) Bumps [actions/stale](https://github.com/actions/stale) from 5 to 6. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 5e9ac844fb..aafb50ceda 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v6 with: days-before-issue-stale: 7 days-before-issue-close: 3 From b0e0da4c0d1bd66bb2ca053f421dc4c7e78d78dc Mon Sep 17 00:00:00 2001 From: Egorka Voronyansky Date: Fri, 21 Oct 2022 13:40:35 +0800 Subject: [PATCH 05/32] Bump version of mockito up to 4.8.1 (#2793) * Bump version of mockito up to 4.8.1 * Change scope of byte buddy dependency to test Co-authored-by: vrnsky --- kafka-ui-api/pom.xml | 6 ++++++ pom.xml | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kafka-ui-api/pom.xml b/kafka-ui-api/pom.xml index ab5fa62bd1..65ab22682f 100644 --- a/kafka-ui-api/pom.xml +++ b/kafka-ui-api/pom.xml @@ -173,6 +173,12 @@ ${mockito.version} test + + net.bytebuddy + byte-buddy + ${byte-buddy.version} + test + org.assertj assertj-core diff --git a/pom.xml b/pom.xml index c01fb507dd..2333b2472f 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,8 @@ 1.17.1 4.10.0 5.7.2 - 2.21.0 + 4.8.1 + 1.12.18 3.19.0 4.7.1 3.0.9 From 40c3bcf82b7c21a367fb4112e7f8e51c843481ea Mon Sep 17 00:00:00 2001 From: Anton Mnykh <44034169+Redbutton18@users.noreply.github.com> Date: Fri, 21 Oct 2022 14:43:21 +0300 Subject: [PATCH 06/32] Issue/2774 2 (#2785) * clear message Test updated issue-2774 * clear message Test created issue-2774 * conflict resolving * conflict resolving * conflict resolving * fix --- .../com/provectus/kafka/ui/models/Topic.java | 2 +- .../kafka/ui/pages/topic/TopicDetails.java | 39 +++++++++++++ .../provectus/kafka/ui/tests/TopicTests.java | 58 +++++++++++++++---- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Topic.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Topic.java index 725db0dd8d..18763e3719 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Topic.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Topic.java @@ -6,5 +6,5 @@ import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Topic { - private String name, compactPolicyValue, timeToRetainData, maxSizeOnDisk, maxMessageBytes, messageKey, messageContent ; + private String name, cleanupPolicyValue, timeToRetainData, maxSizeOnDisk, maxMessageBytes, messageKey, messageContent ; } diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicDetails.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicDetails.java index b455d310fb..5b4f1d3c96 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicDetails.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicDetails.java @@ -17,6 +17,8 @@ public class TopicDetails { protected SelenideElement loadingSpinner = $x("//*[contains(text(),'Loading')]"); protected SelenideElement dotMenuBtn = $$x("//button[@aria-label='Dropdown Toggle']").first(); + protected SelenideElement dotPartitionIdMenuBtn = $(By.cssSelector("button.sc-hOqruk.eYtACj")); + protected SelenideElement clearMessagesBtn = $x(("//div[contains(text(), 'Clear messages')]")); protected SelenideElement overviewTab = $x("//a[contains(text(),'Overview')]"); protected SelenideElement messagesTab = $x("//a[contains(text(),'Messages')]"); protected SelenideElement editSettingsTab = $x("//li[@role][contains(text(),'Edit settings')]"); @@ -45,6 +47,18 @@ public class TopicDetails { return this; } + @Step + public TopicDetails openDotPartitionIdMenu() { + dotPartitionIdMenuBtn.shouldBe(Condition.visible.because("dot menu invisible")).click(); + return this; + } + + @Step + public TopicDetails clickClearMessagesBtn() { + clearMessagesBtn.shouldBe(Condition.visible.because("Clear Messages invisible")).click(); + return this; + } + @Step public TopicDetails deleteTopic() { clickByJavaScript(dotMenuBtn); @@ -70,6 +84,11 @@ public class TopicDetails { return contentMessage.matches(contentMessageTab.getText().trim()); } + @Step + public String MessageCountAmount() { + return $(By.xpath("//table[@class=\"sc-hiSbEG cvnuic\"]/tbody/tr/td[5]")).getText(); + } + private enum DotMenuHeaderItems { EDIT_SETTINGS("Edit settings"), CLEAR_MESSAGES("Clear messages"), @@ -91,6 +110,26 @@ public class TopicDetails { } } + public enum DotPartitionIdMenu { + CLEAR_MESSAGES("Clear messages"); + + + private final String value; + + DotPartitionIdMenu(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "DotPartitionIdMenuItems{" + "value='" + value + '\'' + '}'; + } + } + public enum TopicMenu { OVERVIEW("Overview"), MESSAGES("Messages"), diff --git a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/tests/TopicTests.java b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/tests/TopicTests.java index bb1cfc3427..2b596a9e54 100644 --- a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/tests/TopicTests.java +++ b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/tests/TopicTests.java @@ -6,6 +6,7 @@ import com.provectus.kafka.ui.pages.topic.TopicDetails; import com.provectus.kafka.ui.utilities.qaseIoUtils.annotations.AutomationStatus; import com.provectus.kafka.ui.utilities.qaseIoUtils.annotations.Suite; import com.provectus.kafka.ui.utilities.qaseIoUtils.enums.Status; +import io.qameta.allure.Issue; import io.qase.api.annotation.CaseId; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.*; @@ -14,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import static com.provectus.kafka.ui.pages.NaviSideBar.SideMenuOption.TOPICS; +import static com.provectus.kafka.ui.pages.topic.TopicDetails.DotPartitionIdMenu.CLEAR_MESSAGES; import static com.provectus.kafka.ui.settings.Source.CLUSTER_NAME; import static com.provectus.kafka.ui.utilities.FileUtils.fileToString; @@ -23,18 +25,23 @@ public class TopicTests extends BaseTest { private static final String SUITE_TITLE = "Topics"; private static final Topic TOPIC_FOR_UPDATE = new Topic() .setName("topic-to-update") - .setCompactPolicyValue("Compact") + .setCleanupPolicyValue("Compact") .setTimeToRetainData("604800001") .setMaxSizeOnDisk("20 GB") .setMaxMessageBytes("1000020") .setMessageKey(fileToString(System.getProperty("user.dir") + "/src/test/resources/producedkey.txt")) .setMessageContent(fileToString(System.getProperty("user.dir") + "/src/test/resources/testData.txt")); + private static final Topic TOPIC_FOR_MESSAGES = new Topic() + .setName("topic-with-clean-message-attribute") + .setMessageKey(fileToString(System.getProperty("user.dir") + "/src/test/resources/producedkey.txt")) + .setMessageContent(fileToString(System.getProperty("user.dir") + "/src/test/resources/testData.txt")); + private static final Topic TOPIC_FOR_DELETE = new Topic().setName("topic-to-delete"); private static final List TOPIC_LIST = new ArrayList<>(); @BeforeAll public void beforeAll() { - TOPIC_LIST.addAll(List.of(TOPIC_FOR_UPDATE, TOPIC_FOR_DELETE)); + TOPIC_LIST.addAll(List.of(TOPIC_FOR_UPDATE, TOPIC_FOR_DELETE, TOPIC_FOR_MESSAGES)); TOPIC_LIST.forEach(topic -> apiHelper.createTopic(CLUSTER_NAME, topic.getName())); } @@ -81,7 +88,7 @@ public class TopicTests extends BaseTest { .openEditSettings(); topicCreateEditForm .waitUntilScreenReady() - .selectCleanupPolicy(TOPIC_FOR_UPDATE.getCompactPolicyValue()) + .selectCleanupPolicy(TOPIC_FOR_UPDATE.getCleanupPolicyValue()) .setMinInsyncReplicas(10) .setTimeToRetainDataInMs(TOPIC_FOR_UPDATE.getTimeToRetainData()) .setMaxSizeOnDiskInGB(TOPIC_FOR_UPDATE.getMaxSizeOnDisk()) @@ -98,7 +105,7 @@ public class TopicTests extends BaseTest { .waitUntilScreenReady() .openEditSettings(); SoftAssertions softly = new SoftAssertions(); - softly.assertThat(topicCreateEditForm.getCleanupPolicy()).as("Cleanup Policy").isEqualTo(TOPIC_FOR_UPDATE.getCompactPolicyValue()); + softly.assertThat(topicCreateEditForm.getCleanupPolicy()).as("Cleanup Policy").isEqualTo(TOPIC_FOR_UPDATE.getCleanupPolicyValue()); softly.assertThat(topicCreateEditForm.getTimeToRetain()).as("Time to retain").isEqualTo(TOPIC_FOR_UPDATE.getTimeToRetainData()); softly.assertThat(topicCreateEditForm.getMaxSizeOnDisk()).as("Max size on disk").isEqualTo(TOPIC_FOR_UPDATE.getMaxSizeOnDisk()); softly.assertThat(topicCreateEditForm.getMaxMessageBytes()).as("Max message bytes").isEqualTo(TOPIC_FOR_UPDATE.getMaxMessageBytes()); @@ -126,7 +133,7 @@ public class TopicTests extends BaseTest { Assertions.assertFalse(topicsList.isTopicVisible(TOPIC_FOR_DELETE.getName()), "isTopicVisible"); TOPIC_LIST.remove(TOPIC_FOR_DELETE); } - + @DisplayName("produce message") @Suite(suiteId = SUITE_ID, title = SUITE_TITLE) @AutomationStatus(status = Status.AUTOMATED) @@ -137,24 +144,55 @@ public class TopicTests extends BaseTest { .openSideMenu(TOPICS); topicsList .waitUntilScreenReady() - .openTopic(TOPIC_FOR_UPDATE.getName()); + .openTopic(TOPIC_FOR_MESSAGES.getName()); topicDetails .waitUntilScreenReady() .openTopicMenu(TopicDetails.TopicMenu.MESSAGES) .clickProduceMessageBtn(); produceMessagePanel .waitUntilScreenReady() - .setContentFiled(TOPIC_FOR_UPDATE.getMessageContent()) - .setKeyField(TOPIC_FOR_UPDATE.getMessageKey()) + .setContentFiled(TOPIC_FOR_MESSAGES.getMessageContent()) + .setKeyField(TOPIC_FOR_MESSAGES.getMessageKey()) .submitProduceMessage(); topicDetails .waitUntilScreenReady(); SoftAssertions softly = new SoftAssertions(); - softly.assertThat(topicDetails.isKeyMessageVisible((TOPIC_FOR_UPDATE.getMessageKey()))).withFailMessage("isKeyMessageVisible()").isTrue(); - softly.assertThat(topicDetails.isContentMessageVisible((TOPIC_FOR_UPDATE.getMessageContent()).trim())).withFailMessage("isContentMessageVisible()").isTrue(); + softly.assertThat(topicDetails.isKeyMessageVisible((TOPIC_FOR_MESSAGES.getMessageKey()))).withFailMessage("isKeyMessageVisible()").isTrue(); + softly.assertThat(topicDetails.isContentMessageVisible((TOPIC_FOR_MESSAGES.getMessageContent()).trim())).withFailMessage("isContentMessageVisible()").isTrue(); softly.assertAll(); } + @Issue("Uncomment last assertion after bug https://github.com/provectus/kafka-ui/issues/2778 fix") + @DisplayName("clear message") + @Suite(suiteId = SUITE_ID, title = SUITE_TITLE) + @AutomationStatus(status = Status.AUTOMATED) + @CaseId(19) + @Test + void clearMessage() { + naviSideBar + .openSideMenu(TOPICS); + topicsList + .waitUntilScreenReady() + .openTopic(TOPIC_FOR_MESSAGES.getName()); + topicDetails + .waitUntilScreenReady() + .openTopicMenu(TopicDetails.TopicMenu.OVERVIEW) + .clickProduceMessageBtn(); + produceMessagePanel + .waitUntilScreenReady() + .setContentFiled(TOPIC_FOR_MESSAGES.getMessageContent()) + .setKeyField(TOPIC_FOR_MESSAGES.getMessageKey()) + .submitProduceMessage(); + topicDetails + .waitUntilScreenReady(); + String messageAmount = topicDetails.MessageCountAmount(); + Assertions.assertEquals(messageAmount,topicDetails.MessageCountAmount()); + topicDetails + .openDotPartitionIdMenu() + .clickClearMessagesBtn(); +// Assertions.assertEquals(Integer.toString(Integer.valueOf(messageAmount)-1),topicDetails.MessageCountAmount()); + } + @AfterAll public void afterAll() { TOPIC_LIST.forEach(topic -> apiHelper.deleteTopic(CLUSTER_NAME, topic.getName())); From ae219de36cda741686711b194b83464d045109c3 Mon Sep 17 00:00:00 2001 From: Egorka Voronyansky Date: Sun, 23 Oct 2022 20:54:56 +0800 Subject: [PATCH 07/32] Bump Groovy version up to 3.0.13 (#2806) Co-authored-by: vrnsky --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2333b2472f..d977385b94 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 1.12.18 3.19.0 4.7.1 - 3.0.9 + 3.0.13 3.1.0 ..//kafka-ui-react-app/src/generated-sources From 4558466ff6aec1bd4f26ab4dc8b8b85a52345ba3 Mon Sep 17 00:00:00 2001 From: Ilya Kuramshin Date: Sun, 23 Oct 2022 19:47:21 +0400 Subject: [PATCH 08/32] Emitters logic refactoring (#2729) * Emitters logic refactoring: 1. consumers seeking moved to SeekOperations class 2. offsets info gathering moved to OffsetsInfo class * wip * checkstyle fix * checkstyle fix * minor improvements Co-authored-by: iliax --- .../ui/controller/MessagesController.java | 28 ++- .../ui/emitter/BackwardRecordEmitter.java | 101 ++++----- .../ui/emitter/ForwardRecordEmitter.java | 20 +- .../kafka/ui/emitter/OffsetsInfo.java | 59 ++++++ .../kafka/ui/emitter/SeekOperations.java | 111 ++++++++++ .../kafka/ui/emitter/TailingEmitter.java | 25 ++- .../kafka/ui/model/ConsumerPosition.java | 6 +- .../kafka/ui/service/MessagesService.java | 35 ++-- .../service/analyze/TopicAnalysisService.java | 8 +- .../provectus/kafka/ui/util/OffsetsSeek.java | 143 ------------- .../kafka/ui/util/OffsetsSeekBackward.java | 120 ----------- .../kafka/ui/util/OffsetsSeekForward.java | 61 ------ .../kafka/ui/emitter/OffsetsInfoTest.java | 53 +++++ .../kafka/ui/emitter/SeekOperationsTest.java | 88 ++++++++ .../kafka/ui/emitter/TailingEmitterTest.java | 5 +- .../kafka/ui/service/MessagesServiceTest.java | 2 +- .../kafka/ui/service/RecordEmitterTest.java | 70 +++---- .../kafka/ui/service/SendAndReadTests.java | 5 +- .../kafka/ui/util/OffsetsSeekTest.java | 196 ------------------ 19 files changed, 462 insertions(+), 674 deletions(-) create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/OffsetsInfo.java create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/SeekOperations.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeek.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekBackward.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekForward.java create mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/OffsetsInfoTest.java create mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/SeekOperationsTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/OffsetsSeekTest.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/MessagesController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/MessagesController.java index fab4bcfd5c..79ae59c3b3 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/MessagesController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/MessagesController.java @@ -5,6 +5,7 @@ import static com.provectus.kafka.ui.serde.api.Serde.Target.VALUE; import static java.util.stream.Collectors.toMap; import com.provectus.kafka.ui.api.MessagesApi; +import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.CreateTopicMessageDTO; import com.provectus.kafka.ui.model.MessageFilterTypeDTO; @@ -18,6 +19,7 @@ import com.provectus.kafka.ui.service.MessagesService; import java.util.List; import java.util.Map; import java.util.Optional; +import javax.annotation.Nullable; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -63,18 +65,22 @@ public class MessagesController extends AbstractController implements MessagesAp String keySerde, String valueSerde, ServerWebExchange exchange) { + seekType = seekType != null ? seekType : SeekTypeDTO.BEGINNING; + seekDirection = seekDirection != null ? seekDirection : SeekDirectionDTO.FORWARD; + filterQueryType = filterQueryType != null ? filterQueryType : MessageFilterTypeDTO.STRING_CONTAINS; + int recordsLimit = + Optional.ofNullable(limit).map(s -> Math.min(s, MAX_LOAD_RECORD_LIMIT)).orElse(DEFAULT_LOAD_RECORD_LIMIT); + var positions = new ConsumerPosition( - seekType != null ? seekType : SeekTypeDTO.BEGINNING, - parseSeekTo(topicName, seekTo), - seekDirection + seekType, + topicName, + parseSeekTo(topicName, seekType, seekTo) ); - int recordsLimit = Optional.ofNullable(limit) - .map(s -> Math.min(s, MAX_LOAD_RECORD_LIMIT)) - .orElse(DEFAULT_LOAD_RECORD_LIMIT); return Mono.just( ResponseEntity.ok( messagesService.loadMessages( - getCluster(clusterName), topicName, positions, q, filterQueryType, recordsLimit, keySerde, valueSerde) + getCluster(clusterName), topicName, positions, q, filterQueryType, + recordsLimit, seekDirection, keySerde, valueSerde) ) ); } @@ -92,9 +98,13 @@ public class MessagesController extends AbstractController implements MessagesAp * The format is [partition]::[offset] for specifying offsets * or [partition]::[timestamp in millis] for specifying timestamps. */ - private Map parseSeekTo(String topic, List seekTo) { + @Nullable + private Map parseSeekTo(String topic, SeekTypeDTO seekType, List seekTo) { if (seekTo == null || seekTo.isEmpty()) { - return Map.of(); + if (seekType == SeekTypeDTO.LATEST || seekType == SeekTypeDTO.BEGINNING) { + return null; + } + throw new ValidationException("seekTo should be set if seekType is " + seekType); } return seekTo.stream() .map(p -> { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/BackwardRecordEmitter.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/BackwardRecordEmitter.java index 59db425b33..d2012355db 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/BackwardRecordEmitter.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/BackwardRecordEmitter.java @@ -1,21 +1,18 @@ package com.provectus.kafka.ui.emitter; +import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; -import com.provectus.kafka.ui.util.OffsetsSeekBackward; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Map; -import java.util.SortedMap; import java.util.TreeMap; -import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; @@ -29,80 +26,68 @@ public class BackwardRecordEmitter private static final Duration POLL_TIMEOUT = Duration.ofMillis(200); - private final Function, KafkaConsumer> consumerSupplier; - private final OffsetsSeekBackward offsetsSeek; + private final Supplier> consumerSupplier; + private final ConsumerPosition consumerPosition; + private final int messagesPerPage; public BackwardRecordEmitter( - Function, KafkaConsumer> consumerSupplier, - OffsetsSeekBackward offsetsSeek, + Supplier> consumerSupplier, + ConsumerPosition consumerPosition, + int messagesPerPage, ConsumerRecordDeserializer recordDeserializer) { super(recordDeserializer); - this.offsetsSeek = offsetsSeek; + this.consumerPosition = consumerPosition; + this.messagesPerPage = messagesPerPage; this.consumerSupplier = consumerSupplier; } @Override public void accept(FluxSink sink) { - try (KafkaConsumer configConsumer = consumerSupplier.apply(Map.of())) { - final List requestedPartitions = - offsetsSeek.getRequestedPartitions(configConsumer); - sendPhase(sink, "Request partitions"); - final int msgsPerPartition = offsetsSeek.msgsPerPartition(requestedPartitions.size()); - try (KafkaConsumer consumer = - consumerSupplier.apply( - Map.of(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, msgsPerPartition) - ) - ) { - sendPhase(sink, "Created consumer"); + try (KafkaConsumer consumer = consumerSupplier.get()) { + sendPhase(sink, "Created consumer"); - SortedMap readUntilOffsets = - new TreeMap<>(Comparator.comparingInt(TopicPartition::partition)); - readUntilOffsets.putAll(offsetsSeek.getPartitionsOffsets(consumer)); + var seekOperations = SeekOperations.create(consumer, consumerPosition); + var readUntilOffsets = new TreeMap(Comparator.comparingInt(TopicPartition::partition)); + readUntilOffsets.putAll(seekOperations.getOffsetsForSeek()); - sendPhase(sink, "Requested partitions offsets"); - log.debug("partition offsets: {}", readUntilOffsets); - var waitingOffsets = - offsetsSeek.waitingOffsets(consumer, readUntilOffsets.keySet()); - log.debug("waiting offsets {} {}", - waitingOffsets.getBeginOffsets(), - waitingOffsets.getEndOffsets() - ); + int msgsToPollPerPartition = (int) Math.ceil((double) messagesPerPage / readUntilOffsets.size()); + log.debug("'Until' offsets for polling: {}", readUntilOffsets); - while (!sink.isCancelled() && !waitingOffsets.beginReached()) { - new TreeMap<>(readUntilOffsets).forEach((tp, readToOffset) -> { - long lowestOffset = waitingOffsets.getBeginOffsets().get(tp.partition()); - long readFromOffset = Math.max(lowestOffset, readToOffset - msgsPerPartition); - - partitionPollIteration(tp, readFromOffset, readToOffset, consumer, sink) - .stream() - .filter(r -> !sink.isCancelled()) - .forEach(r -> sendMessage(sink, r)); - - waitingOffsets.markPolled(tp.partition(), readFromOffset); - if (waitingOffsets.getBeginOffsets().get(tp.partition()) == null) { - // we fully read this partition -> removing it from polling iterations - readUntilOffsets.remove(tp); - } else { - readUntilOffsets.put(tp, readFromOffset); - } - }); - - if (waitingOffsets.beginReached()) { - log.debug("begin reached after partitions poll iteration"); - } else if (sink.isCancelled()) { - log.debug("sink is cancelled after partitions poll iteration"); + while (!sink.isCancelled() && !readUntilOffsets.isEmpty()) { + new TreeMap<>(readUntilOffsets).forEach((tp, readToOffset) -> { + if (sink.isCancelled()) { + return; //fast return in case of sink cancellation } + long beginOffset = seekOperations.getBeginOffsets().get(tp); + long readFromOffset = Math.max(beginOffset, readToOffset - msgsToPollPerPartition); + + partitionPollIteration(tp, readFromOffset, readToOffset, consumer, sink) + .stream() + .filter(r -> !sink.isCancelled()) + .forEach(r -> sendMessage(sink, r)); + + if (beginOffset == readFromOffset) { + // we fully read this partition -> removing it from polling iterations + readUntilOffsets.remove(tp); + } else { + // updating 'to' offset for next polling iteration + readUntilOffsets.put(tp, readFromOffset); + } + }); + if (readUntilOffsets.isEmpty()) { + log.debug("begin reached after partitions poll iteration"); + } else if (sink.isCancelled()) { + log.debug("sink is cancelled after partitions poll iteration"); } - sink.complete(); - log.debug("Polling finished"); } + sink.complete(); + log.debug("Polling finished"); } catch (Exception e) { log.error("Error occurred while consuming records", e); sink.error(e); } } - private List> partitionPollIteration( TopicPartition tp, long fromOffset, diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ForwardRecordEmitter.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ForwardRecordEmitter.java index bc636cab7b..69d9801b70 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ForwardRecordEmitter.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ForwardRecordEmitter.java @@ -1,8 +1,8 @@ package com.provectus.kafka.ui.emitter; +import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; -import com.provectus.kafka.ui.util.OffsetsSeek; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -17,34 +17,38 @@ public class ForwardRecordEmitter implements java.util.function.Consumer> { private final Supplier> consumerSupplier; - private final OffsetsSeek offsetsSeek; + private final ConsumerPosition position; public ForwardRecordEmitter( Supplier> consumerSupplier, - OffsetsSeek offsetsSeek, + ConsumerPosition position, ConsumerRecordDeserializer recordDeserializer) { super(recordDeserializer); + this.position = position; this.consumerSupplier = consumerSupplier; - this.offsetsSeek = offsetsSeek; } @Override public void accept(FluxSink sink) { try (KafkaConsumer consumer = consumerSupplier.get()) { sendPhase(sink, "Assigning partitions"); - var waitingOffsets = offsetsSeek.assignAndSeek(consumer); + var seekOperations = SeekOperations.create(consumer, position); + seekOperations.assignAndSeekNonEmptyPartitions(); + // we use empty polls counting to verify that topic was fully read int emptyPolls = 0; - while (!sink.isCancelled() && !waitingOffsets.endReached() && emptyPolls < NO_MORE_DATA_EMPTY_POLLS_COUNT) { + while (!sink.isCancelled() + && !seekOperations.assignedPartitionsFullyPolled() + && emptyPolls < NO_MORE_DATA_EMPTY_POLLS_COUNT) { + sendPhase(sink, "Polling"); ConsumerRecords records = poll(sink, consumer); log.info("{} records polled", records.count()); emptyPolls = records.isEmpty() ? emptyPolls + 1 : 0; for (ConsumerRecord msg : records) { - if (!sink.isCancelled() && !waitingOffsets.endReached()) { + if (!sink.isCancelled()) { sendMessage(sink, msg); - waitingOffsets.markPolled(msg); } else { break; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/OffsetsInfo.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/OffsetsInfo.java new file mode 100644 index 0000000000..1b1381ea70 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/OffsetsInfo.java @@ -0,0 +1,59 @@ +package com.provectus.kafka.ui.emitter; + +import com.google.common.base.Preconditions; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.TopicPartition; + +@Slf4j +@Getter +public class OffsetsInfo { + + private final Consumer consumer; + + private final Map beginOffsets; + private final Map endOffsets; + + private final Set nonEmptyPartitions = new HashSet<>(); + private final Set emptyPartitions = new HashSet<>(); + + public OffsetsInfo(Consumer consumer, String topic) { + this(consumer, + consumer.partitionsFor(topic).stream() + .map(pi -> new TopicPartition(topic, pi.partition())) + .collect(Collectors.toList()) + ); + } + + public OffsetsInfo(Consumer consumer, + Collection targetPartitions) { + this.consumer = consumer; + this.beginOffsets = consumer.beginningOffsets(targetPartitions); + this.endOffsets = consumer.endOffsets(targetPartitions); + endOffsets.forEach((tp, endOffset) -> { + var beginningOffset = beginOffsets.get(tp); + if (endOffset > beginningOffset) { + nonEmptyPartitions.add(tp); + } else { + emptyPartitions.add(tp); + } + }); + } + + public boolean assignedPartitionsFullyPolled() { + for (var tp: consumer.assignment()) { + Preconditions.checkArgument(endOffsets.containsKey(tp)); + if (endOffsets.get(tp) > consumer.position(tp)) { + return false; + } + } + return true; + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/SeekOperations.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/SeekOperations.java new file mode 100644 index 0000000000..014b120757 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/SeekOperations.java @@ -0,0 +1,111 @@ +package com.provectus.kafka.ui.emitter; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.provectus.kafka.ui.model.ConsumerPosition; +import com.provectus.kafka.ui.model.SeekTypeDTO; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.TopicPartition; + +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +class SeekOperations { + + private final Consumer consumer; + private final OffsetsInfo offsetsInfo; + private final Map offsetsForSeek; //only contains non-empty partitions! + + static SeekOperations create(Consumer consumer, ConsumerPosition consumerPosition) { + OffsetsInfo offsetsInfo; + if (consumerPosition.getSeekTo() == null) { + offsetsInfo = new OffsetsInfo(consumer, consumerPosition.getTopic()); + } else { + offsetsInfo = new OffsetsInfo(consumer, consumerPosition.getSeekTo().keySet()); + } + return new SeekOperations( + consumer, + offsetsInfo, + getOffsetsForSeek(consumer, offsetsInfo, consumerPosition.getSeekType(), consumerPosition.getSeekTo()) + ); + } + + void assignAndSeekNonEmptyPartitions() { + consumer.assign(offsetsForSeek.keySet()); + offsetsForSeek.forEach(consumer::seek); + } + + Map getBeginOffsets() { + return offsetsInfo.getBeginOffsets(); + } + + Map getEndOffsets() { + return offsetsInfo.getEndOffsets(); + } + + boolean assignedPartitionsFullyPolled() { + return offsetsInfo.assignedPartitionsFullyPolled(); + } + + // Get offsets to seek to. NOTE: offsets do not contain empty partitions offsets + Map getOffsetsForSeek() { + return offsetsForSeek; + } + + /** + * Finds offsets for ConsumerPosition. Note: will return empty map if no offsets found for desired criteria. + */ + @VisibleForTesting + static Map getOffsetsForSeek(Consumer consumer, + OffsetsInfo offsetsInfo, + SeekTypeDTO seekType, + @Nullable Map seekTo) { + switch (seekType) { + case LATEST: + return consumer.endOffsets(offsetsInfo.getNonEmptyPartitions()); + case BEGINNING: + return consumer.beginningOffsets(offsetsInfo.getNonEmptyPartitions()); + case OFFSET: + Preconditions.checkNotNull(offsetsInfo); + return fixOffsets(offsetsInfo, seekTo); + case TIMESTAMP: + Preconditions.checkNotNull(offsetsInfo); + return offsetsForTimestamp(consumer, offsetsInfo, seekTo); + default: + throw new IllegalStateException(); + } + } + + private static Map fixOffsets(OffsetsInfo offsetsInfo, Map offsets) { + offsets = new HashMap<>(offsets); + offsets.keySet().retainAll(offsetsInfo.getNonEmptyPartitions()); + + Map result = new HashMap<>(); + offsets.forEach((tp, targetOffset) -> { + long endOffset = offsetsInfo.getEndOffsets().get(tp); + long beginningOffset = offsetsInfo.getBeginOffsets().get(tp); + // fixing offsets with min - max bounds + if (targetOffset > endOffset) { + targetOffset = endOffset; + } else if (targetOffset < beginningOffset) { + targetOffset = beginningOffset; + } + result.put(tp, targetOffset); + }); + return result; + } + + private static Map offsetsForTimestamp(Consumer consumer, OffsetsInfo offsetsInfo, + Map timestamps) { + timestamps = new HashMap<>(timestamps); + timestamps.keySet().retainAll(offsetsInfo.getNonEmptyPartitions()); + + return consumer.offsetsForTimes(timestamps).entrySet().stream() + .filter(e -> e.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/TailingEmitter.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/TailingEmitter.java index 852c31038f..12a8ae183d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/TailingEmitter.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/TailingEmitter.java @@ -1,8 +1,9 @@ package com.provectus.kafka.ui.emitter; +import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; -import com.provectus.kafka.ui.util.OffsetsSeek; +import java.util.HashMap; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -15,21 +16,21 @@ public class TailingEmitter extends AbstractEmitter implements java.util.function.Consumer> { private final Supplier> consumerSupplier; - private final OffsetsSeek offsetsSeek; + private final ConsumerPosition consumerPosition; - public TailingEmitter(ConsumerRecordDeserializer recordDeserializer, - Supplier> consumerSupplier, - OffsetsSeek offsetsSeek) { + public TailingEmitter(Supplier> consumerSupplier, + ConsumerPosition consumerPosition, + ConsumerRecordDeserializer recordDeserializer) { super(recordDeserializer); this.consumerSupplier = consumerSupplier; - this.offsetsSeek = offsetsSeek; + this.consumerPosition = consumerPosition; } @Override public void accept(FluxSink sink) { try (KafkaConsumer consumer = consumerSupplier.get()) { log.debug("Starting topic tailing"); - offsetsSeek.assignAndSeek(consumer); + assignAndSeek(consumer); while (!sink.isCancelled()) { sendPhase(sink, "Polling"); var polled = poll(sink, consumer); @@ -40,9 +41,17 @@ public class TailingEmitter extends AbstractEmitter } catch (InterruptException kafkaInterruptException) { sink.complete(); } catch (Exception e) { - log.error("Error consuming {}", offsetsSeek.getConsumerPosition(), e); + log.error("Error consuming {}", consumerPosition, e); sink.error(e); } } + private void assignAndSeek(KafkaConsumer consumer) { + var seekOperations = SeekOperations.create(consumer, consumerPosition); + var seekOffsets = new HashMap<>(seekOperations.getEndOffsets()); // defaulting offsets to topic end + seekOffsets.putAll(seekOperations.getOffsetsForSeek()); // this will only set non-empty partitions + consumer.assign(seekOffsets.keySet()); + seekOffsets.forEach(consumer::seek); + } + } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ConsumerPosition.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ConsumerPosition.java index 7c3f5a6229..9d77923fbc 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ConsumerPosition.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ConsumerPosition.java @@ -1,12 +1,14 @@ package com.provectus.kafka.ui.model; import java.util.Map; +import javax.annotation.Nullable; import lombok.Value; import org.apache.kafka.common.TopicPartition; @Value public class ConsumerPosition { SeekTypeDTO seekType; - Map seekTo; - SeekDirectionDTO seekDirection; + String topic; + @Nullable + Map seekTo; // null if positioning should apply to all tps } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java index 9191a3840e..1f217b1e40 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java @@ -14,12 +14,9 @@ import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import com.provectus.kafka.ui.serdes.ProducerRecordCreator; -import com.provectus.kafka.ui.util.OffsetsSeekBackward; -import com.provectus.kafka.ui.util.OffsetsSeekForward; import com.provectus.kafka.ui.util.ResultSizeLimiter; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -129,58 +126,62 @@ public class MessagesService { } public Flux loadMessages(KafkaCluster cluster, String topic, - ConsumerPosition consumerPosition, String query, + ConsumerPosition consumerPosition, + @Nullable String query, MessageFilterTypeDTO filterQueryType, int limit, + SeekDirectionDTO seekDirection, @Nullable String keySerde, @Nullable String valueSerde) { return withExistingTopic(cluster, topic) .flux() .flatMap(td -> loadMessagesImpl(cluster, topic, consumerPosition, query, - filterQueryType, limit, keySerde, valueSerde)); + filterQueryType, limit, seekDirection, keySerde, valueSerde)); } private Flux loadMessagesImpl(KafkaCluster cluster, String topic, ConsumerPosition consumerPosition, - String query, + @Nullable String query, MessageFilterTypeDTO filterQueryType, int limit, + SeekDirectionDTO seekDirection, @Nullable String keySerde, @Nullable String valueSerde) { java.util.function.Consumer> emitter; ConsumerRecordDeserializer recordDeserializer = deserializationService.deserializerFor(cluster, topic, keySerde, valueSerde); - if (consumerPosition.getSeekDirection().equals(SeekDirectionDTO.FORWARD)) { + if (seekDirection.equals(SeekDirectionDTO.FORWARD)) { emitter = new ForwardRecordEmitter( () -> consumerGroupService.createConsumer(cluster), - new OffsetsSeekForward(topic, consumerPosition), + consumerPosition, recordDeserializer ); - } else if (consumerPosition.getSeekDirection().equals(SeekDirectionDTO.BACKWARD)) { + } else if (seekDirection.equals(SeekDirectionDTO.BACKWARD)) { emitter = new BackwardRecordEmitter( - (Map props) -> consumerGroupService.createConsumer(cluster, props), - new OffsetsSeekBackward(topic, consumerPosition, limit), + () -> consumerGroupService.createConsumer(cluster), + consumerPosition, + limit, recordDeserializer ); } else { emitter = new TailingEmitter( - recordDeserializer, () -> consumerGroupService.createConsumer(cluster), - new OffsetsSeekForward(topic, consumerPosition) + consumerPosition, + recordDeserializer ); } return Flux.create(emitter) .filter(getMsgFilter(query, filterQueryType)) - .takeWhile(createTakeWhilePredicate(consumerPosition, limit)) + .takeWhile(createTakeWhilePredicate(seekDirection, limit)) .subscribeOn(Schedulers.boundedElastic()) .share(); } private Predicate createTakeWhilePredicate( - ConsumerPosition consumerPosition, int limit) { - return consumerPosition.getSeekDirection() == SeekDirectionDTO.TAILING + SeekDirectionDTO seekDirection, int limit) { + return seekDirection == SeekDirectionDTO.TAILING ? evt -> true // no limit for tailing : new ResultSizeLimiter(limit); } @@ -189,8 +190,6 @@ public class MessagesService { if (StringUtils.isEmpty(query)) { return evt -> true; } - filterQueryType = Optional.ofNullable(filterQueryType) - .orElse(MessageFilterTypeDTO.STRING_CONTAINS); var messageFilter = MessageFilters.createMsgFilter(query, filterQueryType); return evt -> { // we only apply filter for message events diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisService.java index 3e72e8a07e..3eb61a56d2 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisService.java @@ -2,12 +2,12 @@ package com.provectus.kafka.ui.service.analyze; import static com.provectus.kafka.ui.emitter.AbstractEmitter.NO_MORE_DATA_EMPTY_POLLS_COUNT; +import com.provectus.kafka.ui.emitter.OffsetsInfo; import com.provectus.kafka.ui.exception.TopicAnalysisException; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.TopicAnalysisDTO; import com.provectus.kafka.ui.service.ConsumerGroupService; import com.provectus.kafka.ui.service.TopicsService; -import com.provectus.kafka.ui.util.OffsetsSeek.WaitingOffsets; import java.io.Closeable; import java.time.Duration; import java.time.Instant; @@ -119,14 +119,14 @@ public class TopicAnalysisService { consumer.assign(topicPartitions); consumer.seekToBeginning(topicPartitions); - var waitingOffsets = new WaitingOffsets(topicId.topicName, consumer, topicPartitions); - for (int emptyPolls = 0; !waitingOffsets.endReached() && emptyPolls < NO_MORE_DATA_EMPTY_POLLS_COUNT;) { + var offsetsInfo = new OffsetsInfo(consumer, topicId.topicName); + for (int emptyPolls = 0; !offsetsInfo.assignedPartitionsFullyPolled() + && emptyPolls < NO_MORE_DATA_EMPTY_POLLS_COUNT;) { var polled = consumer.poll(Duration.ofSeconds(3)); emptyPolls = polled.isEmpty() ? emptyPolls + 1 : 0; polled.forEach(r -> { totalStats.apply(r); partitionStats.get(r.partition()).apply(r); - waitingOffsets.markPolled(r); }); updateProgress(); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeek.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeek.java deleted file mode 100644 index e8d475a65d..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeek.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.provectus.kafka.ui.util; - -import com.provectus.kafka.ui.model.ConsumerPosition; -import com.provectus.kafka.ui.model.SeekTypeDTO; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.Bytes; -import reactor.util.function.Tuple2; -import reactor.util.function.Tuples; - -@Slf4j -public abstract class OffsetsSeek { - protected final String topic; - protected final ConsumerPosition consumerPosition; - - protected OffsetsSeek(String topic, ConsumerPosition consumerPosition) { - this.topic = topic; - this.consumerPosition = consumerPosition; - } - - public ConsumerPosition getConsumerPosition() { - return consumerPosition; - } - - public Map getPartitionsOffsets(Consumer consumer) { - SeekTypeDTO seekType = consumerPosition.getSeekType(); - List partitions = getRequestedPartitions(consumer); - log.info("Positioning consumer for topic {} with {}", topic, consumerPosition); - Map offsets; - switch (seekType) { - case OFFSET: - offsets = offsetsFromPositions(consumer, partitions); - break; - case TIMESTAMP: - offsets = offsetsForTimestamp(consumer); - break; - case BEGINNING: - offsets = offsetsFromBeginning(consumer, partitions); - break; - case LATEST: - offsets = endOffsets(consumer, partitions); - break; - default: - throw new IllegalArgumentException("Unknown seekType: " + seekType); - } - return offsets; - } - - public WaitingOffsets waitingOffsets(Consumer consumer, - Collection partitions) { - return new WaitingOffsets(topic, consumer, partitions); - } - - public WaitingOffsets assignAndSeek(Consumer consumer) { - final Map partitionsOffsets = getPartitionsOffsets(consumer); - consumer.assign(partitionsOffsets.keySet()); - partitionsOffsets.forEach(consumer::seek); - log.info("Assignment: {}", consumer.assignment()); - return waitingOffsets(consumer, partitionsOffsets.keySet()); - } - - - public List getRequestedPartitions(Consumer consumer) { - Map partitionPositions = consumerPosition.getSeekTo(); - return consumer.partitionsFor(topic).stream() - .filter( - p -> partitionPositions.isEmpty() - || partitionPositions.containsKey(new TopicPartition(p.topic(), p.partition())) - ).map(p -> new TopicPartition(p.topic(), p.partition())) - .collect(Collectors.toList()); - } - - protected Map endOffsets( - Consumer consumer, List partitions) { - return consumer.endOffsets(partitions); - } - - protected abstract Map offsetsFromBeginning( - Consumer consumer, List partitions); - - protected abstract Map offsetsForTimestamp( - Consumer consumer); - - protected abstract Map offsetsFromPositions( - Consumer consumer, List partitions); - - public static class WaitingOffsets { - private final Map endOffsets; // partition number -> offset - private final Map beginOffsets; // partition number -> offset - - public WaitingOffsets(String topic, Consumer consumer, - Collection partitions) { - var allBeginningOffsets = consumer.beginningOffsets(partitions); - var allEndOffsets = consumer.endOffsets(partitions); - - this.endOffsets = allEndOffsets.entrySet().stream() - .filter(entry -> !allBeginningOffsets.get(entry.getKey()).equals(entry.getValue())) - .map(e -> Tuples.of(e.getKey().partition(), e.getValue() - 1)) - .collect(Collectors.toMap(Tuple2::getT1, Tuple2::getT2)); - - this.beginOffsets = this.endOffsets.keySet().stream() - .map(p -> Tuples.of(p, allBeginningOffsets.get(new TopicPartition(topic, p)))) - .collect(Collectors.toMap(Tuple2::getT1, Tuple2::getT2)); - } - - public void markPolled(ConsumerRecord rec) { - markPolled(rec.partition(), rec.offset()); - } - - public void markPolled(int partition, long offset) { - Long endWaiting = endOffsets.get(partition); - if (endWaiting != null && endWaiting <= offset) { - endOffsets.remove(partition); - } - Long beginWaiting = beginOffsets.get(partition); - if (beginWaiting != null && beginWaiting >= offset) { - beginOffsets.remove(partition); - } - } - - public boolean endReached() { - return endOffsets.isEmpty(); - } - - public boolean beginReached() { - return beginOffsets.isEmpty(); - } - - public Map getEndOffsets() { - return endOffsets; - } - - public Map getBeginOffsets() { - return beginOffsets; - } - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekBackward.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekBackward.java deleted file mode 100644 index e3d8f1b5b8..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekBackward.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.provectus.kafka.ui.util; - -import com.provectus.kafka.ui.model.ConsumerPosition; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.Bytes; -import reactor.util.function.Tuple2; -import reactor.util.function.Tuples; - -@Slf4j -public class OffsetsSeekBackward extends OffsetsSeek { - - private final int maxMessages; - - public OffsetsSeekBackward(String topic, - ConsumerPosition consumerPosition, int maxMessages) { - super(topic, consumerPosition); - this.maxMessages = maxMessages; - } - - public int msgsPerPartition(int partitionsSize) { - return msgsPerPartition(maxMessages, partitionsSize); - } - - public int msgsPerPartition(long awaitingMessages, int partitionsSize) { - return (int) Math.ceil((double) awaitingMessages / partitionsSize); - } - - - protected Map offsetsFromPositions(Consumer consumer, - List partitions) { - - return findOffsetsInt(consumer, consumerPosition.getSeekTo(), partitions); - } - - protected Map offsetsFromBeginning(Consumer consumer, - List partitions) { - return findOffsets(consumer, Map.of(), partitions); - } - - protected Map offsetsForTimestamp(Consumer consumer) { - Map timestampsToSearch = - consumerPosition.getSeekTo().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue - )); - Map offsetsForTimestamps = consumer.offsetsForTimes(timestampsToSearch) - .entrySet().stream() - .filter(e -> e.getValue() != null) - .map(v -> Tuples.of(v.getKey(), v.getValue().offset())) - .collect(Collectors.toMap(Tuple2::getT1, Tuple2::getT2)); - - if (offsetsForTimestamps.isEmpty()) { - throw new IllegalArgumentException("No offsets were found for requested timestamps"); - } - - log.info("Timestamps: {} to offsets: {}", timestampsToSearch, offsetsForTimestamps); - - return findOffsets(consumer, offsetsForTimestamps, offsetsForTimestamps.keySet()); - } - - protected Map findOffsetsInt( - Consumer consumer, Map seekTo, - List partitions) { - return findOffsets(consumer, seekTo, partitions); - } - - protected Map findOffsets( - Consumer consumer, Map seekTo, - Collection partitions) { - - final Map beginningOffsets = consumer.beginningOffsets(partitions); - final Map endOffsets = consumer.endOffsets(partitions); - - final Map seekMap = new HashMap<>(); - final Set emptyPartitions = new HashSet<>(); - - for (Map.Entry entry : seekTo.entrySet()) { - final Long endOffset = endOffsets.get(entry.getKey()); - final Long beginningOffset = beginningOffsets.get(entry.getKey()); - if (beginningOffset != null - && endOffset != null - && beginningOffset < endOffset - && entry.getValue() > beginningOffset - ) { - final Long value; - if (entry.getValue() > endOffset) { - value = endOffset; - } else { - value = entry.getValue(); - } - - seekMap.put(entry.getKey(), value); - } else { - emptyPartitions.add(entry.getKey()); - } - } - - Set waiting = new HashSet<>(partitions); - waiting.removeAll(emptyPartitions); - waiting.removeAll(seekMap.keySet()); - - for (TopicPartition topicPartition : waiting) { - seekMap.put(topicPartition, endOffsets.get(topicPartition)); - } - - return seekMap; - } - - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekForward.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekForward.java deleted file mode 100644 index 6b6ea735fc..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/OffsetsSeekForward.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.provectus.kafka.ui.util; - -import com.provectus.kafka.ui.model.ConsumerPosition; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.Bytes; - -@Slf4j -public class OffsetsSeekForward extends OffsetsSeek { - - public OffsetsSeekForward(String topic, ConsumerPosition consumerPosition) { - super(topic, consumerPosition); - } - - protected Map offsetsFromPositions(Consumer consumer, - List partitions) { - final Map offsets = - offsetsFromBeginning(consumer, partitions); - - final Map endOffsets = consumer.endOffsets(offsets.keySet()); - final Set set = new HashSet<>(consumerPosition.getSeekTo().keySet()); - final Map collect = consumerPosition.getSeekTo().entrySet().stream() - .filter(e -> e.getValue() < endOffsets.get(e.getKey())) - .filter(e -> endOffsets.get(e.getKey()) > offsets.get(e.getKey())) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue - )); - offsets.putAll(collect); - set.removeAll(collect.keySet()); - set.forEach(offsets::remove); - - return offsets; - } - - protected Map offsetsForTimestamp(Consumer consumer) { - Map offsetsForTimestamps = - consumer.offsetsForTimes(consumerPosition.getSeekTo()) - .entrySet().stream() - .filter(e -> e.getValue() != null) - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); - - if (offsetsForTimestamps.isEmpty()) { - throw new IllegalArgumentException("No offsets were found for requested timestamps"); - } - - return offsetsForTimestamps; - } - - protected Map offsetsFromBeginning(Consumer consumer, - List partitions) { - return consumer.beginningOffsets(partitions); - } - -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/OffsetsInfoTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/OffsetsInfoTest.java new file mode 100644 index 0000000000..156f62846b --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/OffsetsInfoTest.java @@ -0,0 +1,53 @@ +package com.provectus.kafka.ui.emitter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.kafka.clients.consumer.MockConsumer; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Bytes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class OffsetsInfoTest { + + final String topic = "test"; + final TopicPartition tp0 = new TopicPartition(topic, 0); //offsets: start 0, end 0 + final TopicPartition tp1 = new TopicPartition(topic, 1); //offsets: start 10, end 10 + final TopicPartition tp2 = new TopicPartition(topic, 2); //offsets: start 0, end 20 + final TopicPartition tp3 = new TopicPartition(topic, 3); //offsets: start 25, end 30 + + MockConsumer consumer; + + @BeforeEach + void initMockConsumer() { + consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); + consumer.updatePartitions( + topic, + Stream.of(tp0, tp1, tp2, tp3) + .map(tp -> new PartitionInfo(topic, tp.partition(), null, null, null, null)) + .collect(Collectors.toList())); + consumer.updateBeginningOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 0L, tp3, 25L)); + consumer.updateEndOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 20L, tp3, 30L)); + } + + @Test + void fillsInnerFieldsAccordingToTopicState() { + var offsets = new OffsetsInfo(consumer, List.of(tp0, tp1, tp2, tp3)); + + assertThat(offsets.getBeginOffsets()).containsEntry(tp0, 0L).containsEntry(tp1, 10L).containsEntry(tp2, 0L) + .containsEntry(tp3, 25L); + + assertThat(offsets.getEndOffsets()).containsEntry(tp0, 0L).containsEntry(tp1, 10L).containsEntry(tp2, 20L) + .containsEntry(tp3, 30L); + + assertThat(offsets.getEmptyPartitions()).contains(tp0, tp1); + assertThat(offsets.getNonEmptyPartitions()).contains(tp2, tp3); + } + +} \ No newline at end of file diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/SeekOperationsTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/SeekOperationsTest.java new file mode 100644 index 0000000000..affa423123 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/SeekOperationsTest.java @@ -0,0 +1,88 @@ +package com.provectus.kafka.ui.emitter; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.model.SeekTypeDTO; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.kafka.clients.consumer.MockConsumer; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Bytes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class SeekOperationsTest { + + final String topic = "test"; + final TopicPartition tp0 = new TopicPartition(topic, 0); //offsets: start 0, end 0 + final TopicPartition tp1 = new TopicPartition(topic, 1); //offsets: start 10, end 10 + final TopicPartition tp2 = new TopicPartition(topic, 2); //offsets: start 0, end 20 + final TopicPartition tp3 = new TopicPartition(topic, 3); //offsets: start 25, end 30 + + MockConsumer consumer; + + @BeforeEach + void initMockConsumer() { + consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); + consumer.updatePartitions( + topic, + Stream.of(tp0, tp1, tp2, tp3) + .map(tp -> new PartitionInfo(topic, tp.partition(), null, null, null, null)) + .collect(Collectors.toList())); + consumer.updateBeginningOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 0L, tp3, 25L)); + consumer.updateEndOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 20L, tp3, 30L)); + } + + @Nested + class GetOffsetsForSeek { + + @Test + void latest() { + var offsets = SeekOperations.getOffsetsForSeek( + consumer, + new OffsetsInfo(consumer, topic), + SeekTypeDTO.LATEST, + null + ); + assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 20L, tp3, 30L)); + } + + @Test + void beginning() { + var offsets = SeekOperations.getOffsetsForSeek( + consumer, + new OffsetsInfo(consumer, topic), + SeekTypeDTO.BEGINNING, + null + ); + assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 0L, tp3, 25L)); + } + + @Test + void offsets() { + var offsets = SeekOperations.getOffsetsForSeek( + consumer, + new OffsetsInfo(consumer, topic), + SeekTypeDTO.OFFSET, + Map.of(tp1, 10L, tp2, 10L, tp3, 26L) + ); + assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 10L, tp3, 26L)); + } + + @Test + void offsetsWithBoundsFixing() { + var offsets = SeekOperations.getOffsetsForSeek( + consumer, + new OffsetsInfo(consumer, topic), + SeekTypeDTO.OFFSET, + Map.of(tp1, 10L, tp2, 21L, tp3, 24L) + ); + assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 20L, tp3, 25L)); + } + } + +} \ No newline at end of file diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/TailingEmitterTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/TailingEmitterTest.java index 27fbda1e9d..cdb1eaaa54 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/TailingEmitterTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/TailingEmitterTest.java @@ -111,10 +111,11 @@ class TailingEmitterTest extends AbstractIntegrationTest { return applicationContext.getBean(MessagesService.class) .loadMessages(cluster, topicName, - new ConsumerPosition(SeekTypeDTO.LATEST, Map.of(), SeekDirectionDTO.TAILING), + new ConsumerPosition(SeekTypeDTO.LATEST, topic, null), query, MessageFilterTypeDTO.STRING_CONTAINS, 0, + SeekDirectionDTO.TAILING, "String", "String"); } @@ -137,7 +138,7 @@ class TailingEmitterTest extends AbstractIntegrationTest { Awaitility.await() .pollInSameThread() .pollDelay(Duration.ofMillis(100)) - .atMost(Duration.ofSeconds(10)) + .atMost(Duration.ofSeconds(200)) .until(() -> fluxOutput.stream() .anyMatch(msg -> msg.getType() == TopicMessageEventDTO.TypeEnum.CONSUMING)); } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/MessagesServiceTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/MessagesServiceTest.java index 9dfbaed0b1..8c18e20882 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/MessagesServiceTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/MessagesServiceTest.java @@ -45,7 +45,7 @@ class MessagesServiceTest extends AbstractIntegrationTest { @Test void loadMessagesReturnsExceptionWhenTopicNotFound() { StepVerifier.create(messagesService - .loadMessages(cluster, NON_EXISTING_TOPIC, null, null, null, 1, "String", "String")) + .loadMessages(cluster, NON_EXISTING_TOPIC, null, null, null, 1, null, "String", "String")) .expectError(TopicNotFoundException.class) .verify(); } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/RecordEmitterTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/RecordEmitterTest.java index e6c9b3c83a..3289d177d2 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/RecordEmitterTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/RecordEmitterTest.java @@ -1,8 +1,7 @@ package com.provectus.kafka.ui.service; -import static com.provectus.kafka.ui.model.SeekDirectionDTO.BACKWARD; -import static com.provectus.kafka.ui.model.SeekDirectionDTO.FORWARD; import static com.provectus.kafka.ui.model.SeekTypeDTO.BEGINNING; +import static com.provectus.kafka.ui.model.SeekTypeDTO.LATEST; import static com.provectus.kafka.ui.model.SeekTypeDTO.OFFSET; import static com.provectus.kafka.ui.model.SeekTypeDTO.TIMESTAMP; import static org.assertj.core.api.Assertions.assertThat; @@ -17,8 +16,6 @@ import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.builtin.StringSerde; -import com.provectus.kafka.ui.util.OffsetsSeekBackward; -import com.provectus.kafka.ui.util.OffsetsSeekForward; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -112,18 +109,15 @@ class RecordEmitterTest extends AbstractIntegrationTest { void pollNothingOnEmptyTopic() { var forwardEmitter = new ForwardRecordEmitter( this::createConsumer, - new OffsetsSeekForward(EMPTY_TOPIC, - new ConsumerPosition(BEGINNING, Map.of(), FORWARD) - ), RECORD_DESERIALIZER + new ConsumerPosition(BEGINNING, EMPTY_TOPIC, null), + RECORD_DESERIALIZER ); var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward( - EMPTY_TOPIC, - new ConsumerPosition(BEGINNING, Map.of(), BACKWARD), - 100 - ), RECORD_DESERIALIZER + new ConsumerPosition(BEGINNING, EMPTY_TOPIC, null), + 100, + RECORD_DESERIALIZER ); StepVerifier.create( @@ -143,17 +137,15 @@ class RecordEmitterTest extends AbstractIntegrationTest { void pollFullTopicFromBeginning() { var forwardEmitter = new ForwardRecordEmitter( this::createConsumer, - new OffsetsSeekForward(TOPIC, - new ConsumerPosition(BEGINNING, Map.of(), FORWARD) - ), RECORD_DESERIALIZER + new ConsumerPosition(BEGINNING, TOPIC, null), + RECORD_DESERIALIZER ); var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward(TOPIC, - new ConsumerPosition(BEGINNING, Map.of(), BACKWARD), - PARTITIONS * MSGS_PER_PARTITION - ), RECORD_DESERIALIZER + new ConsumerPosition(LATEST, TOPIC, null), + PARTITIONS * MSGS_PER_PARTITION, + RECORD_DESERIALIZER ); List expectedValues = SENT_RECORDS.stream().map(Record::getValue).collect(Collectors.toList()); @@ -172,17 +164,15 @@ class RecordEmitterTest extends AbstractIntegrationTest { var forwardEmitter = new ForwardRecordEmitter( this::createConsumer, - new OffsetsSeekForward(TOPIC, - new ConsumerPosition(OFFSET, targetOffsets, FORWARD) - ), RECORD_DESERIALIZER + new ConsumerPosition(OFFSET, TOPIC, targetOffsets), + RECORD_DESERIALIZER ); var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward(TOPIC, - new ConsumerPosition(OFFSET, targetOffsets, BACKWARD), - PARTITIONS * MSGS_PER_PARTITION - ), RECORD_DESERIALIZER + new ConsumerPosition(OFFSET, TOPIC, targetOffsets), + PARTITIONS * MSGS_PER_PARTITION, + RECORD_DESERIALIZER ); var expectedValues = SENT_RECORDS.stream() @@ -217,17 +207,15 @@ class RecordEmitterTest extends AbstractIntegrationTest { var forwardEmitter = new ForwardRecordEmitter( this::createConsumer, - new OffsetsSeekForward(TOPIC, - new ConsumerPosition(TIMESTAMP, targetTimestamps, FORWARD) - ), RECORD_DESERIALIZER + new ConsumerPosition(TIMESTAMP, TOPIC, targetTimestamps), + RECORD_DESERIALIZER ); var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward(TOPIC, - new ConsumerPosition(TIMESTAMP, targetTimestamps, BACKWARD), - PARTITIONS * MSGS_PER_PARTITION - ), RECORD_DESERIALIZER + new ConsumerPosition(TIMESTAMP, TOPIC, targetTimestamps), + PARTITIONS * MSGS_PER_PARTITION, + RECORD_DESERIALIZER ); var expectedValues = SENT_RECORDS.stream() @@ -255,10 +243,9 @@ class RecordEmitterTest extends AbstractIntegrationTest { var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward(TOPIC, - new ConsumerPosition(OFFSET, targetOffsets, BACKWARD), - numMessages - ), RECORD_DESERIALIZER + new ConsumerPosition(OFFSET, TOPIC, targetOffsets), + numMessages, + RECORD_DESERIALIZER ); var expectedValues = SENT_RECORDS.stream() @@ -281,10 +268,9 @@ class RecordEmitterTest extends AbstractIntegrationTest { var backwardEmitter = new BackwardRecordEmitter( this::createConsumer, - new OffsetsSeekBackward(TOPIC, - new ConsumerPosition(OFFSET, offsets, BACKWARD), - 100 - ), RECORD_DESERIALIZER + new ConsumerPosition(OFFSET, TOPIC, offsets), + 100, + RECORD_DESERIALIZER ); expectEmitter(backwardEmitter, @@ -331,7 +317,7 @@ class RecordEmitterTest extends AbstractIntegrationTest { final Map map = Map.of( ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers(), ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString(), - ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 20, // to check multiple polls + ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 19, // to check multiple polls ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class, ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class ); diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SendAndReadTests.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SendAndReadTests.java index 4de939e033..78c111cdd1 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SendAndReadTests.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SendAndReadTests.java @@ -502,12 +502,13 @@ public class SendAndReadTests extends AbstractIntegrationTest { topic, new ConsumerPosition( SeekTypeDTO.BEGINNING, - Map.of(new TopicPartition(topic, 0), 0L), - SeekDirectionDTO.FORWARD + topic, + Map.of(new TopicPartition(topic, 0), 0L) ), null, null, 1, + SeekDirectionDTO.FORWARD, msgToSend.getKeySerde().get(), msgToSend.getValueSerde().get() ).filter(e -> e.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/OffsetsSeekTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/OffsetsSeekTest.java deleted file mode 100644 index 54c2064c1c..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/OffsetsSeekTest.java +++ /dev/null @@ -1,196 +0,0 @@ -package com.provectus.kafka.ui.util; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.provectus.kafka.ui.model.ConsumerPosition; -import com.provectus.kafka.ui.model.SeekDirectionDTO; -import com.provectus.kafka.ui.model.SeekTypeDTO; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.MockConsumer; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.Bytes; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class OffsetsSeekTest { - - final String topic = "test"; - final TopicPartition tp0 = new TopicPartition(topic, 0); //offsets: start 0, end 0 - final TopicPartition tp1 = new TopicPartition(topic, 1); //offsets: start 10, end 10 - final TopicPartition tp2 = new TopicPartition(topic, 2); //offsets: start 0, end 20 - final TopicPartition tp3 = new TopicPartition(topic, 3); //offsets: start 25, end 30 - - MockConsumer consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); - - @BeforeEach - void initConsumer() { - consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); - consumer.updatePartitions( - topic, - Stream.of(tp0, tp1, tp2, tp3) - .map(tp -> new PartitionInfo(topic, tp.partition(), null, null, null, null)) - .collect(Collectors.toList())); - consumer.updateBeginningOffsets(Map.of( - tp0, 0L, - tp1, 10L, - tp2, 0L, - tp3, 25L - )); - consumer.updateEndOffsets(Map.of( - tp0, 0L, - tp1, 10L, - tp2, 20L, - tp3, 30L - )); - } - - @Test - void forwardSeekToBeginningAllPartitions() { - var seek = new OffsetsSeekForward( - topic, - new ConsumerPosition( - SeekTypeDTO.BEGINNING, - Map.of(tp0, 0L, tp1, 0L), - SeekDirectionDTO.FORWARD - ) - ); - - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp0, tp1); - assertThat(consumer.position(tp0)).isZero(); - assertThat(consumer.position(tp1)).isEqualTo(10L); - } - - @Test - void backwardSeekToBeginningAllPartitions() { - var seek = new OffsetsSeekBackward( - topic, - new ConsumerPosition( - SeekTypeDTO.BEGINNING, - Map.of(tp2, 0L, tp3, 0L), - SeekDirectionDTO.BACKWARD - ), - 10 - ); - - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp2, tp3); - assertThat(consumer.position(tp2)).isEqualTo(20L); - assertThat(consumer.position(tp3)).isEqualTo(30L); - } - - @Test - void forwardSeekToBeginningWithPartitionsList() { - var seek = new OffsetsSeekForward( - topic, - new ConsumerPosition(SeekTypeDTO.BEGINNING, Map.of(), SeekDirectionDTO.FORWARD)); - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp0, tp1, tp2, tp3); - assertThat(consumer.position(tp0)).isZero(); - assertThat(consumer.position(tp1)).isEqualTo(10L); - assertThat(consumer.position(tp2)).isZero(); - assertThat(consumer.position(tp3)).isEqualTo(25L); - } - - @Test - void backwardSeekToBeginningWithPartitionsList() { - var seek = new OffsetsSeekBackward( - topic, - new ConsumerPosition(SeekTypeDTO.BEGINNING, Map.of(), SeekDirectionDTO.BACKWARD), - 10 - ); - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp0, tp1, tp2, tp3); - assertThat(consumer.position(tp0)).isZero(); - assertThat(consumer.position(tp1)).isEqualTo(10L); - assertThat(consumer.position(tp2)).isEqualTo(20L); - assertThat(consumer.position(tp3)).isEqualTo(30L); - } - - - @Test - void forwardSeekToOffset() { - var seek = new OffsetsSeekForward( - topic, - new ConsumerPosition( - SeekTypeDTO.OFFSET, - Map.of(tp0, 0L, tp1, 1L, tp2, 2L), - SeekDirectionDTO.FORWARD - ) - ); - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp2); - assertThat(consumer.position(tp2)).isEqualTo(2L); - } - - @Test - void backwardSeekToOffset() { - var seek = new OffsetsSeekBackward( - topic, - new ConsumerPosition( - SeekTypeDTO.OFFSET, - Map.of(tp0, 0L, tp1, 1L, tp2, 20L), - SeekDirectionDTO.BACKWARD - ), - 2 - ); - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp2); - assertThat(consumer.position(tp2)).isEqualTo(20L); - } - - @Test - void backwardSeekToOffsetOnlyOnePartition() { - var seek = new OffsetsSeekBackward( - topic, - new ConsumerPosition( - SeekTypeDTO.OFFSET, - Map.of(tp2, 20L), - SeekDirectionDTO.BACKWARD - ), - 20 - ); - seek.assignAndSeek(consumer); - assertThat(consumer.assignment()).containsExactlyInAnyOrder(tp2); - assertThat(consumer.position(tp2)).isEqualTo(20L); - } - - - @Nested - class WaitingOffsetsTest { - - OffsetsSeekForward.WaitingOffsets offsets; - - @BeforeEach - void assignAndCreateOffsets() { - consumer.assign(List.of(tp0, tp1, tp2, tp3)); - offsets = new OffsetsSeek.WaitingOffsets(topic, consumer, List.of(tp0, tp1, tp2, tp3)); - } - - @Test - void collectsSignificantOffsetsMinus1ForAssignedPartitions() { - // offsets for partition 0 & 1 should be skipped because they - // effectively contains no data (start offset = end offset) - assertThat(offsets.getEndOffsets()).containsExactlyInAnyOrderEntriesOf( - Map.of(2, 19L, 3, 29L) - ); - } - - @Test - void returnTrueWhenOffsetsReachedReached() { - assertThat(offsets.endReached()).isFalse(); - offsets.markPolled(new ConsumerRecord<>(topic, 2, 19, null, null)); - assertThat(offsets.endReached()).isFalse(); - offsets.markPolled(new ConsumerRecord<>(topic, 3, 29, null, null)); - assertThat(offsets.endReached()).isTrue(); - } - } - -} From d60808a2f268003046da0832d8b02294ea23ca1a Mon Sep 17 00:00:00 2001 From: Ilya Kuramshin Date: Sun, 23 Oct 2022 19:49:11 +0400 Subject: [PATCH 09/32] Drop deprecated ksql api (#2796) Co-authored-by: iliax --- .../provectus/kafka/ui/client/KsqlClient.java | 53 ------ .../kafka/ui/controller/KsqlController.java | 14 +- .../kafka/ui/service/KsqlService.java | 47 ----- .../strategy/ksql/statement/BaseStrategy.java | 166 ------------------ .../ksql/statement/CreateStrategy.java | 20 --- .../ksql/statement/DescribeStrategy.java | 20 --- .../strategy/ksql/statement/DropStrategy.java | 20 --- .../ksql/statement/ExplainStrategy.java | 20 --- .../ksql/statement/SelectStrategy.java | 24 --- .../strategy/ksql/statement/ShowStrategy.java | 67 ------- .../ksql/statement/TerminateStrategy.java | 20 --- .../kafka/ui/service/KsqlServiceTest.java | 104 ----------- .../ksql/statement/CreateStrategyTest.java | 85 --------- .../ksql/statement/DescribeStrategyTest.java | 76 -------- .../ksql/statement/DropStrategyTest.java | 75 -------- .../ksql/statement/ExplainStrategyTest.java | 74 -------- .../ksql/statement/SelectStrategyTest.java | 79 --------- .../ksql/statement/ShowStrategyTest.java | 102 ----------- .../ksql/statement/TerminateStrategyTest.java | 72 -------- .../main/resources/swagger/kafka-ui-api.yaml | 62 ------- 20 files changed, 1 insertion(+), 1199 deletions(-) delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/KsqlClient.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KsqlService.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/BaseStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategy.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategy.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KsqlServiceTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategyTest.java delete mode 100644 kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategyTest.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/KsqlClient.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/KsqlClient.java deleted file mode 100644 index 8d051234ab..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/KsqlClient.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.provectus.kafka.ui.client; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.service.ksql.KsqlApiClient; -import com.provectus.kafka.ui.strategy.ksql.statement.BaseStrategy; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; -import org.springframework.web.reactive.function.BodyInserters; -import org.springframework.web.reactive.function.client.ClientResponse; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.core.publisher.Mono; - -@Service -@RequiredArgsConstructor -@Slf4j -public class KsqlClient { - private final WebClient webClient; - private final ObjectMapper mapper; - - public Mono execute(BaseStrategy ksqlStatement, KafkaCluster cluster) { - return webClient.post() - .uri(ksqlStatement.getUri()) - .headers(httpHeaders -> KsqlApiClient.setBasicAuthIfEnabled(httpHeaders, cluster)) - .accept(new MediaType("application", "vnd.ksql.v1+json")) - .body(BodyInserters.fromValue(ksqlStatement.getKsqlCommand())) - .retrieve() - .onStatus(HttpStatus::isError, this::getErrorMessage) - .bodyToMono(byte[].class) - .map(this::toJson) - .map(ksqlStatement::serializeResponse); - } - - private Mono getErrorMessage(ClientResponse response) { - return response - .bodyToMono(byte[].class) - .map(this::toJson) - .map(jsonNode -> jsonNode.get("message").asText()) - .flatMap(error -> Mono.error(new UnprocessableEntityException(error))); - } - - @SneakyThrows - private JsonNode toJson(byte[] content) { - return this.mapper.readTree(content); - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KsqlController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KsqlController.java index 62dc24fab2..c3d833e2b8 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KsqlController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KsqlController.java @@ -1,15 +1,12 @@ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.KsqlApi; -import com.provectus.kafka.ui.model.KsqlCommandDTO; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; import com.provectus.kafka.ui.model.KsqlCommandV2DTO; import com.provectus.kafka.ui.model.KsqlCommandV2ResponseDTO; import com.provectus.kafka.ui.model.KsqlResponseDTO; import com.provectus.kafka.ui.model.KsqlStreamDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableResponseDTO; -import com.provectus.kafka.ui.service.KsqlService; import com.provectus.kafka.ui.service.ksql.KsqlServiceV2; import java.util.List; import java.util.Map; @@ -27,17 +24,8 @@ import reactor.core.publisher.Mono; @RequiredArgsConstructor @Slf4j public class KsqlController extends AbstractController implements KsqlApi { - private final KsqlService ksqlService; - private final KsqlServiceV2 ksqlServiceV2; - @Override - public Mono> executeKsqlCommand(String clusterName, - Mono - ksqlCommand, - ServerWebExchange exchange) { - return ksqlService.executeKsqlCommand(getCluster(clusterName), ksqlCommand) - .map(ResponseEntity::ok); - } + private final KsqlServiceV2 ksqlServiceV2; @Override public Mono> executeKsql(String clusterName, diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KsqlService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KsqlService.java deleted file mode 100644 index 6f74ede75e..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KsqlService.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.provectus.kafka.ui.service; - -import com.provectus.kafka.ui.client.KsqlClient; -import com.provectus.kafka.ui.exception.ClusterNotFoundException; -import com.provectus.kafka.ui.exception.KsqlDbNotFoundException; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.KsqlCommandDTO; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.strategy.ksql.statement.BaseStrategy; -import java.util.List; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -@Service -@RequiredArgsConstructor -public class KsqlService { - private final KsqlClient ksqlClient; - private final List ksqlStatementStrategies; - - public Mono executeKsqlCommand(KafkaCluster cluster, - Mono ksqlCommand) { - return Mono.justOrEmpty(cluster) - .map(KafkaCluster::getKsqldbServer) - .onErrorResume(e -> { - Throwable throwable = - e instanceof ClusterNotFoundException ? e : new KsqlDbNotFoundException(); - return Mono.error(throwable); - }) - .flatMap(ksqlServer -> getStatementStrategyForKsqlCommand(ksqlCommand) - .map(statement -> statement.host(ksqlServer.getUrl())) - ) - .flatMap(baseStrategy -> ksqlClient.execute(baseStrategy, cluster)); - } - - private Mono getStatementStrategyForKsqlCommand( - Mono ksqlCommand) { - return ksqlCommand - .map(command -> ksqlStatementStrategies.stream() - .filter(s -> s.test(command.getKsql())) - .map(s -> s.ksqlCommand(command)) - .findFirst()) - .flatMap(Mono::justOrEmpty) - .switchIfEmpty(Mono.error(new UnprocessableEntityException("Invalid sql"))); - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/BaseStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/BaseStrategy.java deleted file mode 100644 index fa057116ad..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/BaseStrategy.java +++ /dev/null @@ -1,166 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandDTO; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.model.TableDTO; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -public abstract class BaseStrategy { - protected static final String KSQL_REQUEST_PATH = "/ksql"; - protected static final String QUERY_REQUEST_PATH = "/query"; - private static final String MAPPING_EXCEPTION_ERROR = "KSQL DB response mapping error"; - protected String host = null; - protected KsqlCommandDTO ksqlCommand = null; - - public String getUri() { - if (this.host != null) { - return this.host + this.getRequestPath(); - } - throw new UnprocessableEntityException("Strategy doesn't have host"); - } - - public boolean test(String sql) { - return sql.trim().toLowerCase().matches(getTestRegExp()); - } - - public BaseStrategy host(String host) { - this.host = host; - return this; - } - - public KsqlCommandDTO getKsqlCommand() { - return ksqlCommand; - } - - public BaseStrategy ksqlCommand(KsqlCommandDTO ksqlCommand) { - this.ksqlCommand = ksqlCommand; - return this; - } - - protected String getRequestPath() { - return BaseStrategy.KSQL_REQUEST_PATH; - } - - protected KsqlCommandResponseDTO serializeTableResponse(JsonNode response, String key) { - JsonNode item = getResponseFirstItemValue(response, key); - TableDTO table = item.isArray() ? getTableFromArray(item) : getTableFromObject(item); - return (new KsqlCommandResponseDTO()).data(table); - } - - protected KsqlCommandResponseDTO serializeMessageResponse(JsonNode response, String key) { - JsonNode item = getResponseFirstItemValue(response, key); - return (new KsqlCommandResponseDTO()).message(getMessageFromObject(item)); - } - - protected KsqlCommandResponseDTO serializeQueryResponse(JsonNode response) { - if (response.isArray() && response.size() > 0) { - TableDTO table = (new TableDTO()) - .headers(getQueryResponseHeader(response)) - .rows(getQueryResponseRows(response)); - return (new KsqlCommandResponseDTO()).data(table); - } - throw new UnprocessableEntityException(MAPPING_EXCEPTION_ERROR); - } - - private JsonNode getResponseFirstItemValue(JsonNode response, String key) { - if (response.isArray() && response.size() > 0) { - JsonNode first = response.get(0); - if (first.has(key)) { - return first.path(key); - } - } - throw new UnprocessableEntityException(MAPPING_EXCEPTION_ERROR); - } - - private List getQueryResponseHeader(JsonNode response) { - JsonNode headerRow = response.get(0); - if (headerRow.isObject() && headerRow.has("header")) { - String schema = headerRow.get("header").get("schema").asText(); - return Arrays.stream(schema.split(",")).map(String::trim).collect(Collectors.toList()); - } - return new ArrayList<>(); - } - - private List> getQueryResponseRows(JsonNode node) { - return getStreamForJsonArray(node) - .filter(row -> row.has("row") && row.get("row").has("columns")) - .map(row -> row.get("row").get("columns")) - .map(cellNode -> getStreamForJsonArray(cellNode) - .map(JsonNode::asText) - .collect(Collectors.toList()) - ) - .collect(Collectors.toList()); - } - - private TableDTO getTableFromArray(JsonNode node) { - TableDTO table = new TableDTO(); - table.headers(new ArrayList<>()).rows(new ArrayList<>()); - if (node.size() > 0) { - List keys = getJsonObjectKeys(node.get(0)); - List> rows = getTableRows(node, keys); - table.headers(keys).rows(rows); - } - return table; - } - - private TableDTO getTableFromObject(JsonNode node) { - List keys = getJsonObjectKeys(node); - List values = getJsonObjectValues(node); - List> rows = IntStream - .range(0, keys.size()) - .mapToObj(i -> List.of(keys.get(i), values.get(i))) - .collect(Collectors.toList()); - return (new TableDTO()).headers(List.of("key", "value")).rows(rows); - } - - private String getMessageFromObject(JsonNode node) { - if (node.isObject() && node.has("message")) { - return node.get("message").asText(); - } - throw new UnprocessableEntityException(MAPPING_EXCEPTION_ERROR); - } - - private List> getTableRows(JsonNode node, List keys) { - return getStreamForJsonArray(node) - .map(row -> keys.stream() - .map(header -> row.get(header).asText()) - .collect(Collectors.toList()) - ) - .collect(Collectors.toList()); - } - - private Stream getStreamForJsonArray(JsonNode node) { - if (node.isArray() && node.size() > 0) { - return StreamSupport.stream(node.spliterator(), false); - } - throw new UnprocessableEntityException(MAPPING_EXCEPTION_ERROR); - } - - private List getJsonObjectKeys(JsonNode node) { - if (node.isObject()) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(node.fieldNames(), Spliterator.ORDERED), false - ).collect(Collectors.toList()); - } - throw new UnprocessableEntityException(MAPPING_EXCEPTION_ERROR); - } - - private List getJsonObjectValues(JsonNode node) { - return getJsonObjectKeys(node).stream().map(key -> node.get(key).asText()) - .collect(Collectors.toList()); - } - - public abstract KsqlCommandResponseDTO serializeResponse(JsonNode response); - - protected abstract String getTestRegExp(); -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategy.java deleted file mode 100644 index d26046a0fd..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategy.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class CreateStrategy extends BaseStrategy { - private static final String RESPONSE_VALUE_KEY = "commandStatus"; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeMessageResponse(response, RESPONSE_VALUE_KEY); - } - - @Override - protected String getTestRegExp() { - return "create (table|stream)(.*)(with|as select(.*)from)(.*);"; - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategy.java deleted file mode 100644 index b8d7435bad..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategy.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class DescribeStrategy extends BaseStrategy { - private static final String RESPONSE_VALUE_KEY = "sourceDescription"; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeTableResponse(response, RESPONSE_VALUE_KEY); - } - - @Override - protected String getTestRegExp() { - return "describe (.*);"; - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategy.java deleted file mode 100644 index 95b6884dc1..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategy.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class DropStrategy extends BaseStrategy { - private static final String RESPONSE_VALUE_KEY = "commandStatus"; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeMessageResponse(response, RESPONSE_VALUE_KEY); - } - - @Override - protected String getTestRegExp() { - return "drop (table|stream) (.*);"; - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategy.java deleted file mode 100644 index 221113b7e8..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategy.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class ExplainStrategy extends BaseStrategy { - private static final String RESPONSE_VALUE_KEY = "queryDescription"; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeTableResponse(response, RESPONSE_VALUE_KEY); - } - - @Override - protected String getTestRegExp() { - return "explain (.*);"; - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategy.java deleted file mode 100644 index e535c8107d..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategy.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class SelectStrategy extends BaseStrategy { - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeQueryResponse(response); - } - - @Override - protected String getRequestPath() { - return BaseStrategy.QUERY_REQUEST_PATH; - } - - @Override - protected String getTestRegExp() { - return "select (.*) from (.*);"; - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategy.java deleted file mode 100644 index 93c635b044..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategy.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandDTO; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import java.util.List; -import java.util.Optional; -import org.springframework.stereotype.Component; - -@Component -public class ShowStrategy extends BaseStrategy { - private static final List SHOW_STATEMENTS = - List.of("functions", "topics", "streams", "tables", "queries", "properties"); - private static final List LIST_STATEMENTS = - List.of("functions", "topics", "streams", "tables"); - private String responseValueKey = ""; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeTableResponse(response, responseValueKey); - } - - @Override - public boolean test(String sql) { - Optional statement = SHOW_STATEMENTS.stream() - .filter(s -> testSql(sql, getShowRegExp(s)) || testSql(sql, getListRegExp(s))) - .findFirst(); - if (statement.isPresent()) { - setResponseValueKey(statement.get()); - return true; - } - return false; - } - - @Override - protected String getTestRegExp() { - return ""; - } - - @Override - public BaseStrategy ksqlCommand(KsqlCommandDTO ksqlCommand) { - // return new instance to avoid conflicts for parallel requests - ShowStrategy clone = new ShowStrategy(); - clone.setResponseValueKey(responseValueKey); - clone.ksqlCommand = ksqlCommand; - return clone; - } - - protected String getShowRegExp(String key) { - return "show " + key + ";"; - } - - protected String getListRegExp(String key) { - if (LIST_STATEMENTS.contains(key)) { - return "list " + key + ";"; - } - return ""; - } - - private void setResponseValueKey(String path) { - responseValueKey = path; - } - - private boolean testSql(String sql, String pattern) { - return sql.trim().toLowerCase().matches(pattern); - } -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategy.java deleted file mode 100644 index b043b8c6c9..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategy.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import com.fasterxml.jackson.databind.JsonNode; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import org.springframework.stereotype.Component; - -@Component -public class TerminateStrategy extends BaseStrategy { - private static final String RESPONSE_VALUE_KEY = "commandStatus"; - - @Override - public KsqlCommandResponseDTO serializeResponse(JsonNode response) { - return serializeMessageResponse(response, RESPONSE_VALUE_KEY); - } - - @Override - protected String getTestRegExp() { - return "terminate (.*);"; - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KsqlServiceTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KsqlServiceTest.java deleted file mode 100644 index ed434efba4..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KsqlServiceTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.provectus.kafka.ui.service; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.provectus.kafka.ui.client.KsqlClient; -import com.provectus.kafka.ui.exception.KsqlDbNotFoundException; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.InternalKsqlServer; -import com.provectus.kafka.ui.model.KafkaCluster; -import com.provectus.kafka.ui.model.KsqlCommandDTO; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.strategy.ksql.statement.BaseStrategy; -import com.provectus.kafka.ui.strategy.ksql.statement.DescribeStrategy; -import com.provectus.kafka.ui.strategy.ksql.statement.ShowStrategy; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import reactor.core.publisher.Mono; -import reactor.test.StepVerifier; - -@ExtendWith(MockitoExtension.class) -class KsqlServiceTest { - private KsqlService ksqlService; - private BaseStrategy baseStrategy; - private BaseStrategy alternativeStrategy; - - @Mock - private ClustersStorage clustersStorage; - @Mock - private KsqlClient ksqlClient; - - - @BeforeEach - public void setUp() { - this.baseStrategy = new ShowStrategy(); - this.alternativeStrategy = new DescribeStrategy(); - this.ksqlService = new KsqlService( - this.ksqlClient, - List.of(baseStrategy, alternativeStrategy) - ); - } - - @Test - void shouldThrowKsqlDbNotFoundExceptionOnExecuteKsqlCommand() { - KsqlCommandDTO command = (new KsqlCommandDTO()).ksql("show streams;"); - KafkaCluster kafkaCluster = Mockito.mock(KafkaCluster.class); - when(kafkaCluster.getKsqldbServer()).thenReturn(null); - - StepVerifier.create(ksqlService.executeKsqlCommand(kafkaCluster, Mono.just(command))) - .verifyError(KsqlDbNotFoundException.class); - } - - @Test - void shouldThrowUnprocessableEntityExceptionOnExecuteKsqlCommand() { - KsqlCommandDTO command = - (new KsqlCommandDTO()).ksql("CREATE STREAM users WITH (KAFKA_TOPIC='users');"); - KafkaCluster kafkaCluster = Mockito.mock(KafkaCluster.class); - when(kafkaCluster.getKsqldbServer()).thenReturn(InternalKsqlServer.builder().url("localhost:8088").build()); - - StepVerifier.create(ksqlService.executeKsqlCommand(kafkaCluster, Mono.just(command))) - .verifyError(UnprocessableEntityException.class); - - StepVerifier.create(ksqlService.executeKsqlCommand(kafkaCluster, Mono.just(command))) - .verifyErrorMessage("Invalid sql"); - } - - @Test - void shouldSetHostToStrategy() { - String host = "localhost:8088"; - KsqlCommandDTO command = (new KsqlCommandDTO()).ksql("describe streams;"); - KafkaCluster kafkaCluster = Mockito.mock(KafkaCluster.class); - - when(kafkaCluster.getKsqldbServer()).thenReturn(InternalKsqlServer.builder().url(host).build()); - when(ksqlClient.execute(any(), any())).thenReturn(Mono.just(new KsqlCommandResponseDTO())); - - ksqlService.executeKsqlCommand(kafkaCluster, Mono.just(command)).block(); - assertThat(alternativeStrategy.getUri()).isEqualTo(host + "/ksql"); - } - - @Test - void shouldCallClientAndReturnResponse() { - KsqlCommandDTO command = (new KsqlCommandDTO()).ksql("describe streams;"); - KafkaCluster kafkaCluster = Mockito.mock(KafkaCluster.class); - KsqlCommandResponseDTO response = new KsqlCommandResponseDTO().message("success"); - - when(kafkaCluster.getKsqldbServer()).thenReturn(InternalKsqlServer.builder().url("host").build()); - when(ksqlClient.execute(any(), any())).thenReturn(Mono.just(response)); - - KsqlCommandResponseDTO receivedResponse = - ksqlService.executeKsqlCommand(kafkaCluster, Mono.just(command)).block(); - verify(ksqlClient, times(1)).execute(eq(alternativeStrategy), any()); - assertThat(receivedResponse).isEqualTo(response); - - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategyTest.java deleted file mode 100644 index 257fb36d35..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/CreateStrategyTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class CreateStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private CreateStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new CreateStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("CREATE STREAM stream WITH (KAFKA_TOPIC='topic');")); - assertTrue(strategy.test("CREATE STREAM stream" - + " AS SELECT users.id AS userid FROM users EMIT CHANGES;" - )); - assertTrue(strategy.test( - "CREATE TABLE table (id VARCHAR) WITH (KAFKA_TOPIC='table');" - )); - assertTrue(strategy.test( - "CREATE TABLE pageviews_regions WITH (KEY_FORMAT='JSON')" - + " AS SELECT gender, COUNT(*) AS numbers" - + " FROM pageviews EMIT CHANGES;" - )); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("show streams;")); - assertFalse(strategy.test("show tables;")); - assertFalse(strategy.test("CREATE TABLE test;")); - assertFalse(strategy.test("CREATE STREAM test;")); - } - - @Test - void shouldSerializeResponse() { - String message = "updated successful"; - JsonNode node = getResponseWithMessage(message); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - assertThat(serializedResponse.getMessage()).isEqualTo(message); - - } - - @Test - void shouldSerializeWithException() { - JsonNode commandStatusNode = mapper.createObjectNode().put("commandStatus", "nodeWithMessage"); - JsonNode node = mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithMessage(String message) { - JsonNode nodeWithMessage = mapper.createObjectNode().put("message", message); - JsonNode commandStatusNode = mapper.createObjectNode().set("commandStatus", nodeWithMessage); - return mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategyTest.java deleted file mode 100644 index 51cb0c742a..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DescribeStrategyTest.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.model.TableDTO; -import java.util.List; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class DescribeStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private DescribeStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new DescribeStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("DESCRIBE users;")); - assertTrue(strategy.test("DESCRIBE EXTENDED users;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("list streams;")); - assertFalse(strategy.test("show tables;")); - } - - @Test - void shouldSerializeResponse() { - JsonNode node = getResponseWithObjectNode(); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - TableDTO table = serializedResponse.getData(); - assertThat(table.getHeaders()).isEqualTo(List.of("key", "value")); - assertThat(table.getRows()).isEqualTo(List.of(List.of("name", "kafka"))); - } - - @Test - void shouldSerializeWithException() { - JsonNode sourceDescriptionNode = - mapper.createObjectNode().put("sourceDescription", "nodeWithMessage"); - JsonNode node = mapper.createArrayNode().add(mapper.valueToTree(sourceDescriptionNode)); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithObjectNode() { - JsonNode nodeWithMessage = mapper.createObjectNode().put("name", "kafka"); - JsonNode nodeWithResponse = mapper.createObjectNode().set("sourceDescription", nodeWithMessage); - return mapper.createArrayNode().add(mapper.valueToTree(nodeWithResponse)); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategyTest.java deleted file mode 100644 index 5f2b8fcc84..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/DropStrategyTest.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class DropStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private DropStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new DropStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("drop table table1;")); - assertTrue(strategy.test("drop stream stream2;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("show streams;")); - assertFalse(strategy.test("show tables;")); - assertFalse(strategy.test("create table test;")); - assertFalse(strategy.test("create stream test;")); - } - - @Test - void shouldSerializeResponse() { - String message = "updated successful"; - JsonNode node = getResponseWithMessage(message); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - assertThat(serializedResponse.getMessage()).isEqualTo(message); - - } - - @Test - void shouldSerializeWithException() { - JsonNode commandStatusNode = mapper.createObjectNode().put("commandStatus", "nodeWithMessage"); - JsonNode node = mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithMessage(String message) { - JsonNode nodeWithMessage = mapper.createObjectNode().put("message", message); - JsonNode commandStatusNode = mapper.createObjectNode().set("commandStatus", nodeWithMessage); - return mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategyTest.java deleted file mode 100644 index 2582abedbf..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ExplainStrategyTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.model.TableDTO; -import java.util.List; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class ExplainStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private ExplainStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new ExplainStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("explain users_query_id;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("show queries;")); - } - - @Test - void shouldSerializeResponse() { - JsonNode node = getResponseWithObjectNode(); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - TableDTO table = serializedResponse.getData(); - assertThat(table.getHeaders()).isEqualTo(List.of("key", "value")); - assertThat(table.getRows()).isEqualTo(List.of(List.of("name", "kafka"))); - } - - @Test - void shouldSerializeWithException() { - JsonNode sourceDescriptionNode = - mapper.createObjectNode().put("sourceDescription", "nodeWithMessage"); - JsonNode node = mapper.createArrayNode().add(mapper.valueToTree(sourceDescriptionNode)); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithObjectNode() { - JsonNode nodeWithMessage = mapper.createObjectNode().put("name", "kafka"); - JsonNode nodeWithResponse = mapper.createObjectNode().set("queryDescription", nodeWithMessage); - return mapper.createArrayNode().add(mapper.valueToTree(nodeWithResponse)); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategyTest.java deleted file mode 100644 index efeb87d584..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/SelectStrategyTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.model.TableDTO; -import java.util.List; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class SelectStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private SelectStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new SelectStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/query"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("select * from users;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("show streams;")); - assertFalse(strategy.test("select *;")); - } - - @Test - void shouldSerializeResponse() { - JsonNode node = getResponseWithData(); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - TableDTO table = serializedResponse.getData(); - assertThat(table.getHeaders()).isEqualTo(List.of("header1", "header2")); - assertThat(table.getRows()).isEqualTo(List.of(List.of("value1", "value2"))); - } - - @Test - void shouldSerializeWithException() { - JsonNode node = mapper.createObjectNode(); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithData() { - JsonNode headerNode = mapper.createObjectNode().set( - "header", mapper.createObjectNode().put("schema", "header1, header2") - ); - JsonNode row = mapper.createObjectNode().set( - "row", mapper.createObjectNode().set( - "columns", mapper.createArrayNode().add("value1").add("value2") - ) - ); - return mapper.createArrayNode().add(headerNode).add(row); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategyTest.java deleted file mode 100644 index 3b12afa71a..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/ShowStrategyTest.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import com.provectus.kafka.ui.model.TableDTO; -import java.util.List; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DynamicTest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestFactory; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class ShowStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private ShowStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new ShowStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("SHOW STREAMS;")); - assertTrue(strategy.test("SHOW TABLES;")); - assertTrue(strategy.test("SHOW TOPICS;")); - assertTrue(strategy.test("SHOW QUERIES;")); - assertTrue(strategy.test("SHOW PROPERTIES;")); - assertTrue(strategy.test("SHOW FUNCTIONS;")); - assertTrue(strategy.test("LIST STREAMS;")); - assertTrue(strategy.test("LIST TABLES;")); - assertTrue(strategy.test("LIST TOPICS;")); - assertTrue(strategy.test("LIST FUNCTIONS;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("LIST QUERIES;")); - assertFalse(strategy.test("LIST PROPERTIES;")); - } - - @TestFactory - public Iterable shouldSerialize() { - return List.of( - shouldSerializeGenerate("streams", "show streams;"), - shouldSerializeGenerate("tables", "show tables;"), - shouldSerializeGenerate("topics", "show topics;"), - shouldSerializeGenerate("properties", "show properties;"), - shouldSerializeGenerate("functions", "show functions;"), - shouldSerializeGenerate("queries", "show queries;") - ); - } - - public DynamicTest shouldSerializeGenerate(final String key, final String sql) { - return DynamicTest.dynamicTest("Should serialize " + key, - () -> { - JsonNode node = getResponseWithData(key); - strategy.test(sql); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - TableDTO table = serializedResponse.getData(); - assertThat(table.getHeaders()).isEqualTo(List.of("header")); - assertThat(table.getRows()).isEqualTo(List.of(List.of("value"))); - } - ); - } - - @Test - void shouldSerializeWithException() { - JsonNode node = getResponseWithData("streams"); - strategy.test("show tables;"); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithData(String key) { - JsonNode nodeWithDataItem = mapper.createObjectNode().put("header", "value"); - JsonNode nodeWithData = mapper.createArrayNode().add(nodeWithDataItem); - JsonNode nodeWithResponse = mapper.createObjectNode().set(key, nodeWithData); - return mapper.createArrayNode().add(mapper.valueToTree(nodeWithResponse)); - } -} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategyTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategyTest.java deleted file mode 100644 index 2f3b8756a1..0000000000 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/strategy/ksql/statement/TerminateStrategyTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.provectus.kafka.ui.strategy.ksql.statement; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.provectus.kafka.ui.exception.UnprocessableEntityException; -import com.provectus.kafka.ui.model.KsqlCommandResponseDTO; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class TerminateStrategyTest { - private final ObjectMapper mapper = new ObjectMapper(); - private TerminateStrategy strategy; - - @BeforeEach - void setUp() { - strategy = new TerminateStrategy(); - } - - @Test - void shouldReturnUri() { - strategy.host("ksqldb-server:8088"); - assertThat(strategy.getUri()).isEqualTo("ksqldb-server:8088/ksql"); - } - - @Test - void shouldReturnTrueInTest() { - assertTrue(strategy.test("terminate query_id;")); - } - - @Test - void shouldReturnFalseInTest() { - assertFalse(strategy.test("show streams;")); - assertFalse(strategy.test("create table test;")); - } - - @Test - void shouldSerializeResponse() { - String message = "query terminated."; - JsonNode node = getResponseWithMessage(message); - KsqlCommandResponseDTO serializedResponse = strategy.serializeResponse(node); - assertThat(serializedResponse.getMessage()).isEqualTo(message); - - } - - @Test - void shouldSerializeWithException() { - JsonNode commandStatusNode = mapper.createObjectNode().put("commandStatus", "nodeWithMessage"); - JsonNode node = mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - Exception exception = assertThrows( - UnprocessableEntityException.class, - () -> strategy.serializeResponse(node) - ); - - assertThat(exception.getMessage()).isEqualTo("KSQL DB response mapping error"); - } - - @SneakyThrows - private JsonNode getResponseWithMessage(String message) { - JsonNode nodeWithMessage = mapper.createObjectNode().put("message", message); - JsonNode commandStatusNode = mapper.createObjectNode().set("commandStatus", nodeWithMessage); - return mapper.createArrayNode().add(mapper.valueToTree(commandStatusNode)); - } -} diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index d07c24a61b..55ea795f03 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -1561,31 +1561,6 @@ paths: 200: description: OK - /api/clusters/{clusterName}/ksql: - description: Deprecated - use ksql/v2 instead! - post: - tags: - - Ksql - summary: executeKsqlCommand - operationId: executeKsqlCommand - parameters: - - name: clusterName - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/KsqlCommand' - responses: - 200: - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/KsqlCommandResponse' /api/clusters/{clusterName}/ksql/v2: post: @@ -2986,18 +2961,6 @@ components: items: $ref: '#/components/schemas/ConnectorPluginConfig' - KsqlCommand: - type: object - properties: - ksql: - type: string - streamsProperties: - type: object - additionalProperties: - type: string - required: - - ksql - KsqlCommandV2: type: object properties: @@ -3044,31 +3007,6 @@ components: valueFormat: type: string - KsqlCommandResponse: - type: object - properties: - data: - $ref: '#/components/schemas/Table' - message: - type: string - - Table: - type: object - properties: - headers: - type: array - items: - type: string - rows: - type: array - items: - type: array - items: - type: string - required: - - headers - - rows - KsqlResponse: type: object properties: From a2e87cc8d5ba21e2b01f2f240926298f382db63c Mon Sep 17 00:00:00 2001 From: Vlad Senyuta <66071557+VladSenyuta@users.noreply.github.com> Date: Mon, 24 Oct 2022 11:13:06 +0300 Subject: [PATCH 10/32] [e2e] Fix openTopic() method to wait element (#2809) --- .../provectus/kafka/ui/pages/connector/KafkaConnectList.java | 3 ++- .../provectus/kafka/ui/pages/schema/SchemaRegistryList.java | 3 ++- .../java/com/provectus/kafka/ui/pages/topic/TopicsList.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connector/KafkaConnectList.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connector/KafkaConnectList.java index 6f878bdd2e..36a905d2d9 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connector/KafkaConnectList.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connector/KafkaConnectList.java @@ -35,7 +35,8 @@ public class KafkaConnectList { @Step public KafkaConnectList openConnector(String connectorName) { - $x(String.format(tabElementLocator,connectorName)).shouldBe(Condition.visible).click(); + $x(String.format(tabElementLocator,connectorName)) + .shouldBe(Condition.enabled).click(); return this; } diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schema/SchemaRegistryList.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schema/SchemaRegistryList.java index 9638d830f3..b79fa307c4 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schema/SchemaRegistryList.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schema/SchemaRegistryList.java @@ -30,7 +30,8 @@ public class SchemaRegistryList { @Step public SchemaRegistryList openSchema(String schemaName) { - $x(String.format(schemaTabElementLocator,schemaName)).shouldBe(Condition.visible).click(); + $x(String.format(schemaTabElementLocator,schemaName)) + .shouldBe(Condition.enabled).click(); return this; } diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicsList.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicsList.java index 2f13933484..793353cfad 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicsList.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topic/TopicsList.java @@ -42,7 +42,8 @@ public class TopicsList { @Step public TopicsList openTopic(String topicName) { - $(By.linkText(topicName)).click(); + $(By.linkText(topicName)) + .shouldBe(Condition.enabled).click(); return this; } } From dabe2878c1610fc6631d3ab81bb7aa7048617a47 Mon Sep 17 00:00:00 2001 From: Narekmat <47845266+Narekmat@users.noreply.github.com> Date: Mon, 24 Oct 2022 23:18:18 +0400 Subject: [PATCH 11/32] Fix helm charts validation (#2797) --- .github/workflows/helm.yaml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index d8c2875aed..39a21548b6 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -19,11 +19,8 @@ jobs: - name: Check version shell: bash run: | - git fetch - git checkout master - helm_version_old=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') - git checkout $GITHUB_HEAD_REF helm_version_new=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') + helm_version_old=$(curl -s https://raw.githubusercontent.com/provectus/kafka-ui/master/charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}' ) echo $helm_version_old echo $helm_version_new if [[ "$helm_version_new" > "$helm_version_old" ]]; then exit 0 ; else exit 1 ; fi @@ -39,17 +36,3 @@ jobs: echo $version; helm template --kube-version $version --set ingress.enabled=true charts/kafka-ui -f charts/kafka-ui/values.yaml | kubeval --additional-schema-locations https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master --strict -v $version; done - #check, was helm version increased in Chart.yaml? - - name: Check version - shell: bash - run: | - git fetch - git checkout master - helm_version_old=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') - git checkout $GITHUB_HEAD_REF - helm_version_new=$(cat charts/kafka-ui/Chart.yaml | grep version | awk '{print $2}') - echo $helm_version_old - echo $helm_version_new - if [[ "$helm_version_new" > "$helm_version_old" ]]; then exit 0 ; else exit 1 ; fi - - From f7f2e1dd1f6a146d7371e77bd497f82f21b03444 Mon Sep 17 00:00:00 2001 From: Hrant Abrahamyan <113341474+habrahamyanpro@users.noreply.github.com> Date: Mon, 24 Oct 2022 23:32:51 +0400 Subject: [PATCH 12/32] Fix wrong actuator URL with custom path (#2782) Co-authored-by: Oleg Shur --- kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts b/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts index 8373bf53d3..7e1835d907 100644 --- a/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts +++ b/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts @@ -2,9 +2,10 @@ import { useQuery } from '@tanstack/react-query'; import { BASE_PARAMS, QUERY_REFETCH_OFF_OPTIONS } from 'lib/constants'; const fetchActuatorInfo = async () => { - const data = await fetch('/actuator/info', BASE_PARAMS).then((res) => - res.json() - ); + const data = await fetch( + `${BASE_PARAMS.basePath}/actuator/info`, + BASE_PARAMS + ).then((res) => res.json()); return data; }; From 5a67adbf3e8fd5f71957b847a162d29fc3e5c12f Mon Sep 17 00:00:00 2001 From: David <58771979+David-DB88@users.noreply.github.com> Date: Mon, 24 Oct 2022 23:34:20 +0400 Subject: [PATCH 13/32] Removed @types/yup package due to being deprecated also updated yup version to 0.32.11 (#2799) Co-authored-by: davitbejanyan Co-authored-by: Oleg Shur --- kafka-ui-react-app/package.json | 3 +-- kafka-ui-react-app/pnpm-lock.yaml | 8 +------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/kafka-ui-react-app/package.json b/kafka-ui-react-app/package.json index 22c9047db4..7420b8bddf 100644 --- a/kafka-ui-react-app/package.json +++ b/kafka-ui-react-app/package.json @@ -16,7 +16,6 @@ "@tanstack/react-table": "^8.5.10", "@testing-library/react": "^13.2.0", "@types/testing-library__jest-dom": "^5.14.5", - "@types/yup": "^0.29.13", "@vitejs/plugin-react": "^2.0.0", "ace-builds": "^1.7.1", "ajv": "^8.6.3", @@ -47,7 +46,7 @@ "vite": "^3.0.2", "vite-tsconfig-paths": "^3.5.0", "whatwg-fetch": "^3.6.2", - "yup": "^0.32.9", + "yup": "^0.32.11", "zustand": "^4.1.1" }, "lint-staged": { diff --git a/kafka-ui-react-app/pnpm-lock.yaml b/kafka-ui-react-app/pnpm-lock.yaml index c789e1b048..aabb45ce34 100644 --- a/kafka-ui-react-app/pnpm-lock.yaml +++ b/kafka-ui-react-app/pnpm-lock.yaml @@ -30,7 +30,6 @@ specifiers: '@types/react-router-dom': ^5.3.3 '@types/styled-components': ^5.1.13 '@types/testing-library__jest-dom': ^5.14.5 - '@types/yup': ^0.29.13 '@typescript-eslint/eslint-plugin': ^5.29.0 '@typescript-eslint/parser': ^5.29.0 '@vitejs/plugin-react': ^2.0.0 @@ -88,7 +87,7 @@ specifiers: vite: ^3.0.2 vite-tsconfig-paths: ^3.5.0 whatwg-fetch: ^3.6.2 - yup: ^0.32.9 + yup: ^0.32.11 zustand: ^4.1.1 dependencies: @@ -104,7 +103,6 @@ dependencies: '@tanstack/react-table': 8.5.10_ef5jwxihqo6n7gxfmzogljlgcm '@testing-library/react': 13.2.0_ef5jwxihqo6n7gxfmzogljlgcm '@types/testing-library__jest-dom': 5.14.5 - '@types/yup': 0.29.13 '@vitejs/plugin-react': 2.0.0_vite@3.0.2 ace-builds: 1.7.1 ajv: 8.8.2 @@ -3546,10 +3544,6 @@ packages: dependencies: '@types/yargs-parser': 20.2.0 - /@types/yup/0.29.13: - resolution: {integrity: sha512-qRyuv+P/1t1JK1rA+elmK1MmCL1BapEzKKfbEhDBV/LMMse4lmhZ/XbgETI39JveDJRpLjmToOI6uFtMW/WR2g==} - dev: false - /@typescript-eslint/eslint-plugin/5.29.0_uaxwak76nssfibsnotx5epygnu: resolution: {integrity: sha512-kgTsISt9pM53yRFQmLZ4npj99yGl3x3Pl7z4eA66OuTzAGC4bQB5H5fuLwPnqTKU3yyrrg4MIhjF17UYnL4c0w==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} From 1b8ee3b6723d2159217c39a43166e96035c4c311 Mon Sep 17 00:00:00 2001 From: David <58771979+David-DB88@users.noreply.github.com> Date: Tue, 25 Oct 2022 23:14:40 +0400 Subject: [PATCH 14/32] [UI] Topic search not working properly with pagination #2705 (#2766) * added reset page to a search component * changed search file Co-authored-by: davitbejanyan Co-authored-by: Oleg Shur Co-authored-by: Roman Zabaluev --- kafka-ui-react-app/src/components/common/Search/Search.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kafka-ui-react-app/src/components/common/Search/Search.tsx b/kafka-ui-react-app/src/components/common/Search/Search.tsx index 9baf1ed3b2..66c0e95030 100644 --- a/kafka-ui-react-app/src/components/common/Search/Search.tsx +++ b/kafka-ui-react-app/src/components/common/Search/Search.tsx @@ -22,6 +22,9 @@ const Search: React.FC = ({ onChange(e.target.value); } else { searchParams.set('q', e.target.value); + if (searchParams.get('page')) { + searchParams.set('page', '1'); + } setSearchParams(searchParams); } }, 500); From a12380100e061b55c6e4636e2899da0da38006b7 Mon Sep 17 00:00:00 2001 From: Artem Tanyhin <71770433+devils2ndself@users.noreply.github.com> Date: Tue, 25 Oct 2022 15:41:09 -0400 Subject: [PATCH 15/32] [UI] Fix ksql panes overlap (#2804) Co-authored-by: Roman Zabaluev --- .../components/KsqlDb/Query/QueryForm/QueryForm.styled.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/QueryForm.styled.ts b/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/QueryForm.styled.ts index a8fa1cf7af..3fc59fcecf 100644 --- a/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/QueryForm.styled.ts +++ b/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/QueryForm.styled.ts @@ -34,17 +34,22 @@ export const StreamPropertiesContainer = styled.label` `; export const InputsContainer = styled.div` + overflow: hidden; + width: 100%; display: flex; justify-content: center; gap: 10px; `; export const StreamPropertiesInputWrapper = styled.div` + & { + width: 100%; + } & > input { + width: 100%; height: 40px; border: 1px solid grey; border-radius: 4px; - min-width: 300px; font-size: 16px; padding-left: 15px; } From 25111085be6712aaab02a69384bc2c65b6c03a94 Mon Sep 17 00:00:00 2001 From: Narekmat <47845266+Narekmat@users.noreply.github.com> Date: Tue, 25 Oct 2022 23:43:44 +0400 Subject: [PATCH 16/32] Change app version in charts.yaml (#2821) * change app version in charts.yaml * change setup-helm-version --- .github/workflows/helm.yaml | 2 +- charts/kafka-ui/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index 39a21548b6..b8c88a4305 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Helm tool installer - uses: Azure/setup-helm@v1 + uses: Azure/setup-helm@v3 - name: Setup Kubeval uses: lra/setup-kubeval@v1.0.1 #check, was helm version increased in Chart.yaml? diff --git a/charts/kafka-ui/Chart.yaml b/charts/kafka-ui/Chart.yaml index e0d9b0b161..8998267401 100644 --- a/charts/kafka-ui/Chart.yaml +++ b/charts/kafka-ui/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: kafka-ui description: A Helm chart for kafka-UI type: application -version: 0.4.3 -appVersion: latest +version: 0.4.4 +appVersion: 0.4.0 icon: https://github.com/provectus/kafka-ui/raw/master/documentation/images/kafka-ui-logo.png From e87178136c86be37dbf1ff08eb74142dd53ccd99 Mon Sep 17 00:00:00 2001 From: Aditya Bhattad <93488388+adityabhattad2021@users.noreply.github.com> Date: Wed, 26 Oct 2022 01:22:19 +0530 Subject: [PATCH 17/32] [UI] Changed Content to Value and updated a Test (#2812) * Changed Content to Value and Update a Test * Corrected content->value in tests Co-authored-by: Roman Zabaluev --- .../Topic/Messages/MessageContent/MessageContent.tsx | 4 ++-- .../MessageContent/__tests__/MessageContent.spec.tsx | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx index 6e385916b5..85f2dcd27c 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx @@ -82,7 +82,7 @@ const MessageContent: React.FC = ({ type="button" onClick={handleContentTabClick} > - Content + Value = ({ - Content + Value {messageContentFormat} diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx index 8159a8f727..e7fbd2b44a 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx @@ -61,7 +61,7 @@ describe('MessageContent screen', () => { }); describe('when switched to display the headers', () => { - it('makes Headers tab active', () => { + it('makes headers tab active', () => { userEvent.click(screen.getByText('Headers')); expect(screen.getByText('Headers')).toHaveStyleRule( 'background-color', @@ -70,9 +70,9 @@ describe('MessageContent screen', () => { }); }); - describe('when switched to display the content', () => { - it('makes content tab active', () => { - const contentTab = screen.getAllByText('Content'); + describe('when switched to display the value', () => { + it('makes value tab active', () => { + const contentTab = screen.getAllByText('Value'); userEvent.click(contentTab[0]); expect(contentTab[0]).toHaveStyleRule( 'background-color', From e67d940981612753b406aa6273514c0b37c04788 Mon Sep 17 00:00:00 2001 From: Hrant Abrahamyan <113341474+habrahamyanpro@users.noreply.github.com> Date: Wed, 26 Oct 2022 12:07:03 +0400 Subject: [PATCH 18/32] added shouldDirty (#2776) --- .../src/components/Topics/shared/Form/TimeToRetainBtn.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kafka-ui-react-app/src/components/Topics/shared/Form/TimeToRetainBtn.tsx b/kafka-ui-react-app/src/components/Topics/shared/Form/TimeToRetainBtn.tsx index 2364d90516..8746eef792 100644 --- a/kafka-ui-react-app/src/components/Topics/shared/Form/TimeToRetainBtn.tsx +++ b/kafka-ui-react-app/src/components/Topics/shared/Form/TimeToRetainBtn.tsx @@ -18,7 +18,11 @@ const TimeToRetainBtn: React.FC = ({ inputName, text, value }) => { setValue(inputName, value)} + onClick={() => + setValue(inputName, value, { + shouldDirty: true, + }) + } > {text} From b43ba68405dabd5ebf125fbe3681e2947f17e83c Mon Sep 17 00:00:00 2001 From: David <58771979+David-DB88@users.noreply.github.com> Date: Wed, 26 Oct 2022 23:15:25 +0400 Subject: [PATCH 19/32] Testing library update to 14.4.3 (#2741) * Update RTL to v14.4.3 * fixed test cases QueryForm.spec.tsx, SchemaVersion.spec.tsx Co-authored-by: Oleg Shur Co-authored-by: davitbejanyan --- kafka-ui-react-app/package.json | 2 +- kafka-ui-react-app/pnpm-lock.yaml | 11 +- .../__test__/BrokerLogdir.spec.tsx | 19 ++- .../Broker/Configs/__test__/Configs.spec.tsx | 11 +- .../BrokersList/__test__/BrokersList.spec.tsx | 6 +- .../Cluster/__tests__/Cluster.spec.tsx | 15 +-- .../Actions/__tests__/Actions.spec.tsx | 54 ++++---- .../Details/Tasks/__tests__/Tasks.spec.tsx | 16 +-- .../Connect/List/__tests__/List.spec.tsx | 16 ++- .../Connect/New/__tests__/New.spec.tsx | 18 ++- .../__test__/ResetOffsets.spec.tsx | 43 ++++--- .../Details/__tests__/Details.spec.tsx | 13 +- .../Details/__tests__/ListItem.spec.tsx | 4 +- .../List/__test__/List.spec.tsx | 4 +- .../__test__/ClustersWidget.spec.tsx | 10 +- .../KsqlDbItem/__test__/KsqlDbItem.spec.tsx | 44 +++---- .../QueryForm/__test__/QueryForm.spec.tsx | 97 ++++++--------- .../KsqlDb/Query/__test__/Query.spec.tsx | 35 +++--- .../Nav/__tests__/ClusterMenu.spec.tsx | 12 +- .../src/components/Nav/__tests__/Nav.spec.tsx | 15 +-- .../Schemas/Details/__test__/Details.spec.tsx | 4 +- .../Details/__test__/SchemaVersion.spec.tsx | 2 +- .../Schemas/Edit/__tests__/Edit.spec.tsx | 4 +- .../__test__/GlobalSchemaSelector.spec.tsx | 16 +-- .../Schemas/List/__test__/List.spec.tsx | 4 +- .../Schemas/New/__test__/New.spec.tsx | 16 +-- .../Topics/List/__tests__/ListPage.spec.tsx | 6 +- .../Topics/List/__tests__/TopicTable.spec.tsx | 38 +++--- .../Topics/New/__test__/New.spec.tsx | 22 ++-- .../DangerZone/__test__/DangerZone.spec.tsx | 46 +++---- .../__tests__/AddEditFilterContainer.spec.tsx | 83 ++++++------- .../Filters/__tests__/AddFilter.spec.tsx | 117 +++++++++++------- .../Filters/__tests__/EditFilter.spec.tsx | 14 +-- .../Filters/__tests__/FilterModal.spec.tsx | 6 +- .../Filters/__tests__/Filters.spec.tsx | 69 ++++++----- .../Filters/__tests__/InfoModal.spec.tsx | 4 +- .../Filters/__tests__/SavedFilters.spec.tsx | 54 ++++---- .../__tests__/MessageContent.spec.tsx | 12 +- .../Topic/Messages/__test__/Message.spec.tsx | 10 +- .../Topic/Messages/__test__/Messages.spec.tsx | 12 +- .../Topic/Overview/__test__/Overview.spec.tsx | 4 +- .../SendMessage/__test__/SendMessage.spec.tsx | 30 ++--- .../Statistics/__test__/Metrics.spec.tsx | 8 +- .../Topics/Topic/__test__/Topic.spec.tsx | 32 ++--- .../__test__/CustomParamField.spec.tsx | 8 +- .../__test__/CustomParams.spec.tsx | 42 +++++-- .../Form/__tests__/TimeToRetainBtn.spec.tsx | 4 +- .../shared/Form/__tests__/TopicForm.spec.tsx | 9 +- .../src/components/__tests__/App.spec.tsx | 4 +- .../common/Alert/__tests__/Alert.spec.tsx | 4 +- .../common/Input/__tests__/Input.spec.tsx | 8 +- .../common/NewTable/__test__/Table.spec.tsx | 59 +++++---- .../common/Search/__tests__/Search.spec.tsx | 6 +- .../common/Select/__tests__/Select.spec.tsx | 20 +-- .../table/__tests__/TableHeaderCell.spec.tsx | 16 +-- 55 files changed, 612 insertions(+), 626 deletions(-) diff --git a/kafka-ui-react-app/package.json b/kafka-ui-react-app/package.json index 7420b8bddf..fa1842dedb 100644 --- a/kafka-ui-react-app/package.json +++ b/kafka-ui-react-app/package.json @@ -82,7 +82,7 @@ "@openapitools/openapi-generator-cli": "^2.5.1", "@testing-library/dom": "^8.11.1", "@testing-library/jest-dom": "^5.16.4", - "@testing-library/user-event": "^13.5.0", + "@testing-library/user-event": "^14.4.3", "@types/eventsource": "^1.1.8", "@types/jest": "^29.0.1", "@types/lodash": "^4.14.172", diff --git a/kafka-ui-react-app/pnpm-lock.yaml b/kafka-ui-react-app/pnpm-lock.yaml index aabb45ce34..88fcfac42e 100644 --- a/kafka-ui-react-app/pnpm-lock.yaml +++ b/kafka-ui-react-app/pnpm-lock.yaml @@ -19,7 +19,7 @@ specifiers: '@testing-library/dom': ^8.11.1 '@testing-library/jest-dom': ^5.16.4 '@testing-library/react': ^13.2.0 - '@testing-library/user-event': ^13.5.0 + '@testing-library/user-event': ^14.4.3 '@types/eventsource': ^1.1.8 '@types/jest': ^29.0.1 '@types/lodash': ^4.14.172 @@ -144,7 +144,7 @@ devDependencies: '@openapitools/openapi-generator-cli': 2.5.1 '@testing-library/dom': 8.13.0 '@testing-library/jest-dom': 5.16.4 - '@testing-library/user-event': 13.5.0_tlwynutqiyp5mns3woioasuxnq + '@testing-library/user-event': 14.4.3_tlwynutqiyp5mns3woioasuxnq '@types/eventsource': 1.1.8 '@types/jest': 29.0.1 '@types/lodash': 4.14.177 @@ -3337,13 +3337,12 @@ packages: react-dom: 18.1.0_react@18.1.0 dev: false - /@testing-library/user-event/13.5.0_tlwynutqiyp5mns3woioasuxnq: - resolution: {integrity: sha512-5Kwtbo3Y/NowpkbRuSepbyMFkZmHgD+vPzYB/RJ4oxt5Gj/avFFBYjhw27cqSVPVw/3a67NK1PbiIr9k4Gwmdg==} - engines: {node: '>=10', npm: '>=6'} + /@testing-library/user-event/14.4.3_tlwynutqiyp5mns3woioasuxnq: + resolution: {integrity: sha512-kCUc5MEwaEMakkO5x7aoD+DLi02ehmEM2QCGWvNqAS1dV/fAvORWEjnjsEIvml59M7Y5kCkWN6fCCyPOe8OL6Q==} + engines: {node: '>=12', npm: '>=6'} peerDependencies: '@testing-library/dom': '>=7.21.4' dependencies: - '@babel/runtime': 7.17.9 '@testing-library/dom': 8.13.0 dev: true diff --git a/kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/__test__/BrokerLogdir.spec.tsx b/kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/__test__/BrokerLogdir.spec.tsx index bb48fde32a..b8ae5bcd58 100644 --- a/kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/__test__/BrokerLogdir.spec.tsx +++ b/kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/__test__/BrokerLogdir.spec.tsx @@ -2,7 +2,6 @@ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/dom'; import { clusterBrokerPath } from 'lib/paths'; -import { act } from '@testing-library/react'; import { brokerLogDirsPayload } from 'lib/fixtures/brokers'; import { useBrokerLogDirs } from 'lib/hooks/api/brokers'; import { BrokerLogdirs } from 'generated-sources'; @@ -20,16 +19,14 @@ describe('BrokerLogdir Component', () => { (useBrokerLogDirs as jest.Mock).mockImplementation(() => ({ data: payload, })); - await act(() => { - render( - - - , - { - initialEntries: [clusterBrokerPath(clusterName, brokerId)], - } - ); - }); + await render( + + + , + { + initialEntries: [clusterBrokerPath(clusterName, brokerId)], + } + ); }; it('shows warning when server returns undefined logDirs response', async () => { diff --git a/kafka-ui-react-app/src/components/Brokers/Broker/Configs/__test__/Configs.spec.tsx b/kafka-ui-react-app/src/components/Brokers/Broker/Configs/__test__/Configs.spec.tsx index 500e65cfdc..d82065eb32 100644 --- a/kafka-ui-react-app/src/components/Brokers/Broker/Configs/__test__/Configs.spec.tsx +++ b/kafka-ui-react-app/src/components/Brokers/Broker/Configs/__test__/Configs.spec.tsx @@ -6,7 +6,6 @@ import { useBrokerConfig } from 'lib/hooks/api/brokers'; import { brokerConfigPayload } from 'lib/fixtures/brokers'; import Configs from 'components/Brokers/Broker/Configs/Configs'; import userEvent from '@testing-library/user-event'; -import { act } from '@testing-library/react'; const clusterName = 'Cluster_Name'; const brokerId = 'Broker_Id'; @@ -42,9 +41,7 @@ describe('Configs', () => { }); it('updates textbox value', async () => { - await act(() => { - userEvent.click(screen.getAllByLabelText('editAction')[0]); - }); + await userEvent.click(screen.getAllByLabelText('editAction')[0]); const textbox = screen.getByLabelText('inputValue'); expect(textbox).toBeInTheDocument(); @@ -59,9 +56,9 @@ describe('Configs', () => { screen.getByRole('button', { name: 'cancelAction' }) ).toBeInTheDocument(); - await act(() => { - userEvent.click(screen.getByRole('button', { name: 'confirmAction' })); - }); + await userEvent.click( + screen.getByRole('button', { name: 'confirmAction' }) + ); expect( screen.getByText('Are you sure you want to change the value?') diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/__test__/BrokersList.spec.tsx b/kafka-ui-react-app/src/components/Brokers/BrokersList/__test__/BrokersList.spec.tsx index b90cef0a43..b11d477b60 100644 --- a/kafka-ui-react-app/src/components/Brokers/BrokersList/__test__/BrokersList.spec.tsx +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/__test__/BrokersList.spec.tsx @@ -2,7 +2,6 @@ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen, waitFor } from '@testing-library/dom'; import { clusterBrokerPath, clusterBrokersPath } from 'lib/paths'; -import { act } from '@testing-library/react'; import BrokersList from 'components/Brokers/BrokersList/BrokersList'; import userEvent from '@testing-library/user-event'; import { useBrokers } from 'lib/hooks/api/brokers'; @@ -57,9 +56,8 @@ describe('BrokersList Component', () => { }); it('opens broker when row clicked', async () => { renderComponent(); - await act(() => { - userEvent.click(screen.getByRole('cell', { name: '0' })); - }); + await userEvent.click(screen.getByRole('cell', { name: '0' })); + await waitFor(() => expect(mockedUsedNavigate).toBeCalledWith( clusterBrokerPath(clusterName, '0') diff --git a/kafka-ui-react-app/src/components/Cluster/__tests__/Cluster.spec.tsx b/kafka-ui-react-app/src/components/Cluster/__tests__/Cluster.spec.tsx index b718db8b29..9fcb77a79a 100644 --- a/kafka-ui-react-app/src/components/Cluster/__tests__/Cluster.spec.tsx +++ b/kafka-ui-react-app/src/components/Cluster/__tests__/Cluster.spec.tsx @@ -13,7 +13,6 @@ import { clusterSchemasPath, clusterTopicsPath, } from 'lib/paths'; -import { act } from 'react-dom/test-utils'; import { useClusters } from 'lib/hooks/api/clusters'; import { onlineClusterPayload } from 'lib/fixtures/clusters'; @@ -54,14 +53,12 @@ describe('Cluster', () => { (useClusters as jest.Mock).mockImplementation(() => ({ data: payload, })); - await act(() => { - render( - - - , - { initialEntries: [pathname] } - ); - }); + await render( + + + , + { initialEntries: [pathname] } + ); }; it('renders Brokers', async () => { diff --git a/kafka-ui-react-app/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx b/kafka-ui-react-app/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx index 878e8bcee6..c3c4cff8f1 100644 --- a/kafka-ui-react-app/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx +++ b/kafka-ui-react-app/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx @@ -33,10 +33,10 @@ const expectActionButtonsExists = () => { expect(screen.getByText('Restart Failed Tasks')).toBeInTheDocument(); expect(screen.getByText('Delete')).toBeInTheDocument(); }; -const afterClickDropDownButton = () => { +const afterClickDropDownButton = async () => { const dropDownButton = screen.getAllByRole('button'); expect(dropDownButton.length).toEqual(1); - userEvent.click(dropDownButton[0]); + await userEvent.click(dropDownButton[0]); }; describe('Actions', () => { afterEach(() => { @@ -61,48 +61,48 @@ describe('Actions', () => { { initialEntries: [path] } ); - it('renders buttons when paused', () => { + it('renders buttons when paused', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.PAUSED), })); renderComponent(); - afterClickDropDownButton(); + await afterClickDropDownButton(); expect(screen.getAllByRole('menuitem').length).toEqual(5); expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); - it('renders buttons when failed', () => { + it('renders buttons when failed', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.FAILED), })); renderComponent(); - afterClickDropDownButton(); + await afterClickDropDownButton(); expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); - it('renders buttons when unassigned', () => { + it('renders buttons when unassigned', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.UNASSIGNED), })); renderComponent(); - afterClickDropDownButton(); + await afterClickDropDownButton(); expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); - it('renders buttons when running connector action', () => { + it('renders buttons when running connector action', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.RUNNING), })); renderComponent(); - afterClickDropDownButton(); + await afterClickDropDownButton(); expect(screen.getAllByRole('menuitem').length).toEqual(5); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); @@ -118,34 +118,34 @@ describe('Actions', () => { it('opens confirmation modal when delete button clicked', async () => { renderComponent(); - afterClickDropDownButton(); - await waitFor(() => + await afterClickDropDownButton(); + await waitFor(async () => userEvent.click(screen.getByRole('menuitem', { name: 'Delete' })) ); expect(screen.getByRole('dialog')).toBeInTheDocument(); }); - it('calls restartConnector when restart button clicked', () => { + it('calls restartConnector when restart button clicked', async () => { const restartConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartConnector, })); renderComponent(); - afterClickDropDownButton(); - userEvent.click( + await afterClickDropDownButton(); + await userEvent.click( screen.getByRole('menuitem', { name: 'Restart Connector' }) ); expect(restartConnector).toHaveBeenCalledWith(ConnectorAction.RESTART); }); - it('calls restartAllTasks', () => { + it('calls restartAllTasks', async () => { const restartAllTasks = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartAllTasks, })); renderComponent(); - afterClickDropDownButton(); - userEvent.click( + await afterClickDropDownButton(); + await userEvent.click( screen.getByRole('menuitem', { name: 'Restart All Tasks' }) ); expect(restartAllTasks).toHaveBeenCalledWith( @@ -153,14 +153,14 @@ describe('Actions', () => { ); }); - it('calls restartFailedTasks', () => { + it('calls restartFailedTasks', async () => { const restartFailedTasks = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartFailedTasks, })); renderComponent(); - afterClickDropDownButton(); - userEvent.click( + await afterClickDropDownButton(); + await userEvent.click( screen.getByRole('menuitem', { name: 'Restart Failed Tasks' }) ); expect(restartFailedTasks).toHaveBeenCalledWith( @@ -168,18 +168,18 @@ describe('Actions', () => { ); }); - it('calls pauseConnector when pause button clicked', () => { + it('calls pauseConnector when pause button clicked', async () => { const pauseConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: pauseConnector, })); renderComponent(); - afterClickDropDownButton(); - userEvent.click(screen.getByRole('menuitem', { name: 'Pause' })); + await afterClickDropDownButton(); + await userEvent.click(screen.getByRole('menuitem', { name: 'Pause' })); expect(pauseConnector).toHaveBeenCalledWith(ConnectorAction.PAUSE); }); - it('calls resumeConnector when resume button clicked', () => { + it('calls resumeConnector when resume button clicked', async () => { const resumeConnector = jest.fn(); (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.PAUSED), @@ -188,8 +188,8 @@ describe('Actions', () => { mutateAsync: resumeConnector, })); renderComponent(); - afterClickDropDownButton(); - userEvent.click(screen.getByRole('menuitem', { name: 'Resume' })); + await afterClickDropDownButton(); + await userEvent.click(screen.getByRole('menuitem', { name: 'Resume' })); expect(resumeConnector).toHaveBeenCalledWith(ConnectorAction.RESUME); }); }); diff --git a/kafka-ui-react-app/src/components/Connect/Details/Tasks/__tests__/Tasks.spec.tsx b/kafka-ui-react-app/src/components/Connect/Details/Tasks/__tests__/Tasks.spec.tsx index da38068a20..dba12d4b0e 100644 --- a/kafka-ui-react-app/src/components/Connect/Details/Tasks/__tests__/Tasks.spec.tsx +++ b/kafka-ui-react-app/src/components/Connect/Details/Tasks/__tests__/Tasks.spec.tsx @@ -57,7 +57,7 @@ describe('Tasks', () => { ).toBeInTheDocument(); }); - it('renders truncates long trace and expands', () => { + it('renders truncates long trace and expands', async () => { renderComponent(tasks); const trace = tasks[2]?.status?.trace || ''; @@ -72,7 +72,7 @@ describe('Tasks', () => { // Full trace is not visible expect(expandedDetails).not.toBeInTheDocument(); - userEvent.click(thirdRow); + await userEvent.click(thirdRow); expect( screen.getByRole('row', { @@ -82,7 +82,7 @@ describe('Tasks', () => { }); describe('Action button', () => { - const expectDropdownExists = () => { + const expectDropdownExists = async () => { const firstTaskRow = screen.getByRole('row', { name: '1 kafka-connect0:8083 RUNNING', }); @@ -91,13 +91,13 @@ describe('Tasks', () => { name: 'Dropdown Toggle', }); expect(extBtn).toBeEnabled(); - userEvent.click(extBtn); + await userEvent.click(extBtn); expect(screen.getByRole('menu')).toBeInTheDocument(); }; - it('renders action button', () => { + it('renders action button', async () => { renderComponent(tasks); - expectDropdownExists(); + await expectDropdownExists(); expect( screen.getAllByRole('button', { name: 'Dropdown Toggle' }).length ).toEqual(tasks.length); @@ -108,11 +108,11 @@ describe('Tasks', () => { it('works as expected', async () => { renderComponent(tasks); - expectDropdownExists(); + await expectDropdownExists(); const actionBtn = screen.getAllByRole('menuitem'); expect(actionBtn[0]).toHaveTextContent('Restart task'); - userEvent.click(actionBtn[0]); + await userEvent.click(actionBtn[0]); expect( screen.getByText('Are you sure you want to restart the task?') ).toBeInTheDocument(); diff --git a/kafka-ui-react-app/src/components/Connect/List/__tests__/List.spec.tsx b/kafka-ui-react-app/src/components/Connect/List/__tests__/List.spec.tsx index 04a7ba8150..9de28f38ff 100644 --- a/kafka-ui-react-app/src/components/Connect/List/__tests__/List.spec.tsx +++ b/kafka-ui-react-app/src/components/Connect/List/__tests__/List.spec.tsx @@ -5,7 +5,7 @@ import ClusterContext, { initialValue, } from 'components/contexts/ClusterContext'; import List from 'components/Connect/List/List'; -import { act, screen, waitFor } from '@testing-library/react'; +import { screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorPath, clusterConnectorsPath } from 'lib/paths'; @@ -52,13 +52,11 @@ describe('Connectors List', () => { it('opens broker when row clicked', async () => { renderComponent(); - await act(() => { - userEvent.click( - screen.getByRole('row', { - name: 'hdfs-source-connector first SOURCE FileStreamSource a b c RUNNING 2 of 2', - }) - ); - }); + await userEvent.click( + screen.getByRole('row', { + name: 'hdfs-source-connector first SOURCE FileStreamSource a b c RUNNING 2 of 2', + }) + ); await waitFor(() => expect(mockedUsedNavigate).toBeCalledWith( clusterConnectConnectorPath( @@ -105,7 +103,7 @@ describe('Connectors List', () => { const submitButton = screen.getAllByRole('button', { name: 'Confirm', })[0]; - await act(() => userEvent.click(submitButton)); + await userEvent.click(submitButton); expect(mockDelete).toHaveBeenCalledWith(); }); diff --git a/kafka-ui-react-app/src/components/Connect/New/__tests__/New.spec.tsx b/kafka-ui-react-app/src/components/Connect/New/__tests__/New.spec.tsx index bbb710a8af..3b93c2d86c 100644 --- a/kafka-ui-react-app/src/components/Connect/New/__tests__/New.spec.tsx +++ b/kafka-ui-react-app/src/components/Connect/New/__tests__/New.spec.tsx @@ -31,16 +31,14 @@ jest.mock('lib/hooks/api/kafkaConnect', () => ({ describe('New', () => { const clusterName = 'my-cluster'; const simulateFormSubmit = async () => { - await act(() => { - userEvent.type( - screen.getByPlaceholderText('Connector Name'), - 'my-connector' - ); - userEvent.type( - screen.getByPlaceholderText('json'), - '{"class":"MyClass"}'.replace(/[{[]/g, '$&$&') - ); - }); + await userEvent.type( + screen.getByPlaceholderText('Connector Name'), + 'my-connector' + ); + await userEvent.type( + screen.getByPlaceholderText('json'), + '{"class":"MyClass"}'.replace(/[{[]/g, '$&$&') + ); expect(screen.getByPlaceholderText('json')).toHaveValue( '{"class":"MyClass"}' diff --git a/kafka-ui-react-app/src/components/ConsumerGroups/Details/ResetOffsets/__test__/ResetOffsets.spec.tsx b/kafka-ui-react-app/src/components/ConsumerGroups/Details/ResetOffsets/__test__/ResetOffsets.spec.tsx index 963600b797..9b0682f04f 100644 --- a/kafka-ui-react-app/src/components/ConsumerGroups/Details/ResetOffsets/__test__/ResetOffsets.spec.tsx +++ b/kafka-ui-react-app/src/components/ConsumerGroups/Details/ResetOffsets/__test__/ResetOffsets.spec.tsx @@ -33,25 +33,24 @@ const resetConsumerGroupOffsetsMockCalled = () => ).toBeTruthy(); const selectresetTypeAndPartitions = async (resetType: string) => { - userEvent.click(screen.getByLabelText('Reset Type')); - userEvent.click(screen.getByText(resetType)); - userEvent.click(screen.getByText('Select...')); - await waitFor(() => { - userEvent.click(screen.getByText('Partition #0')); - }); + await userEvent.click(screen.getByLabelText('Reset Type')); + await userEvent.click(screen.getByText(resetType)); + await userEvent.click(screen.getByText('Select...')); + + await userEvent.click(screen.getByText('Partition #0')); }; const resetConsumerGroupOffsetsWith = async ( resetType: string, offset: null | number = null ) => { - userEvent.click(screen.getByLabelText('Reset Type')); + await userEvent.click(screen.getByLabelText('Reset Type')); const options = screen.getAllByText(resetType); - userEvent.click(options.length > 1 ? options[1] : options[0]); - userEvent.click(screen.getByText('Select...')); - await waitFor(() => { - userEvent.click(screen.getByText('Partition #0')); - }); + await userEvent.click(options.length > 1 ? options[1] : options[0]); + await userEvent.click(screen.getByText('Select...')); + + await userEvent.click(screen.getByText('Partition #0')); + fetchMock.postOnce( `/api/clusters/${clusterName}/consumer-groups/${groupId}/offsets`, 200, @@ -64,7 +63,7 @@ const resetConsumerGroupOffsetsWith = async ( }, } ); - userEvent.click(screen.getByText('Submit')); + await userEvent.click(screen.getByText('Submit')); await waitFor(() => resetConsumerGroupOffsetsMockCalled()); }; @@ -116,14 +115,14 @@ describe('ResetOffsets', () => { }, } ); - await waitFor(() => { - userEvent.click(screen.getAllByLabelText('Partition #0')[1]); - }); - await waitFor(() => { - userEvent.keyboard('10'); - }); - userEvent.click(screen.getByText('Submit')); - await waitFor(() => resetConsumerGroupOffsetsMockCalled()); + + await userEvent.click(screen.getAllByLabelText('Partition #0')[1]); + + await userEvent.keyboard('10'); + + await userEvent.click(screen.getByText('Submit')); + + await resetConsumerGroupOffsetsMockCalled(); }); it('calls resetConsumerGroupOffsets with TIMESTAMP', async () => { await selectresetTypeAndPartitions('TIMESTAMP'); @@ -139,7 +138,7 @@ describe('ResetOffsets', () => { }, } ); - userEvent.click(screen.getByText('Submit')); + await userEvent.click(screen.getByText('Submit')); await waitFor(() => expect( screen.getByText("This field shouldn't be empty!") diff --git a/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/Details.spec.tsx b/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/Details.spec.tsx index e086de8b63..195e71411a 100644 --- a/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/Details.spec.tsx +++ b/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/Details.spec.tsx @@ -13,7 +13,6 @@ import { waitForElementToBeRemoved, } from '@testing-library/dom'; import userEvent from '@testing-library/user-event'; -import { act } from '@testing-library/react'; const clusterName = 'cluster1'; const { groupId } = consumerGroupPayload; @@ -71,7 +70,7 @@ describe('Details component', () => { }); it('handles [Reset offset] click', async () => { - userEvent.click(screen.getByText('Reset offset')); + await userEvent.click(screen.getByText('Reset offset')); expect(mockNavigate).toHaveBeenLastCalledWith( clusterConsumerGroupResetRelativePath ); @@ -86,19 +85,19 @@ describe('Details component', () => { it('shows confirmation modal on consumer group delete', async () => { expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); - userEvent.click(screen.getByText('Delete consumer group')); + await userEvent.click(screen.getByText('Delete consumer group')); await waitFor(() => expect(screen.queryByRole('dialog')).toBeInTheDocument() ); - userEvent.click(screen.getByText('Cancel')); + await userEvent.click(screen.getByText('Cancel')); expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); }); it('handles [Delete consumer group] click', async () => { expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); - await act(() => { - userEvent.click(screen.getByText('Delete consumer group')); - }); + + await userEvent.click(screen.getByText('Delete consumer group')); + expect(screen.queryByRole('dialog')).toBeInTheDocument(); const deleteConsumerGroupMock = fetchMock.deleteOnce( `/api/clusters/${clusterName}/consumer-groups/${groupId}`, diff --git a/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/ListItem.spec.tsx b/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/ListItem.spec.tsx index 5bbce7a927..c4906e9209 100644 --- a/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/ListItem.spec.tsx +++ b/kafka-ui-react-app/src/components/ConsumerGroups/Details/__tests__/ListItem.spec.tsx @@ -39,8 +39,8 @@ describe('ListItem', () => { expect(screen.getByRole('row')).toBeInTheDocument(); }); - it('should renders list item with topic content open', () => { - userEvent.click(screen.getAllByRole('cell')[0].children[0]); + it('should renders list item with topic content open', async () => { + await userEvent.click(screen.getAllByRole('cell')[0].children[0]); expect(screen.getByText('Consumer ID')).toBeInTheDocument(); }); }); diff --git a/kafka-ui-react-app/src/components/ConsumerGroups/List/__test__/List.spec.tsx b/kafka-ui-react-app/src/components/ConsumerGroups/List/__test__/List.spec.tsx index 500549c0aa..a1393c2ccd 100644 --- a/kafka-ui-react-app/src/components/ConsumerGroups/List/__test__/List.spec.tsx +++ b/kafka-ui-react-app/src/components/ConsumerGroups/List/__test__/List.spec.tsx @@ -48,10 +48,10 @@ describe('List', () => { expect(screen.getByText('groupId2')).toBeInTheDocument(); }); - it('handles onRowClick', () => { + it('handles onRowClick', async () => { const row = screen.getByRole('row', { name: 'groupId1 0 1 1' }); expect(row).toBeInTheDocument(); - userEvent.click(row); + await userEvent.click(row); expect(mockedUsedNavigate).toHaveBeenCalledWith( clusterConsumerGroupDetailsPath(':clusterName', 'groupId1') ); diff --git a/kafka-ui-react-app/src/components/Dashboard/ClustersWidget/__test__/ClustersWidget.spec.tsx b/kafka-ui-react-app/src/components/Dashboard/ClustersWidget/__test__/ClustersWidget.spec.tsx index dfdcb34179..2d6f967e2c 100644 --- a/kafka-ui-react-app/src/components/Dashboard/ClustersWidget/__test__/ClustersWidget.spec.tsx +++ b/kafka-ui-react-app/src/components/Dashboard/ClustersWidget/__test__/ClustersWidget.spec.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { act, screen } from '@testing-library/react'; +import { screen } from '@testing-library/react'; import ClustersWidget from 'components/Dashboard/ClustersWidget/ClustersWidget'; import userEvent from '@testing-library/user-event'; import { render } from 'lib/testHelpers'; @@ -16,18 +16,16 @@ describe('ClustersWidget', () => { data: clustersPayload, isSuccess: true, })); - await act(() => { - render(); - }); + await render(); }); it('renders clusterWidget list', () => { expect(screen.getAllByRole('row').length).toBe(3); }); - it('hides online cluster widgets', () => { + it('hides online cluster widgets', async () => { expect(screen.getAllByRole('row').length).toBe(3); - userEvent.click(screen.getByRole('checkbox')); + await userEvent.click(screen.getByRole('checkbox')); expect(screen.getAllByRole('row').length).toBe(2); }); diff --git a/kafka-ui-react-app/src/components/KsqlDb/List/KsqlDbItem/__test__/KsqlDbItem.spec.tsx b/kafka-ui-react-app/src/components/KsqlDb/List/KsqlDbItem/__test__/KsqlDbItem.spec.tsx index 366f01c020..ea6705b6a4 100644 --- a/kafka-ui-react-app/src/components/KsqlDb/List/KsqlDbItem/__test__/KsqlDbItem.spec.tsx +++ b/kafka-ui-react-app/src/components/KsqlDb/List/KsqlDbItem/__test__/KsqlDbItem.spec.tsx @@ -7,7 +7,6 @@ import KsqlDbItem, { } from 'components/KsqlDb/List/KsqlDbItem/KsqlDbItem'; import { screen } from '@testing-library/dom'; import { fetchKsqlDbTablesPayload } from 'redux/reducers/ksqlDb/__test__/fixtures'; -import { act } from '@testing-library/react'; describe('KsqlDbItem', () => { const tablesPathname = clusterKsqlDbTablesPath(); @@ -27,37 +26,34 @@ describe('KsqlDbItem', () => { ); }; - it('renders progressbar when fetching tables and streams', async () => { - await act(() => renderComponent({ fetching: true })); + it('renders progressbar when fetching tables and streams', () => { + renderComponent({ fetching: true }); expect(screen.getByRole('progressbar')).toBeInTheDocument(); }); - it('show no text if no data found', async () => { - await act(() => renderComponent({})); + it('show no text if no data found', () => { + renderComponent({}); expect(screen.getByText('No tables or streams found')).toBeInTheDocument(); }); - it('renders with tables', async () => { - await act(() => - renderComponent({ - rows: { - tables: fetchKsqlDbTablesPayload.tables, - streams: [], - }, - }) - ); + it('renders with tables', () => { + renderComponent({ + rows: { + tables: fetchKsqlDbTablesPayload.tables, + streams: [], + }, + }); + expect(screen.getByRole('table').querySelectorAll('td')).toHaveLength(10); }); - it('renders with streams', async () => { - await act(() => - renderComponent({ - type: KsqlDbItemType.Streams, - rows: { - tables: [], - streams: fetchKsqlDbTablesPayload.streams, - }, - }) - ); + it('renders with streams', () => { + renderComponent({ + type: KsqlDbItemType.Streams, + rows: { + tables: [], + streams: fetchKsqlDbTablesPayload.streams, + }, + }); expect(screen.getByRole('table').querySelectorAll('td')).toHaveLength(10); }); }); diff --git a/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/__test__/QueryForm.spec.tsx b/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/__test__/QueryForm.spec.tsx index 0b8aa60b78..76f8b21335 100644 --- a/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/__test__/QueryForm.spec.tsx +++ b/kafka-ui-react-app/src/components/KsqlDb/Query/QueryForm/__test__/QueryForm.spec.tsx @@ -3,7 +3,6 @@ import React from 'react'; import QueryForm, { Props } from 'components/KsqlDb/Query/QueryForm/QueryForm'; import { screen, waitFor, within } from '@testing-library/dom'; import userEvent from '@testing-library/user-event'; -import { act } from '@testing-library/react'; const renderComponent = (props: Props) => render(); @@ -57,10 +56,9 @@ describe('QueryForm', () => { submitHandler: submitFn, }); - await act(() => - userEvent.click(screen.getByRole('button', { name: 'Execute' })) - ); - waitFor(() => { + await userEvent.click(screen.getByRole('button', { name: 'Execute' })); + + await waitFor(() => { expect(screen.getByText('ksql is a required field')).toBeInTheDocument(); expect(submitFn).not.toBeCalled(); }); @@ -76,12 +74,16 @@ describe('QueryForm', () => { submitHandler: submitFn, }); - await act(() => { - userEvent.paste(screen.getAllByRole('textbox')[0], 'show tables;'); - userEvent.paste(screen.getByRole('textbox', { name: 'key' }), 'test'); - userEvent.paste(screen.getByRole('textbox', { name: 'value' }), 'test'); - userEvent.click(screen.getByRole('button', { name: 'Execute' })); - }); + const textbox = screen.getAllByRole('textbox'); + textbox[0].focus(); + await userEvent.paste('show tables;'); + const key = screen.getByRole('textbox', { name: 'key' }); + key.focus(); + await userEvent.paste('test'); + const value = screen.getByRole('textbox', { name: 'value' }); + value.focus(); + await userEvent.paste('test'); + await userEvent.click(screen.getByRole('button', { name: 'Execute' })); expect( screen.queryByText('ksql is a required field') @@ -106,8 +108,8 @@ describe('QueryForm', () => { expect(screen.getByRole('button', { name: 'Clear results' })).toBeEnabled(); - await act(() => - userEvent.click(screen.getByRole('button', { name: 'Clear results' })) + await userEvent.click( + screen.getByRole('button', { name: 'Clear results' }) ); expect(clearFn).toBeCalled(); @@ -125,39 +127,12 @@ describe('QueryForm', () => { expect(screen.getByRole('button', { name: 'Stop query' })).toBeEnabled(); - await act(() => - userEvent.click(screen.getByRole('button', { name: 'Stop query' })) - ); + await userEvent.click(screen.getByRole('button', { name: 'Stop query' })); expect(cancelFn).toBeCalled(); }); - it('submits form with ctrl+enter on KSQL editor', async () => { - const submitFn = jest.fn(); - renderComponent({ - fetching: false, - hasResults: false, - handleClearResults: jest.fn(), - handleSSECancel: jest.fn(), - submitHandler: submitFn, - }); - - await act(() => { - userEvent.paste( - within(screen.getByLabelText('KSQL')).getByRole('textbox'), - 'show tables;' - ); - - userEvent.type( - within(screen.getByLabelText('KSQL')).getByRole('textbox'), - '{ctrl}{enter}' - ); - }); - - expect(submitFn.mock.calls.length).toBe(1); - }); - - it('adds new property', async () => { + it('add new property', async () => { renderComponent({ fetching: false, hasResults: false, @@ -168,11 +143,9 @@ describe('QueryForm', () => { const textbox = screen.getByLabelText('key'); await userEvent.type(textbox, 'prop_name'); - await act(() => { - userEvent.click( - screen.getByRole('button', { name: 'Add Stream Property' }) - ); - }); + await userEvent.click( + screen.getByRole('button', { name: 'Add Stream Property' }) + ); expect(screen.getAllByRole('textbox', { name: 'key' }).length).toEqual(2); }); @@ -185,11 +158,9 @@ describe('QueryForm', () => { submitHandler: jest.fn(), }); - await act(() => { - userEvent.click( - screen.getByRole('button', { name: 'Add Stream Property' }) - ); - }); + await userEvent.click( + screen.getByRole('button', { name: 'Add Stream Property' }) + ); expect(screen.getAllByRole('textbox', { name: 'key' }).length).toEqual(1); }); @@ -201,16 +172,18 @@ describe('QueryForm', () => { handleSSECancel: jest.fn(), submitHandler: jest.fn(), }); + const textBoxes = screen.getAllByRole('textbox', { name: 'key' }); + textBoxes[0].focus(); + await userEvent.paste('test'); + await userEvent.click( + screen.getByRole('button', { name: 'Add Stream Property' }) + ); + await userEvent.click(screen.getAllByLabelText('deleteProperty')[0]); - await act(() => { - userEvent.paste(screen.getByRole('textbox', { name: 'key' }), 'test'); - userEvent.click( - screen.getByRole('button', { name: 'Add Stream Property' }) - ); - }); - await act(() => { - userEvent.click(screen.getAllByLabelText('deleteProperty')[0]); - }); - expect(screen.getAllByRole('textbox', { name: 'key' }).length).toEqual(1); + await screen.getByRole('button', { name: 'Add Stream Property' }); + + await userEvent.click(screen.getAllByLabelText('deleteProperty')[0]); + + expect(textBoxes.length).toEqual(1); }); }); diff --git a/kafka-ui-react-app/src/components/KsqlDb/Query/__test__/Query.spec.tsx b/kafka-ui-react-app/src/components/KsqlDb/Query/__test__/Query.spec.tsx index 985ebde5e5..705d86be5f 100644 --- a/kafka-ui-react-app/src/components/KsqlDb/Query/__test__/Query.spec.tsx +++ b/kafka-ui-react-app/src/components/KsqlDb/Query/__test__/Query.spec.tsx @@ -6,7 +6,6 @@ import Query, { import { screen } from '@testing-library/dom'; import fetchMock from 'fetch-mock'; import { clusterKsqlDbQueryPath } from 'lib/paths'; -import { act } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; const clusterName = 'testLocal'; @@ -41,10 +40,10 @@ describe('Query', () => { }); const inputs = screen.getAllByRole('textbox'); const textAreaElement = inputs[0] as HTMLTextAreaElement; - await act(() => { - userEvent.paste(textAreaElement, 'show tables;'); - userEvent.click(screen.getByRole('button', { name: 'Execute' })); - }); + + textAreaElement.focus(); + await userEvent.paste('show tables;'); + await userEvent.click(screen.getByRole('button', { name: 'Execute' })); expect(mock.calls().length).toBe(1); }); @@ -59,18 +58,20 @@ describe('Query', () => { Object.defineProperty(window, 'EventSource', { value: EventSourceMock, }); - await act(() => { - const inputs = screen.getAllByRole('textbox'); - const textAreaElement = inputs[0] as HTMLTextAreaElement; - userEvent.paste(textAreaElement, 'show tables;'); - }); - await act(() => { - userEvent.paste(screen.getByLabelText('key'), 'key'); - userEvent.paste(screen.getByLabelText('value'), 'value'); - }); - await act(() => { - userEvent.click(screen.getByRole('button', { name: 'Execute' })); - }); + + const inputs = screen.getAllByRole('textbox'); + const textAreaElement = inputs[0] as HTMLTextAreaElement; + textAreaElement.focus(); + await userEvent.paste('show tables;'); + + const key = screen.getByLabelText('key'); + key.focus(); + await userEvent.paste('key'); + const value = screen.getByLabelText('value'); + value.focus(); + await userEvent.paste('value'); + + await userEvent.click(screen.getByRole('button', { name: 'Execute' })); expect(mock.calls().length).toBe(1); }); diff --git a/kafka-ui-react-app/src/components/Nav/__tests__/ClusterMenu.spec.tsx b/kafka-ui-react-app/src/components/Nav/__tests__/ClusterMenu.spec.tsx index 11d273c6ec..22bc1eabf5 100644 --- a/kafka-ui-react-app/src/components/Nav/__tests__/ClusterMenu.spec.tsx +++ b/kafka-ui-react-app/src/components/Nav/__tests__/ClusterMenu.spec.tsx @@ -19,19 +19,19 @@ describe('ClusterMenu', () => { const getKafkaConnect = () => screen.getByTitle('Kafka Connect'); const getCluster = () => screen.getByText(onlineClusterPayload.name); - it('renders cluster menu with default set of features', () => { + it('renders cluster menu with default set of features', async () => { render(setupComponent(onlineClusterPayload)); expect(getCluster()).toBeInTheDocument(); expect(getMenuItems().length).toEqual(1); - userEvent.click(getMenuItem()); + await userEvent.click(getMenuItem()); expect(getMenuItems().length).toEqual(4); expect(getBrokers()).toBeInTheDocument(); expect(getTopics()).toBeInTheDocument(); expect(getConsumers()).toBeInTheDocument(); }); - it('renders cluster menu with correct set of features', () => { + it('renders cluster menu with correct set of features', async () => { render( setupComponent({ ...onlineClusterPayload, @@ -43,7 +43,7 @@ describe('ClusterMenu', () => { }) ); expect(getMenuItems().length).toEqual(1); - userEvent.click(getMenuItem()); + await userEvent.click(getMenuItem()); expect(getMenuItems().length).toEqual(7); expect(getBrokers()).toBeInTheDocument(); @@ -64,7 +64,7 @@ describe('ClusterMenu', () => { expect(getTopics()).toBeInTheDocument(); expect(getConsumers()).toBeInTheDocument(); }); - it('makes Kafka Connect link active', () => { + it('makes Kafka Connect link active', async () => { render( setupComponent({ ...onlineClusterPayload, @@ -73,7 +73,7 @@ describe('ClusterMenu', () => { { initialEntries: [clusterConnectorsPath(onlineClusterPayload.name)] } ); expect(getMenuItems().length).toEqual(1); - userEvent.click(getMenuItem()); + await userEvent.click(getMenuItem()); expect(getMenuItems().length).toEqual(5); const kafkaConnect = getKafkaConnect(); diff --git a/kafka-ui-react-app/src/components/Nav/__tests__/Nav.spec.tsx b/kafka-ui-react-app/src/components/Nav/__tests__/Nav.spec.tsx index 023fdfdba7..582c341411 100644 --- a/kafka-ui-react-app/src/components/Nav/__tests__/Nav.spec.tsx +++ b/kafka-ui-react-app/src/components/Nav/__tests__/Nav.spec.tsx @@ -2,7 +2,6 @@ import React from 'react'; import Nav from 'components/Nav/Nav'; import { screen } from '@testing-library/react'; import { render } from 'lib/testHelpers'; -import { act } from 'react-dom/test-utils'; import { Cluster } from 'generated-sources'; import { useClusters } from 'lib/hooks/api/clusters'; import { @@ -15,28 +14,26 @@ jest.mock('lib/hooks/api/clusters', () => ({ })); describe('Nav', () => { - const renderComponent = async (payload: Cluster[] = []) => { + const renderComponent = (payload: Cluster[] = []) => { (useClusters as jest.Mock).mockImplementation(() => ({ data: payload, isSuccess: true, })); - await act(() => { - render(