627: tests on crud kafka connectors (#866)

* #627 add class for Connectors List page

* #627 add class for testing of Connectors functionality

* #627 add step for create a Connector

* #627 add class for Connector View page

* #627 add test for updating connector

* #627 add test for delete connector

* #627 add class for create connector view

* #627 add connector creation steps

* #627 add step for visibility check

* #627 add api interaction to connector test

* #627 update connector config file

* #627 add api helpers to create connector test

* #627 add creating the connector

* #627 update delete Connector test

* #627 add api helpers to send messages

* #627 add before all and after all methods to use one topic for tests

* #627 get rid of duplicate Lombok version

* #627 add check that connector is visible in the connector's list

* #627 add class for update connector config

* #627 add steps to update connector test

* #627 add check that connector is updated

* #627 add separate config file to connector deletion test

* #627 add different tables for each config files

* #627 move api helpers to test from BeforeAll and AfterAll

* #627 refactor config names

* #627 add support for different operational systems

* #622 change command for docker image composing until issue #819 will be fixed

* #622 return the test because the topic deleting bug has been fixed

* merge conflict fixes

* #627 extract methods to the before all/after all

* #627 fix github action file

* #627 fix github action file

* #627 refactor connector update method

* #627 update api helpers

* #627 fix update connector config

* #627 merge conflict fixes

* #627 move delete topic instance to end of list

* #627 refactor FileUtils to use Apache Commons lib

* #627 reduce code duplication

* #627 add waiting step to update topic test

* #627 return of test skip

* save container logs on fail

* #627 testing that creating the connector works in PR check

* #627 change element for checking connector creation

* #627 check that connector deletion works in PR check

* #627 add check during connector creation

* #627 disable connector creation

* #627 move check to the view page

* #627 add steps to check that connector is created

* #627 pr check

* #627 add check after connector is created

* #627 add logging

* #627 add check before topic deletion

* #627 rename package with tests

Co-authored-by: Roman Zabaluev <rzabaluev@provectus.com>
This commit is contained in:
Anna Antipova 2021-12-09 12:55:26 +03:00 committed by GitHub
parent 4227d3cdf3
commit 24b401d5ad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 439 additions and 15 deletions

View file

@ -1,7 +1,7 @@
name: e2e-checks
on:
pull_request:
types: ['opened', 'edited', 'reopened', 'synchronize']
types: [ 'opened', 'edited', 'reopened', 'synchronize' ]
paths:
- 'kafka-ui-api/**'
- 'kafka-ui-contract/**'
@ -37,8 +37,9 @@ jobs:
mvn clean package -DskipTests ${{ github.event.inputs.extraMavenOptions }}
- name: compose app
id: compose_app
# use the following command until #819 will be fixed
run: |
docker-compose -f ./docker/kafka-ui.yaml up -d
docker-compose -f ./docker/kafka-ui-connectors.yaml up -d
- name: e2e run
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
@ -69,4 +70,7 @@ jobs:
context: 'Test report'
state: 'success'
sha: ${{ github.event.pull_request.head.sha || github.sha }}
target_url: https://${{ github.repository_owner }}.github.io/kafka-ui/allure/allure-results/${{ github.run_number }}
target_url: https://${{ github.repository_owner }}.github.io/kafka-ui/allure/allure-results/${{ github.run_number }}
- name: Dump docker logs on failure
if: failure()
uses: jwalton/gh-docker-logs@v1

View file

@ -304,6 +304,7 @@
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven-resources-plugin.version}</version>
<executions>
<execution>
<id>copy-resources</id>

View file

@ -181,7 +181,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${org.projectlombok.e2e-checks.version}</version>
<version>${org.projectlombok.version}</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>

View file

@ -0,0 +1,17 @@
{
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"connection.url": "jdbc:postgresql://postgres-db:5432/test",
"connection.user": "dev_user",
"connection.password": "12345",
"topics": "topic_for_connector",
"table.name.format": "sink_activities_e2e_test_connector_creating",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"key.converter.schema.registry.url": "http://schemaregistry0:8085",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schema.registry.url": "http://schemaregistry0:8085",
"auto.create": "true",
"pk.mode": "record_value",
"pk.fields": "id",
"insert.mode": "upsert",
"errors.log.enable": "true",
"errors.log.include.messages": "true"

View file

@ -0,0 +1,18 @@
{
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"connection.url": "jdbc:postgresql://postgres-db:5432/test",
"connection.user": "dev_user",
"connection.password": "12345",
"topics": "topic_for_connector",
"table.name.format": "sink_activities_e2e_test_connector_updating",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"key.converter.schema.registry.url": "http://schemaregistry0:8085",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schema.registry.url": "http://schemaregistry0:8085",
"auto.create": "true",
"pk.mode": "record_value",
"pk.fields": "id",
"insert.mode": "upsert",
"errors.log.enable": "true",
"errors.log.include.messages": "true"
}

View file

@ -0,0 +1,17 @@
{
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"connection.url": "jdbc:postgresql://postgres-db:5432/test",
"connection.user": "dev_user",
"connection.password": "12345",
"topics": "topic_for_update_connector",
"table.name.format": "sink_activities_e2e_test_connector_updating",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"key.converter.schema.registry.url": "http://schemaregistry0:8085",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schema.registry.url": "http://schemaregistry0:8085",
"auto.create": "true",
"pk.mode": "record_value",
"pk.fields": "id",
"insert.mode": "upsert",
"errors.log.enable": "true",
"errors.log.include.messages": "true"

View file

@ -0,0 +1,18 @@
{
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"connection.url": "jdbc:postgresql://postgres-db:5432/test",
"connection.user": "dev_user",
"connection.password": "12345",
"topics": "topic_for_delete_connector",
"table.name.format": "sink_activities_e2e_test_connector_deleting",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"key.converter.schema.registry.url": "http://schemaregistry0:8085",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schema.registry.url": "http://schemaregistry0:8085",
"auto.create": "true",
"pk.mode": "record_value",
"pk.fields": "id",
"insert.mode": "upsert",
"errors.log.enable": "true",
"errors.log.include.messages": "true"
}

View file

@ -0,0 +1,24 @@
{
"schema":
{
"type":"struct",
"fields": [
{
"type":"string",
"optional":false,
"field":"id"
},{
"type":"string",
"optional":false,
"field":"value"
}
],
"optional":false,
"name":"test"
},
"payload":
{
"id":"1",
"value":"kafka"
}
}

View file

@ -0,0 +1,15 @@
package com.provectus.kafka.ui.extensions;
import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
public class FileUtils {
public static String getResourceAsString(String resourceFileName) throws IOException {
return IOUtils.resourceToString("/" + resourceFileName, StandardCharsets.UTF_8);
}
}

View file

@ -1,29 +1,32 @@
package com.provectus.kafka.ui.helpers;
import com.provectus.kafka.ui.api.api.KafkaConnectApi;
import com.provectus.kafka.ui.api.api.MessagesApi;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.provectus.kafka.ui.api.ApiClient;
import com.provectus.kafka.ui.api.api.TopicsApi;
import com.provectus.kafka.ui.api.model.CreateTopicMessage;
import com.provectus.kafka.ui.api.model.NewConnector;
import com.provectus.kafka.ui.api.model.TopicCreation;
import lombok.SneakyThrows;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import java.util.HashMap;
import java.util.Map;
public class ApiHelper {
int partitions = 1;
int replicationFactor = 1;
String newTopic = "new-topic";
String baseURL = "http://localhost:8080/";
@SneakyThrows
private TopicsApi topicApi(){
private TopicsApi topicApi() {
ApiClient defaultClient = new ApiClient();
defaultClient.setBasePath(baseURL);
TopicsApi topicsApi = new TopicsApi(defaultClient);
return topicsApi;
}
}
@SneakyThrows
public void createTopic(String clusterName, String topicName) {
@ -44,4 +47,46 @@ public class ApiHelper {
}
}
@SneakyThrows
private KafkaConnectApi connectorApi(){
ApiClient defaultClient = new ApiClient();
defaultClient.setBasePath(baseURL);
KafkaConnectApi connectorsApi = new KafkaConnectApi(defaultClient);
return connectorsApi;
}
@SneakyThrows
public void deleteConnector(String clusterName, String connectName, String connectorName) {
try {
connectorApi().deleteConnector(clusterName, connectName, connectorName).block();
} catch (WebClientResponseException ex) {
if (ex.getRawStatusCode() != 404)
throw ex;
}
}
@SneakyThrows
public void createConnector(String clusterName, String connectName, String connectorName, String configJson) {
NewConnector connector = new NewConnector();
connector.setName(connectorName);
Map<String, Object> configMap = new ObjectMapper().readValue(configJson, HashMap.class);
connector.setConfig(configMap);
connectorApi().createConnector(clusterName, connectName, connector).block();
}
@SneakyThrows
private MessagesApi messageApi() {
ApiClient defaultClient = new ApiClient();
defaultClient.setBasePath(baseURL);
MessagesApi messagesApi = new MessagesApi(defaultClient);
return messagesApi;
}
@SneakyThrows
public void sendMessage(String clusterName, String topicName, String messageContentJson, String messageKey){
CreateTopicMessage createMessage = new CreateTopicMessage();
createMessage.setContent(messageContentJson);
createMessage.setKey(messageKey);
messageApi().sendTopicMessages(clusterName, topicName, createMessage).block();
}
}

View file

@ -0,0 +1,36 @@
package com.provectus.kafka.ui.pages;
import com.codeborne.selenide.Condition;
import com.provectus.kafka.ui.extensions.WaitUtils;
import io.qameta.allure.Step;
import lombok.experimental.ExtensionMethod;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
import static com.codeborne.selenide.Selenide.$;
import static com.provectus.kafka.ui.screenshots.Screenshooter.log;
import static java.lang.Thread.sleep;
@ExtensionMethod(WaitUtils.class)
public class ConnectorCreateView {
private static final String path = "ui/clusters/secondLocal/connectors/create_new";
@Step
public ConnectorsView setConnectorConfig(String connectName, String configJson) throws InterruptedException {
$(By.xpath("//input[@name='name']")).sendKeys(connectName);
$(".ace_text-input").sendKeys(Keys.BACK_SPACE);
$(".ace_text-input").sendKeys(Keys.BACK_SPACE);
$(".ace_text-input").sendKeys(String.valueOf(configJson.toCharArray()));
$(By.xpath("//input[@name='name']")).click();
$(By.xpath("//input[@type='submit']")).click();
sleep(2000);
log.info("Connector config is submitted");
return new ConnectorsView();
}
@Step
public ConnectorCreateView isOnConnectorCreatePage() {
$(By.xpath("//input[@name='name']")).shouldBe(Condition.visible);
return this;
}
}

View file

@ -0,0 +1,25 @@
package com.provectus.kafka.ui.pages;
import io.qameta.allure.Step;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
import static com.codeborne.selenide.Selenide.$;
import static org.openqa.selenium.Keys.*;
public class ConnectorUpdateView {
@Step
public ConnectorUpdateView updateConnectorConfig(String configJson) {
String os = System.getProperty("os.name");
Keys CMD = os.equalsIgnoreCase("Mac OS X") ? COMMAND : CONTROL;
$(".ace_text-input").sendKeys(CMD, "a");
$(".ace_text-input").sendKeys(Keys.BACK_SPACE);
$(".ace_text-input").sendKeys(String.valueOf(configJson.toCharArray()));
$(".ace_text-input").sendKeys(CMD, "a");
$(".ace_text-input").sendKeys(SHIFT, TAB);
$("div.ace_content").click();
$(By.xpath("//input[@type='submit']")).click();
return this;
}
}

View file

@ -0,0 +1,63 @@
package com.provectus.kafka.ui.pages;
import com.codeborne.selenide.Condition;
import com.codeborne.selenide.Selenide;
import com.provectus.kafka.ui.base.TestConfiguration;
import com.provectus.kafka.ui.extensions.WaitUtils;
import io.qameta.allure.Step;
import lombok.SneakyThrows;
import lombok.experimental.ExtensionMethod;
import org.openqa.selenium.By;
import static com.codeborne.selenide.Selenide.$;
@ExtensionMethod(WaitUtils.class)
public class ConnectorsList {
private static final String path = "ui/clusters/%s/connectors";
@Step
public ConnectorsList goTo(String cluster) {
Selenide.open(TestConfiguration.BASE_URL+path.formatted(cluster));
return this;
}
@Step
public ConnectorsList isOnPage() {
$(By.xpath("//*[contains(text(),'Loading')]")).shouldBe(Condition.disappear);
$(By.xpath("//span[text()='All Connectors']")).shouldBe(Condition.visible);
return this;
}
@Step
public ConnectorCreateView clickCreateConnectorButton() {
$(By.xpath("//a[text()='Create Connector']")).click();
return new ConnectorCreateView();
}
@SneakyThrows
public ConnectorsList openConnector(String connectorName) {
$(By.xpath("//*/tr/td[1]/a[text()='%s']".formatted(connectorName)))
.click();
return this;
}
@SneakyThrows
public ConnectorsList isNotVisible(String connectorName) {
By.xpath("//div[contains(@class,'section')]//table").refreshUntil(Condition.visible);
$(By.xpath("//a[text()='%s']".formatted(connectorName))).shouldNotBe(Condition.visible);
return this;
}
@Step
public ConnectorsList connectorIsVisibleInList(String connectorName, String topicName) {
By.xpath("//a[text() = '%s']".formatted(connectorName)).refreshUntil(Condition.visible);
By.xpath("//a[text() = '%s']".formatted(topicName)).refreshUntil(Condition.visible);
return this;
}
public ConnectorsList connectorIsUpdatedInList(String connectorName, String topicName) {
$(By.xpath("//a[text() = '%s']".formatted(connectorName))).shouldBe(Condition.visible);
By.xpath("//a[text() = '%s']".formatted(topicName)).refreshUntil(Condition.visible);
return this;
}
}

View file

@ -0,0 +1,41 @@
package com.provectus.kafka.ui.pages;
import com.codeborne.selenide.Condition;
import com.codeborne.selenide.Selenide;
import com.provectus.kafka.ui.base.TestConfiguration;
import com.provectus.kafka.ui.extensions.WaitUtils;
import io.qameta.allure.Step;
import lombok.experimental.ExtensionMethod;
import org.openqa.selenium.By;
import static com.codeborne.selenide.Selenide.$;
@ExtensionMethod(WaitUtils.class)
public class ConnectorsView {
private static final String path = "ui/clusters/%s/connects/first/connectors/%s";
@Step
public ConnectorsView goTo(String cluster, String connector) {
Selenide.open(TestConfiguration.BASE_URL + path.formatted(cluster, connector));
return this;
}
@Step
public ConnectorUpdateView openEditConfig() {
$(By.xpath("//a/span[text()='Edit config']")).click();
return new ConnectorUpdateView();
}
@Step
public void clickDeleteButton() {
$(By.xpath("//span[text()='Delete']")).click();
$(By.xpath("//button[text()='Confirm']")).click();
}
@Step
public void connectorIsVisibleOnOverview() {
$(By.xpath("//a[text() ='Tasks']")).click();
$(By.xpath("//a[text() ='Config']")).click();
$(By.xpath("//span[text()='Edit config']")).waitUntil(Condition.visible, 100);
}
}

View file

@ -7,6 +7,8 @@ public class Pages {
public MainPage mainPage = new MainPage();
public TopicsList topicsList = new TopicsList();
public TopicView topicView = new TopicView();
public ConnectorsList connectorsList = new ConnectorsList();
public ConnectorsView connectorsView = new ConnectorsView();
public MainPage open() {
return openMainPage();
@ -24,4 +26,12 @@ public class Pages {
return topicView.goTo(clusterName, topicName);
}
public ConnectorsList openConnectorsList(String clusterName) {
return connectorsList.goTo(clusterName);
}
public ConnectorsView openConnectorsView(String clusterName, String connectorName) {
return connectorsView.goTo(clusterName, connectorName);
}
}

View file

@ -27,6 +27,13 @@ public class TopicView {
return this;
}
@Step
public TopicsList isOnTopicViewPage() {
$(By.xpath("//*[contains(text(),'Loading')]")).shouldBe(Condition.disappear);
$(By.xpath("//a[text()='All Topics']")).shouldBe(Condition.visible);
return new TopicsList();
}
@Step
public TopicsList isOnTopicListPage() {
$(By.xpath("//*[contains(text(),'Loading')]")).shouldBe(Condition.disappear);
@ -42,6 +49,7 @@ public class TopicView {
@SneakyThrows
public TopicView clickDeleteTopicButton() {
By.xpath("//*[text()='Delete Topic']").refreshUntil(Condition.visible);
$(By.xpath("//*[text()='Delete Topic']")).click();
$(By.xpath("//*[text()='Confirm']")).click();
return this;
@ -76,8 +84,9 @@ public class TopicView {
}
@SneakyThrows
public void submitSettingChanges() {
public TopicView submitSettingChanges() {
$(By.xpath("//input[@type='submit']")).click();
return this;
}
public TopicView cleanupPolicyIs(String value) {

View file

@ -0,0 +1,81 @@
package com.provectus.kafka.ui.tests;
import com.provectus.kafka.ui.base.BaseTest;
import com.provectus.kafka.ui.extensions.FileUtils;
import com.provectus.kafka.ui.helpers.ApiHelper;
import com.provectus.kafka.ui.helpers.Helpers;
import lombok.SneakyThrows;
import org.junit.jupiter.api.*;
public class ConnectorsTests extends BaseTest {
public static final String LOCAL = "local";
public static final String SINK_CONNECTOR = "sink_postgres_activities_e2e_checks";
public static final String TOPIC_FOR_CONNECTOR = "topic_for_connector";
public static final String TOPIC_FOR_UPDATE_CONNECTOR = "topic_for_update_connector";
public static final String FIRST = "first";
public static final String CONNECTOR_FOR_DELETE = "sink_postgres_activities_e2e_checks_for_delete";
public static final String CONNECTOR_FOR_UPDATE = "sink_postgres_activities_e2e_checks_for_update";
@BeforeAll
@SneakyThrows
public static void beforeAll() {
ApiHelper apiHelper = Helpers.INSTANCE.apiHelper;
String message = FileUtils.getResourceAsString("message_content_create_topic.json");
apiHelper.createTopic(LOCAL, TOPIC_FOR_CONNECTOR);
apiHelper.sendMessage(LOCAL, TOPIC_FOR_CONNECTOR, message, " ");
}
@AfterAll
@SneakyThrows
public static void afterAll() {
ApiHelper apiHelper = Helpers.INSTANCE.apiHelper;
apiHelper.deleteConnector(LOCAL, FIRST, SINK_CONNECTOR);
apiHelper.deleteTopic(LOCAL, TOPIC_FOR_CONNECTOR);
}
@SneakyThrows
@DisplayName("should create a connector")
@Test
void createConnector() {
pages.openConnectorsList(LOCAL)
.isOnPage()
.clickCreateConnectorButton()
.isOnConnectorCreatePage()
.setConnectorConfig(
SINK_CONNECTOR,
FileUtils.getResourceAsString("config_for_create_connector.json")
);
pages.openConnectorsList(LOCAL).connectorIsVisibleInList(SINK_CONNECTOR, TOPIC_FOR_CONNECTOR);
}
//disable test due 500 error during create connector via api
@SneakyThrows
@DisplayName("should update a connector")
@Test
@Disabled
void updateConnector() {
pages.openConnectorsList(LOCAL)
.isOnPage()
.openConnector(CONNECTOR_FOR_UPDATE);
pages.openConnectorsView(LOCAL, CONNECTOR_FOR_UPDATE)
.openEditConfig()
.updateConnectorConfig(
FileUtils.getResourceAsString("config_for_update_connector.json"));
pages.openConnectorsList(LOCAL).connectorIsVisibleInList(CONNECTOR_FOR_UPDATE, TOPIC_FOR_UPDATE_CONNECTOR);
}
@SneakyThrows
@DisplayName("should delete connector")
@Test
@Disabled
void deleteConnector() {
pages.openConnectorsList(LOCAL)
.isOnPage()
.openConnector(CONNECTOR_FOR_DELETE);
pages.openConnectorsView(LOCAL, CONNECTOR_FOR_DELETE)
.clickDeleteButton();
pages.openConnectorsList(LOCAL).isNotVisible(CONNECTOR_FOR_DELETE);
}
}

View file

@ -1,6 +1,5 @@
package com.provectus.kafka.ui.topics;
package com.provectus.kafka.ui.tests;
import com.codeborne.selenide.Selenide;
import com.provectus.kafka.ui.base.BaseTest;
import com.provectus.kafka.ui.helpers.Helpers;
import com.provectus.kafka.ui.pages.MainPage;
@ -63,7 +62,8 @@ public class TopicTests extends BaseTest {
.changeTimeToRetainValue(UPDATED_TIME_TO_RETAIN_VALUE)
.changeMaxSizeOnDisk(UPDATED_MAX_SIZE_ON_DISK)
.changeMaxMessageBytes(UPDATED_MAX_MESSAGE_BYTES)
.submitSettingChanges();
.submitSettingChanges()
.isOnTopicViewPage();
pages.openTopicView(SECOND_LOCAL, TOPIC_TO_UPDATE)
.openEditSettings()
// Assertions