diff --git a/documentation/compose/jaas/kafka_server.conf b/documentation/compose/jaas/kafka_server.conf index 25388be5aa..0c1fb34652 100644 --- a/documentation/compose/jaas/kafka_server.conf +++ b/documentation/compose/jaas/kafka_server.conf @@ -11,4 +11,8 @@ KafkaClient { user_admin="admin-secret"; }; -Client {}; +Client { + org.apache.zookeeper.server.auth.DigestLoginModule required + username="zkuser" + password="zkuserpassword"; +}; diff --git a/documentation/compose/jaas/zookeeper_jaas.conf b/documentation/compose/jaas/zookeeper_jaas.conf new file mode 100644 index 0000000000..2d7fd1b1c2 --- /dev/null +++ b/documentation/compose/jaas/zookeeper_jaas.conf @@ -0,0 +1,4 @@ +Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_zkuser="zkuserpassword"; +}; diff --git a/documentation/compose/kafka-ui-acl-with-zk.yaml b/documentation/compose/kafka-ui-acl-with-zk.yaml new file mode 100644 index 0000000000..e1d70b2970 --- /dev/null +++ b/documentation/compose/kafka-ui-acl-with-zk.yaml @@ -0,0 +1,59 @@ +--- +version: '2' +services: + + kafka-ui: + container_name: kafka-ui + image: provectuslabs/kafka-ui:latest + ports: + - 8080:8080 + depends_on: + - zookeeper + - kafka + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092 + KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT + KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM: PLAIN + KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";' + + zookeeper: + image: wurstmeister/zookeeper:3.4.6 + environment: + JVMFLAGS: "-Djava.security.auth.login.config=/etc/zookeeper/zookeeper_jaas.conf" + volumes: + - ./jaas/zookeeper_jaas.conf:/etc/zookeeper/zookeeper_jaas.conf + ports: + - 2181:2181 + + kafka: + image: confluentinc/cp-kafka:7.2.1 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + - "9997:9997" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server.conf" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_JMX_PORT: 9997 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_NODE_ID: 1 + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' + KAFKA_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'SASL_PLAINTEXT' + KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN' + KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' + KAFKA_SECURITY_PROTOCOL: 'SASL_PLAINTEXT' + KAFKA_SUPER_USERS: 'User:admin' + volumes: + - ./scripts/update_run.sh:/tmp/update_run.sh + - ./jaas:/etc/kafka/jaas diff --git a/kafka-ui-api/pom.xml b/kafka-ui-api/pom.xml index 7f2d4c16be..5ebefe31df 100644 --- a/kafka-ui-api/pom.xml +++ b/kafka-ui-api/pom.xml @@ -12,7 +12,7 @@ kafka-ui-api - 0.8.8 + 0.8.10 jacoco reuseReports ${project.basedir}/target/jacoco.exec diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java index 1d5cc5393c..64ec894cd5 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java @@ -131,8 +131,9 @@ public class ClustersProperties { @Data public static class Masking { Type type; - List fields; //if null or empty list - policy will be applied to all fields - List pattern; //used when type=MASK + List fields; + String fieldsNamePattern; + List maskingCharsReplacement; //used when type=MASK String replacement; //used when type=REPLACE String topicKeysPattern; String topicValuesPattern; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java new file mode 100644 index 0000000000..13119b3bb9 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java @@ -0,0 +1,26 @@ +package com.provectus.kafka.ui.config.auth; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("spring.ldap") +@Data +public class LdapProperties { + + private String urls; + private String base; + private String adminUser; + private String adminPassword; + private String userFilterSearchBase; + private String userFilterSearchFilter; + + @Value("${oauth2.ldap.activeDirectory:false}") + private boolean isActiveDirectory; + @Value("${oauth2.ldap.aсtiveDirectory.domain:@null}") + private String activeDirectoryDomain; + + @Value("${oauth2.ldap.groupRoleAttribute:cn}") + private String groupRoleAttribute; + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java index 0ba5c231f4..fae1125239 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java @@ -1,13 +1,23 @@ package com.provectus.kafka.ui.config.auth; +import static com.provectus.kafka.ui.config.auth.AbstractAuthSecurityConfig.AUTH_WHITELIST; + +import com.provectus.kafka.ui.service.rbac.AccessControlService; +import com.provectus.kafka.ui.service.rbac.extractor.RbacLdapAuthoritiesExtractor; +import java.util.Collection; import java.util.List; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.ldap.core.support.LdapContextSource; import org.springframework.security.authentication.AuthenticationManager; @@ -16,70 +26,71 @@ import org.springframework.security.authentication.ReactiveAuthenticationManager import org.springframework.security.authentication.ReactiveAuthenticationManagerAdapter; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.ldap.authentication.AbstractLdapAuthenticationProvider; import org.springframework.security.ldap.authentication.BindAuthenticator; import org.springframework.security.ldap.authentication.LdapAuthenticationProvider; import org.springframework.security.ldap.authentication.ad.ActiveDirectoryLdapAuthenticationProvider; import org.springframework.security.ldap.search.FilterBasedLdapUserSearch; import org.springframework.security.ldap.search.LdapUserSearch; +import org.springframework.security.ldap.userdetails.LdapUserDetailsMapper; import org.springframework.security.web.server.SecurityWebFilterChain; @Configuration @EnableWebFluxSecurity @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") @Import(LdapAutoConfiguration.class) +@EnableConfigurationProperties(LdapProperties.class) +@RequiredArgsConstructor @Slf4j -public class LdapSecurityConfig extends AbstractAuthSecurityConfig { +public class LdapSecurityConfig { - @Value("${spring.ldap.urls}") - private String ldapUrls; - @Value("${spring.ldap.dn.pattern:#{null}}") - private String ldapUserDnPattern; - @Value("${spring.ldap.adminUser:#{null}}") - private String adminUser; - @Value("${spring.ldap.adminPassword:#{null}}") - private String adminPassword; - @Value("${spring.ldap.userFilter.searchBase:#{null}}") - private String userFilterSearchBase; - @Value("${spring.ldap.userFilter.searchFilter:#{null}}") - private String userFilterSearchFilter; - - @Value("${oauth2.ldap.activeDirectory:false}") - private boolean isActiveDirectory; - @Value("${oauth2.ldap.aсtiveDirectory.domain:#{null}}") - private String activeDirectoryDomain; + private final LdapProperties props; @Bean - public ReactiveAuthenticationManager authenticationManager(BaseLdapPathContextSource contextSource) { + public ReactiveAuthenticationManager authenticationManager(BaseLdapPathContextSource contextSource, + ApplicationContext context, + @Nullable AccessControlService acs) { + var rbacEnabled = acs != null && acs.isRbacEnabled(); BindAuthenticator ba = new BindAuthenticator(contextSource); - if (ldapUserDnPattern != null) { - ba.setUserDnPatterns(new String[] {ldapUserDnPattern}); + if (props.getBase() != null) { + ba.setUserDnPatterns(new String[] {props.getBase()}); } - if (userFilterSearchFilter != null) { + if (props.getUserFilterSearchFilter() != null) { LdapUserSearch userSearch = - new FilterBasedLdapUserSearch(userFilterSearchBase, userFilterSearchFilter, contextSource); + new FilterBasedLdapUserSearch(props.getUserFilterSearchBase(), props.getUserFilterSearchFilter(), + contextSource); ba.setUserSearch(userSearch); } AbstractLdapAuthenticationProvider authenticationProvider; - if (!isActiveDirectory) { - authenticationProvider = new LdapAuthenticationProvider(ba); + if (!props.isActiveDirectory()) { + authenticationProvider = rbacEnabled + ? new LdapAuthenticationProvider(ba, new RbacLdapAuthoritiesExtractor(context)) + : new LdapAuthenticationProvider(ba); } else { - authenticationProvider = new ActiveDirectoryLdapAuthenticationProvider(activeDirectoryDomain, ldapUrls); + authenticationProvider = new ActiveDirectoryLdapAuthenticationProvider(props.getActiveDirectoryDomain(), + props.getUrls()); // TODO Issue #3741 authenticationProvider.setUseAuthenticationRequestCredentials(true); } + if (rbacEnabled) { + authenticationProvider.setUserDetailsContextMapper(new UserDetailsMapper()); + } + AuthenticationManager am = new ProviderManager(List.of(authenticationProvider)); return new ReactiveAuthenticationManagerAdapter(am); } @Bean + @Primary public BaseLdapPathContextSource contextSource() { LdapContextSource ctx = new LdapContextSource(); - ctx.setUrl(ldapUrls); - ctx.setUserDn(adminUser); - ctx.setPassword(adminPassword); + ctx.setUrl(props.getUrls()); + ctx.setUserDn(props.getAdminUser()); + ctx.setPassword(props.getAdminPassword()); ctx.afterPropertiesSet(); return ctx; } @@ -87,20 +98,35 @@ public class LdapSecurityConfig extends AbstractAuthSecurityConfig { @Bean public SecurityWebFilterChain configureLdap(ServerHttpSecurity http) { log.info("Configuring LDAP authentication."); - if (isActiveDirectory) { + if (props.isActiveDirectory()) { log.info("Active Directory support for LDAP has been enabled."); } - http + return http .authorizeExchange() .pathMatchers(AUTH_WHITELIST) .permitAll() .anyExchange() .authenticated() - .and() - .httpBasic(); - return http.csrf().disable().build(); + .and() + .formLogin() + + .and() + .logout() + + .and() + .csrf().disable() + .build(); + } + + private static class UserDetailsMapper extends LdapUserDetailsMapper { + @Override + public UserDetails mapUserFromContext(DirContextOperations ctx, String username, + Collection authorities) { + UserDetails userDetails = super.mapUserFromContext(ctx, username, authorities); + return new RbacLdapUser(userDetails); + } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java index 1d237e0173..5db612f256 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java @@ -115,7 +115,7 @@ public class OAuthSecurityConfig extends AbstractAuthSecurityConfig { @Nullable private ProviderAuthorityExtractor getExtractor(final String providerId, AccessControlService acs) { final String provider = getProviderByProviderId(providerId); - Optional extractor = acs.getExtractors() + Optional extractor = acs.getOauthExtractors() .stream() .filter(e -> e.isApplicable(provider)) .findFirst(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java new file mode 100644 index 0000000000..037d2fd302 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java @@ -0,0 +1,60 @@ +package com.provectus.kafka.ui.config.auth; + +import java.util.Collection; +import java.util.stream.Collectors; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.userdetails.UserDetails; + +public class RbacLdapUser implements UserDetails, RbacUser { + + private final UserDetails userDetails; + + public RbacLdapUser(UserDetails userDetails) { + this.userDetails = userDetails; + } + + @Override + public String name() { + return userDetails.getUsername(); + } + + @Override + public Collection groups() { + return userDetails.getAuthorities().stream().map(GrantedAuthority::getAuthority).collect(Collectors.toSet()); + } + + @Override + public Collection getAuthorities() { + return userDetails.getAuthorities(); + } + + @Override + public String getPassword() { + return userDetails.getPassword(); + } + + @Override + public String getUsername() { + return userDetails.getUsername(); + } + + @Override + public boolean isAccountNonExpired() { + return userDetails.isAccountNonExpired(); + } + + @Override + public boolean isAccountNonLocked() { + return userDetails.isAccountNonLocked(); + } + + @Override + public boolean isCredentialsNonExpired() { + return userDetails.isCredentialsNonExpired(); + } + + @Override + public boolean isEnabled() { + return userDetails.isEnabled(); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java new file mode 100644 index 0000000000..c38e83238a --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java @@ -0,0 +1,21 @@ +package com.provectus.kafka.ui.config.auth.condition; + +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +public class ActiveDirectoryCondition extends AllNestedConditions { + + public ActiveDirectoryCondition() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") + public static class OnAuthType { + + } + + @ConditionalOnProperty(value = "${oauth2.ldap.activeDirectory}:false", havingValue = "true", matchIfMissing = false) + public static class OnActiveDirectory { + + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AclsController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AclsController.java new file mode 100644 index 0000000000..83d2ef553e --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AclsController.java @@ -0,0 +1,115 @@ +package com.provectus.kafka.ui.controller; + +import com.provectus.kafka.ui.api.AclsApi; +import com.provectus.kafka.ui.mapper.ClusterMapper; +import com.provectus.kafka.ui.model.KafkaAclDTO; +import com.provectus.kafka.ui.model.KafkaAclNamePatternTypeDTO; +import com.provectus.kafka.ui.model.KafkaAclResourceTypeDTO; +import com.provectus.kafka.ui.model.rbac.AccessContext; +import com.provectus.kafka.ui.model.rbac.permission.AclAction; +import com.provectus.kafka.ui.service.acl.AclsService; +import com.provectus.kafka.ui.service.rbac.AccessControlService; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePatternFilter; +import org.apache.kafka.common.resource.ResourceType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@RestController +@RequiredArgsConstructor +public class AclsController extends AbstractController implements AclsApi { + + private final AclsService aclsService; + private final AccessControlService accessControlService; + + @Override + public Mono> createAcl(String clusterName, Mono kafkaAclDto, + ServerWebExchange exchange) { + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .aclActions(AclAction.EDIT) + .build(); + + return accessControlService.validateAccess(context) + .then(kafkaAclDto) + .map(ClusterMapper::toAclBinding) + .flatMap(binding -> aclsService.createAcl(getCluster(clusterName), binding)) + .thenReturn(ResponseEntity.ok().build()); + } + + @Override + public Mono> deleteAcl(String clusterName, Mono kafkaAclDto, + ServerWebExchange exchange) { + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .aclActions(AclAction.EDIT) + .build(); + + return accessControlService.validateAccess(context) + .then(kafkaAclDto) + .map(ClusterMapper::toAclBinding) + .flatMap(binding -> aclsService.deleteAcl(getCluster(clusterName), binding)) + .thenReturn(ResponseEntity.ok().build()); + } + + @Override + public Mono>> listAcls(String clusterName, + KafkaAclResourceTypeDTO resourceTypeDto, + String resourceName, + KafkaAclNamePatternTypeDTO namePatternTypeDto, + ServerWebExchange exchange) { + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .aclActions(AclAction.VIEW) + .build(); + + var resourceType = Optional.ofNullable(resourceTypeDto) + .map(ClusterMapper::mapAclResourceTypeDto) + .orElse(ResourceType.ANY); + + var namePatternType = Optional.ofNullable(namePatternTypeDto) + .map(ClusterMapper::mapPatternTypeDto) + .orElse(PatternType.ANY); + + var filter = new ResourcePatternFilter(resourceType, resourceName, namePatternType); + + return accessControlService.validateAccess(context).then( + Mono.just( + ResponseEntity.ok( + aclsService.listAcls(getCluster(clusterName), filter) + .map(ClusterMapper::toKafkaAclDto))) + ); + } + + @Override + public Mono> getAclAsCsv(String clusterName, ServerWebExchange exchange) { + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .aclActions(AclAction.VIEW) + .build(); + + return accessControlService.validateAccess(context).then( + aclsService.getAclAsCsvString(getCluster(clusterName)) + .map(ResponseEntity::ok) + .flatMap(Mono::just) + ); + } + + @Override + public Mono> syncAclsCsv(String clusterName, Mono csvMono, ServerWebExchange exchange) { + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .aclActions(AclAction.EDIT) + .build(); + + return accessControlService.validateAccess(context) + .then(csvMono) + .flatMap(csv -> aclsService.syncAclWithAclCsv(getCluster(clusterName), csv)) + .thenReturn(ResponseEntity.ok().build()); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ApplicationConfigController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ApplicationConfigController.java index 571250ba94..df04b40fab 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ApplicationConfigController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ApplicationConfigController.java @@ -27,6 +27,7 @@ import org.mapstruct.Mapper; import org.mapstruct.factory.Mappers; import org.springframework.http.ResponseEntity; import org.springframework.http.codec.multipart.FilePart; +import org.springframework.http.codec.multipart.Part; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; @@ -92,16 +93,19 @@ public class ApplicationConfigController implements ApplicationConfigApi { } @Override - public Mono> uploadConfigRelatedFile(FilePart file, ServerWebExchange exchange) { + public Mono> uploadConfigRelatedFile(Flux fileFlux, + ServerWebExchange exchange) { return accessControlService .validateAccess( AccessContext.builder() .applicationConfigActions(EDIT) .build() ) - .then(dynamicConfigOperations.uploadConfigRelatedFile(file)) - .map(path -> new UploadedFileInfoDTO().location(path.toString())) - .map(ResponseEntity::ok); + .then(fileFlux.single()) + .flatMap(file -> + dynamicConfigOperations.uploadConfigRelatedFile((FilePart) file) + .map(path -> new UploadedFileInfoDTO().location(path.toString())) + .map(ResponseEntity::ok)); } @Override diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java index d989ce93ba..a122a269a4 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java @@ -20,6 +20,9 @@ import com.provectus.kafka.ui.model.InternalPartition; import com.provectus.kafka.ui.model.InternalReplica; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.InternalTopicConfig; +import com.provectus.kafka.ui.model.KafkaAclDTO; +import com.provectus.kafka.ui.model.KafkaAclNamePatternTypeDTO; +import com.provectus.kafka.ui.model.KafkaAclResourceTypeDTO; import com.provectus.kafka.ui.model.MetricDTO; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.PartitionDTO; @@ -27,12 +30,18 @@ import com.provectus.kafka.ui.model.ReplicaDTO; import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; -import com.provectus.kafka.ui.service.masking.DataMasking; import com.provectus.kafka.ui.service.metrics.RawMetric; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @@ -109,8 +118,74 @@ public interface ClusterMapper { return brokerDiskUsage; } - default DataMasking map(List maskingProperties) { - return DataMasking.create(maskingProperties); + static KafkaAclDTO.OperationEnum mapAclOperation(AclOperation operation) { + return switch (operation) { + case ALL -> KafkaAclDTO.OperationEnum.ALL; + case READ -> KafkaAclDTO.OperationEnum.READ; + case WRITE -> KafkaAclDTO.OperationEnum.WRITE; + case CREATE -> KafkaAclDTO.OperationEnum.CREATE; + case DELETE -> KafkaAclDTO.OperationEnum.DELETE; + case ALTER -> KafkaAclDTO.OperationEnum.ALTER; + case DESCRIBE -> KafkaAclDTO.OperationEnum.DESCRIBE; + case CLUSTER_ACTION -> KafkaAclDTO.OperationEnum.CLUSTER_ACTION; + case DESCRIBE_CONFIGS -> KafkaAclDTO.OperationEnum.DESCRIBE_CONFIGS; + case ALTER_CONFIGS -> KafkaAclDTO.OperationEnum.ALTER_CONFIGS; + case IDEMPOTENT_WRITE -> KafkaAclDTO.OperationEnum.IDEMPOTENT_WRITE; + case CREATE_TOKENS -> KafkaAclDTO.OperationEnum.CREATE_TOKENS; + case DESCRIBE_TOKENS -> KafkaAclDTO.OperationEnum.DESCRIBE_TOKENS; + case ANY -> throw new IllegalArgumentException("ANY operation can be only part of filter"); + case UNKNOWN -> KafkaAclDTO.OperationEnum.UNKNOWN; + }; + } + + static KafkaAclResourceTypeDTO mapAclResourceType(ResourceType resourceType) { + return switch (resourceType) { + case CLUSTER -> KafkaAclResourceTypeDTO.CLUSTER; + case TOPIC -> KafkaAclResourceTypeDTO.TOPIC; + case GROUP -> KafkaAclResourceTypeDTO.GROUP; + case DELEGATION_TOKEN -> KafkaAclResourceTypeDTO.DELEGATION_TOKEN; + case TRANSACTIONAL_ID -> KafkaAclResourceTypeDTO.TRANSACTIONAL_ID; + case USER -> KafkaAclResourceTypeDTO.USER; + case ANY -> throw new IllegalArgumentException("ANY type can be only part of filter"); + case UNKNOWN -> KafkaAclResourceTypeDTO.UNKNOWN; + }; + } + + static ResourceType mapAclResourceTypeDto(KafkaAclResourceTypeDTO dto) { + return ResourceType.valueOf(dto.name()); + } + + static PatternType mapPatternTypeDto(KafkaAclNamePatternTypeDTO dto) { + return PatternType.valueOf(dto.name()); + } + + static AclBinding toAclBinding(KafkaAclDTO dto) { + return new AclBinding( + new ResourcePattern( + mapAclResourceTypeDto(dto.getResourceType()), + dto.getResourceName(), + mapPatternTypeDto(dto.getNamePatternType()) + ), + new AccessControlEntry( + dto.getPrincipal(), + dto.getHost(), + AclOperation.valueOf(dto.getOperation().name()), + AclPermissionType.valueOf(dto.getPermission().name()) + ) + ); + } + + static KafkaAclDTO toKafkaAclDto(AclBinding binding) { + var pattern = binding.pattern(); + var filter = binding.toFilter().entryFilter(); + return new KafkaAclDTO() + .resourceType(mapAclResourceType(pattern.resourceType())) + .resourceName(pattern.name()) + .namePatternType(KafkaAclNamePatternTypeDTO.fromValue(pattern.patternType().name())) + .principal(filter.principal()) + .host(filter.host()) + .operation(mapAclOperation(filter.operation())) + .permission(KafkaAclDTO.PermissionEnum.fromValue(filter.permissionType().name())); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ClusterFeature.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ClusterFeature.java index 9731492f00..2973e5500d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ClusterFeature.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ClusterFeature.java @@ -4,5 +4,7 @@ public enum ClusterFeature { KAFKA_CONNECT, KSQL_DB, SCHEMA_REGISTRY, - TOPIC_DELETION + TOPIC_DELETION, + KAFKA_ACL_VIEW, + KAFKA_ACL_EDIT } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java index b625533d1d..46efc67000 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java @@ -1,7 +1,7 @@ package com.provectus.kafka.ui.model; import java.math.BigDecimal; -import java.math.MathContext; +import java.math.RoundingMode; import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -21,8 +21,6 @@ public class PartitionDistributionStats { // avg skew will show unuseful results on low number of partitions private static final int MIN_PARTITIONS_FOR_SKEW_CALCULATION = 50; - private static final MathContext ROUNDING_MATH_CTX = new MathContext(3); - private final Map partitionLeaders; private final Map partitionsCount; private final Map inSyncPartitions; @@ -88,6 +86,7 @@ public class PartitionDistributionStats { return null; } value = value == null ? 0 : value; - return new BigDecimal((value - avgValue) / avgValue * 100.0).round(ROUNDING_MATH_CTX); + return new BigDecimal((value - avgValue) / avgValue * 100.0) + .setScale(1, RoundingMode.HALF_UP); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/AccessContext.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/AccessContext.java index 0c2587d681..45858093a7 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/AccessContext.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/AccessContext.java @@ -1,5 +1,6 @@ package com.provectus.kafka.ui.model.rbac; +import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; @@ -37,6 +38,8 @@ public class AccessContext { Collection ksqlActions; + Collection aclActions; + public static AccessContextBuilder builder() { return new AccessContextBuilder(); } @@ -55,6 +58,7 @@ public class AccessContext { private String schema; private Collection schemaActions = Collections.emptySet(); private Collection ksqlActions = Collections.emptySet(); + private Collection aclActions = Collections.emptySet(); private AccessContextBuilder() { } @@ -131,6 +135,12 @@ public class AccessContext { return this; } + public AccessContextBuilder aclActions(AclAction... actions) { + Assert.isTrue(actions.length > 0, "actions not present"); + this.aclActions = List.of(actions); + return this; + } + public AccessContext build() { return new AccessContext( applicationConfigActions, @@ -140,7 +150,7 @@ public class AccessContext { connect, connectActions, connector, schema, schemaActions, - ksqlActions); + ksqlActions, aclActions); } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Permission.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Permission.java index afdcf0ca15..16f01f60e6 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Permission.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Permission.java @@ -4,6 +4,7 @@ import static com.provectus.kafka.ui.model.rbac.Resource.APPLICATIONCONFIG; import static com.provectus.kafka.ui.model.rbac.Resource.CLUSTERCONFIG; import static com.provectus.kafka.ui.model.rbac.Resource.KSQL; +import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; @@ -76,6 +77,7 @@ public class Permission { case SCHEMA -> Arrays.stream(SchemaAction.values()).map(Enum::toString).toList(); case CONNECT -> Arrays.stream(ConnectAction.values()).map(Enum::toString).toList(); case KSQL -> Arrays.stream(KsqlAction.values()).map(Enum::toString).toList(); + case ACL -> Arrays.stream(AclAction.values()).map(Enum::toString).toList(); }; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Resource.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Resource.java index 4b2c66361f..f71dfb2979 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Resource.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Resource.java @@ -11,7 +11,8 @@ public enum Resource { CONSUMER, SCHEMA, CONNECT, - KSQL; + KSQL, + ACL; @Nullable public static Resource fromString(String name) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/AclAction.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/AclAction.java new file mode 100644 index 0000000000..c86af7e72d --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/AclAction.java @@ -0,0 +1,15 @@ +package com.provectus.kafka.ui.model.rbac.permission; + +import org.apache.commons.lang3.EnumUtils; +import org.jetbrains.annotations.Nullable; + +public enum AclAction implements PermissibleAction { + + VIEW, + EDIT; + + @Nullable + public static AclAction fromString(String name) { + return EnumUtils.getEnum(AclAction.class, name); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/FeatureService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/FeatureService.java index ec749abd14..7ba3f036e9 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/FeatureService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/FeatureService.java @@ -2,16 +2,19 @@ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.KafkaCluster; +import com.provectus.kafka.ui.service.ReactiveAdminClient.ClusterDescription; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.Node; +import org.apache.kafka.common.acl.AclOperation; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -26,7 +29,7 @@ public class FeatureService { private final AdminClientService adminClientService; public Mono> getAvailableFeatures(KafkaCluster cluster, - ReactiveAdminClient.ClusterDescription clusterDescription) { + ClusterDescription clusterDescription) { List> features = new ArrayList<>(); if (Optional.ofNullable(cluster.getConnectsClients()) @@ -44,6 +47,8 @@ public class FeatureService { } features.add(topicDeletionEnabled(cluster, clusterDescription.getController())); + features.add(aclView(cluster)); + features.add(aclEdit(clusterDescription)); return Flux.fromIterable(features).flatMap(m -> m).collectList(); } @@ -65,4 +70,20 @@ public class FeatureService { ? Mono.just(ClusterFeature.TOPIC_DELETION) : Mono.empty()); } + + private Mono aclEdit(ClusterDescription clusterDescription) { + var authorizedOps = Optional.ofNullable(clusterDescription.getAuthorizedOperations()).orElse(Set.of()); + boolean canEdit = authorizedOps.contains(AclOperation.ALL) || authorizedOps.contains(AclOperation.ALTER); + return canEdit + ? Mono.just(ClusterFeature.KAFKA_ACL_EDIT) + : Mono.empty(); + } + + private Mono aclView(KafkaCluster cluster) { + return adminClientService.get(cluster).flatMap( + ac -> ac.getClusterFeatures().contains(ReactiveAdminClient.SupportedFeature.AUTHORIZED_SECURITY_ENABLED) + ? Mono.just(ClusterFeature.KAFKA_ACL_VIEW) + : Mono.empty() + ); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java index 39332da39e..8451a89f97 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java @@ -5,6 +5,7 @@ import static java.util.stream.Collectors.toMap; import static org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Iterables; import com.google.common.collect.Table; @@ -15,7 +16,6 @@ import com.provectus.kafka.ui.util.KafkaVersion; import com.provectus.kafka.ui.util.annotation.KafkaClientInternalsDependant; import java.io.Closeable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -61,16 +61,22 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.acl.AccessControlEntryFilter; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.ClusterAuthorizationException; import org.apache.kafka.common.errors.GroupIdNotFoundException; import org.apache.kafka.common.errors.GroupNotEmptyException; import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.SecurityDisabledException; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.requests.DescribeLogDirsResponse; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourcePatternFilter; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -82,26 +88,29 @@ import reactor.util.function.Tuples; @RequiredArgsConstructor public class ReactiveAdminClient implements Closeable { - private enum SupportedFeature { + public enum SupportedFeature { INCREMENTAL_ALTER_CONFIGS(2.3f), CONFIG_DOCUMENTATION_RETRIEVAL(2.6f), - DESCRIBE_CLUSTER_INCLUDE_AUTHORIZED_OPERATIONS(2.3f); + DESCRIBE_CLUSTER_INCLUDE_AUTHORIZED_OPERATIONS(2.3f), + AUTHORIZED_SECURITY_ENABLED(ReactiveAdminClient::isAuthorizedSecurityEnabled); - private final float sinceVersion; + private final BiFunction> predicate; - SupportedFeature(float sinceVersion) { - this.sinceVersion = sinceVersion; + SupportedFeature(BiFunction> predicate) { + this.predicate = predicate; } - static Set forVersion(float kafkaVersion) { - return Arrays.stream(SupportedFeature.values()) - .filter(f -> kafkaVersion >= f.sinceVersion) + SupportedFeature(float fromVersion) { + this.predicate = (admin, ver) -> Mono.just(ver != null && ver >= fromVersion); + } + + static Mono> forVersion(AdminClient ac, @Nullable Float kafkaVersion) { + return Flux.fromArray(SupportedFeature.values()) + .flatMap(f -> f.predicate.apply(ac, kafkaVersion).map(enabled -> Tuples.of(f, enabled))) + .filter(Tuple2::getT2) + .map(Tuple2::getT1) .collect(Collectors.toSet()); } - - static Set defaultFeatures() { - return Set.of(); - } } @Value @@ -110,25 +119,31 @@ public class ReactiveAdminClient implements Closeable { Node controller; String clusterId; Collection nodes; + @Nullable // null, if ACL is disabled Set authorizedOperations; } public static Mono create(AdminClient adminClient) { return getClusterVersion(adminClient) - .map(ver -> - new ReactiveAdminClient( - adminClient, - ver, - getSupportedUpdateFeaturesForVersion(ver))); + .flatMap(ver -> + getSupportedUpdateFeaturesForVersion(adminClient, ver) + .map(features -> + new ReactiveAdminClient(adminClient, ver, features))); } - private static Set getSupportedUpdateFeaturesForVersion(String versionStr) { - try { - float version = KafkaVersion.parse(versionStr); - return SupportedFeature.forVersion(version); - } catch (NumberFormatException e) { - return SupportedFeature.defaultFeatures(); - } + private static Mono> getSupportedUpdateFeaturesForVersion(AdminClient ac, String versionStr) { + @Nullable Float kafkaVersion = KafkaVersion.parse(versionStr).orElse(null); + return SupportedFeature.forVersion(ac, kafkaVersion); + } + + private static Mono isAuthorizedSecurityEnabled(AdminClient ac, @Nullable Float kafkaVersion) { + return toMono(ac.describeAcls(AclBindingFilter.ANY).values()) + .thenReturn(true) + .doOnError(th -> !(th instanceof SecurityDisabledException) + && !(th instanceof InvalidRequestException) + && !(th instanceof UnsupportedVersionException), + th -> log.warn("Error checking if security enabled", th)) + .onErrorReturn(false); } // NOTE: if KafkaFuture returns null, that Mono will be empty(!), since Reactor does not support nullable results @@ -162,6 +177,10 @@ public class ReactiveAdminClient implements Closeable { private final String version; private final Set features; + public Set getClusterFeatures() { + return features; + } + public Mono> listTopics(boolean listInternal) { return toMono(client.listTopics(new ListTopicsOptions().listInternal(listInternal)).names()); } @@ -576,6 +595,22 @@ public class ReactiveAdminClient implements Closeable { ); } + public Mono> listAcls(ResourcePatternFilter filter) { + Preconditions.checkArgument(features.contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); + return toMono(client.describeAcls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).values()); + } + + public Mono createAcls(Collection aclBindings) { + Preconditions.checkArgument(features.contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); + return toMono(client.createAcls(aclBindings).all()); + } + + public Mono deleteAcls(Collection aclBindings) { + Preconditions.checkArgument(features.contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); + var filters = aclBindings.stream().map(AclBinding::toFilter).collect(Collectors.toSet()); + return toMono(client.deleteAcls(filters).all()).then(); + } + public Mono updateBrokerConfigByName(Integer brokerId, String name, String value) { ConfigResource cr = new ConfigResource(ConfigResource.Type.BROKER, String.valueOf(brokerId)); AlterConfigOp op = new AlterConfigOp(new ConfigEntry(name, value), AlterConfigOp.OpType.SET); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclCsv.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclCsv.java new file mode 100644 index 0000000000..673b17ee1f --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclCsv.java @@ -0,0 +1,81 @@ +package com.provectus.kafka.ui.service.acl; + +import com.provectus.kafka.ui.exception.ValidationException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; + +public class AclCsv { + + private static final String LINE_SEPARATOR = System.lineSeparator(); + private static final String VALUES_SEPARATOR = ","; + private static final String HEADER = "Principal,ResourceType,PatternType,ResourceName,Operation,PermissionType,Host"; + + public static String transformToCsvString(Collection acls) { + return Stream.concat(Stream.of(HEADER), acls.stream().map(AclCsv::createAclString)) + .collect(Collectors.joining(System.lineSeparator())); + } + + public static String createAclString(AclBinding binding) { + var pattern = binding.pattern(); + var filter = binding.toFilter().entryFilter(); + return String.format( + "%s,%s,%s,%s,%s,%s,%s", + filter.principal(), + pattern.resourceType(), + pattern.patternType(), + pattern.name(), + filter.operation(), + filter.permissionType(), + filter.host() + ); + } + + private static AclBinding parseCsvLine(String csv, int line) { + String[] values = csv.split(VALUES_SEPARATOR); + if (values.length != 7) { + throw new ValidationException("Input csv is not valid - there should be 7 columns in line " + line); + } + for (int i = 0; i < values.length; i++) { + if ((values[i] = values[i].trim()).isBlank()) { + throw new ValidationException("Input csv is not valid - blank value in colum " + i + ", line " + line); + } + } + try { + return new AclBinding( + new ResourcePattern( + ResourceType.valueOf(values[1]), values[3], PatternType.valueOf(values[2])), + new AccessControlEntry( + values[0], values[6], AclOperation.valueOf(values[4]), AclPermissionType.valueOf(values[5])) + ); + } catch (IllegalArgumentException enumParseError) { + throw new ValidationException("Error parsing enum value in line " + line); + } + } + + public static Collection parseCsv(String csvString) { + String[] lines = csvString.split(LINE_SEPARATOR); + if (lines.length == 0) { + throw new ValidationException("Error parsing ACL csv file: no lines in file"); + } + boolean firstLineIsHeader = HEADER.equalsIgnoreCase(lines[0].trim().replace(" ", "")); + Set result = new HashSet<>(); + for (int i = firstLineIsHeader ? 1 : 0; i < lines.length; i++) { + String line = lines[i]; + if (!line.isBlank()) { + AclBinding aclBinding = parseCsvLine(line, i); + result.add(aclBinding); + } + } + return result; + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclsService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclsService.java new file mode 100644 index 0000000000..8c5a8dab06 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclsService.java @@ -0,0 +1,93 @@ +package com.provectus.kafka.ui.service.acl; + +import com.google.common.collect.Sets; +import com.provectus.kafka.ui.model.KafkaCluster; +import com.provectus.kafka.ui.service.AdminClientService; +import java.util.List; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.resource.ResourcePatternFilter; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@Slf4j +@Service +@RequiredArgsConstructor +public class AclsService { + + private final AdminClientService adminClientService; + + public Mono createAcl(KafkaCluster cluster, AclBinding aclBinding) { + var aclString = AclCsv.createAclString(aclBinding); + log.info("CREATING ACL: [{}]", aclString); + return adminClientService.get(cluster) + .flatMap(ac -> ac.createAcls(List.of(aclBinding))) + .doOnSuccess(v -> log.info("ACL CREATED: [{}]", aclString)); + } + + public Mono deleteAcl(KafkaCluster cluster, AclBinding aclBinding) { + var aclString = AclCsv.createAclString(aclBinding); + log.info("DELETING ACL: [{}]", aclString); + return adminClientService.get(cluster) + .flatMap(ac -> ac.deleteAcls(List.of(aclBinding))) + .doOnSuccess(v -> log.info("ACL DELETED: [{}]", aclString)); + } + + public Flux listAcls(KafkaCluster cluster, ResourcePatternFilter filter) { + return adminClientService.get(cluster) + .flatMap(c -> c.listAcls(filter)) + .flatMapIterable(acls -> acls); + } + + public Mono getAclAsCsvString(KafkaCluster cluster) { + return adminClientService.get(cluster) + .flatMap(c -> c.listAcls(ResourcePatternFilter.ANY)) + .map(AclCsv::transformToCsvString); + } + + public Mono syncAclWithAclCsv(KafkaCluster cluster, String csv) { + return adminClientService.get(cluster) + .flatMap(ac -> ac.listAcls(ResourcePatternFilter.ANY).flatMap(existingAclList -> { + var existingSet = Set.copyOf(existingAclList); + var newAcls = Set.copyOf(AclCsv.parseCsv(csv)); + var toDelete = Sets.difference(existingSet, newAcls); + var toAdd = Sets.difference(newAcls, existingSet); + logAclSyncPlan(cluster, toAdd, toDelete); + if (toAdd.isEmpty() && toDelete.isEmpty()) { + return Mono.empty(); + } + log.info("Starting new ACLs creation"); + return ac.createAcls(toAdd) + .doOnSuccess(v -> { + log.info("{} new ACLs created", toAdd.size()); + log.info("Starting ACLs deletion"); + }) + .then(ac.deleteAcls(toDelete) + .doOnSuccess(v -> log.info("{} ACLs deleted", toDelete.size()))); + })); + } + + private void logAclSyncPlan(KafkaCluster cluster, Set toBeAdded, Set toBeDeleted) { + log.info("'{}' cluster ACL sync plan: ", cluster.getName()); + if (toBeAdded.isEmpty() && toBeDeleted.isEmpty()) { + log.info("Nothing to do, ACL is already in sync"); + return; + } + if (!toBeAdded.isEmpty()) { + log.info("ACLs to be added ({}): ", toBeAdded.size()); + for (AclBinding aclBinding : toBeAdded) { + log.info(" " + AclCsv.createAclString(aclBinding)); + } + } + if (!toBeDeleted.isEmpty()) { + log.info("ACLs to be deleted ({}): ", toBeDeleted.size()); + for (AclBinding aclBinding : toBeDeleted) { + log.info(" " + AclCsv.createAclString(aclBinding)); + } + } + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/DataMasking.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/DataMasking.java index 78e74f3332..ad1c2da31e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/DataMasking.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/DataMasking.java @@ -44,7 +44,7 @@ public class DataMasking { public static DataMasking create(@Nullable List config) { return new DataMasking( Optional.ofNullable(config).orElse(List.of()).stream().map(property -> { - Preconditions.checkNotNull(property.getType(), "masking type not specifed"); + Preconditions.checkNotNull(property.getType(), "masking type not specified"); Preconditions.checkArgument( StringUtils.isNotEmpty(property.getTopicKeysPattern()) || StringUtils.isNotEmpty(property.getTopicValuesPattern()), diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelector.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelector.java new file mode 100644 index 0000000000..9956394398 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelector.java @@ -0,0 +1,28 @@ +package com.provectus.kafka.ui.service.masking.policies; + +import com.provectus.kafka.ui.config.ClustersProperties; +import com.provectus.kafka.ui.exception.ValidationException; +import java.util.regex.Pattern; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +interface FieldsSelector { + + static FieldsSelector create(ClustersProperties.Masking property) { + if (StringUtils.hasText(property.getFieldsNamePattern()) && !CollectionUtils.isEmpty(property.getFields())) { + throw new ValidationException("You can't provide both fieldNames & fieldsNamePattern for masking"); + } + if (StringUtils.hasText(property.getFieldsNamePattern())) { + Pattern pattern = Pattern.compile(property.getFieldsNamePattern()); + return f -> pattern.matcher(f).matches(); + } + if (!CollectionUtils.isEmpty(property.getFields())) { + return f -> property.getFields().contains(f); + } + //no pattern, no field names - mean all fields should be masked + return fieldName -> true; + } + + boolean shouldBeMasked(String fieldName); + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Mask.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Mask.java index dbbc5d131a..e6a469f2c0 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Mask.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Mask.java @@ -15,8 +15,8 @@ class Mask extends MaskingPolicy { private final UnaryOperator masker; - Mask(List fieldNames, List maskingChars) { - super(fieldNames); + Mask(FieldsSelector fieldsSelector, List maskingChars) { + super(fieldsSelector); this.masker = createMasker(maskingChars); } @@ -38,22 +38,13 @@ class Mask extends MaskingPolicy { for (int i = 0; i < input.length(); i++) { int cp = input.codePointAt(i); switch (Character.getType(cp)) { - case Character.SPACE_SEPARATOR: - case Character.LINE_SEPARATOR: - case Character.PARAGRAPH_SEPARATOR: - sb.appendCodePoint(cp); // keeping separators as-is - break; - case Character.UPPERCASE_LETTER: - sb.append(maskingChars.get(0)); - break; - case Character.LOWERCASE_LETTER: - sb.append(maskingChars.get(1)); - break; - case Character.DECIMAL_DIGIT_NUMBER: - sb.append(maskingChars.get(2)); - break; - default: - sb.append(maskingChars.get(3)); + case Character.SPACE_SEPARATOR, + Character.LINE_SEPARATOR, + Character.PARAGRAPH_SEPARATOR -> sb.appendCodePoint(cp); // keeping separators as-is + case Character.UPPERCASE_LETTER -> sb.append(maskingChars.get(0)); + case Character.LOWERCASE_LETTER -> sb.append(maskingChars.get(1)); + case Character.DECIMAL_DIGIT_NUMBER -> sb.append(maskingChars.get(2)); + default -> sb.append(maskingChars.get(3)); } } return sb.toString(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/MaskingPolicy.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/MaskingPolicy.java index 7a75338210..9b80da0cb1 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/MaskingPolicy.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/MaskingPolicy.java @@ -2,46 +2,36 @@ package com.provectus.kafka.ui.service.masking.policies; import com.fasterxml.jackson.databind.node.ContainerNode; import com.provectus.kafka.ui.config.ClustersProperties; -import java.util.List; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public abstract class MaskingPolicy { - public static MaskingPolicy create(ClustersProperties.Masking property) { - List fields = property.getFields() == null - ? List.of() // empty list means that policy will be applied to all fields - : property.getFields(); - switch (property.getType()) { - case REMOVE: - return new Remove(fields); - case REPLACE: - return new Replace( - fields, - property.getReplacement() == null - ? Replace.DEFAULT_REPLACEMENT - : property.getReplacement() - ); - case MASK: - return new Mask( - fields, - property.getPattern() == null - ? Mask.DEFAULT_PATTERN - : property.getPattern() - ); - default: - throw new IllegalStateException("Unknown policy type: " + property.getType()); - } + FieldsSelector fieldsSelector = FieldsSelector.create(property); + return switch (property.getType()) { + case REMOVE -> new Remove(fieldsSelector); + case REPLACE -> new Replace( + fieldsSelector, + property.getReplacement() == null + ? Replace.DEFAULT_REPLACEMENT + : property.getReplacement() + ); + case MASK -> new Mask( + fieldsSelector, + property.getMaskingCharsReplacement() == null + ? Mask.DEFAULT_PATTERN + : property.getMaskingCharsReplacement() + ); + }; } //---------------------------------------------------------------- - // empty list means policy will be applied to all fields - private final List fieldNames; + private final FieldsSelector fieldsSelector; protected boolean fieldShouldBeMasked(String fieldName) { - return fieldNames.isEmpty() || fieldNames.contains(fieldName); + return fieldsSelector.shouldBeMasked(fieldName); } public abstract ContainerNode applyToJsonContainer(ContainerNode node); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Remove.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Remove.java index eb38b0d3e3..cc5cdd1415 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Remove.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Remove.java @@ -4,12 +4,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.List; + class Remove extends MaskingPolicy { - Remove(List fieldNames) { - super(fieldNames); + Remove(FieldsSelector fieldsSelector) { + super(fieldsSelector); } @Override diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Replace.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Replace.java index 3af645cb11..1cf91793d2 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Replace.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Replace.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Preconditions; -import java.util.List; class Replace extends MaskingPolicy { @@ -14,8 +13,8 @@ class Replace extends MaskingPolicy { private final String replacement; - Replace(List fieldNames, String replacementString) { - super(fieldNames); + Replace(FieldsSelector fieldsSelector, String replacementString) { + super(fieldsSelector); this.replacement = Preconditions.checkNotNull(replacementString); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java index 06304365c7..fa84fc361c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java @@ -61,7 +61,9 @@ class JmxSslSocketFactory extends javax.net.ssl.SSLSocketFactory { } catch (Exception e) { log.error("----------------------------------"); log.error("SSL can't be enabled for JMX retrieval. " - + "Make sure your java app run with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg.", e); + + "Make sure your java app run with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", + e.getMessage()); + log.trace("SSL can't be enabled for JMX retrieval", e); log.error("----------------------------------"); } SSL_JMX_SUPPORTED = sslJmxSupported; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java index 3178feae34..e964f64a9b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java @@ -12,6 +12,7 @@ import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.Permission; import com.provectus.kafka.ui.model.rbac.Resource; import com.provectus.kafka.ui.model.rbac.Role; +import com.provectus.kafka.ui.model.rbac.Subject; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; @@ -19,11 +20,11 @@ import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import com.provectus.kafka.ui.service.rbac.extractor.CognitoAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GithubAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GoogleAuthorityExtractor; -import com.provectus.kafka.ui.service.rbac.extractor.LdapAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.ProviderAuthorityExtractor; import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.core.env.Environment; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.context.ReactiveSecurityContextHolder; import org.springframework.security.core.context.SecurityContext; @@ -50,10 +52,11 @@ public class AccessControlService { @Nullable private final InMemoryReactiveClientRegistrationRepository clientRegistrationRepository; + private final RoleBasedAccessControlProperties properties; + private final Environment environment; private boolean rbacEnabled = false; - private Set extractors = Collections.emptySet(); - private final RoleBasedAccessControlProperties properties; + private Set oauthExtractors = Collections.emptySet(); @PostConstruct public void init() { @@ -63,21 +66,26 @@ public class AccessControlService { } rbacEnabled = true; - this.extractors = properties.getRoles() + this.oauthExtractors = properties.getRoles() .stream() .map(role -> role.getSubjects() .stream() - .map(provider -> switch (provider.getProvider()) { + .map(Subject::getProvider) + .distinct() + .map(provider -> switch (provider) { case OAUTH_COGNITO -> new CognitoAuthorityExtractor(); case OAUTH_GOOGLE -> new GoogleAuthorityExtractor(); case OAUTH_GITHUB -> new GithubAuthorityExtractor(); - case LDAP, LDAP_AD -> new LdapAuthorityExtractor(); - }).collect(Collectors.toSet())) + default -> null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet())) .flatMap(Set::stream) .collect(Collectors.toSet()); - if ((clientRegistrationRepository == null || !clientRegistrationRepository.iterator().hasNext()) - && !properties.getRoles().isEmpty()) { + if (!properties.getRoles().isEmpty() + && "oauth2".equalsIgnoreCase(environment.getProperty("auth.type")) + && (clientRegistrationRepository == null || !clientRegistrationRepository.iterator().hasNext())) { log.error("Roles are configured but no authentication methods are present. Authentication might fail."); } } @@ -354,8 +362,8 @@ public class AccessControlService { return isAccessible(Resource.KSQL, null, user, context, requiredActions); } - public Set getExtractors() { - return extractors; + public Set getOauthExtractors() { + return oauthExtractors; } public List getRoles() { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java deleted file mode 100644 index 6284bb2923..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.provectus.kafka.ui.service.rbac.extractor; - -import com.provectus.kafka.ui.service.rbac.AccessControlService; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; - -@Slf4j -public class LdapAuthorityExtractor implements ProviderAuthorityExtractor { - - @Override - public boolean isApplicable(String provider) { - return false; // TODO #2752 - } - - @Override - public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { - return Mono.just(Collections.emptySet()); // TODO #2752 - } - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java new file mode 100644 index 0000000000..e24fc0aeda --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java @@ -0,0 +1,70 @@ +package com.provectus.kafka.ui.service.rbac.extractor; + +import com.provectus.kafka.ui.config.auth.LdapProperties; +import com.provectus.kafka.ui.model.rbac.Role; +import com.provectus.kafka.ui.model.rbac.provider.Provider; +import com.provectus.kafka.ui.service.rbac.AccessControlService; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationContext; +import org.springframework.ldap.core.DirContextOperations; +import org.springframework.ldap.core.support.BaseLdapPathContextSource; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; +import org.springframework.util.Assert; + +@Slf4j +public class RbacLdapAuthoritiesExtractor extends DefaultLdapAuthoritiesPopulator { + + private final AccessControlService acs; + private final LdapProperties props; + + private final Function>, GrantedAuthority> authorityMapper = (record) -> { + String role = record.get(getGroupRoleAttribute()).get(0); + return new SimpleGrantedAuthority(role); + }; + + public RbacLdapAuthoritiesExtractor(ApplicationContext context) { + super(context.getBean(BaseLdapPathContextSource.class), null); + this.acs = context.getBean(AccessControlService.class); + this.props = context.getBean(LdapProperties.class); + } + + @Override + public Set getAdditionalRoles(DirContextOperations user, String username) { + return acs.getRoles() + .stream() + .map(Role::getSubjects) + .flatMap(List::stream) + .filter(s -> s.getProvider().equals(Provider.LDAP)) + .filter(s -> s.getType().equals("group")) + .flatMap(subject -> getRoles(subject.getValue(), user.getNameInNamespace(), username).stream()) + .collect(Collectors.toSet()); + } + + private Set getRoles(String groupSearchBase, String userDn, String username) { + Assert.notNull(groupSearchBase, "groupSearchBase is empty"); + + log.trace( + "Searching for roles for user [{}] with DN [{}], groupRoleAttribute [{}] and filter [{}] in search base [{}]", + username, userDn, props.getGroupRoleAttribute(), getGroupSearchFilter(), groupSearchBase); + + var ldapTemplate = getLdapTemplate(); + ldapTemplate.setIgnoreNameNotFoundException(true); + + Set>> userRoles = ldapTemplate.searchForMultipleAttributeValues( + groupSearchBase, getGroupSearchFilter(), new String[] {userDn, username}, + new String[] {props.getGroupRoleAttribute()}); + + return userRoles.stream() + .map(authorityMapper) + .peek(a -> log.debug("Mapped role [{}] for user [{}]", a, username)) + .collect(Collectors.toSet()); + } + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/DynamicConfigOperations.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/DynamicConfigOperations.java index 75c6d25f95..68f826bd0f 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/DynamicConfigOperations.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/DynamicConfigOperations.java @@ -90,6 +90,7 @@ public class DynamicConfigOperations { } public PropertiesStructure getCurrentProperties() { + checkIfDynamicConfigEnabled(); return PropertiesStructure.builder() .kafka(getNullableBean(ClustersProperties.class)) .rbac(getNullableBean(RoleBasedAccessControlProperties.class)) @@ -112,11 +113,7 @@ public class DynamicConfigOperations { } public void persist(PropertiesStructure properties) { - if (!dynamicConfigEnabled()) { - throw new ValidationException( - "Dynamic config change is not allowed. " - + "Set dynamic.config.enabled property to 'true' to enabled it."); - } + checkIfDynamicConfigEnabled(); properties.initAndValidate(); String yaml = serializeToYaml(properties); @@ -124,8 +121,9 @@ public class DynamicConfigOperations { } public Mono uploadConfigRelatedFile(FilePart file) { - String targetDirStr = (String) ctx.getEnvironment().getSystemEnvironment() - .getOrDefault(CONFIG_RELATED_UPLOADS_DIR_PROPERTY, CONFIG_RELATED_UPLOADS_DIR_DEFAULT); + checkIfDynamicConfigEnabled(); + String targetDirStr = ctx.getEnvironment() + .getProperty(CONFIG_RELATED_UPLOADS_DIR_PROPERTY, CONFIG_RELATED_UPLOADS_DIR_DEFAULT); Path targetDir = Path.of(targetDirStr); if (!Files.exists(targetDir)) { @@ -149,6 +147,14 @@ public class DynamicConfigOperations { .onErrorMap(th -> new FileUploadException(targetFilePath, th)); } + private void checkIfDynamicConfigEnabled() { + if (!dynamicConfigEnabled()) { + throw new ValidationException( + "Dynamic config change is not allowed. " + + "Set dynamic.config.enabled property to 'true' to enabled it."); + } + } + @SneakyThrows private void writeYamlToFile(String yaml, Path path) { if (Files.isDirectory(path)) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaVersion.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaVersion.java index 5ed21c6a6e..3d6b2ca40e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaVersion.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaVersion.java @@ -1,24 +1,21 @@ package com.provectus.kafka.ui.util; -import lombok.extern.slf4j.Slf4j; +import java.util.Optional; -@Slf4j public final class KafkaVersion { private KafkaVersion() { } - public static float parse(String version) throws NumberFormatException { - log.trace("Parsing cluster version [{}]", version); + public static Optional parse(String version) throws NumberFormatException { try { final String[] parts = version.split("\\."); if (parts.length > 2) { version = parts[0] + "." + parts[1]; } - return Float.parseFloat(version.split("-")[0]); + return Optional.of(Float.parseFloat(version.split("-")[0])); } catch (Exception e) { - log.error("Conversion clusterVersion [{}] to float value failed", version, e); - throw e; + return Optional.empty(); } } } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java index dbdfb67fd5..1938f93044 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java @@ -2,6 +2,7 @@ package com.provectus.kafka.ui; import com.provectus.kafka.ui.container.KafkaConnectContainer; import com.provectus.kafka.ui.container.SchemaRegistryContainer; +import java.nio.file.Path; import java.util.List; import java.util.Properties; import org.apache.kafka.clients.admin.AdminClient; @@ -9,6 +10,7 @@ import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.function.ThrowingConsumer; +import org.junit.jupiter.api.io.TempDir; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; import org.springframework.boot.test.context.SpringBootTest; @@ -47,6 +49,9 @@ public abstract class AbstractIntegrationTest { .dependsOn(kafka) .dependsOn(schemaRegistry); + @TempDir + public static Path tmpDir; + static { kafka.start(); schemaRegistry.start(); @@ -76,6 +81,9 @@ public abstract class AbstractIntegrationTest { System.setProperty("kafka.clusters.1.schemaRegistry", schemaRegistry.getUrl()); System.setProperty("kafka.clusters.1.kafkaConnect.0.name", "kafka-connect"); System.setProperty("kafka.clusters.1.kafkaConnect.0.address", kafkaConnect.getTarget()); + + System.setProperty("dynamic.config.enabled", "true"); + System.setProperty("config.related.uploads.dir", tmpDir.toString()); } } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/controller/ApplicationConfigControllerTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/controller/ApplicationConfigControllerTest.java new file mode 100644 index 0000000000..7840760868 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/controller/ApplicationConfigControllerTest.java @@ -0,0 +1,49 @@ +package com.provectus.kafka.ui.controller; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.provectus.kafka.ui.AbstractIntegrationTest; +import com.provectus.kafka.ui.model.UploadedFileInfoDTO; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.client.MultipartBodyBuilder; +import org.springframework.test.web.reactive.server.WebTestClient; +import org.springframework.util.MultiValueMap; + +class ApplicationConfigControllerTest extends AbstractIntegrationTest { + + @Autowired + private WebTestClient webTestClient; + + @Test + public void testUpload() throws IOException { + var fileToUpload = new ClassPathResource("/fileForUploadTest.txt", this.getClass()); + + UploadedFileInfoDTO result = webTestClient + .post() + .uri("/api/config/relatedfiles") + .bodyValue(generateBody(fileToUpload)) + .exchange() + .expectStatus() + .isOk() + .expectBody(UploadedFileInfoDTO.class) + .returnResult() + .getResponseBody(); + + assertThat(result).isNotNull(); + assertThat(result.getLocation()).isNotNull(); + assertThat(Path.of(result.getLocation())) + .hasSameBinaryContentAs(fileToUpload.getFile().toPath()); + } + + private MultiValueMap> generateBody(ClassPathResource resource) { + MultipartBodyBuilder builder = new MultipartBodyBuilder(); + builder.part("file", resource); + return builder.build(); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclCsvTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclCsvTest.java new file mode 100644 index 0000000000..08ca4d1507 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclCsvTest.java @@ -0,0 +1,70 @@ +package com.provectus.kafka.ui.service.acl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.provectus.kafka.ui.exception.ValidationException; +import java.util.Collection; +import java.util.List; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class AclCsvTest { + + private static final List TEST_BINDINGS = List.of( + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "*", PatternType.LITERAL), + new AccessControlEntry("User:test1", "*", AclOperation.READ, AclPermissionType.ALLOW)), + new AclBinding( + new ResourcePattern(ResourceType.GROUP, "group1", PatternType.PREFIXED), + new AccessControlEntry("User:test2", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)) + ); + + @ParameterizedTest + @ValueSource(strings = { + "Principal,ResourceType, PatternType, ResourceName,Operation,PermissionType,Host\n" + + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + + "User:test2,GROUP,PREFIXED,group1,DESCRIBE,DENY,localhost", + + //without header + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + + "\n" + + "User:test2,GROUP,PREFIXED,group1,DESCRIBE,DENY,localhost" + + "\n" + }) + void parsesValidInputCsv(String csvString) { + Collection parsed = AclCsv.parseCsv(csvString); + assertThat(parsed).containsExactlyInAnyOrderElementsOf(TEST_BINDINGS); + } + + @ParameterizedTest + @ValueSource(strings = { + // columns > 7 + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*,1,2,3,4", + // columns < 7 + "User:test1,TOPIC,LITERAL,*", + // enum values are illegal + "User:test1,ILLEGAL,LITERAL,*,READ,ALLOW,*", + "User:test1,TOPIC,LITERAL,*,READ,ILLEGAL,*" + }) + void throwsExceptionForInvalidInputCsv(String csvString) { + assertThatThrownBy(() -> AclCsv.parseCsv(csvString)) + .isInstanceOf(ValidationException.class); + } + + @Test + void transformAndParseUseSameFormat() { + String csv = AclCsv.transformToCsvString(TEST_BINDINGS); + Collection parsedBindings = AclCsv.parseCsv(csv); + assertThat(parsedBindings).containsExactlyInAnyOrderElementsOf(TEST_BINDINGS); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclsServiceTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclsServiceTest.java new file mode 100644 index 0000000000..5791bb2041 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclsServiceTest.java @@ -0,0 +1,82 @@ +package com.provectus.kafka.ui.service.acl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.provectus.kafka.ui.model.KafkaCluster; +import com.provectus.kafka.ui.service.AdminClientService; +import com.provectus.kafka.ui.service.ReactiveAdminClient; +import java.util.Collection; +import java.util.List; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourcePatternFilter; +import org.apache.kafka.common.resource.ResourceType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import reactor.core.publisher.Mono; + +class AclsServiceTest { + + private static final KafkaCluster CLUSTER = KafkaCluster.builder().build(); + + private final ReactiveAdminClient adminClientMock = mock(ReactiveAdminClient.class); + private final AdminClientService adminClientService = mock(AdminClientService.class); + + private final AclsService aclsService = new AclsService(adminClientService); + + @BeforeEach + void initMocks() { + when(adminClientService.get(CLUSTER)).thenReturn(Mono.just(adminClientMock)); + } + + @Test + void testSyncAclWithAclCsv() { + var existingBinding1 = new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "*", PatternType.LITERAL), + new AccessControlEntry("User:test1", "*", AclOperation.READ, AclPermissionType.ALLOW)); + + var existingBinding2 = new AclBinding( + new ResourcePattern(ResourceType.GROUP, "group1", PatternType.PREFIXED), + new AccessControlEntry("User:test2", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)); + + var newBindingToBeAdded = new AclBinding( + new ResourcePattern(ResourceType.GROUP, "groupNew", PatternType.PREFIXED), + new AccessControlEntry("User:test3", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)); + + when(adminClientMock.listAcls(ResourcePatternFilter.ANY)) + .thenReturn(Mono.just(List.of(existingBinding1, existingBinding2))); + + ArgumentCaptor createdCaptor = ArgumentCaptor.forClass(Collection.class); + when(adminClientMock.createAcls((Collection) createdCaptor.capture())) + .thenReturn(Mono.empty()); + + ArgumentCaptor deletedCaptor = ArgumentCaptor.forClass(Collection.class); + when(adminClientMock.deleteAcls((Collection) deletedCaptor.capture())) + .thenReturn(Mono.empty()); + + aclsService.syncAclWithAclCsv( + CLUSTER, + "Principal,ResourceType, PatternType, ResourceName,Operation,PermissionType,Host\n" + + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + + "User:test3,GROUP,PREFIXED,groupNew,DESCRIBE,DENY,localhost" + ).block(); + + Collection createdBindings = (Collection) createdCaptor.getValue(); + assertThat(createdBindings) + .hasSize(1) + .contains(newBindingToBeAdded); + + Collection deletedBindings = (Collection) deletedCaptor.getValue(); + assertThat(deletedBindings) + .hasSize(1) + .contains(existingBinding2); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelectorTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelectorTest.java new file mode 100644 index 0000000000..497a9365d7 --- /dev/null +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelectorTest.java @@ -0,0 +1,53 @@ +package com.provectus.kafka.ui.service.masking.policies; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.provectus.kafka.ui.config.ClustersProperties; +import com.provectus.kafka.ui.exception.ValidationException; +import java.util.List; +import org.junit.jupiter.api.Test; + +class FieldsSelectorTest { + + @Test + void selectsFieldsDueToProvidedPattern() { + var properties = new ClustersProperties.Masking(); + properties.setFieldsNamePattern("f1|f2"); + + var selector = FieldsSelector.create(properties); + assertThat(selector.shouldBeMasked("f1")).isTrue(); + assertThat(selector.shouldBeMasked("f2")).isTrue(); + assertThat(selector.shouldBeMasked("doesNotMatchPattern")).isFalse(); + } + + @Test + void selectsFieldsDueToProvidedFieldNames() { + var properties = new ClustersProperties.Masking(); + properties.setFields(List.of("f1", "f2")); + + var selector = FieldsSelector.create(properties); + assertThat(selector.shouldBeMasked("f1")).isTrue(); + assertThat(selector.shouldBeMasked("f2")).isTrue(); + assertThat(selector.shouldBeMasked("notInAList")).isFalse(); + } + + @Test + void selectAllFieldsIfNoPatternAndNoNamesProvided() { + var properties = new ClustersProperties.Masking(); + + var selector = FieldsSelector.create(properties); + assertThat(selector.shouldBeMasked("anyPropertyName")).isTrue(); + } + + @Test + void throwsExceptionIfBothFieldListAndPatternProvided() { + var properties = new ClustersProperties.Masking(); + properties.setFieldsNamePattern("f1|f2"); + properties.setFields(List.of("f3", "f4")); + + assertThatThrownBy(() -> FieldsSelector.create(properties)) + .isInstanceOf(ValidationException.class); + } + +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/MaskTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/MaskTest.java index 9cb9701245..b33a26f300 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/MaskTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/MaskTest.java @@ -15,35 +15,35 @@ import org.junit.jupiter.params.provider.MethodSource; class MaskTest { - private static final List TARGET_FIELDS = List.of("id", "name"); + private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); private static final List PATTERN = List.of("X", "x", "n", "-"); @ParameterizedTest @MethodSource - void testApplyToJsonContainer(List fields, ContainerNode original, ContainerNode expected) { - Mask policy = new Mask(fields, PATTERN); + void testApplyToJsonContainer(FieldsSelector selector, ContainerNode original, ContainerNode expected) { + Mask policy = new Mask(selector, PATTERN); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{ \"id\": \"nnn\", \"name\": { \"first\": \"Xxxxx\", \"surname\": \"Xxxxnnn-\"}}") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"id\": \"nnn\", \"f2\": 234}, { \"name\": \"n-n\", \"f2\": 345} ]") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Xxxxnnn-\"}}") ), Arguments.of( - List.of(), + (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"Xxxxx\", \"name\": \"Xxxxnnn-\"}}") ) @@ -57,7 +57,7 @@ class MaskTest { "null, xxxx" }) void testApplyToString(String original, String expected) { - Mask policy = new Mask(List.of(), PATTERN); + Mask policy = new Mask(fieldName -> true, PATTERN); assertThat(policy.applyToString(original)).isEqualTo(expected); } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/RemoveTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/RemoveTest.java index 31ef4eb3c3..9393ea1c62 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/RemoveTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/RemoveTest.java @@ -15,39 +15,39 @@ import org.junit.jupiter.params.provider.MethodSource; class RemoveTest { - private static final List TARGET_FIELDS = List.of("id", "name"); + private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); @ParameterizedTest @MethodSource - void testApplyToJsonContainer(List fields, ContainerNode original, ContainerNode expected) { - var policy = new Remove(fields); + void testApplyToJsonContainer(FieldsSelector fieldsSelector, ContainerNode original, ContainerNode expected) { + var policy = new Remove(fieldsSelector); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{}") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"f2\": 234}, { \"f2\": 345} ]") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\"}}") ), Arguments.of( - List.of(), + (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"v1\", \"f2\": \"v2\", \"inner\" : {\"if1\": \"iv1\"}}}"), parse("{}") ), Arguments.of( - List.of(), + (FieldsSelector) (fieldName -> true), parse("[{ \"f1\": 123}, { \"f2\": \"1.2\"} ]"), parse("[{}, {}]") ) @@ -66,7 +66,7 @@ class RemoveTest { "null, null" }) void testApplyToString(String original, String expected) { - var policy = new Remove(List.of()); + var policy = new Remove(fieldName -> true); assertThat(policy.applyToString(original)).isEqualTo(expected); } -} \ No newline at end of file +} diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/ReplaceTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/ReplaceTest.java index f3ac69747b..9f2fcd90c4 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/ReplaceTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/ReplaceTest.java @@ -15,35 +15,35 @@ import org.junit.jupiter.params.provider.MethodSource; class ReplaceTest { - private static final List TARGET_FIELDS = List.of("id", "name"); + private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); private static final String REPLACEMENT_STRING = "***"; @ParameterizedTest @MethodSource - void testApplyToJsonContainer(List fields, ContainerNode original, ContainerNode expected) { - var policy = new Replace(fields, REPLACEMENT_STRING); + void testApplyToJsonContainer(FieldsSelector fieldsSelector, ContainerNode original, ContainerNode expected) { + var policy = new Replace(fieldsSelector, REPLACEMENT_STRING); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{ \"id\": \"***\", \"name\": { \"first\": \"***\", \"surname\": \"***\"}}") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"id\": \"***\", \"f2\": 234}, { \"name\": \"***\", \"f2\": 345} ]") ), Arguments.of( - TARGET_FIELDS, + FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"***\"}}") ), Arguments.of( - List.of(), + (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"v1\", \"f2\": \"v2\", \"inner\" : {\"if1\": \"iv1\"}}}"), parse("{ \"outer\": { \"f1\": \"***\", \"f2\": \"***\", \"inner\" : {\"if1\": \"***\"}}}}") ) @@ -62,7 +62,7 @@ class ReplaceTest { "null, ***" }) void testApplyToString(String original, String expected) { - var policy = new Replace(List.of(), REPLACEMENT_STRING); + var policy = new Replace(fieldName -> true, REPLACEMENT_STRING); assertThat(policy.applyToString(original)).isEqualTo(expected); } -} \ No newline at end of file +} diff --git a/kafka-ui-api/src/test/resources/fileForUploadTest.txt b/kafka-ui-api/src/test/resources/fileForUploadTest.txt new file mode 100644 index 0000000000..cc58280d07 --- /dev/null +++ b/kafka-ui-api/src/test/resources/fileForUploadTest.txt @@ -0,0 +1 @@ +some content goes here diff --git a/kafka-ui-contract/pom.xml b/kafka-ui-contract/pom.xml index f99f20d3d8..0d8e238368 100644 --- a/kafka-ui-contract/pom.xml +++ b/kafka-ui-contract/pom.xml @@ -101,9 +101,6 @@ true java8 - - filepart=org.springframework.http.codec.multipart.FilePart - diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 78c7cf3bf5..b89f8d0963 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -1730,6 +1730,125 @@ paths: 404: description: Not found + /api/clusters/{clusterName}/acls: + get: + tags: + - Acls + summary: listKafkaAcls + operationId: listAcls + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + - name: resourceType + in: query + required: false + schema: + $ref: '#/components/schemas/KafkaAclResourceType' + - name: resourceName + in: query + required: false + schema: + type: string + - name: namePatternType + in: query + required: false + schema: + $ref: '#/components/schemas/KafkaAclNamePatternType' + responses: + 200: + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/KafkaAcl' + + /api/clusters/{clusterName}/acl/csv: + get: + tags: + - Acls + summary: getAclAsCsv + operationId: getAclAsCsv + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + responses: + 200: + description: OK + content: + text/plain: + schema: + type: string + post: + tags: + - Acls + summary: syncAclsCsv + operationId: syncAclsCsv + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + text/plain: + schema: + type: string + responses: + 200: + description: OK + + /api/clusters/{clusterName}/acl: + post: + tags: + - Acls + summary: createAcl + operationId: createAcl + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/KafkaAcl' + responses: + 200: + description: OK + + delete: + tags: + - Acls + summary: deleteAcl + operationId: deleteAcl + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/KafkaAcl' + responses: + 200: + description: OK + 404: + description: Acl not found + /api/authorization: get: tags: @@ -1819,7 +1938,7 @@ paths: properties: file: type: string - format: filepart + format: binary responses: 200: description: OK @@ -1972,6 +2091,8 @@ components: - KAFKA_CONNECT - KSQL_DB - TOPIC_DELETION + - KAFKA_ACL_VIEW # get ACLs listing + - KAFKA_ACL_EDIT # create & delete ACLs required: - id - name @@ -3342,6 +3463,62 @@ components: - SCHEMA - CONNECT - KSQL + - ACL + + KafkaAcl: + type: object + required: [resourceType, resourceName, namePatternType, principal, host, operation, permission] + properties: + resourceType: + $ref: '#/components/schemas/KafkaAclResourceType' + resourceName: + type: string # "*" if acl can be applied to any resource of given type + namePatternType: + $ref: '#/components/schemas/KafkaAclNamePatternType' + principal: + type: string + host: + type: string # "*" if acl can be applied to any resource of given type + operation: + type: string + enum: + - UNKNOWN # Unknown operation, need to update mapping code on BE + - ALL # Cluster, Topic, Group + - READ # Topic, Group + - WRITE # Topic, TransactionalId + - CREATE # Cluster, Topic + - DELETE # Topic, Group + - ALTER # Cluster, Topic, + - DESCRIBE # Cluster, Topic, Group, TransactionalId, DelegationToken + - CLUSTER_ACTION # Cluster + - DESCRIBE_CONFIGS # Cluster, Topic + - ALTER_CONFIGS # Cluster, Topic + - IDEMPOTENT_WRITE # Cluster + - CREATE_TOKENS + - DESCRIBE_TOKENS + permission: + type: string + enum: + - ALLOW + - DENY + + KafkaAclResourceType: + type: string + enum: + - UNKNOWN # Unknown operation, need to update mapping code on BE + - TOPIC + - GROUP + - CLUSTER + - TRANSACTIONAL_ID + - DELEGATION_TOKEN + - USER + + KafkaAclNamePatternType: + type: string + enum: + - MATCH + - LITERAL + - PREFIXED RestartRequest: type: object @@ -3632,7 +3809,9 @@ components: type: array items: type: string - pattern: + fieldsNamePattern: + type: string + maskingCharsReplacement: type: array items: type: string diff --git a/kafka-ui-e2e-checks/pom.xml b/kafka-ui-e2e-checks/pom.xml index c93f6bcabb..cfd1414fd4 100644 --- a/kafka-ui-e2e-checks/pom.xml +++ b/kafka-ui-e2e-checks/pom.xml @@ -18,7 +18,7 @@ 5.2.1 4.8.1 6.12.3 - 7.7.0 + 7.7.1 2.21.0 3.0.4 1.9.9.1 diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersList.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersList.java index 50ecdff359..9e81a0795c 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersList.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersList.java @@ -48,7 +48,8 @@ public class BrokersList extends BasePage { } private List getEnabledColumnHeaders() { - return Stream.of("Broker ID", "Segment Size", "Segment Count", "Port", "Host") + return Stream.of("Broker ID", "Disk usage", "Partitions skew", + "Leaders", "Leader skew", "Online partitions", "Port", "Host") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } diff --git a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlQueryForm.java b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlQueryForm.java index 4ce282b6cc..ff57de39b2 100644 --- a/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlQueryForm.java +++ b/kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlQueryForm.java @@ -17,11 +17,12 @@ import java.util.List; public class KsqlQueryForm extends BasePage { protected SelenideElement clearBtn = $x("//div/button[text()='Clear']"); protected SelenideElement executeBtn = $x("//div/button[text()='Execute']"); - protected SelenideElement stopQueryBtn = $x("//div/button[text()='Stop query']"); protected SelenideElement clearResultsBtn = $x("//div/button[text()='Clear results']"); protected SelenideElement addStreamPropertyBtn = $x("//button[text()='Add Stream Property']"); protected SelenideElement queryAreaValue = $x("//div[@class='ace_content']"); protected SelenideElement queryArea = $x("//div[@id='ksql']/textarea[@class='ace_text-input']"); + protected SelenideElement abortButton = $x("//div[@role='status']/div[text()='Abort']"); + protected SelenideElement cancelledAlert = $x("//div[@role='status'][text()='Cancelled']"); protected ElementsCollection ksqlGridItems = $$x("//tbody//tr"); protected ElementsCollection keyField = $$x("//input[@aria-label='key']"); protected ElementsCollection valueField = $$x("//input[@aria-label='value']"); @@ -48,7 +49,7 @@ public class KsqlQueryForm extends BasePage { public KsqlQueryForm clickExecuteBtn(String query) { clickByActions(executeBtn); if (query.contains("EMIT CHANGES")) { - loadingSpinner.shouldBe(Condition.visible); + abortButton.shouldBe(Condition.visible); } else { waitUntilSpinnerDisappear(); } @@ -56,12 +57,21 @@ public class KsqlQueryForm extends BasePage { } @Step - public KsqlQueryForm clickStopQueryBtn() { - clickByActions(stopQueryBtn); - waitUntilSpinnerDisappear(); + public boolean isAbortBtnVisible() { + return isVisible(abortButton); + } + + @Step + public KsqlQueryForm clickAbortBtn() { + clickByActions(abortButton); return this; } + @Step + public boolean isCancelledAlertVisible() { + return isVisible(cancelledAlert); + } + @Step public KsqlQueryForm clickClearResultsBtn() { clickByActions(clearResultsBtn); diff --git a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SmokeBacklog.java b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SmokeBacklog.java index 3ce086ee7b..7e663f5893 100644 --- a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SmokeBacklog.java +++ b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SmokeBacklog.java @@ -21,80 +21,66 @@ public class SmokeBacklog extends BaseManualTest { public void testCaseA() { } - @Automation(state = TO_BE_AUTOMATED) - @Suite(id = KSQL_DB_SUITE_ID) - @QaseId(277) - @Test - public void testCaseB() { - } - - @Automation(state = TO_BE_AUTOMATED) - @Suite(id = KSQL_DB_SUITE_ID) - @QaseId(278) - @Test - public void testCaseC() { - } - @Automation(state = TO_BE_AUTOMATED) @Suite(id = BROKERS_SUITE_ID) @QaseId(331) @Test - public void testCaseD() { + public void testCaseB() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = BROKERS_SUITE_ID) @QaseId(332) @Test - public void testCaseE() { + public void testCaseC() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(335) @Test - public void testCaseF() { + public void testCaseD() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(336) @Test - public void testCaseG() { + public void testCaseE() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(343) @Test - public void testCaseH() { + public void testCaseF() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = KSQL_DB_SUITE_ID) @QaseId(344) @Test - public void testCaseI() { + public void testCaseG() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = SCHEMAS_SUITE_ID) @QaseId(345) @Test - public void testCaseJ() { + public void testCaseH() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = SCHEMAS_SUITE_ID) @QaseId(346) @Test - public void testCaseK() { + public void testCaseI() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(347) @Test - public void testCaseL() { + public void testCaseJ() { } } diff --git a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/ksqldb/KsqlDbTest.java b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/ksqldb/KsqlDbTest.java index 22ef931bf1..0504a8a31a 100644 --- a/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/ksqldb/KsqlDbTest.java +++ b/kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/ksqldb/KsqlDbTest.java @@ -1,6 +1,8 @@ package com.provectus.kafka.ui.smokesuite.ksqldb; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlMenuTabs.STREAMS; +import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SELECT_ALL_FROM; +import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SHOW_STREAMS; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SHOW_TABLES; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KSQL_DB; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; @@ -80,8 +82,19 @@ public class KsqlDbTest extends BaseTest { softly.assertAll(); } - @QaseId(86) + @QaseId(278) @Test(priority = 4) + public void checkShowStreamsRequestExecution() { + navigateToKsqlDbAndExecuteRequest(SHOW_STREAMS.getQuery()); + SoftAssert softly = new SoftAssert(); + softly.assertTrue(ksqlQueryForm.areResultsVisible(), "areResultsVisible()"); + softly.assertTrue(ksqlQueryForm.getItemByName(DEFAULT_STREAM.getName()).isVisible(), + String.format("getItemByName(%s)", FIRST_TABLE.getName())); + softly.assertAll(); + } + + @QaseId(86) + @Test(priority = 5) public void clearResultsForExecutedRequest() { navigateToKsqlDbAndExecuteRequest(SHOW_TABLES.getQuery()); SoftAssert softly = new SoftAssert(); @@ -93,6 +106,16 @@ public class KsqlDbTest extends BaseTest { softly.assertAll(); } + @QaseId(277) + @Test(priority = 6) + public void stopQueryFunctionalCheck() { + navigateToKsqlDbAndExecuteRequest(String.format(SELECT_ALL_FROM.getQuery(), FIRST_TABLE.getName())); + Assert.assertTrue(ksqlQueryForm.isAbortBtnVisible(), "isAbortBtnVisible()"); + ksqlQueryForm + .clickAbortBtn(); + Assert.assertTrue(ksqlQueryForm.isCancelledAlertVisible(), "isCancelledAlertVisible()"); + } + @AfterClass(alwaysRun = true) public void afterClass() { TOPIC_NAMES_LIST.forEach(topicName -> apiService.deleteTopic(topicName)); diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx index 966edecf1f..d8cd0a2f76 100644 --- a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx @@ -11,7 +11,9 @@ import CheckMarkRoundIcon from 'components/common/Icons/CheckMarkRoundIcon'; import { ColumnDef } from '@tanstack/react-table'; import { clusterBrokerPath } from 'lib/paths'; import Tooltip from 'components/common/Tooltip/Tooltip'; +import ColoredCell from 'components/common/NewTable/ColoredCell'; +import SkewHeader from './SkewHeader/SkewHeader'; import * as S from './BrokersList.styled'; const NA = 'N/A'; @@ -57,11 +59,15 @@ const BrokersList: React.FC = () => { count: segmentCount || NA, port: broker?.port, host: broker?.host, + partitionsLeader: broker?.partitionsLeader, + partitionsSkew: broker?.partitionsSkew, + leadersSkew: broker?.leadersSkew, + inSyncPartitions: broker?.inSyncPartitions, }; }); }, [diskUsage, brokers]); - const columns = React.useMemo[]>( + const columns = React.useMemo[]>( () => [ { header: 'Broker ID', @@ -84,7 +90,7 @@ const BrokersList: React.FC = () => { ), }, { - header: 'Segment Size', + header: 'Disk usage', accessorKey: 'size', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue, table, cell, column, renderValue, row }) => @@ -98,10 +104,56 @@ const BrokersList: React.FC = () => { cell={cell} getValue={getValue} renderValue={renderValue} + renderSegments /> ), }, - { header: 'Segment Count', accessorKey: 'count' }, + { + // eslint-disable-next-line react/no-unstable-nested-components + header: () => , + accessorKey: 'partitionsSkew', + // eslint-disable-next-line react/no-unstable-nested-components + cell: ({ getValue }) => { + const value = getValue(); + return ( + = 10 && value < 20} + attention={value >= 20} + /> + ); + }, + }, + { header: 'Leaders', accessorKey: 'partitionsLeader' }, + { + header: 'Leader skew', + accessorKey: 'leadersSkew', + // eslint-disable-next-line react/no-unstable-nested-components + cell: ({ getValue }) => { + const value = getValue(); + return ( + = 10 && value < 20} + attention={value >= 20} + /> + ); + }, + }, + { + header: 'Online partitions', + accessorKey: 'inSyncPartitions', + // eslint-disable-next-line react/no-unstable-nested-components + cell: ({ getValue, row }) => { + const value = getValue(); + return ( + + ); + }, + }, { header: 'Port', accessorKey: 'port' }, { header: 'Host', diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.styled.ts b/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.styled.ts new file mode 100644 index 0000000000..eea2fa3cd9 --- /dev/null +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.styled.ts @@ -0,0 +1,11 @@ +import styled from 'styled-components'; +import { MessageTooltip } from 'components/common/Tooltip/Tooltip.styled'; + +export const CellWrapper = styled.div` + display: flex; + gap: 10px; + + ${MessageTooltip} { + max-height: unset; + } +`; diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.tsx b/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.tsx new file mode 100644 index 0000000000..978d1768dd --- /dev/null +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import Tooltip from 'components/common/Tooltip/Tooltip'; +import InfoIcon from 'components/common/Icons/InfoIcon'; + +import * as S from './SkewHeader.styled'; + +const SkewHeader: React.FC = () => ( + + Partitions skew + } + content="The divergence from the average brokers' value" + /> + +); + +export default SkewHeader; diff --git a/kafka-ui-react-app/src/components/Connect/Details/Config/Config.tsx b/kafka-ui-react-app/src/components/Connect/Details/Config/Config.tsx index 0e86d48940..8a372e9d12 100644 --- a/kafka-ui-react-app/src/components/Connect/Details/Config/Config.tsx +++ b/kafka-ui-react-app/src/components/Connect/Details/Config/Config.tsx @@ -37,7 +37,7 @@ const Config: React.FC = () => { formState: { isDirty, isSubmitting, isValid, errors }, setValue, } = useForm({ - mode: 'onTouched', + mode: 'onChange', resolver: yupResolver(validationSchema), defaultValues: { config: JSON.stringify(config, null, '\t'), diff --git a/kafka-ui-react-app/src/components/Dashboard/ClusterTableActionsCell.tsx b/kafka-ui-react-app/src/components/Dashboard/ClusterTableActionsCell.tsx index cb41ab06a8..19fefd784c 100644 --- a/kafka-ui-react-app/src/components/Dashboard/ClusterTableActionsCell.tsx +++ b/kafka-ui-react-app/src/components/Dashboard/ClusterTableActionsCell.tsx @@ -1,17 +1,31 @@ -import React from 'react'; -import { Cluster } from 'generated-sources'; +import React, { useMemo } from 'react'; +import { Cluster, ResourceType } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; -import { Button } from 'components/common/Button/Button'; import { clusterConfigPath } from 'lib/paths'; +import { useGetUserInfo } from 'lib/hooks/api/roles'; +import { ActionCanButton } from 'components/common/ActionComponent'; type Props = CellContext; const ClusterTableActionsCell: React.FC = ({ row }) => { const { name } = row.original; + const { data } = useGetUserInfo(); + + const isApplicationConfig = useMemo(() => { + return !!data?.userInfo?.permissions.some( + (permission) => permission.resource === ResourceType.APPLICATIONCONFIG + ); + }, [data]); + return ( - + ); }; diff --git a/kafka-ui-react-app/src/components/Dashboard/Dashboard.tsx b/kafka-ui-react-app/src/components/Dashboard/Dashboard.tsx index 7eab4c1d2f..c7b64aef1c 100644 --- a/kafka-ui-react-app/src/components/Dashboard/Dashboard.tsx +++ b/kafka-ui-react-app/src/components/Dashboard/Dashboard.tsx @@ -1,23 +1,25 @@ -import React, { useEffect } from 'react'; +import React, { useEffect, useMemo } from 'react'; import PageHeading from 'components/common/PageHeading/PageHeading'; import * as Metrics from 'components/common/Metrics'; import { Tag } from 'components/common/Tag/Tag.styled'; import Switch from 'components/common/Switch/Switch'; import { useClusters } from 'lib/hooks/api/clusters'; -import { Cluster, ServerStatus } from 'generated-sources'; +import { Cluster, ResourceType, ServerStatus } from 'generated-sources'; import { ColumnDef } from '@tanstack/react-table'; import Table, { SizeCell } from 'components/common/NewTable'; import useBoolean from 'lib/hooks/useBoolean'; -import { Button } from 'components/common/Button/Button'; import { clusterNewConfigPath } from 'lib/paths'; import { GlobalSettingsContext } from 'components/contexts/GlobalSettingsContext'; import { useNavigate } from 'react-router-dom'; +import { ActionCanButton } from 'components/common/ActionComponent'; +import { useGetUserInfo } from 'lib/hooks/api/roles'; import * as S from './Dashboard.styled'; import ClusterName from './ClusterName'; import ClusterTableActionsCell from './ClusterTableActionsCell'; const Dashboard: React.FC = () => { + const { data } = useGetUserInfo(); const clusters = useClusters(); const { value: showOfflineOnly, toggle } = useBoolean(false); const appInfo = React.useContext(GlobalSettingsContext); @@ -62,6 +64,11 @@ const Dashboard: React.FC = () => { } }, [clusters, appInfo.hasDynamicConfig]); + const isApplicationConfig = useMemo(() => { + return !!data?.userInfo?.permissions.some( + (permission) => permission.resource === ResourceType.APPLICATIONCONFIG + ); + }, [data]); return ( <> @@ -87,9 +94,14 @@ const Dashboard: React.FC = () => { {appInfo.hasDynamicConfig && ( - + )} = ({ versions, areVersionsFetched }) => { backText="Schema Registry" backTo={clusterSchemasPath(clusterName)} /> + navigate(-1)} + > + Back + {areVersionsFetched ? ( diff --git a/kafka-ui-react-app/src/components/Schemas/Diff/__test__/Diff.spec.tsx b/kafka-ui-react-app/src/components/Schemas/Diff/__test__/Diff.spec.tsx index 0c614cf661..2a9429eef1 100644 --- a/kafka-ui-react-app/src/components/Schemas/Diff/__test__/Diff.spec.tsx +++ b/kafka-ui-react-app/src/components/Schemas/Diff/__test__/Diff.spec.tsx @@ -3,6 +3,7 @@ import Diff, { DiffProps } from 'components/Schemas/Diff/Diff'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/react'; import { clusterSchemaComparePath } from 'lib/paths'; +import userEvent from '@testing-library/user-event'; import { versions } from './fixtures'; @@ -142,4 +143,24 @@ describe('Diff', () => { expect(select).toHaveTextContent(versions[0].version); }); }); + + describe('Back button', () => { + beforeEach(() => { + setupComponent({ + areVersionsFetched: true, + versions, + }); + }); + + it('back button is appear', () => { + const backButton = screen.getAllByRole('button', { name: 'Back' }); + expect(backButton[0]).toBeInTheDocument(); + }); + + it('click on back button', () => { + const backButton = screen.getAllByRole('button', { name: 'Back' }); + userEvent.click(backButton[0]); + expect(screen.queryByRole('Back')).not.toBeInTheDocument(); + }); + }); }); diff --git a/kafka-ui-react-app/src/components/Schemas/Edit/Form.tsx b/kafka-ui-react-app/src/components/Schemas/Edit/Form.tsx index 2fce1ad7d7..56d7bdc817 100644 --- a/kafka-ui-react-app/src/components/Schemas/Edit/Form.tsx +++ b/kafka-ui-react-app/src/components/Schemas/Edit/Form.tsx @@ -55,7 +55,7 @@ const Form: React.FC = () => { yup.object().shape({ newSchema: schema?.schemaType === SchemaType.PROTOBUF - ? yup.string().required().isEnum('Schema syntax is not valid') + ? yup.string().required() : yup.string().required().isJsonObject('Schema syntax is not valid'), }); const methods = useForm({ diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx index dd5cfae748..af76db6739 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx @@ -8,6 +8,7 @@ import { formatTimestamp } from 'lib/dateTimeHelpers'; import { JSONPath } from 'jsonpath-plus'; import Ellipsis from 'components/common/Ellipsis/Ellipsis'; import WarningRedIcon from 'components/common/Icons/WarningRedIcon'; +import Tooltip from 'components/common/Tooltip/Tooltip'; import MessageContent from './MessageContent/MessageContent'; import * as S from './MessageContent/MessageContent.styled'; @@ -110,14 +111,26 @@ const Message: React.FC = ({ - {keySerde === 'Fallback' && } + {keySerde === 'Fallback' && ( + } + content="Fallback serde was used" + placement="left" + /> + )} - {valueSerde === 'Fallback' && } + {valueSerde === 'Fallback' && ( + } + content="Fallback serde was used" + placement="left" + /> + )} diff --git a/kafka-ui-react-app/src/components/Topics/Topic/SendMessage/SendMessage.tsx b/kafka-ui-react-app/src/components/Topics/Topic/SendMessage/SendMessage.tsx index bacfa76c93..b7f31a230b 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/SendMessage/SendMessage.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/SendMessage/SendMessage.tsx @@ -210,6 +210,7 @@ const SendMessage: React.FC<{ closeSidebar: () => void }> = ({ name={name} onChange={onChange} value={value} + height="40px" /> )} /> @@ -225,6 +226,7 @@ const SendMessage: React.FC<{ closeSidebar: () => void }> = ({ name={name} onChange={onChange} value={value} + height="280px" /> )} /> @@ -242,7 +244,7 @@ const SendMessage: React.FC<{ closeSidebar: () => void }> = ({ defaultValue="{}" name={name} onChange={onChange} - height="200px" + height="40px" /> )} /> diff --git a/kafka-ui-react-app/src/components/common/Input/Input.styled.ts b/kafka-ui-react-app/src/components/common/Input/Input.styled.ts index 9495aaecbe..f21962fe6b 100644 --- a/kafka-ui-react-app/src/components/common/Input/Input.styled.ts +++ b/kafka-ui-react-app/src/components/common/Input/Input.styled.ts @@ -29,6 +29,16 @@ export const Wrapper = styled.div` width: 16px; fill: ${({ theme }) => theme.input.icon.color}; } + svg:last-child { + position: absolute; + top: 8px; + line-height: 0; + z-index: 1; + left: unset; + right: 12px; + height: 16px; + width: 16px; + } `; export const Input = styled.input( diff --git a/kafka-ui-react-app/src/components/common/Input/Input.tsx b/kafka-ui-react-app/src/components/common/Input/Input.tsx index ae76bc4717..4d04b730e5 100644 --- a/kafka-ui-react-app/src/components/common/Input/Input.tsx +++ b/kafka-ui-react-app/src/components/common/Input/Input.tsx @@ -16,6 +16,7 @@ export interface InputProps withError?: boolean; label?: React.ReactNode; hint?: React.ReactNode; + clearIcon?: React.ReactNode; // Some may only accept integer, like `Number of Partitions` // some may accept decimal @@ -99,19 +100,22 @@ function pasteNumberCheck( return value; } -const Input: React.FC = ({ - name, - hookFormOptions, - search, - inputSize = 'L', - type, - positiveOnly, - integerOnly, - withError = false, - label, - hint, - ...rest -}) => { +const Input = React.forwardRef((props, ref) => { + const { + name, + hookFormOptions, + search, + inputSize = 'L', + type, + positiveOnly, + integerOnly, + withError = false, + label, + hint, + clearIcon, + ...rest + } = props; + const methods = useFormContext(); const fieldId = React.useId(); @@ -168,7 +172,6 @@ const Input: React.FC = ({ // if the field is a part of react-hook-form form inputOptions = { ...rest, ...methods.register(name, hookFormOptions) }; } - return (
{label && {label}} @@ -181,8 +184,11 @@ const Input: React.FC = ({ type={type} onKeyPress={keyPressEventHandler} onPaste={pasteEventHandler} + ref={ref} {...inputOptions} /> + {clearIcon} + {withError && isHookFormField && ( @@ -192,6 +198,6 @@ const Input: React.FC = ({
); -}; +}); export default Input; diff --git a/kafka-ui-react-app/src/components/common/NewTable/ColoredCell.tsx b/kafka-ui-react-app/src/components/common/NewTable/ColoredCell.tsx new file mode 100644 index 0000000000..df8ab2d6a8 --- /dev/null +++ b/kafka-ui-react-app/src/components/common/NewTable/ColoredCell.tsx @@ -0,0 +1,41 @@ +import React from 'react'; +import styled from 'styled-components'; + +interface CellProps { + isWarning?: boolean; + isAttention?: boolean; +} + +interface ColoredCellProps { + value: number | string; + warn?: boolean; + attention?: boolean; +} + +const Cell = styled.div` + color: ${(props) => { + if (props.isAttention) { + return props.theme.table.colored.color.attention; + } + + if (props.isWarning) { + return props.theme.table.colored.color.warning; + } + + return 'inherit'; + }}; +`; + +const ColoredCell: React.FC = ({ + value, + warn, + attention, +}) => { + return ( + + {value} + + ); +}; + +export default ColoredCell; diff --git a/kafka-ui-react-app/src/components/common/NewTable/SizeCell.tsx b/kafka-ui-react-app/src/components/common/NewTable/SizeCell.tsx index 00a60086d9..24485342aa 100644 --- a/kafka-ui-react-app/src/components/common/NewTable/SizeCell.tsx +++ b/kafka-ui-react-app/src/components/common/NewTable/SizeCell.tsx @@ -3,8 +3,15 @@ import { CellContext } from '@tanstack/react-table'; import BytesFormatted from 'components/common/BytesFormatted/BytesFormatted'; // eslint-disable-next-line @typescript-eslint/no-explicit-any -const SizeCell: React.FC> = ({ getValue }) => ( - ()} /> +type AsAny = any; + +const SizeCell: React.FC< + CellContext & { renderSegments?: boolean } +> = ({ getValue, row, renderSegments = false }) => ( + <> + ()} /> + {renderSegments ? `, ${row?.original.count} segment(s)` : null} + ); export default SizeCell; diff --git a/kafka-ui-react-app/src/components/common/Search/Search.tsx b/kafka-ui-react-app/src/components/common/Search/Search.tsx index 66c0e95030..65116d645a 100644 --- a/kafka-ui-react-app/src/components/common/Search/Search.tsx +++ b/kafka-ui-react-app/src/components/common/Search/Search.tsx @@ -1,7 +1,9 @@ -import React from 'react'; +import React, { useRef } from 'react'; import { useDebouncedCallback } from 'use-debounce'; import Input from 'components/common/Input/Input'; import { useSearchParams } from 'react-router-dom'; +import CloseIcon from 'components/common/Icons/CloseIcon'; +import styled from 'styled-components'; interface SearchProps { placeholder?: string; @@ -10,6 +12,16 @@ interface SearchProps { value?: string; } +const IconButtonWrapper = styled.span.attrs(() => ({ + role: 'button', + tabIndex: '0', +}))` + height: 16px !important; + display: inline-block; + &:hover { + cursor: pointer; + } +`; const Search: React.FC = ({ placeholder = 'Search', disabled = false, @@ -17,7 +29,11 @@ const Search: React.FC = ({ onChange, }) => { const [searchParams, setSearchParams] = useSearchParams(); + const ref = useRef(null); const handleChange = useDebouncedCallback((e) => { + if (ref.current != null) { + ref.current.value = e.target.value; + } if (onChange) { onChange(e.target.value); } else { @@ -28,6 +44,15 @@ const Search: React.FC = ({ setSearchParams(searchParams); } }, 500); + const clearSearchValue = () => { + if (searchParams.get('q')) { + searchParams.set('q', ''); + setSearchParams(searchParams); + } + if (ref.current != null) { + ref.current.value = ''; + } + }; return ( = ({ defaultValue={value || searchParams.get('q') || ''} inputSize="M" disabled={disabled} + ref={ref} search + clearIcon={ + + + + } /> ); }; diff --git a/kafka-ui-react-app/src/components/common/Search/__tests__/Search.spec.tsx b/kafka-ui-react-app/src/components/common/Search/__tests__/Search.spec.tsx index 808f229317..2103d22336 100644 --- a/kafka-ui-react-app/src/components/common/Search/__tests__/Search.spec.tsx +++ b/kafka-ui-react-app/src/components/common/Search/__tests__/Search.spec.tsx @@ -41,4 +41,24 @@ describe('Search', () => { render(); expect(screen.queryByPlaceholderText('Search')).toBeInTheDocument(); }); + + it('Clear button is visible', () => { + render(); + + const clearButton = screen.getByRole('button'); + expect(clearButton).toBeInTheDocument(); + }); + + it('Clear button should clear text from input', async () => { + render(); + + const searchField = screen.getAllByRole('textbox')[0]; + await userEvent.type(searchField, 'some text'); + expect(searchField).toHaveValue('some text'); + + const clearButton = screen.getByRole('button'); + await userEvent.click(clearButton); + + expect(searchField).toHaveValue(''); + }); }); diff --git a/kafka-ui-react-app/src/lib/__test__/yupExtended.spec.ts b/kafka-ui-react-app/src/lib/__test__/yupExtended.spec.ts index 8100b9a326..bd43dd3f72 100644 --- a/kafka-ui-react-app/src/lib/__test__/yupExtended.spec.ts +++ b/kafka-ui-react-app/src/lib/__test__/yupExtended.spec.ts @@ -1,19 +1,5 @@ -import { isValidEnum, isValidJsonObject } from 'lib/yupExtended'; +import { isValidJsonObject } from 'lib/yupExtended'; -const invalidEnum = ` -ennum SchemType { - AVRO = 0; - JSON = 1; - PROTOBUF = 3; -} -`; -const validEnum = ` -enum SchemType { - AVRO = 0; - JSON = 1; - PROTOBUF = 3; -} -`; describe('yup extended', () => { describe('isValidJsonObject', () => { it('returns false for no value', () => { @@ -35,21 +21,4 @@ describe('yup extended', () => { expect(isValidJsonObject('{ "foo": "bar" }')).toBeTruthy(); }); }); - - describe('isValidEnum', () => { - it('returns false for invalid enum', () => { - expect(isValidEnum(invalidEnum)).toBeFalsy(); - }); - it('returns false for no value', () => { - expect(isValidEnum()).toBeFalsy(); - }); - it('returns true should trim value', () => { - expect( - isValidEnum(` enum SchemType {AVRO = 0; PROTOBUF = 3;} `) - ).toBeTruthy(); - }); - it('returns true for valid enum', () => { - expect(isValidEnum(validEnum)).toBeTruthy(); - }); - }); }); diff --git a/kafka-ui-react-app/src/lib/dateTimeHelpers.ts b/kafka-ui-react-app/src/lib/dateTimeHelpers.ts index 3dce0edd78..148a70d2a3 100644 --- a/kafka-ui-react-app/src/lib/dateTimeHelpers.ts +++ b/kafka-ui-react-app/src/lib/dateTimeHelpers.ts @@ -1,6 +1,6 @@ export const formatTimestamp = ( timestamp?: number | string | Date, - format: Intl.DateTimeFormatOptions = { hour12: false } + format: Intl.DateTimeFormatOptions = { hourCycle: 'h23' } ): string => { if (!timestamp) { return ''; @@ -8,7 +8,6 @@ export const formatTimestamp = ( // empty array gets the default one from the browser const date = new Date(timestamp); - // invalid date if (Number.isNaN(date.getTime())) { return ''; diff --git a/kafka-ui-react-app/src/lib/yupExtended.ts b/kafka-ui-react-app/src/lib/yupExtended.ts index 4c662ca822..241dac9770 100644 --- a/kafka-ui-react-app/src/lib/yupExtended.ts +++ b/kafka-ui-react-app/src/lib/yupExtended.ts @@ -10,7 +10,6 @@ declare module 'yup' { TFlags extends yup.Flags = '' > extends yup.Schema { isJsonObject(message?: string): StringSchema; - isEnum(message?: string): StringSchema; } } @@ -40,32 +39,6 @@ const isJsonObject = (message?: string) => { isValidJsonObject ); }; - -export const isValidEnum = (value?: string) => { - try { - if (!value) return false; - const trimmedValue = value.trim(); - if ( - trimmedValue.indexOf('enum') === 0 && - trimmedValue.lastIndexOf('}') === trimmedValue.length - 1 - ) { - return true; - } - } catch { - // do nothing - } - return false; -}; - -const isEnum = (message?: string) => { - return yup.string().test( - 'isEnum', - // eslint-disable-next-line no-template-curly-in-string - message || '${path} is not Enum object', - isValidEnum - ); -}; - /** * due to yup rerunning all the object validiation during any render, * it makes sense to cache the async results @@ -88,7 +61,6 @@ export function cacheTest( } yup.addMethod(yup.StringSchema, 'isJsonObject', isJsonObject); -yup.addMethod(yup.StringSchema, 'isEnum', isEnum); export const topicFormValidationSchema = yup.object().shape({ name: yup diff --git a/kafka-ui-react-app/src/theme/theme.ts b/kafka-ui-react-app/src/theme/theme.ts index 09ff73510d..21a7746163 100644 --- a/kafka-ui-react-app/src/theme/theme.ts +++ b/kafka-ui-react-app/src/theme/theme.ts @@ -544,6 +544,12 @@ export const theme = { active: Colors.neutral[90], }, }, + colored: { + color: { + attention: Colors.red[50], + warning: Colors.yellow[20], + }, + }, expander: { normal: Colors.brand[30], hover: Colors.brand[40], @@ -942,6 +948,12 @@ export const darkTheme: ThemeType = { active: Colors.neutral[0], }, }, + colored: { + color: { + attention: Colors.red[50], + warning: Colors.yellow[20], + }, + }, expander: { normal: Colors.brand[30], hover: Colors.brand[40],