|
@@ -1,21 +1,22 @@
|
|
|
package com.provectus.kafka.ui.service;
|
|
|
|
|
|
-import com.provectus.kafka.ui.model.ConsumerGroupDTO;
|
|
|
-import com.provectus.kafka.ui.model.ConsumerGroupDetailsDTO;
|
|
|
+import com.provectus.kafka.ui.model.ConsumerGroupOrderingDTO;
|
|
|
import com.provectus.kafka.ui.model.InternalConsumerGroup;
|
|
|
import com.provectus.kafka.ui.model.KafkaCluster;
|
|
|
-import com.provectus.kafka.ui.util.ClusterUtil;
|
|
|
-import java.util.Collection;
|
|
|
-import java.util.Collections;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.Comparator;
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
-import java.util.Optional;
|
|
|
import java.util.Properties;
|
|
|
-import java.util.Set;
|
|
|
import java.util.UUID;
|
|
|
+import java.util.function.Function;
|
|
|
import java.util.stream.Collectors;
|
|
|
+import javax.annotation.Nullable;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
+import lombok.Value;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.apache.kafka.clients.admin.ConsumerGroupDescription;
|
|
|
import org.apache.kafka.clients.admin.OffsetSpec;
|
|
|
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
|
|
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
|
@@ -25,6 +26,8 @@ import org.apache.kafka.common.utils.Bytes;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import reactor.core.publisher.Flux;
|
|
|
import reactor.core.publisher.Mono;
|
|
|
+import reactor.util.function.Tuple2;
|
|
|
+import reactor.util.function.Tuples;
|
|
|
|
|
|
|
|
|
@Service
|
|
@@ -33,88 +36,145 @@ public class ConsumerGroupService {
|
|
|
|
|
|
private final AdminClientService adminClientService;
|
|
|
|
|
|
- private Mono<List<InternalConsumerGroup>> getConsumerGroupsInternal(KafkaCluster cluster) {
|
|
|
- return adminClientService.get(cluster).flatMap(ac ->
|
|
|
- ac.listConsumerGroups()
|
|
|
- .flatMap(groupIds -> getConsumerGroupsInternal(cluster, groupIds)));
|
|
|
+ private Mono<List<InternalConsumerGroup>> getConsumerGroups(
|
|
|
+ ReactiveAdminClient ac,
|
|
|
+ List<ConsumerGroupDescription> descriptions) {
|
|
|
+ return Flux.fromIterable(descriptions)
|
|
|
+ // 1. getting committed offsets for all groups
|
|
|
+ .flatMap(desc -> ac.listConsumerGroupOffsets(desc.groupId())
|
|
|
+ .map(offsets -> Tuples.of(desc, offsets)))
|
|
|
+ .collectMap(Tuple2::getT1, Tuple2::getT2)
|
|
|
+ .flatMap((Map<ConsumerGroupDescription, Map<TopicPartition, Long>> groupOffsetsMap) -> {
|
|
|
+ var tpsFromGroupOffsets = groupOffsetsMap.values().stream()
|
|
|
+ .flatMap(v -> v.keySet().stream())
|
|
|
+ .collect(Collectors.toSet());
|
|
|
+ // 2. getting end offsets for partitions with in committed offsets
|
|
|
+ return ac.listOffsets(tpsFromGroupOffsets, OffsetSpec.latest())
|
|
|
+ .map(endOffsets ->
|
|
|
+ descriptions.stream()
|
|
|
+ .map(desc -> {
|
|
|
+ var groupOffsets = groupOffsetsMap.get(desc);
|
|
|
+ var endOffsetsForGroup = new HashMap<>(endOffsets);
|
|
|
+ endOffsetsForGroup.keySet().retainAll(groupOffsets.keySet());
|
|
|
+ // 3. gathering description & offsets
|
|
|
+ return InternalConsumerGroup.create(desc, groupOffsets, endOffsetsForGroup);
|
|
|
+ })
|
|
|
+ .collect(Collectors.toList()));
|
|
|
+ });
|
|
|
}
|
|
|
|
|
|
- private Mono<List<InternalConsumerGroup>> getConsumerGroupsInternal(KafkaCluster cluster,
|
|
|
- List<String> groupIds) {
|
|
|
- return adminClientService.get(cluster).flatMap(ac ->
|
|
|
- ac.describeConsumerGroups(groupIds)
|
|
|
- .map(Map::values)
|
|
|
- .flatMap(descriptions ->
|
|
|
- Flux.fromIterable(descriptions)
|
|
|
- .parallel()
|
|
|
- .flatMap(d ->
|
|
|
- ac.listConsumerGroupOffsets(d.groupId())
|
|
|
- .map(offsets -> ClusterUtil.convertToInternalConsumerGroup(d, offsets))
|
|
|
- )
|
|
|
- .sequential()
|
|
|
- .collectList()));
|
|
|
+ @Deprecated // need to migrate to pagination
|
|
|
+ public Mono<List<InternalConsumerGroup>> getAllConsumerGroups(KafkaCluster cluster) {
|
|
|
+ return adminClientService.get(cluster)
|
|
|
+ .flatMap(ac -> describeConsumerGroups(ac, null)
|
|
|
+ .flatMap(descriptions -> getConsumerGroups(ac, descriptions)));
|
|
|
}
|
|
|
|
|
|
- public Mono<List<InternalConsumerGroup>> getConsumerGroups(
|
|
|
- KafkaCluster cluster, Optional<String> topic, List<String> groupIds) {
|
|
|
- final Mono<List<InternalConsumerGroup>> consumerGroups;
|
|
|
-
|
|
|
- if (groupIds.isEmpty()) {
|
|
|
- consumerGroups = getConsumerGroupsInternal(cluster);
|
|
|
- } else {
|
|
|
- consumerGroups = getConsumerGroupsInternal(cluster, groupIds);
|
|
|
- }
|
|
|
-
|
|
|
- return consumerGroups.flatMap(c -> {
|
|
|
- final List<InternalConsumerGroup> groups = c.stream()
|
|
|
- .map(d -> ClusterUtil.filterConsumerGroupTopic(d, topic))
|
|
|
- .filter(Optional::isPresent)
|
|
|
- .map(Optional::get)
|
|
|
- .collect(Collectors.toList());
|
|
|
-
|
|
|
- final Set<TopicPartition> topicPartitions =
|
|
|
- groups.stream().flatMap(g -> g.getOffsets().keySet().stream())
|
|
|
- .collect(Collectors.toSet());
|
|
|
-
|
|
|
- return topicPartitionsEndOffsets(cluster, topicPartitions).map(offsets ->
|
|
|
- groups.stream().map(g -> {
|
|
|
- Map<TopicPartition, Long> offsetsCopy = new HashMap<>(offsets);
|
|
|
- offsetsCopy.keySet().retainAll(g.getOffsets().keySet());
|
|
|
- return g.toBuilder().endOffsets(offsetsCopy).build();
|
|
|
- }).collect(Collectors.toList())
|
|
|
- );
|
|
|
- });
|
|
|
+ public Mono<List<InternalConsumerGroup>> getConsumerGroupsForTopic(KafkaCluster cluster,
|
|
|
+ String topic) {
|
|
|
+ return adminClientService.get(cluster)
|
|
|
+ // 1. getting topic's end offsets
|
|
|
+ .flatMap(ac -> ac.listOffsets(topic, OffsetSpec.latest())
|
|
|
+ .flatMap(endOffsets -> {
|
|
|
+ var tps = new ArrayList<>(endOffsets.keySet());
|
|
|
+ // 2. getting all consumer groups
|
|
|
+ return ac.listConsumerGroups()
|
|
|
+ .flatMap((List<String> groups) ->
|
|
|
+ Flux.fromIterable(groups)
|
|
|
+ // 3. for each group trying to find committed offsets for topic
|
|
|
+ .flatMap(g ->
|
|
|
+ ac.listConsumerGroupOffsets(g, tps)
|
|
|
+ .map(offsets -> Tuples.of(g, offsets)))
|
|
|
+ .filter(t -> !t.getT2().isEmpty())
|
|
|
+ .collectMap(Tuple2::getT1, Tuple2::getT2)
|
|
|
+ )
|
|
|
+ .flatMap((Map<String, Map<TopicPartition, Long>> groupOffsets) ->
|
|
|
+ // 4. getting description for groups with non-emtpy offsets
|
|
|
+ ac.describeConsumerGroups(new ArrayList<>(groupOffsets.keySet()))
|
|
|
+ .map((Map<String, ConsumerGroupDescription> descriptions) ->
|
|
|
+ descriptions.values().stream().map(desc ->
|
|
|
+ // 5. gathering and filter non-target-topic data
|
|
|
+ InternalConsumerGroup.create(
|
|
|
+ desc, groupOffsets.get(desc.groupId()), endOffsets)
|
|
|
+ .retainDataForPartitions(p -> p.topic().equals(topic))
|
|
|
+ )
|
|
|
+ .collect(Collectors.toList())));
|
|
|
+ }));
|
|
|
}
|
|
|
|
|
|
- public Mono<List<ConsumerGroupDTO>> getConsumerGroups(KafkaCluster cluster) {
|
|
|
- return getConsumerGroups(cluster, Optional.empty());
|
|
|
+ @Value
|
|
|
+ public static class ConsumerGroupsPage {
|
|
|
+ List<InternalConsumerGroup> consumerGroups;
|
|
|
+ int totalPages;
|
|
|
}
|
|
|
|
|
|
- public Mono<List<ConsumerGroupDTO>> getConsumerGroups(KafkaCluster cluster,
|
|
|
- Optional<String> topic) {
|
|
|
- return getConsumerGroups(cluster, topic, Collections.emptyList())
|
|
|
- .map(c ->
|
|
|
- c.stream().map(ClusterUtil::convertToConsumerGroup).collect(Collectors.toList())
|
|
|
- );
|
|
|
+ public Mono<ConsumerGroupsPage> getConsumerGroupsPage(
|
|
|
+ KafkaCluster cluster,
|
|
|
+ int page,
|
|
|
+ int perPage,
|
|
|
+ @Nullable String search,
|
|
|
+ ConsumerGroupOrderingDTO orderBy) {
|
|
|
+ return adminClientService.get(cluster).flatMap(ac ->
|
|
|
+ describeConsumerGroups(ac, search).flatMap(descriptions ->
|
|
|
+ getConsumerGroups(
|
|
|
+ ac,
|
|
|
+ descriptions.stream()
|
|
|
+ .sorted(getPaginationComparator(orderBy))
|
|
|
+ .skip((long) (page - 1) * perPage)
|
|
|
+ .limit(perPage)
|
|
|
+ .collect(Collectors.toList())
|
|
|
+ ).map(cgs -> new ConsumerGroupsPage(
|
|
|
+ cgs,
|
|
|
+ (descriptions.size() / perPage) + (descriptions.size() % perPage == 0 ? 0 : 1))))
|
|
|
+ );
|
|
|
}
|
|
|
|
|
|
- private Mono<Map<TopicPartition, Long>> topicPartitionsEndOffsets(
|
|
|
- KafkaCluster cluster, Collection<TopicPartition> topicPartitions) {
|
|
|
+ private Comparator<ConsumerGroupDescription> getPaginationComparator(ConsumerGroupOrderingDTO
|
|
|
+ orderBy) {
|
|
|
+ switch (orderBy) {
|
|
|
+ case NAME:
|
|
|
+ return Comparator.comparing(ConsumerGroupDescription::groupId);
|
|
|
+ case STATE:
|
|
|
+ Function<ConsumerGroupDescription, Integer> statesPriorities = cg -> {
|
|
|
+ switch (cg.state()) {
|
|
|
+ case STABLE: return 0;
|
|
|
+ case COMPLETING_REBALANCE: return 1;
|
|
|
+ case PREPARING_REBALANCE: return 2;
|
|
|
+ case EMPTY: return 3;
|
|
|
+ case DEAD: return 4;
|
|
|
+ case UNKNOWN: return 5;
|
|
|
+ default: return 100;
|
|
|
+ }
|
|
|
+ };
|
|
|
+ return Comparator.comparingInt(statesPriorities::apply);
|
|
|
+ case MEMBERS:
|
|
|
+ return Comparator.comparingInt(cg -> -cg.members().size());
|
|
|
+ default:
|
|
|
+ throw new IllegalStateException("Unsupported order by: " + orderBy);
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- return adminClientService.get(cluster).flatMap(ac ->
|
|
|
- ac.listOffsets(topicPartitions, OffsetSpec.latest())
|
|
|
- );
|
|
|
+ private Mono<List<ConsumerGroupDescription>> describeConsumerGroups(ReactiveAdminClient ac,
|
|
|
+ @Nullable String search) {
|
|
|
+ return ac.listConsumerGroups()
|
|
|
+ .map(groupIds -> groupIds
|
|
|
+ .stream()
|
|
|
+ .filter(groupId -> search == null || StringUtils.containsIgnoreCase(groupId, search))
|
|
|
+ .collect(Collectors.toList()))
|
|
|
+ .flatMap(ac::describeConsumerGroups)
|
|
|
+ .map(cgs -> new ArrayList<>(cgs.values()));
|
|
|
}
|
|
|
|
|
|
- public Mono<ConsumerGroupDetailsDTO> getConsumerGroupDetail(KafkaCluster cluster,
|
|
|
- String consumerGroupId) {
|
|
|
- return getConsumerGroups(
|
|
|
- cluster,
|
|
|
- Optional.empty(),
|
|
|
- Collections.singletonList(consumerGroupId)
|
|
|
- ).filter(groups -> !groups.isEmpty()).map(groups -> groups.get(0)).map(
|
|
|
- ClusterUtil::convertToConsumerGroupDetails
|
|
|
- );
|
|
|
+ public Mono<InternalConsumerGroup> getConsumerGroupDetail(KafkaCluster cluster,
|
|
|
+ String consumerGroupId) {
|
|
|
+ return adminClientService.get(cluster)
|
|
|
+ .flatMap(ac -> ac.describeConsumerGroups(List.of(consumerGroupId))
|
|
|
+ .filter(m -> m.containsKey(consumerGroupId))
|
|
|
+ .map(r -> r.get(consumerGroupId))
|
|
|
+ .flatMap(descr ->
|
|
|
+ getConsumerGroups(ac, List.of(descr))
|
|
|
+ .filter(groups -> !groups.isEmpty())
|
|
|
+ .map(groups -> groups.get(0))));
|
|
|
}
|
|
|
|
|
|
public Mono<Void> deleteConsumerGroupById(KafkaCluster cluster,
|