0ct0pu5/kafka-ui
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changed consumer groups request logic for better performance (#965)

Browse source 
* Changed consumer groups request logic for better performance

* Fixed checkstyle
This commit is contained in:
German Osin 2021-10-14 11:52:58 +03:00 • committed by GitHub
parent d0f63aeaa0
commit 4339841771
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ConsumerGroupService.java
View file

@ -7,13 +7,16 @@ import com.provectus.kafka.ui.model.KafkaCluster;
import com.provectus.kafka.ui.util.ClusterUtil;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import org.apache.kafka.clients.admin.OffsetSpec;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
@ -62,18 +65,25 @@ public class ConsumerGroupService {
consumerGroups = getConsumerGroupsInternal(cluster, groupIds);
}
return consumerGroups.map(c ->
c.stream()
.map(d -> ClusterUtil.filterConsumerGroupTopic(d, topic))
.filter(Optional::isPresent)
.map(Optional::get)
.map(g ->
g.toBuilder().endOffsets(
topicPartitionsEndOffsets(cluster, g.getOffsets().keySet())
).build()
)
.collect(Collectors.toList())
);
return consumerGroups.flatMap(c -> {
final List<InternalConsumerGroup> groups = c.stream()
.map(d -> ClusterUtil.filterConsumerGroupTopic(d, topic))
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toList());
final Set<TopicPartition> topicPartitions =
groups.stream().flatMap(g -> g.getOffsets().keySet().stream())
.collect(Collectors.toSet());
return topicPartitionsEndOffsets(cluster, topicPartitions).map(offsets ->
groups.stream().map(g -> {
Map<TopicPartition, Long> offsetsCopy = new HashMap<>(offsets);
offsetsCopy.keySet().retainAll(g.getOffsets().keySet());
return g.toBuilder().endOffsets(offsetsCopy).build();
}).collect(Collectors.toList())
);
});
}
public Mono<List<ConsumerGroupDTO>> getConsumerGroups(KafkaCluster cluster) {
@ -88,11 +98,12 @@ public class ConsumerGroupService {
);
}
private Map<TopicPartition, Long> topicPartitionsEndOffsets(
private Mono<Map<TopicPartition, Long>> topicPartitionsEndOffsets(
KafkaCluster cluster, Collection<TopicPartition> topicPartitions) {
try (KafkaConsumer<Bytes, Bytes> consumer = createConsumer(cluster)) {
return consumer.endOffsets(topicPartitions);
}
return adminClientService.get(cluster).flatMap(ac ->
ac.listOffsets(topicPartitions, OffsetSpec.latest())
);
}
public Mono<ConsumerGroupDetailsDTO> getConsumerGroupDetail(KafkaCluster cluster,

2
kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java
View file

@ -249,7 +249,7 @@ public class ReactiveAdminClient implements Closeable {
return topicPartitions(topic).flatMap(tps -> listOffsets(tps, offsetSpec));
}
public Mono<Map<TopicPartition, Long>> listOffsets(Set<TopicPartition> partitions,
public Mono<Map<TopicPartition, Long>> listOffsets(Collection<TopicPartition> partitions,
OffsetSpec offsetSpec) {
return toMono(
client.listOffsets(partitions.stream().collect(toMap(tp -> tp, tp -> offsetSpec))).all())