diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java index 7fc9f82780..30c5d9263c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java @@ -17,7 +17,8 @@ abstract class AbstractAuthSecurityConfig { "/login", "/logout", "/oauth2/**", - "/static/**" + "/static/**", + "/api/clusters/**/prometheus/expose/**" }; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AbstractController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AbstractController.java index fd323d55a1..ec7b229783 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AbstractController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AbstractController.java @@ -7,7 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; public abstract class AbstractController { - private ClustersStorage clustersStorage; + protected ClustersStorage clustersStorage; protected KafkaCluster getCluster(String name) { return clustersStorage.getClusterByName(name) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java index f4ecf26b57..593fd66cb0 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/PrometheusExposeController.java @@ -7,7 +7,7 @@ import com.provectus.kafka.ui.api.PrometheusExposeApi; import com.provectus.kafka.ui.service.StatisticsCache; import io.prometheus.client.exporter.common.TextFormat; import java.io.StringWriter; -import java.io.Writer; +import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; @@ -25,11 +25,16 @@ public class PrometheusExposeController extends AbstractController implements Pr private final StatisticsCache statisticsCache; @Override - public Mono> getAllMetrics(String clusterName, ServerWebExchange exchange) { + public Mono> getAllMetrics(ServerWebExchange exchange) { + return constructResponse(getSummarizedMetricsWithClusterLbl()); + } + + @Override + public Mono> getAllClusterMetrics(String clusterName, ServerWebExchange exchange) { return constructResponse( statisticsCache.get(getCluster(clusterName)) .getMetrics() - .getSummarizedBrokersMetrics() + .getSummarizedMetrics() ); } @@ -37,6 +42,7 @@ public class PrometheusExposeController extends AbstractController implements Pr public Mono> getBrokerMetrics(String clusterName, Long brokerId, ServerWebExchange exchange) { + //TODO: discuss - do we need to append broker_id lbl ? return constructResponse( statisticsCache.get(getCluster(clusterName)) .getMetrics() @@ -46,9 +52,42 @@ public class PrometheusExposeController extends AbstractController implements Pr ); } + private Stream getSummarizedMetricsWithClusterLbl() { + return clustersStorage.getKafkaClusters() + .stream() + .flatMap(c -> statisticsCache.get(c) + .getMetrics() + .getSummarizedMetrics() + .map(mfs -> appendClusterLbl(mfs, c.getName()))); + } + + private static MetricFamilySamples appendClusterLbl(MetricFamilySamples mfs, String clusterName) { + return new MetricFamilySamples( + mfs.name, + mfs.unit, + mfs.type, + mfs.help, + mfs.samples.stream() + .map(sample -> + new MetricFamilySamples.Sample( + sample.name, + prependToList(sample.labelNames, "cluster"), + prependToList(sample.labelValues, clusterName), + sample.value + )).toList() + ); + } + + private static List prependToList(List lst, T toPrepend) { + var result = new ArrayList(lst.size() + 1); + result.add(toPrepend); + result.addAll(lst); + return result; + } + @SneakyThrows - private Mono> constructResponse(Stream metrics) { - Writer writer = new StringWriter(); + private static Mono> constructResponse(Stream metrics) { + StringWriter writer = new StringWriter(); TextFormat.writeOpenMetrics100(writer, Iterators.asEnumeration(metrics.iterator())); HttpHeaders responseHeaders = new HttpHeaders(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java index 2c6bf1610d..256421f1fb 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java @@ -32,10 +32,7 @@ import com.provectus.kafka.ui.model.ReplicaDTO; import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; -import com.provectus.kafka.ui.service.metrics.RawMetric; -import io.prometheus.client.Collector; import java.math.BigDecimal; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -62,7 +59,7 @@ public interface ClusterMapper { @Deprecated default ClusterMetricsDTO toClusterMetrics(Metrics metrics) { return new ClusterMetricsDTO() - .items(convert(metrics.getSummarizedBrokersMetrics()).toList()); + .items(convert(metrics.getSummarizedMetrics()).toList()); } private Stream convert(Stream metrics) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java index 3ac277288d..2c5041d85b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java @@ -47,7 +47,7 @@ public class Metrics { } } - public Stream getSummarizedBrokersMetrics() { + public Stream getSummarizedMetrics() { return Streams.concat( inferredMetrics.asList().stream(), perBrokerScrapedMetrics diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java similarity index 93% rename from kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetrics.java rename to kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java index c082a22332..01de6c4d3d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -10,9 +10,8 @@ import java.math.BigDecimal; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.kafka.common.Node; -public class WellKnownMetrics { +public class IoRatesMetricsScanner { // per broker final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); @@ -22,7 +21,7 @@ public class WellKnownMetrics { final Map bytesInFifteenMinuteRate = new HashMap<>(); final Map bytesOutFifteenMinuteRate = new HashMap<>(); - public WellKnownMetrics(Map> perBrokerMetrics) { + public IoRatesMetricsScanner(Map> perBrokerMetrics) { perBrokerMetrics.forEach((nodeId, metrics) -> { metrics.forEach(m -> { RawMetric.create(m).forEach(rawMetric -> { @@ -33,7 +32,7 @@ public class WellKnownMetrics { }); } - public Metrics.IoRates ioRates() { + public Metrics.IoRates get() { return Metrics.IoRates.builder() .topicBytesInPerSec(bytesInFifteenMinuteRate) .topicBytesOutPerSec(bytesOutFifteenMinuteRate) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java index 25ccc44a90..38186447f7 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java @@ -18,8 +18,7 @@ public class PerBrokerScrapedMetrics { } Metrics.IoRates ioRates() { - //TODO: rename WKMetrics - return new WellKnownMetrics(perBrokerMetrics).ioRates(); + return new IoRatesMetricsScanner(perBrokerMetrics).get(); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java index ec2546a0cd..12816b2b17 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -101,7 +101,7 @@ public class InferredMetricsScraper { state.logDirSpaceStats().totalBytes() ); } - //TODO: maybe add per-directory stats also + //TODO: maybe add per-directory stats also? } }); } diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetricsTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java similarity index 86% rename from kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetricsTest.java rename to kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java index abc33c1e49..cf7964589d 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/WellKnownMetricsTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java @@ -6,9 +6,9 @@ import java.math.BigDecimal; import org.apache.kafka.common.Node; import org.junit.jupiter.api.Test; -class WellKnownMetricsTest { +class IoRatesMetricsScannerTest { - private WellKnownMetrics wellKnownMetrics; + private IoRatesMetricsScanner ioRatesMetricsScanner; @Test void bytesIoTopicMetricsPopulated() { @@ -21,9 +21,9 @@ class WellKnownMetricsTest { "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0" ); - assertThat(wellKnownMetrics.bytesInFifteenMinuteRate) + assertThat(ioRatesMetricsScanner.bytesInFifteenMinuteRate) .containsEntry("test-topic", new BigDecimal("3.0")); - assertThat(wellKnownMetrics.bytesOutFifteenMinuteRate) + assertThat(ioRatesMetricsScanner.bytesOutFifteenMinuteRate) .containsEntry("test-topic", new BigDecimal("6.0")); } @@ -40,12 +40,12 @@ class WellKnownMetricsTest { "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" ); - assertThat(wellKnownMetrics.brokerBytesInFifteenMinuteRate) + assertThat(ioRatesMetricsScanner.brokerBytesInFifteenMinuteRate) .hasSize(2) .containsEntry(1, new BigDecimal("1.0")) .containsEntry(2, new BigDecimal("10.0")); - assertThat(wellKnownMetrics.brokerBytesOutFifteenMinuteRate) + assertThat(ioRatesMetricsScanner.brokerBytesOutFifteenMinuteRate) .hasSize(2) .containsEntry(1, new BigDecimal("2.0")) .containsEntry(2, new BigDecimal("20.0")); diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 590e324abd..79a8c65fb6 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -157,6 +157,66 @@ paths: schema: $ref: '#/components/schemas/ClusterMetrics' + /api/prometheus/expose/all: + get: + tags: + - PrometheusExpose + summary: getAllMetrics + operationId: getAllMetrics + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + + /api/clusters/{clusterName}/prometheus/expose/all: + get: + tags: + - PrometheusExpose + summary: getAllClusterMetrics + operationId: getAllClusterMetrics + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + + /api/clusters/{clusterName}/prometheus/expose/broker/{brokerId}: + get: + tags: + - PrometheusExpose + summary: getBrokerMetrics + operationId: getBrokerMetrics + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + - name: brokerId + in: path + required: true + schema: + type: integer + format: int64 + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + /api/clusters/{clusterName}/stats: get: tags: