From 601bd6bbf50e833d79eaef63120a32b277143e36 Mon Sep 17 00:00:00 2001 From: Ilya Kuramshin Date: Wed, 26 Apr 2023 14:46:57 +0400 Subject: [PATCH 1/4] Minor BE fixes and improvements (#3719) 1. Not setting default key serde to SR/Proto if it is not explicitly set. 2. enabling searching connector by connect name 3. switch using serializedKeySize/serializedValueSize methods instead using byte array len check --- .../kafka/ui/serdes/ConsumerRecordDeserializer.java | 4 ++-- .../com/provectus/kafka/ui/serdes/SerdesInitializer.java | 2 -- .../provectus/kafka/ui/service/KafkaConnectService.java | 1 + .../kafka/ui/service/analyze/TopicAnalysisStats.java | 7 +++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializer.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializer.java index 8c7a3024ed..f5b7018034 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializer.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializer.java @@ -123,11 +123,11 @@ public class ConsumerRecordDeserializer { } private static Long getKeySize(ConsumerRecord consumerRecord) { - return consumerRecord.key() != null ? (long) consumerRecord.key().get().length : null; + return consumerRecord.key() != null ? (long) consumerRecord.serializedKeySize() : null; } private static Long getValueSize(ConsumerRecord consumerRecord) { - return consumerRecord.value() != null ? (long) consumerRecord.value().get().length : null; + return consumerRecord.value() != null ? (long) consumerRecord.serializedValueSize() : null; } private static int headerSize(Header header) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java index 40ea320b2e..66692894a6 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java @@ -122,8 +122,6 @@ public class SerdesInitializer { registeredSerdes, Optional.ofNullable(clusterProperties.getDefaultKeySerde()) .map(name -> Preconditions.checkNotNull(registeredSerdes.get(name), "Default key serde not found")) - .or(() -> Optional.ofNullable(registeredSerdes.get(SchemaRegistrySerde.name()))) - .or(() -> Optional.ofNullable(registeredSerdes.get(ProtobufFileSerde.name()))) .orElse(null), Optional.ofNullable(clusterProperties.getDefaultValueSerde()) .map(name -> Preconditions.checkNotNull(registeredSerdes.get(name), "Default value serde not found")) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java index d07ef7ed2d..98b61541c5 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java @@ -109,6 +109,7 @@ public class KafkaConnectService { private Stream getStringsForSearch(FullConnectorInfoDTO fullConnectorInfo) { return Stream.of( fullConnectorInfo.getName(), + fullConnectorInfo.getConnect(), fullConnectorInfo.getStatus().getState().getValue(), fullConnectorInfo.getType().getValue()); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java index d5b4400807..f36d3bec4d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java @@ -43,8 +43,7 @@ class TopicAnalysisStats { Long max; final UpdateDoublesSketch sizeSketch = DoublesSketch.builder().build(); - void apply(byte[] bytes) { - int len = bytes.length; + void apply(int len) { sum += len; min = minNullable(min, len); max = maxNullable(max, len); @@ -98,7 +97,7 @@ class TopicAnalysisStats { if (rec.key() != null) { byte[] keyBytes = rec.key().get(); - keysSize.apply(keyBytes); + keysSize.apply(rec.serializedKeySize()); uniqKeys.update(keyBytes); } else { nullKeys++; @@ -106,7 +105,7 @@ class TopicAnalysisStats { if (rec.value() != null) { byte[] valueBytes = rec.value().get(); - valuesSize.apply(valueBytes); + valuesSize.apply(rec.serializedValueSize()); uniqValues.update(valueBytes); } else { nullValues++; From 4e255220788f39c15239fb1dda4e723339940ce9 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Thu, 27 Apr 2023 09:59:37 +0800 Subject: [PATCH 2/4] FE: Display message key/value serdes (#3718) Co-authored-by: davitbejanyan --- .../Topics/Topic/Messages/Message.tsx | 2 + .../MessageContent/MessageContent.tsx | 13 ++-- .../__tests__/MessageContent.spec.tsx | 73 ++----------------- 3 files changed, 15 insertions(+), 73 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx index fb4e258cca..dd5cfae748 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/Message.tsx @@ -142,6 +142,8 @@ const Message: React.FC = ({ timestampType={timestampType} keySize={keySize} contentSize={valueSize} + keySerde={keySerde} + valueSerde={valueSerde} /> )} diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx index 93616ca432..d1237ba0d4 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/MessageContent.tsx @@ -3,7 +3,6 @@ import EditorViewer from 'components/common/EditorViewer/EditorViewer'; import BytesFormatted from 'components/common/BytesFormatted/BytesFormatted'; import { SchemaType, TopicMessageTimestampTypeEnum } from 'generated-sources'; import { formatTimestamp } from 'lib/dateTimeHelpers'; -import { useSearchParams } from 'react-router-dom'; import * as S from './MessageContent.styled'; @@ -17,6 +16,8 @@ export interface MessageContentProps { timestampType?: TopicMessageTimestampTypeEnum; keySize?: number; contentSize?: number; + keySerde?: string; + valueSerde?: string; } const MessageContent: React.FC = ({ @@ -27,12 +28,10 @@ const MessageContent: React.FC = ({ timestampType, keySize, contentSize, + keySerde, + valueSerde, }) => { const [activeTab, setActiveTab] = React.useState('content'); - const [searchParams] = useSearchParams(); - const keyFormat = searchParams.get('keySerde') || ''; - const valueFormat = searchParams.get('valueSerde') || ''; - const activeTabContent = () => { switch (activeTab) { case 'content': @@ -110,7 +109,7 @@ const MessageContent: React.FC = ({ Key Serde - {keyFormat} + {keySerde} Size: @@ -120,7 +119,7 @@ const MessageContent: React.FC = ({ Value Serde - {valueFormat} + {valueSerde} Size: diff --git a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx index 91310a30e4..d76455242c 100644 --- a/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/Topic/Messages/MessageContent/__tests__/MessageContent.spec.tsx @@ -20,6 +20,8 @@ const setupWrapper = (props?: Partial) => { headers={{ header: 'test' }} timestamp={new Date(0)} timestampType={TopicMessageTimestampTypeEnum.CREATE_TIME} + keySerde="SchemaRegistry" + valueSerde="Avro" {...props} /> @@ -27,42 +29,20 @@ const setupWrapper = (props?: Partial) => { ); }; -const proto = - 'syntax = "proto3";\npackage com.provectus;\n\nmessage TestProtoRecord {\n string f1 = 1;\n int32 f2 = 2;\n}\n'; - global.TextEncoder = TextEncoder; -const searchParamsContentAVRO = new URLSearchParams({ - keySerde: 'SchemaRegistry', - valueSerde: 'AVRO', - limit: '100', -}); - -const searchParamsContentJSON = new URLSearchParams({ - keySerde: 'SchemaRegistry', - valueSerde: 'JSON', - limit: '100', -}); - -const searchParamsContentPROTOBUF = new URLSearchParams({ - keySerde: 'SchemaRegistry', - valueSerde: 'PROTOBUF', - limit: '100', -}); describe('MessageContent screen', () => { beforeEach(() => { - render(setupWrapper(), { - initialEntries: [`/messages?${searchParamsContentAVRO}`], - }); + render(setupWrapper()); }); - describe('renders', () => { - it('key format in document', () => { + describe('Checking keySerde and valueSerde', () => { + it('keySerde in document', () => { expect(screen.getByText('SchemaRegistry')).toBeInTheDocument(); }); - it('content format in document', () => { - expect(screen.getByText('AVRO')).toBeInTheDocument(); + it('valueSerde in document', () => { + expect(screen.getByText('Avro')).toBeInTheDocument(); }); }); @@ -98,42 +78,3 @@ describe('MessageContent screen', () => { }); }); }); - -describe('checking content type depend on message type', () => { - it('renders component with message having JSON type', () => { - render( - setupWrapper({ - messageContent: '{"data": "test"}', - }), - { initialEntries: [`/messages?${searchParamsContentJSON}`] } - ); - expect(screen.getByText('JSON')).toBeInTheDocument(); - }); - it('renders component with message having AVRO type', () => { - render( - setupWrapper({ - messageContent: '{"data": "test"}', - }), - { initialEntries: [`/messages?${searchParamsContentAVRO}`] } - ); - expect(screen.getByText('AVRO')).toBeInTheDocument(); - }); - it('renders component with message having PROTOBUF type', () => { - render( - setupWrapper({ - messageContent: proto, - }), - { initialEntries: [`/messages?${searchParamsContentPROTOBUF}`] } - ); - expect(screen.getByText('PROTOBUF')).toBeInTheDocument(); - }); - it('renders component with message having no type which is equal to having PROTOBUF type', () => { - render( - setupWrapper({ - messageContent: '', - }), - { initialEntries: [`/messages?${searchParamsContentPROTOBUF}`] } - ); - expect(screen.getByText('PROTOBUF')).toBeInTheDocument(); - }); -}); From da3932e3422e45bdb0dc27cf538b79c8bf872602 Mon Sep 17 00:00:00 2001 From: David Bejanyan <58771979+David-DB88@users.noreply.github.com> Date: Thu, 27 Apr 2023 06:01:38 +0400 Subject: [PATCH 3/4] FE: Impl a possibility to opt out of version check (#3672) --- .../src/components/Version/Version.tsx | 30 ++---- .../Version/__tests__/Version.spec.tsx | 97 +++++-------------- .../components/common/Icons/WarningIcon.tsx | 1 + .../src/lib/fixtures/actuatorInfo.ts | 12 --- .../src/lib/fixtures/latestVersion.ts | 17 +++- .../hooks/api/__tests__/actuatorInfo.spec.ts | 17 ---- .../hooks/api/__tests__/latestVersion.spec.ts | 12 +-- .../src/lib/hooks/api/actuatorInfo.ts | 19 ---- .../src/lib/hooks/api/latestVersion.ts | 18 ++-- 9 files changed, 62 insertions(+), 161 deletions(-) delete mode 100644 kafka-ui-react-app/src/lib/fixtures/actuatorInfo.ts delete mode 100644 kafka-ui-react-app/src/lib/hooks/api/__tests__/actuatorInfo.spec.ts delete mode 100644 kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts diff --git a/kafka-ui-react-app/src/components/Version/Version.tsx b/kafka-ui-react-app/src/components/Version/Version.tsx index 7a820b116a..6788605d92 100644 --- a/kafka-ui-react-app/src/components/Version/Version.tsx +++ b/kafka-ui-react-app/src/components/Version/Version.tsx @@ -1,52 +1,38 @@ import React from 'react'; import WarningIcon from 'components/common/Icons/WarningIcon'; import { gitCommitPath } from 'lib/paths'; -import { useActuatorInfo } from 'lib/hooks/api/actuatorInfo'; -import { BUILD_VERSION_PATTERN } from 'lib/constants'; import { useLatestVersion } from 'lib/hooks/api/latestVersion'; import { formatTimestamp } from 'lib/dateTimeHelpers'; import * as S from './Version.styled'; -import compareVersions from './compareVersions'; const Version: React.FC = () => { - const { data: actuatorInfo = {} } = useActuatorInfo(); const { data: latestVersionInfo = {} } = useLatestVersion(); - - const tag = actuatorInfo?.build?.version; - const commit = actuatorInfo?.git?.commit.id; - const { tag_name: latestTag } = latestVersionInfo; - - const outdated = compareVersions(tag, latestTag); - - const currentVersion = tag?.match(BUILD_VERSION_PATTERN) - ? tag - : formatTimestamp(actuatorInfo?.build?.time); - - if (!tag) return null; + const { buildTime, commitId, isLatestRelease } = latestVersionInfo.build; + const { versionTag } = latestVersionInfo?.latestRelease || ''; return ( - {!!outdated && ( + {!isLatestRelease && ( )} - {commit && ( + {commitId && (
- {commit} + {commitId}
)} - {currentVersion} + {formatTimestamp(buildTime)} ); }; diff --git a/kafka-ui-react-app/src/components/Version/__tests__/Version.spec.tsx b/kafka-ui-react-app/src/components/Version/__tests__/Version.spec.tsx index d407966058..2700dac894 100644 --- a/kafka-ui-react-app/src/components/Version/__tests__/Version.spec.tsx +++ b/kafka-ui-react-app/src/components/Version/__tests__/Version.spec.tsx @@ -2,87 +2,40 @@ import React from 'react'; import { screen } from '@testing-library/dom'; import Version from 'components/Version/Version'; import { render } from 'lib/testHelpers'; -import { formatTimestamp } from 'lib/dateTimeHelpers'; -import { useActuatorInfo } from 'lib/hooks/api/actuatorInfo'; import { useLatestVersion } from 'lib/hooks/api/latestVersion'; -import { actuatorInfoPayload } from 'lib/fixtures/actuatorInfo'; -import { latestVersionPayload } from 'lib/fixtures/latestVersion'; +import { + deprecatedVersionPayload, + latestVersionPayload, +} from 'lib/fixtures/latestVersion'; -jest.mock('lib/hooks/api/actuatorInfo', () => ({ - useActuatorInfo: jest.fn(), -})); jest.mock('lib/hooks/api/latestVersion', () => ({ useLatestVersion: jest.fn(), })); - describe('Version Component', () => { - const versionTag = 'v0.5.0'; - const snapshotTag = 'test-SNAPSHOT'; - const commitTag = 'befd3b328e2c9c7df57b0c5746561b2f7fee8813'; + const commitId = '96a577a'; - const actuatorVersionPayload = actuatorInfoPayload(versionTag); - const formattedTimestamp = formatTimestamp(actuatorVersionPayload.build.time); + describe('render latest version', () => { + beforeEach(() => { + (useLatestVersion as jest.Mock).mockImplementation(() => ({ + data: latestVersionPayload, + })); + }); + it('renders latest release version as current version', async () => { + render(); + expect(screen.getByText(commitId)).toBeInTheDocument(); + }); - beforeEach(() => { - (useActuatorInfo as jest.Mock).mockImplementation(() => ({ - data: actuatorVersionPayload, - })); + it('should not show warning icon if it is last release', async () => { + render(); + expect(screen.queryByRole('img')).not.toBeInTheDocument(); + }); + }); + + it('show warning icon if it is not last release', async () => { (useLatestVersion as jest.Mock).mockImplementation(() => ({ - data: latestVersionPayload, + data: deprecatedVersionPayload, })); - }); - - describe('tag does not exist', () => { - it('does not render component', async () => { - (useActuatorInfo as jest.Mock).mockImplementation(() => ({ - data: null, - })); - const { container } = render(); - expect(container.firstChild).toBeEmptyDOMElement(); - }); - }); - - describe('renders current version', () => { - it('renders release build version as current version', async () => { - render(); - expect(screen.getByText(versionTag)).toBeInTheDocument(); - }); - it('renders formatted timestamp as current version when version is commit', async () => { - (useActuatorInfo as jest.Mock).mockImplementation(() => ({ - data: actuatorInfoPayload(commitTag), - })); - render(); - expect(screen.getByText(formattedTimestamp)).toBeInTheDocument(); - }); - it('renders formatted timestamp as current version when version contains -SNAPSHOT', async () => { - (useActuatorInfo as jest.Mock).mockImplementation(() => ({ - data: actuatorInfoPayload(snapshotTag), - })); - render(); - expect(screen.getByText(formattedTimestamp)).toBeInTheDocument(); - }); - }); - - describe('outdated build version', () => { - it('renders warning message', async () => { - (useActuatorInfo as jest.Mock).mockImplementation(() => ({ - data: actuatorInfoPayload('v0.3.0'), - })); - render(); - expect( - screen.getByTitle( - `Your app version is outdated. Current latest version is ${latestVersionPayload.tag_name}` - ) - ).toBeInTheDocument(); - }); - }); - - describe('current commit id with link', () => { - it('renders', async () => { - render(); - expect( - screen.getByText(actuatorVersionPayload.git.commit.id) - ).toBeInTheDocument(); - }); + render(); + expect(screen.getByRole('img')).toBeInTheDocument(); }); }); diff --git a/kafka-ui-react-app/src/components/common/Icons/WarningIcon.tsx b/kafka-ui-react-app/src/components/common/Icons/WarningIcon.tsx index ab2f8dee75..1bffe0db53 100644 --- a/kafka-ui-react-app/src/components/common/Icons/WarningIcon.tsx +++ b/kafka-ui-react-app/src/components/common/Icons/WarningIcon.tsx @@ -13,6 +13,7 @@ const WarningIcon: React.FC = () => { return ( ({ - git: { commit: { id: 'befd3b3' } }, - build: { - artifact: 'kafka-ui-api', - name: 'kafka-ui-api', - time: '2022-09-15T09:52:21.753Z', - version, - group: 'com.provectus', - }, -}); diff --git a/kafka-ui-react-app/src/lib/fixtures/latestVersion.ts b/kafka-ui-react-app/src/lib/fixtures/latestVersion.ts index a6c9eca856..d1e62da6b8 100644 --- a/kafka-ui-react-app/src/lib/fixtures/latestVersion.ts +++ b/kafka-ui-react-app/src/lib/fixtures/latestVersion.ts @@ -1,3 +1,16 @@ -export const latestVersionPayload = { - tag_name: 'v0.4.0', +export const deprecatedVersionPayload = { + build: { + buildTime: '2023-04-14T09:47:35.463Z', + commitId: '96a577a', + isLatestRelease: false, + version: '96a577a98c6069376c5d22ed49cffd3739f1bbdc', + }, +}; +export const latestVersionPayload = { + build: { + buildTime: '2023-04-14T09:47:35.463Z', + commitId: '96a577a', + isLatestRelease: true, + version: '96a577a98c6069376c5d22ed49cffd3739f1bbdc', + }, }; diff --git a/kafka-ui-react-app/src/lib/hooks/api/__tests__/actuatorInfo.spec.ts b/kafka-ui-react-app/src/lib/hooks/api/__tests__/actuatorInfo.spec.ts deleted file mode 100644 index c4e639680e..0000000000 --- a/kafka-ui-react-app/src/lib/hooks/api/__tests__/actuatorInfo.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import fetchMock from 'fetch-mock'; -import * as hooks from 'lib/hooks/api/actuatorInfo'; -import { expectQueryWorks, renderQueryHook } from 'lib/testHelpers'; -import { actuatorInfoPayload } from 'lib/fixtures/actuatorInfo'; - -const actuatorInfoPath = '/actuator/info'; - -describe('Actuator info hooks', () => { - beforeEach(() => fetchMock.restore()); - describe('useActuatorInfo', () => { - it('returns the correct data', async () => { - const mock = fetchMock.getOnce(actuatorInfoPath, actuatorInfoPayload()); - const { result } = renderQueryHook(() => hooks.useActuatorInfo()); - await expectQueryWorks(mock, result); - }); - }); -}); diff --git a/kafka-ui-react-app/src/lib/hooks/api/__tests__/latestVersion.spec.ts b/kafka-ui-react-app/src/lib/hooks/api/__tests__/latestVersion.spec.ts index 0770bd4606..a12f262995 100644 --- a/kafka-ui-react-app/src/lib/hooks/api/__tests__/latestVersion.spec.ts +++ b/kafka-ui-react-app/src/lib/hooks/api/__tests__/latestVersion.spec.ts @@ -1,18 +1,16 @@ import fetchMock from 'fetch-mock'; import { expectQueryWorks, renderQueryHook } from 'lib/testHelpers'; -import * as hooks from 'lib/hooks/api/latestVersion'; -import { GIT_REPO_LATEST_RELEASE_LINK } from 'lib/constants'; import { latestVersionPayload } from 'lib/fixtures/latestVersion'; +import { useLatestVersion } from 'lib/hooks/api/latestVersion'; + +const latestVersionPath = '/api/info'; describe('Latest version hooks', () => { beforeEach(() => fetchMock.restore()); describe('useLatestVersion', () => { it('returns the correct data', async () => { - const mock = fetchMock.getOnce( - GIT_REPO_LATEST_RELEASE_LINK, - latestVersionPayload - ); - const { result } = renderQueryHook(() => hooks.useLatestVersion()); + const mock = fetchMock.getOnce(latestVersionPath, latestVersionPayload); + const { result } = renderQueryHook(() => useLatestVersion()); await expectQueryWorks(mock, result); }); }); diff --git a/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts b/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts deleted file mode 100644 index 7e1835d907..0000000000 --- a/kafka-ui-react-app/src/lib/hooks/api/actuatorInfo.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { useQuery } from '@tanstack/react-query'; -import { BASE_PARAMS, QUERY_REFETCH_OFF_OPTIONS } from 'lib/constants'; - -const fetchActuatorInfo = async () => { - const data = await fetch( - `${BASE_PARAMS.basePath}/actuator/info`, - BASE_PARAMS - ).then((res) => res.json()); - - return data; -}; - -export function useActuatorInfo() { - return useQuery( - ['actuatorInfo'], - fetchActuatorInfo, - QUERY_REFETCH_OFF_OPTIONS - ); -} diff --git a/kafka-ui-react-app/src/lib/hooks/api/latestVersion.ts b/kafka-ui-react-app/src/lib/hooks/api/latestVersion.ts index 1087fb572e..0711ad34d9 100644 --- a/kafka-ui-react-app/src/lib/hooks/api/latestVersion.ts +++ b/kafka-ui-react-app/src/lib/hooks/api/latestVersion.ts @@ -1,21 +1,19 @@ import { useQuery } from '@tanstack/react-query'; -import { - QUERY_REFETCH_OFF_OPTIONS, - GIT_REPO_LATEST_RELEASE_LINK, -} from 'lib/constants'; +import { BASE_PARAMS, QUERY_REFETCH_OFF_OPTIONS } from 'lib/constants'; -const fetchLatestVersion = async () => { - const data = await fetch(GIT_REPO_LATEST_RELEASE_LINK).then((res) => - res.json() - ); +const fetchLatestVersionInfo = async () => { + const data = await fetch( + `${BASE_PARAMS.basePath}/api/info`, + BASE_PARAMS + ).then((res) => res.json()); return data; }; export function useLatestVersion() { return useQuery( - ['latestVersion'], - fetchLatestVersion, + ['versionInfo'], + fetchLatestVersionInfo, QUERY_REFETCH_OFF_OPTIONS ); } From 744bdb32a310306eefe8641923d712db697b1c70 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 1 May 2023 07:56:28 +0800 Subject: [PATCH 4/4] BE: RBAC: LDAP support implemented (#3700) --- .../kafka/ui/config/auth/LdapProperties.java | 26 +++++ .../ui/config/auth/LdapSecurityConfig.java | 96 ++++++++++++------- .../ui/config/auth/OAuthSecurityConfig.java | 2 +- .../kafka/ui/config/auth/RbacLdapUser.java | 60 ++++++++++++ .../condition/ActiveDirectoryCondition.java | 21 ++++ .../ui/service/rbac/AccessControlService.java | 30 +++--- .../extractor/LdapAuthorityExtractor.java | 23 ----- .../RbacLdapAuthoritiesExtractor.java | 70 ++++++++++++++ 8 files changed, 258 insertions(+), 70 deletions(-) create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java new file mode 100644 index 0000000000..13119b3bb9 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java @@ -0,0 +1,26 @@ +package com.provectus.kafka.ui.config.auth; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("spring.ldap") +@Data +public class LdapProperties { + + private String urls; + private String base; + private String adminUser; + private String adminPassword; + private String userFilterSearchBase; + private String userFilterSearchFilter; + + @Value("${oauth2.ldap.activeDirectory:false}") + private boolean isActiveDirectory; + @Value("${oauth2.ldap.aсtiveDirectory.domain:@null}") + private String activeDirectoryDomain; + + @Value("${oauth2.ldap.groupRoleAttribute:cn}") + private String groupRoleAttribute; + +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java index 0ba5c231f4..fae1125239 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java @@ -1,13 +1,23 @@ package com.provectus.kafka.ui.config.auth; +import static com.provectus.kafka.ui.config.auth.AbstractAuthSecurityConfig.AUTH_WHITELIST; + +import com.provectus.kafka.ui.service.rbac.AccessControlService; +import com.provectus.kafka.ui.service.rbac.extractor.RbacLdapAuthoritiesExtractor; +import java.util.Collection; import java.util.List; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.ldap.core.support.LdapContextSource; import org.springframework.security.authentication.AuthenticationManager; @@ -16,70 +26,71 @@ import org.springframework.security.authentication.ReactiveAuthenticationManager import org.springframework.security.authentication.ReactiveAuthenticationManagerAdapter; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.ldap.authentication.AbstractLdapAuthenticationProvider; import org.springframework.security.ldap.authentication.BindAuthenticator; import org.springframework.security.ldap.authentication.LdapAuthenticationProvider; import org.springframework.security.ldap.authentication.ad.ActiveDirectoryLdapAuthenticationProvider; import org.springframework.security.ldap.search.FilterBasedLdapUserSearch; import org.springframework.security.ldap.search.LdapUserSearch; +import org.springframework.security.ldap.userdetails.LdapUserDetailsMapper; import org.springframework.security.web.server.SecurityWebFilterChain; @Configuration @EnableWebFluxSecurity @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") @Import(LdapAutoConfiguration.class) +@EnableConfigurationProperties(LdapProperties.class) +@RequiredArgsConstructor @Slf4j -public class LdapSecurityConfig extends AbstractAuthSecurityConfig { +public class LdapSecurityConfig { - @Value("${spring.ldap.urls}") - private String ldapUrls; - @Value("${spring.ldap.dn.pattern:#{null}}") - private String ldapUserDnPattern; - @Value("${spring.ldap.adminUser:#{null}}") - private String adminUser; - @Value("${spring.ldap.adminPassword:#{null}}") - private String adminPassword; - @Value("${spring.ldap.userFilter.searchBase:#{null}}") - private String userFilterSearchBase; - @Value("${spring.ldap.userFilter.searchFilter:#{null}}") - private String userFilterSearchFilter; - - @Value("${oauth2.ldap.activeDirectory:false}") - private boolean isActiveDirectory; - @Value("${oauth2.ldap.aсtiveDirectory.domain:#{null}}") - private String activeDirectoryDomain; + private final LdapProperties props; @Bean - public ReactiveAuthenticationManager authenticationManager(BaseLdapPathContextSource contextSource) { + public ReactiveAuthenticationManager authenticationManager(BaseLdapPathContextSource contextSource, + ApplicationContext context, + @Nullable AccessControlService acs) { + var rbacEnabled = acs != null && acs.isRbacEnabled(); BindAuthenticator ba = new BindAuthenticator(contextSource); - if (ldapUserDnPattern != null) { - ba.setUserDnPatterns(new String[] {ldapUserDnPattern}); + if (props.getBase() != null) { + ba.setUserDnPatterns(new String[] {props.getBase()}); } - if (userFilterSearchFilter != null) { + if (props.getUserFilterSearchFilter() != null) { LdapUserSearch userSearch = - new FilterBasedLdapUserSearch(userFilterSearchBase, userFilterSearchFilter, contextSource); + new FilterBasedLdapUserSearch(props.getUserFilterSearchBase(), props.getUserFilterSearchFilter(), + contextSource); ba.setUserSearch(userSearch); } AbstractLdapAuthenticationProvider authenticationProvider; - if (!isActiveDirectory) { - authenticationProvider = new LdapAuthenticationProvider(ba); + if (!props.isActiveDirectory()) { + authenticationProvider = rbacEnabled + ? new LdapAuthenticationProvider(ba, new RbacLdapAuthoritiesExtractor(context)) + : new LdapAuthenticationProvider(ba); } else { - authenticationProvider = new ActiveDirectoryLdapAuthenticationProvider(activeDirectoryDomain, ldapUrls); + authenticationProvider = new ActiveDirectoryLdapAuthenticationProvider(props.getActiveDirectoryDomain(), + props.getUrls()); // TODO Issue #3741 authenticationProvider.setUseAuthenticationRequestCredentials(true); } + if (rbacEnabled) { + authenticationProvider.setUserDetailsContextMapper(new UserDetailsMapper()); + } + AuthenticationManager am = new ProviderManager(List.of(authenticationProvider)); return new ReactiveAuthenticationManagerAdapter(am); } @Bean + @Primary public BaseLdapPathContextSource contextSource() { LdapContextSource ctx = new LdapContextSource(); - ctx.setUrl(ldapUrls); - ctx.setUserDn(adminUser); - ctx.setPassword(adminPassword); + ctx.setUrl(props.getUrls()); + ctx.setUserDn(props.getAdminUser()); + ctx.setPassword(props.getAdminPassword()); ctx.afterPropertiesSet(); return ctx; } @@ -87,20 +98,35 @@ public class LdapSecurityConfig extends AbstractAuthSecurityConfig { @Bean public SecurityWebFilterChain configureLdap(ServerHttpSecurity http) { log.info("Configuring LDAP authentication."); - if (isActiveDirectory) { + if (props.isActiveDirectory()) { log.info("Active Directory support for LDAP has been enabled."); } - http + return http .authorizeExchange() .pathMatchers(AUTH_WHITELIST) .permitAll() .anyExchange() .authenticated() - .and() - .httpBasic(); - return http.csrf().disable().build(); + .and() + .formLogin() + + .and() + .logout() + + .and() + .csrf().disable() + .build(); + } + + private static class UserDetailsMapper extends LdapUserDetailsMapper { + @Override + public UserDetails mapUserFromContext(DirContextOperations ctx, String username, + Collection authorities) { + UserDetails userDetails = super.mapUserFromContext(ctx, username, authorities); + return new RbacLdapUser(userDetails); + } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java index 1d237e0173..5db612f256 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java @@ -115,7 +115,7 @@ public class OAuthSecurityConfig extends AbstractAuthSecurityConfig { @Nullable private ProviderAuthorityExtractor getExtractor(final String providerId, AccessControlService acs) { final String provider = getProviderByProviderId(providerId); - Optional extractor = acs.getExtractors() + Optional extractor = acs.getOauthExtractors() .stream() .filter(e -> e.isApplicable(provider)) .findFirst(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java new file mode 100644 index 0000000000..037d2fd302 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java @@ -0,0 +1,60 @@ +package com.provectus.kafka.ui.config.auth; + +import java.util.Collection; +import java.util.stream.Collectors; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.userdetails.UserDetails; + +public class RbacLdapUser implements UserDetails, RbacUser { + + private final UserDetails userDetails; + + public RbacLdapUser(UserDetails userDetails) { + this.userDetails = userDetails; + } + + @Override + public String name() { + return userDetails.getUsername(); + } + + @Override + public Collection groups() { + return userDetails.getAuthorities().stream().map(GrantedAuthority::getAuthority).collect(Collectors.toSet()); + } + + @Override + public Collection getAuthorities() { + return userDetails.getAuthorities(); + } + + @Override + public String getPassword() { + return userDetails.getPassword(); + } + + @Override + public String getUsername() { + return userDetails.getUsername(); + } + + @Override + public boolean isAccountNonExpired() { + return userDetails.isAccountNonExpired(); + } + + @Override + public boolean isAccountNonLocked() { + return userDetails.isAccountNonLocked(); + } + + @Override + public boolean isCredentialsNonExpired() { + return userDetails.isCredentialsNonExpired(); + } + + @Override + public boolean isEnabled() { + return userDetails.isEnabled(); + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java new file mode 100644 index 0000000000..c38e83238a --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java @@ -0,0 +1,21 @@ +package com.provectus.kafka.ui.config.auth.condition; + +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +public class ActiveDirectoryCondition extends AllNestedConditions { + + public ActiveDirectoryCondition() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") + public static class OnAuthType { + + } + + @ConditionalOnProperty(value = "${oauth2.ldap.activeDirectory}:false", havingValue = "true", matchIfMissing = false) + public static class OnActiveDirectory { + + } +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java index 3178feae34..e964f64a9b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java @@ -12,6 +12,7 @@ import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.Permission; import com.provectus.kafka.ui.model.rbac.Resource; import com.provectus.kafka.ui.model.rbac.Role; +import com.provectus.kafka.ui.model.rbac.Subject; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; @@ -19,11 +20,11 @@ import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import com.provectus.kafka.ui.service.rbac.extractor.CognitoAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GithubAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GoogleAuthorityExtractor; -import com.provectus.kafka.ui.service.rbac.extractor.LdapAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.ProviderAuthorityExtractor; import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.core.env.Environment; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.context.ReactiveSecurityContextHolder; import org.springframework.security.core.context.SecurityContext; @@ -50,10 +52,11 @@ public class AccessControlService { @Nullable private final InMemoryReactiveClientRegistrationRepository clientRegistrationRepository; + private final RoleBasedAccessControlProperties properties; + private final Environment environment; private boolean rbacEnabled = false; - private Set extractors = Collections.emptySet(); - private final RoleBasedAccessControlProperties properties; + private Set oauthExtractors = Collections.emptySet(); @PostConstruct public void init() { @@ -63,21 +66,26 @@ public class AccessControlService { } rbacEnabled = true; - this.extractors = properties.getRoles() + this.oauthExtractors = properties.getRoles() .stream() .map(role -> role.getSubjects() .stream() - .map(provider -> switch (provider.getProvider()) { + .map(Subject::getProvider) + .distinct() + .map(provider -> switch (provider) { case OAUTH_COGNITO -> new CognitoAuthorityExtractor(); case OAUTH_GOOGLE -> new GoogleAuthorityExtractor(); case OAUTH_GITHUB -> new GithubAuthorityExtractor(); - case LDAP, LDAP_AD -> new LdapAuthorityExtractor(); - }).collect(Collectors.toSet())) + default -> null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet())) .flatMap(Set::stream) .collect(Collectors.toSet()); - if ((clientRegistrationRepository == null || !clientRegistrationRepository.iterator().hasNext()) - && !properties.getRoles().isEmpty()) { + if (!properties.getRoles().isEmpty() + && "oauth2".equalsIgnoreCase(environment.getProperty("auth.type")) + && (clientRegistrationRepository == null || !clientRegistrationRepository.iterator().hasNext())) { log.error("Roles are configured but no authentication methods are present. Authentication might fail."); } } @@ -354,8 +362,8 @@ public class AccessControlService { return isAccessible(Resource.KSQL, null, user, context, requiredActions); } - public Set getExtractors() { - return extractors; + public Set getOauthExtractors() { + return oauthExtractors; } public List getRoles() { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java deleted file mode 100644 index 6284bb2923..0000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/LdapAuthorityExtractor.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.provectus.kafka.ui.service.rbac.extractor; - -import com.provectus.kafka.ui.service.rbac.AccessControlService; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; - -@Slf4j -public class LdapAuthorityExtractor implements ProviderAuthorityExtractor { - - @Override - public boolean isApplicable(String provider) { - return false; // TODO #2752 - } - - @Override - public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { - return Mono.just(Collections.emptySet()); // TODO #2752 - } - -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java new file mode 100644 index 0000000000..e24fc0aeda --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java @@ -0,0 +1,70 @@ +package com.provectus.kafka.ui.service.rbac.extractor; + +import com.provectus.kafka.ui.config.auth.LdapProperties; +import com.provectus.kafka.ui.model.rbac.Role; +import com.provectus.kafka.ui.model.rbac.provider.Provider; +import com.provectus.kafka.ui.service.rbac.AccessControlService; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationContext; +import org.springframework.ldap.core.DirContextOperations; +import org.springframework.ldap.core.support.BaseLdapPathContextSource; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; +import org.springframework.util.Assert; + +@Slf4j +public class RbacLdapAuthoritiesExtractor extends DefaultLdapAuthoritiesPopulator { + + private final AccessControlService acs; + private final LdapProperties props; + + private final Function>, GrantedAuthority> authorityMapper = (record) -> { + String role = record.get(getGroupRoleAttribute()).get(0); + return new SimpleGrantedAuthority(role); + }; + + public RbacLdapAuthoritiesExtractor(ApplicationContext context) { + super(context.getBean(BaseLdapPathContextSource.class), null); + this.acs = context.getBean(AccessControlService.class); + this.props = context.getBean(LdapProperties.class); + } + + @Override + public Set getAdditionalRoles(DirContextOperations user, String username) { + return acs.getRoles() + .stream() + .map(Role::getSubjects) + .flatMap(List::stream) + .filter(s -> s.getProvider().equals(Provider.LDAP)) + .filter(s -> s.getType().equals("group")) + .flatMap(subject -> getRoles(subject.getValue(), user.getNameInNamespace(), username).stream()) + .collect(Collectors.toSet()); + } + + private Set getRoles(String groupSearchBase, String userDn, String username) { + Assert.notNull(groupSearchBase, "groupSearchBase is empty"); + + log.trace( + "Searching for roles for user [{}] with DN [{}], groupRoleAttribute [{}] and filter [{}] in search base [{}]", + username, userDn, props.getGroupRoleAttribute(), getGroupSearchFilter(), groupSearchBase); + + var ldapTemplate = getLdapTemplate(); + ldapTemplate.setIgnoreNameNotFoundException(true); + + Set>> userRoles = ldapTemplate.searchForMultipleAttributeValues( + groupSearchBase, getGroupSearchFilter(), new String[] {userDn, username}, + new String[] {props.getGroupRoleAttribute()}); + + return userRoles.stream() + .map(authorityMapper) + .peek(a -> log.debug("Mapped role [{}] for user [{}]", a, username)) + .collect(Collectors.toSet()); + } + +}