From b7173346d2949269b762b023da9ea67b7f327c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Fri, 5 Jan 2024 11:30:15 +0100 Subject: [PATCH] feat(syncing-server): add traffic abuse checks (#1014) --- packages/syncing-server/bin/statistics.ts | 4 +- .../syncing-server/src/Bootstrap/Container.ts | 43 ++++++++ .../syncing-server/src/Bootstrap/Types.ts | 10 ++ .../Domain/Metrics/MetricsStoreInterface.ts | 32 +++--- .../src/Domain/Metrics/MetricsSummary.ts | 6 ++ .../CheckForTrafficAbuse.spec.ts | 97 +++++++++++++++++++ .../CheckForTrafficAbuse.ts | 44 +++++++++ .../CheckForTrafficAbuseDTO.ts | 6 ++ .../Syncing/SaveNewItem/SaveNewItem.ts | 2 +- .../UpdateExistingItem/UpdateExistingItem.ts | 2 +- .../src/Infra/Dummy/DummyMetricStore.ts | 30 ++++-- .../AnnotatedItemsController.ts | 26 ++++- .../Base/BaseItemsController.ts | 41 ++++++++ .../src/Infra/Redis/RedisMetricStore.ts | 81 +++++++++++++--- 14 files changed, 377 insertions(+), 47 deletions(-) create mode 100644 packages/syncing-server/src/Domain/Metrics/MetricsSummary.ts create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.spec.ts create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.ts create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuseDTO.ts diff --git a/packages/syncing-server/bin/statistics.ts b/packages/syncing-server/bin/statistics.ts index ed15af818..021b5af79 100644 --- a/packages/syncing-server/bin/statistics.ts +++ b/packages/syncing-server/bin/statistics.ts @@ -28,7 +28,7 @@ const sendStatistics = async ( const dateNMinutesAgo = timer.getUTCDateNMinutesAgo(minutesToProcess - i) const timestamp = timer.convertDateToMicroseconds(dateNMinutesAgo) - const statistics = await metricsStore.getStatistics( + const statistics = await metricsStore.getMetricsSummary( metricToProcess, timestamp, timestamp + Time.MicrosecondsInAMinute, @@ -64,7 +64,7 @@ const sendStatistics = async ( const dateNMinutesAgo = timer.getUTCDateNMinutesAgo(minutesToProcess - i) const timestamp = timer.convertDateToMicroseconds(dateNMinutesAgo) - const statistics = await metricsStore.getUserBasedStatistics(metricToProcess, timestamp) + const statistics = await metricsStore.getUserBasedMetricsSummary(metricToProcess, timestamp) if (statistics.sampleCount === 0) { continue diff --git a/packages/syncing-server/src/Bootstrap/Container.ts b/packages/syncing-server/src/Bootstrap/Container.ts index 7aeb89012..1422834b5 100644 --- a/packages/syncing-server/src/Bootstrap/Container.ts +++ b/packages/syncing-server/src/Bootstrap/Container.ts @@ -166,6 +166,7 @@ import { SendEventToClients } from '../Domain/UseCase/Syncing/SendEventToClients import { MetricsStoreInterface } from '../Domain/Metrics/MetricsStoreInterface' import { RedisMetricStore } from '../Infra/Redis/RedisMetricStore' import { DummyMetricStore } from '../Infra/Dummy/DummyMetricStore' +import { CheckForTrafficAbuse } from '../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse' export class ContainerConfigLoader { private readonly DEFAULT_CONTENT_SIZE_TRANSFER_LIMIT = 10_000_000 @@ -472,6 +473,33 @@ export class ContainerConfigLoader { }) // env vars + container + .bind(TYPES.Sync_STRICT_ABUSE_PROTECTION) + .toConstantValue(env.get('STRICT_ABUSE_PROTECTION', true) === 'true') + container + .bind(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD) + .toConstantValue( + env.get('ITEM_OPERATIONS_ABUSE_THRESHOLD', true) ? +env.get('ITEM_OPERATIONS_ABUSE_THRESHOLD', true) : 500, + ) + container + .bind(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES) + .toConstantValue( + env.get('ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true) + ? +env.get('ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true) + : 5, + ) + container + .bind(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD) + .toConstantValue( + env.get('PAYLOAD_SIZE_ABUSE_THRESHOLD', true) ? +env.get('PAYLOAD_SIZE_ABUSE_THRESHOLD', true) : 20_000_000, + ) + container + .bind(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES) + .toConstantValue( + env.get('PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true) + ? +env.get('PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', true) + : 5, + ) container.bind(TYPES.Sync_AUTH_JWT_SECRET).toConstantValue(env.get('AUTH_JWT_SECRET')) container .bind(TYPES.Sync_REVISIONS_FREQUENCY) @@ -933,6 +961,14 @@ export class ContainerConfigLoader { context.container.get(TYPES.Sync_SyncResponseFactory20200115), ) }) + container + .bind(TYPES.Sync_CheckForTrafficAbuse) + .toConstantValue( + new CheckForTrafficAbuse( + container.get(TYPES.Sync_MetricsStore), + container.get(TYPES.Sync_Timer), + ), + ) // Handlers container @@ -1094,11 +1130,18 @@ export class ContainerConfigLoader { .bind(TYPES.Sync_BaseItemsController) .toConstantValue( new BaseItemsController( + container.get(TYPES.Sync_CheckForTrafficAbuse), container.get(TYPES.Sync_SyncItems), container.get(TYPES.Sync_CheckIntegrity), container.get(TYPES.Sync_GetItem), container.get>(TYPES.Sync_ItemHttpMapper), container.get(TYPES.Sync_SyncResponseFactoryResolver), + container.get(TYPES.Sync_Logger), + container.get(TYPES.Sync_STRICT_ABUSE_PROTECTION), + container.get(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES), + container.get(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD), + container.get(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD), + container.get(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES), container.get(TYPES.Sync_ControllerContainer), ), ) diff --git a/packages/syncing-server/src/Bootstrap/Types.ts b/packages/syncing-server/src/Bootstrap/Types.ts index 5b1f12620..d1f45b9fb 100644 --- a/packages/syncing-server/src/Bootstrap/Types.ts +++ b/packages/syncing-server/src/Bootstrap/Types.ts @@ -42,6 +42,15 @@ const TYPES = { Sync_VALET_TOKEN_SECRET: Symbol.for('Sync_VALET_TOKEN_SECRET'), Sync_VALET_TOKEN_TTL: Symbol.for('Sync_VALET_TOKEN_TTL'), Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING: Symbol.for('Sync_IS_CONFIGURED_FOR_HOME_SERVER_OR_SELF_HOSTING'), + Sync_STRICT_ABUSE_PROTECTION: Symbol.for('Sync_STRICT_ABUSE_PROTECTION'), + Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES: Symbol.for( + 'Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', + ), + Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD: Symbol.for('Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD'), + Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD: Symbol.for('Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD'), + Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES: Symbol.for( + 'Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES', + ), // use cases Sync_SyncItems: Symbol.for('Sync_SyncItems'), Sync_CheckIntegrity: Symbol.for('Sync_CheckIntegrity'), @@ -85,6 +94,7 @@ const TYPES = { Sync_TransferSharedVault: Symbol.for('Sync_TransferSharedVault'), Sync_TransferSharedVaultItems: Symbol.for('Sync_TransferSharedVaultItems'), Sync_DumpItem: Symbol.for('Sync_DumpItem'), + Sync_CheckForTrafficAbuse: Symbol.for('Sync_CheckForTrafficAbuse'), // Handlers Sync_AccountDeletionRequestedEventHandler: Symbol.for('Sync_AccountDeletionRequestedEventHandler'), Sync_AccountDeletionVerificationRequestedEventHandler: Symbol.for( diff --git a/packages/syncing-server/src/Domain/Metrics/MetricsStoreInterface.ts b/packages/syncing-server/src/Domain/Metrics/MetricsStoreInterface.ts index 6d91e288d..343a13e9c 100644 --- a/packages/syncing-server/src/Domain/Metrics/MetricsStoreInterface.ts +++ b/packages/syncing-server/src/Domain/Metrics/MetricsStoreInterface.ts @@ -1,25 +1,17 @@ +import { Uuid } from '@standardnotes/domain-core' + import { Metric } from './Metric' +import { MetricsSummary } from './MetricsSummary' export interface MetricsStoreInterface { - storeUserBasedMetric(metric: Metric, value: number, userUuid: string): Promise - getUserBasedStatistics( - name: string, - timestamp: number, - ): Promise<{ - sum: number - max: number - min: number - sampleCount: number - }> storeMetric(metric: Metric): Promise - getStatistics( - name: string, - from: number, - to: number, - ): Promise<{ - sum: number - max: number - min: number - sampleCount: number - }> + storeUserBasedMetric(metric: Metric, value: number, userUuid: Uuid): Promise + getUserBasedMetricsSummaryWithinTimeRange(dto: { + metricName: string + userUuid: Uuid + from: Date + to: Date + }): Promise + getUserBasedMetricsSummary(name: string, timestamp: number): Promise + getMetricsSummary(name: string, from: number, to: number): Promise } diff --git a/packages/syncing-server/src/Domain/Metrics/MetricsSummary.ts b/packages/syncing-server/src/Domain/Metrics/MetricsSummary.ts new file mode 100644 index 000000000..a8c773fa1 --- /dev/null +++ b/packages/syncing-server/src/Domain/Metrics/MetricsSummary.ts @@ -0,0 +1,6 @@ +export interface MetricsSummary { + sum: number + max: number + min: number + sampleCount: number +} diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.spec.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.spec.ts new file mode 100644 index 000000000..c5de78ced --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.spec.ts @@ -0,0 +1,97 @@ +import { TimerInterface } from '@standardnotes/time' +import { MetricsStoreInterface } from '../../../Metrics/MetricsStoreInterface' +import { CheckForTrafficAbuse } from './CheckForTrafficAbuse' +import { MetricsSummary } from '../../../Metrics/MetricsSummary' +import { Metric } from '../../../Metrics/Metric' + +describe('CheckForTrafficAbuse', () => { + let metricsStore: MetricsStoreInterface + let timer: TimerInterface + let timeframeLengthInMinutes: number + let threshold: number + + const createUseCase = () => new CheckForTrafficAbuse(metricsStore, timer) + + beforeEach(() => { + const metricsSummary: MetricsSummary = { + sum: 101, + max: 0, + min: 0, + sampleCount: 0, + } + + metricsStore = {} as jest.Mocked + metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary) + + timer = {} as TimerInterface + timer.getTimestampInMicroseconds = jest.fn().mockReturnValue(0) + timer.getUTCDateNMinutesAgo = jest.fn().mockReturnValue(new Date(0)) + timer.getUTCDate = jest.fn().mockReturnValue(new Date(10)) + + timeframeLengthInMinutes = 5 + + threshold = 100 + }) + + it('returns a failure result if the user uuid is invalid', async () => { + const result = await createUseCase().execute({ + userUuid: 'invalid', + metricToCheck: Metric.NAMES.ItemOperation, + timeframeLengthInMinutes, + threshold, + }) + + expect(result.isFailed()).toBeTruthy() + }) + + it('return a failure result if the metric name is invalid', async () => { + const result = await createUseCase().execute({ + userUuid: '00000000-0000-0000-0000-000000000000', + metricToCheck: 'invalid', + timeframeLengthInMinutes, + threshold, + }) + + expect(result.isFailed()).toBeTruthy() + }) + + it('returns a failure result if the metric summary is above the threshold', async () => { + const metricsSummary: MetricsSummary = { + sum: 101, + max: 0, + min: 0, + sampleCount: 0, + } + + metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary) + + const result = await createUseCase().execute({ + userUuid: '00000000-0000-0000-0000-000000000000', + metricToCheck: Metric.NAMES.ItemOperation, + timeframeLengthInMinutes, + threshold, + }) + + expect(result.isFailed()).toBeTruthy() + }) + + it('returns a success result if the metric summary is below the threshold', async () => { + const metricsSummary: MetricsSummary = { + sum: 99, + max: 0, + min: 0, + sampleCount: 0, + } + + metricsStore.getUserBasedMetricsSummaryWithinTimeRange = jest.fn().mockReturnValue(metricsSummary) + + const result = await createUseCase().execute({ + userUuid: '00000000-0000-0000-0000-000000000000', + metricToCheck: Metric.NAMES.ItemOperation, + timeframeLengthInMinutes, + threshold, + }) + + expect(result.isFailed()).toBeFalsy() + }) +}) diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.ts new file mode 100644 index 000000000..5c1e6d189 --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse.ts @@ -0,0 +1,44 @@ +import { Result, UseCaseInterface, Uuid } from '@standardnotes/domain-core' +import { TimerInterface } from '@standardnotes/time' + +import { CheckForTrafficAbuseDTO } from './CheckForTrafficAbuseDTO' +import { MetricsStoreInterface } from '../../../Metrics/MetricsStoreInterface' +import { Metric } from '../../../Metrics/Metric' +import { MetricsSummary } from '../../../Metrics/MetricsSummary' + +export class CheckForTrafficAbuse implements UseCaseInterface { + constructor( + private metricsStore: MetricsStoreInterface, + private timer: TimerInterface, + ) {} + + async execute(dto: CheckForTrafficAbuseDTO): Promise> { + const userUuidOrError = Uuid.create(dto.userUuid) + if (userUuidOrError.isFailed()) { + return Result.fail(userUuidOrError.getError()) + } + const userUuid = userUuidOrError.getValue() + + const metricToCheckOrError = Metric.create({ + name: dto.metricToCheck, + timestamp: this.timer.getTimestampInMicroseconds(), + }) + if (metricToCheckOrError.isFailed()) { + return Result.fail(metricToCheckOrError.getError()) + } + const metricToCheck = metricToCheckOrError.getValue() + + const metricsSummary = await this.metricsStore.getUserBasedMetricsSummaryWithinTimeRange({ + metricName: metricToCheck.props.name, + userUuid, + from: this.timer.getUTCDateNMinutesAgo(dto.timeframeLengthInMinutes), + to: this.timer.getUTCDate(), + }) + + if (metricsSummary.sum > dto.threshold) { + return Result.fail(`Traffic abuse detected for metric: ${metricToCheck.props.name}`) + } + + return Result.ok(metricsSummary) + } +} diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuseDTO.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuseDTO.ts new file mode 100644 index 000000000..6d0406c6d --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuseDTO.ts @@ -0,0 +1,6 @@ +export interface CheckForTrafficAbuseDTO { + userUuid: string + metricToCheck: string + timeframeLengthInMinutes: number + threshold: number +} diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.ts index a6553deb1..a47bd05b9 100644 --- a/packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.ts +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/SaveNewItem/SaveNewItem.ts @@ -145,7 +145,7 @@ export class SaveNewItem implements UseCaseInterface { timestamp: this.timer.getTimestampInMicroseconds(), }).getValue(), newItem.props.contentSize, - userUuid.value, + userUuid, ) await this.metricsStore.storeMetric( diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.ts index a2cfa535f..9b0b430bd 100644 --- a/packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.ts +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/UpdateExistingItem/UpdateExistingItem.ts @@ -186,7 +186,7 @@ export class UpdateExistingItem implements UseCaseInterface { timestamp: this.timer.getTimestampInMicroseconds(), }).getValue(), dto.existingItem.props.contentSize, - userUuid.value, + userUuid, ) /* istanbul ignore next */ diff --git a/packages/syncing-server/src/Infra/Dummy/DummyMetricStore.ts b/packages/syncing-server/src/Infra/Dummy/DummyMetricStore.ts index 3c8ee31ed..75472219b 100644 --- a/packages/syncing-server/src/Infra/Dummy/DummyMetricStore.ts +++ b/packages/syncing-server/src/Infra/Dummy/DummyMetricStore.ts @@ -1,11 +1,16 @@ +import { Uuid } from '@standardnotes/domain-core' + import { MetricsStoreInterface } from '../../Domain/Metrics/MetricsStoreInterface' import { Metric } from '../../Domain/Metrics/Metric' +import { MetricsSummary } from '../../Domain/Metrics/MetricsSummary' export class DummyMetricStore implements MetricsStoreInterface { - async getUserBasedStatistics( - _name: string, - _timestamp: number, - ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> { + async getUserBasedMetricsSummaryWithinTimeRange(_dto: { + metricName: string + userUuid: Uuid + from: Date + to: Date + }): Promise { return { sum: 0, max: 0, @@ -14,7 +19,16 @@ export class DummyMetricStore implements MetricsStoreInterface { } } - async storeUserBasedMetric(_metric: Metric, _value: number, _userUuid: string): Promise { + async getUserBasedMetricsSummary(_name: string, _timestamp: number): Promise { + return { + sum: 0, + max: 0, + min: 0, + sampleCount: 0, + } + } + + async storeUserBasedMetric(_metric: Metric, _value: number, _userUuid: Uuid): Promise { // do nothing } @@ -22,11 +36,7 @@ export class DummyMetricStore implements MetricsStoreInterface { // do nothing } - async getStatistics( - _name: string, - _from: number, - _to: number, - ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> { + async getMetricsSummary(_name: string, _from: number, _to: number): Promise { return { sum: 0, max: 0, diff --git a/packages/syncing-server/src/Infra/InversifyExpressUtils/AnnotatedItemsController.ts b/packages/syncing-server/src/Infra/InversifyExpressUtils/AnnotatedItemsController.ts index ee86356d3..d981bd91a 100644 --- a/packages/syncing-server/src/Infra/InversifyExpressUtils/AnnotatedItemsController.ts +++ b/packages/syncing-server/src/Infra/InversifyExpressUtils/AnnotatedItemsController.ts @@ -11,18 +11,42 @@ import { SyncItems } from '../../Domain/UseCase/Syncing/SyncItems/SyncItems' import { BaseItemsController } from './Base/BaseItemsController' import { MapperInterface } from '@standardnotes/domain-core' import { ItemHttpRepresentation } from '../../Mapping/Http/ItemHttpRepresentation' +import { CheckForTrafficAbuse } from '../../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse' +import { Logger } from 'winston' @controller('/items', TYPES.Sync_AuthMiddleware) export class AnnotatedItemsController extends BaseItemsController { constructor( + @inject(TYPES.Sync_CheckForTrafficAbuse) override checkForTrafficAbuse: CheckForTrafficAbuse, @inject(TYPES.Sync_SyncItems) override syncItems: SyncItems, @inject(TYPES.Sync_CheckIntegrity) override checkIntegrity: CheckIntegrity, @inject(TYPES.Sync_GetItem) override getItem: GetItem, @inject(TYPES.Sync_ItemHttpMapper) override itemHttpMapper: MapperInterface, @inject(TYPES.Sync_SyncResponseFactoryResolver) override syncResponseFactoryResolver: SyncResponseFactoryResolverInterface, + @inject(TYPES.Sync_Logger) override logger: Logger, + @inject(TYPES.Sync_STRICT_ABUSE_PROTECTION) override strictAbuseProtection: boolean, + @inject(TYPES.Sync_ITEM_OPERATIONS_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES) + override itemOperationsAbuseTimeframeLengthInMinutes: number, + @inject(TYPES.Sync_ITEM_OPERATIONS_ABUSE_THRESHOLD) override itemOperationsAbuseThreshold: number, + @inject(TYPES.Sync_PAYLOAD_SIZE_ABUSE_THRESHOLD) override payloadSizeAbuseThreshold: number, + @inject(TYPES.Sync_PAYLOAD_SIZE_ABUSE_TIMEFRAME_LENGTH_IN_MINUTES) + override payloadSizeAbuseTimeframeLengthInMinutes: number, ) { - super(syncItems, checkIntegrity, getItem, itemHttpMapper, syncResponseFactoryResolver) + super( + checkForTrafficAbuse, + syncItems, + checkIntegrity, + getItem, + itemHttpMapper, + syncResponseFactoryResolver, + logger, + strictAbuseProtection, + itemOperationsAbuseTimeframeLengthInMinutes, + itemOperationsAbuseThreshold, + payloadSizeAbuseThreshold, + payloadSizeAbuseTimeframeLengthInMinutes, + ) } @httpPost('/sync') diff --git a/packages/syncing-server/src/Infra/InversifyExpressUtils/Base/BaseItemsController.ts b/packages/syncing-server/src/Infra/InversifyExpressUtils/Base/BaseItemsController.ts index 3ea6f4147..cc5375b7c 100644 --- a/packages/syncing-server/src/Infra/InversifyExpressUtils/Base/BaseItemsController.ts +++ b/packages/syncing-server/src/Infra/InversifyExpressUtils/Base/BaseItemsController.ts @@ -11,14 +11,24 @@ import { ApiVersion } from '../../../Domain/Api/ApiVersion' import { SyncItems } from '../../../Domain/UseCase/Syncing/SyncItems/SyncItems' import { ItemHttpRepresentation } from '../../../Mapping/Http/ItemHttpRepresentation' import { ItemHash } from '../../../Domain/Item/ItemHash' +import { CheckForTrafficAbuse } from '../../../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse' +import { Metric } from '../../../Domain/Metrics/Metric' +import { Logger } from 'winston' export class BaseItemsController extends BaseHttpController { constructor( + protected checkForTrafficAbuse: CheckForTrafficAbuse, protected syncItems: SyncItems, protected checkIntegrity: CheckIntegrity, protected getItem: GetItem, protected itemHttpMapper: MapperInterface, protected syncResponseFactoryResolver: SyncResponseFactoryResolverInterface, + protected logger: Logger, + protected strictAbuseProtection: boolean, + protected itemOperationsAbuseTimeframeLengthInMinutes: number, + protected itemOperationsAbuseThreshold: number, + protected payloadSizeAbuseThreshold: number, + protected payloadSizeAbuseTimeframeLengthInMinutes: number, private controllerContainer?: ControllerContainerInterface, ) { super() @@ -31,6 +41,37 @@ export class BaseItemsController extends BaseHttpController { } async sync(request: Request, response: Response): Promise { + const checkForItemOperationsAbuseResult = await this.checkForTrafficAbuse.execute({ + metricToCheck: Metric.NAMES.ItemOperation, + userUuid: response.locals.user.uuid, + threshold: this.itemOperationsAbuseThreshold, + timeframeLengthInMinutes: this.itemOperationsAbuseTimeframeLengthInMinutes, + }) + if (checkForItemOperationsAbuseResult.isFailed()) { + this.logger.warn(checkForItemOperationsAbuseResult.getError(), { + userId: response.locals.user.uuid, + }) + if (this.strictAbuseProtection) { + return this.json({ error: { message: checkForItemOperationsAbuseResult.getError() } }, 429) + } + } + + const checkForPayloadSizeAbuseResult = await this.checkForTrafficAbuse.execute({ + metricToCheck: Metric.NAMES.ContentSizeUtilized, + userUuid: response.locals.user.uuid, + threshold: this.payloadSizeAbuseThreshold, + timeframeLengthInMinutes: this.payloadSizeAbuseTimeframeLengthInMinutes, + }) + if (checkForPayloadSizeAbuseResult.isFailed()) { + this.logger.warn(checkForPayloadSizeAbuseResult.getError(), { + userId: response.locals.user.uuid, + }) + + if (this.strictAbuseProtection) { + return this.json({ error: { message: checkForPayloadSizeAbuseResult.getError() } }, 429) + } + } + const itemHashes: ItemHash[] = [] if ('items' in request.body) { for (const itemHashInput of request.body.items) { diff --git a/packages/syncing-server/src/Infra/Redis/RedisMetricStore.ts b/packages/syncing-server/src/Infra/Redis/RedisMetricStore.ts index b41024a25..ce861c788 100644 --- a/packages/syncing-server/src/Infra/Redis/RedisMetricStore.ts +++ b/packages/syncing-server/src/Infra/Redis/RedisMetricStore.ts @@ -3,6 +3,8 @@ import { TimerInterface } from '@standardnotes/time' import { MetricsStoreInterface } from '../../Domain/Metrics/MetricsStoreInterface' import { Metric } from '../../Domain/Metrics/Metric' +import { Uuid } from '@standardnotes/domain-core' +import { MetricsSummary } from '../../Domain/Metrics/MetricsSummary' export class RedisMetricStore implements MetricsStoreInterface { private readonly METRIC_PREFIX = 'metric' @@ -13,15 +15,61 @@ export class RedisMetricStore implements MetricsStoreInterface { private timer: TimerInterface, ) {} - async storeUserBasedMetric(metric: Metric, value: number, userUuid: string): Promise { + async getUserBasedMetricsSummaryWithinTimeRange(dto: { + metricName: string + userUuid: Uuid + from: Date + to: Date + }): Promise { + const keys = this.getKeysRepresentingMinutesBetweenFromAndTo(dto.from, dto.to) + + let sum = 0 + let max = 0 + let min = 0 + let sampleCount = 0 + + const values = await this.redisClient.mget( + keys.map((key) => `${this.METRIC_PER_USER_PREFIX}:${dto.userUuid.value}:${dto.metricName}:${key}`), + ) + + for (const value of values) { + if (!value) { + continue + } + + const valueAsNumber = Number(value) + + sum += valueAsNumber + sampleCount++ + + if (valueAsNumber > max) { + max = valueAsNumber + } + + if (valueAsNumber < min) { + min = valueAsNumber + } + } + + return { + sum, + max, + min, + sampleCount, + } + } + + async storeUserBasedMetric(metric: Metric, value: number, userUuid: Uuid): Promise { const date = this.timer.convertMicrosecondsToDate(metric.props.timestamp) const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm') - const key = `${this.METRIC_PER_USER_PREFIX}:${userUuid}:${metric.props.name}:${dateToTheMinuteString}` + const key = `${this.METRIC_PER_USER_PREFIX}:${userUuid.value}:${metric.props.name}:${dateToTheMinuteString}` const pipeline = this.redisClient.pipeline() pipeline.incrbyfloat(key, value) - pipeline.incr(`${this.METRIC_PER_USER_PREFIX}:${userUuid}:${Metric.NAMES.ItemOperation}:${dateToTheMinuteString}`) + pipeline.incr( + `${this.METRIC_PER_USER_PREFIX}:${userUuid.value}:${Metric.NAMES.ItemOperation}:${dateToTheMinuteString}`, + ) const expirationTime = 60 * 60 * 24 pipeline.expire(key, expirationTime) @@ -29,10 +77,7 @@ export class RedisMetricStore implements MetricsStoreInterface { await pipeline.exec() } - async getUserBasedStatistics( - name: string, - timestamp: number, - ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> { + async getUserBasedMetricsSummary(name: string, timestamp: number): Promise { const date = this.timer.convertMicrosecondsToDate(timestamp) const dateToTheMinuteString = this.timer.convertDateToFormattedString(date, 'YYYY-MM-DD HH:mm') @@ -74,11 +119,7 @@ export class RedisMetricStore implements MetricsStoreInterface { } } - async getStatistics( - name: string, - from: number, - to: number, - ): Promise<{ sum: number; max: number; min: number; sampleCount: number }> { + async getMetricsSummary(name: string, from: number, to: number): Promise { const keysRepresentingSecondsBetweenFromAndTo = this.getKeysRepresentingSecondsBetweenFromAndTo(from, to) let sum = 0 @@ -132,6 +173,22 @@ export class RedisMetricStore implements MetricsStoreInterface { await pipeline.exec() } + private getKeysRepresentingMinutesBetweenFromAndTo(from: Date, to: Date): string[] { + const keys: string[] = [] + + let currentMinute = from + + while (currentMinute <= to) { + const dateToTheMinuteString = this.timer.convertDateToFormattedString(currentMinute, 'YYYY-MM-DD HH:mm') + + keys.push(dateToTheMinuteString) + + currentMinute = new Date(currentMinute.getTime() + 60 * 1000) + } + + return keys + } + private getKeysRepresentingSecondsBetweenFromAndTo(from: number, to: number): string[] { const keys: string[] = []