From 70bbf11db504ed6305b9e9922bc38bd4b632b273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Wed, 17 Jan 2024 11:27:26 +0100 Subject: [PATCH] feat(syncing-server): add procedure to recalculate content sizes (#1027) --- packages/syncing-server/bin/content_size.ts | 50 ++++++++++ .../docker/entrypoint-content-size.js | 11 +++ packages/syncing-server/docker/entrypoint.sh | 5 + .../syncing-server/src/Bootstrap/Container.ts | 9 ++ .../syncing-server/src/Bootstrap/Types.ts | 1 + .../src/Domain/Item/Item.spec.ts | 21 +++++ .../syncing-server/src/Domain/Item/Item.ts | 4 + .../FixContentSizes/FixContentSizes.spec.ts | 92 +++++++++++++++++++ .../FixContentSizes/FixContentSizes.ts | 65 +++++++++++++ .../FixContentSizes/FixContentSizesDTO.ts | 3 + 10 files changed, 261 insertions(+) create mode 100644 packages/syncing-server/bin/content_size.ts create mode 100644 packages/syncing-server/docker/entrypoint-content-size.js create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.spec.ts create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.ts create mode 100644 packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizesDTO.ts diff --git a/packages/syncing-server/bin/content_size.ts b/packages/syncing-server/bin/content_size.ts new file mode 100644 index 000000000..699507ba1 --- /dev/null +++ b/packages/syncing-server/bin/content_size.ts @@ -0,0 +1,50 @@ +import 'reflect-metadata' + +import { Logger } from 'winston' + +import { ContainerConfigLoader } from '../src/Bootstrap/Container' +import TYPES from '../src/Bootstrap/Types' +import { Env } from '../src/Bootstrap/Env' +import { FixContentSizes } from '../src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes' +import { Result } from '@standardnotes/domain-core' + +const inputArgs = process.argv.slice(2) +const userUuid = inputArgs[0] + +const container = new ContainerConfigLoader('worker') +void container.load().then((container) => { + const env: Env = new Env() + env.load() + + const logger: Logger = container.get(TYPES.Sync_Logger) + + logger.info('Starting fixing of content sizes', { + userId: userUuid, + }) + + const fixContentSizes = container.get(TYPES.Sync_FixContentSizes) + + Promise.resolve(fixContentSizes.execute({ userUuid })) + .then((result: Result) => { + if (result.isFailed()) { + logger.error(`Error while fixing content sizes: ${result.getError()}`, { + userId: userUuid, + }) + + process.exit(1) + } + + logger.info('Finished fixing of content sizes', { + userId: userUuid, + }) + + process.exit(0) + }) + .catch((error) => { + logger.error(`Error while fixing content sizes: ${error.message}`, { + userId: userUuid, + }) + + process.exit(1) + }) +}) diff --git a/packages/syncing-server/docker/entrypoint-content-size.js b/packages/syncing-server/docker/entrypoint-content-size.js new file mode 100644 index 000000000..170d5ab19 --- /dev/null +++ b/packages/syncing-server/docker/entrypoint-content-size.js @@ -0,0 +1,11 @@ +'use strict' + +const path = require('path') + +const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup() + +const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/content_size.js'))) + +Object.defineProperty(exports, '__esModule', { value: true }) + +exports.default = index diff --git a/packages/syncing-server/docker/entrypoint.sh b/packages/syncing-server/docker/entrypoint.sh index e70472133..380f7546b 100755 --- a/packages/syncing-server/docker/entrypoint.sh +++ b/packages/syncing-server/docker/entrypoint.sh @@ -16,6 +16,11 @@ case "$COMMAND" in exec node docker/entrypoint-statistics.js ;; + 'content-size' ) + EMAIL=$1 && shift 1 + exec node docker/entrypoint-content-size.js $EMAIL + ;; + * ) echo "[Docker] Unknown command" ;; diff --git a/packages/syncing-server/src/Bootstrap/Container.ts b/packages/syncing-server/src/Bootstrap/Container.ts index cf8fc11d6..8cd7e541d 100644 --- a/packages/syncing-server/src/Bootstrap/Container.ts +++ b/packages/syncing-server/src/Bootstrap/Container.ts @@ -167,6 +167,7 @@ import { MetricsStoreInterface } from '../Domain/Metrics/MetricsStoreInterface' import { RedisMetricStore } from '../Infra/Redis/RedisMetricStore' import { DummyMetricStore } from '../Infra/Dummy/DummyMetricStore' import { CheckForTrafficAbuse } from '../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse' +import { FixContentSizes } from '../Domain/UseCase/Syncing/FixContentSizes/FixContentSizes' export class ContainerConfigLoader { private readonly DEFAULT_CONTENT_SIZE_TRANSFER_LIMIT = 10_000_000 @@ -955,6 +956,14 @@ export class ContainerConfigLoader { container.get(TYPES.Sync_DomainEventPublisher), ), ) + container + .bind(TYPES.Sync_FixContentSizes) + .toConstantValue( + new FixContentSizes( + container.get(TYPES.Sync_SQLItemRepository), + container.get(TYPES.Sync_Logger), + ), + ) // Services container diff --git a/packages/syncing-server/src/Bootstrap/Types.ts b/packages/syncing-server/src/Bootstrap/Types.ts index 36b5a933e..ca803b928 100644 --- a/packages/syncing-server/src/Bootstrap/Types.ts +++ b/packages/syncing-server/src/Bootstrap/Types.ts @@ -97,6 +97,7 @@ const TYPES = { Sync_TransferSharedVaultItems: Symbol.for('Sync_TransferSharedVaultItems'), Sync_DumpItem: Symbol.for('Sync_DumpItem'), Sync_CheckForTrafficAbuse: Symbol.for('Sync_CheckForTrafficAbuse'), + Sync_FixContentSizes: Symbol.for('Sync_FixContentSizes'), // Handlers Sync_AccountDeletionRequestedEventHandler: Symbol.for('Sync_AccountDeletionRequestedEventHandler'), Sync_AccountDeletionVerificationRequestedEventHandler: Symbol.for( diff --git a/packages/syncing-server/src/Domain/Item/Item.spec.ts b/packages/syncing-server/src/Domain/Item/Item.spec.ts index cc4ec2674..849969955 100644 --- a/packages/syncing-server/src/Domain/Item/Item.spec.ts +++ b/packages/syncing-server/src/Domain/Item/Item.spec.ts @@ -249,4 +249,25 @@ describe('Item', () => { expect(entity.isIdenticalTo(otherEntity)).toBeFalsy() }) + + it('should calculate content size of the item', () => { + const entity = Item.create( + { + duplicateOf: null, + itemsKeyId: 'items-key-id', + content: 'content', + contentType: ContentType.create(ContentType.TYPES.Note).getValue(), + encItemKey: 'enc-item-key', + authHash: 'auth-hash', + userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(), + deleted: false, + updatedWithSession: null, + dates: Dates.create(new Date(123), new Date(123)).getValue(), + timestamps: Timestamps.create(123, 123).getValue(), + }, + new UniqueEntityId('00000000-0000-0000-0000-000000000000'), + ).getValue() + + expect(entity.calculateContentSize()).toEqual(943) + }) }) diff --git a/packages/syncing-server/src/Domain/Item/Item.ts b/packages/syncing-server/src/Domain/Item/Item.ts index 542e651ae..79d732df7 100644 --- a/packages/syncing-server/src/Domain/Item/Item.ts +++ b/packages/syncing-server/src/Domain/Item/Item.ts @@ -16,6 +16,10 @@ export class Item extends Aggregate { return Result.ok(new Item(props, id)) } + calculateContentSize(): number { + return Buffer.byteLength(JSON.stringify(this)) + } + get uuid(): Uuid { const uuidOrError = Uuid.create(this._id.toString()) if (uuidOrError.isFailed()) { diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.spec.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.spec.ts new file mode 100644 index 000000000..883517055 --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.spec.ts @@ -0,0 +1,92 @@ +import { Logger } from 'winston' +import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface' +import { FixContentSizes } from './FixContentSizes' +import { Uuid, ContentType, Dates, Timestamps, UniqueEntityId } from '@standardnotes/domain-core' +import { Item } from '../../../Item/Item' + +describe('FixContentSizes', () => { + let itemRepository: ItemRepositoryInterface + let logger: Logger + + const createUseCase = () => new FixContentSizes(itemRepository, logger) + + beforeEach(() => { + const existingItem = Item.create( + { + userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(), + updatedWithSession: null, + content: 'foobar', + contentType: ContentType.create(ContentType.TYPES.Note).getValue(), + encItemKey: null, + authHash: null, + itemsKeyId: null, + duplicateOf: null, + deleted: false, + dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(), + timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(), + }, + new UniqueEntityId('00000000-0000-0000-0000-000000000000'), + ).getValue() + + itemRepository = {} as jest.Mocked + itemRepository.findAll = jest.fn().mockReturnValue([existingItem]) + itemRepository.countAll = jest.fn().mockReturnValue(1) + itemRepository.updateContentSize = jest.fn() + + logger = {} as jest.Mocked + logger.info = jest.fn() + }) + + it('should fix content sizes', async () => { + const useCase = createUseCase() + + const result = await useCase.execute({ + userUuid: '00000000-0000-0000-0000-000000000000', + }) + + expect(result.isFailed()).toBeFalsy() + expect(itemRepository.updateContentSize).toHaveBeenCalledTimes(1) + expect(itemRepository.updateContentSize).toHaveBeenCalledWith('00000000-0000-0000-0000-000000000000', 947) + }) + + it('should return an error if user uuid is invalid', async () => { + const useCase = createUseCase() + + const result = await useCase.execute({ + userUuid: 'invalid', + }) + + expect(result.isFailed()).toBeTruthy() + expect(result.getError()).toEqual('Given value is not a valid uuid: invalid') + }) + + it('should do nothing if the content size is correct', async () => { + const existingItem = Item.create( + { + userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(), + updatedWithSession: null, + content: 'foobar', + contentType: ContentType.create(ContentType.TYPES.Note).getValue(), + contentSize: 947, + encItemKey: null, + authHash: null, + itemsKeyId: null, + duplicateOf: null, + deleted: false, + dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(), + timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(), + }, + new UniqueEntityId('00000000-0000-0000-0000-000000000000'), + ).getValue() + itemRepository.findAll = jest.fn().mockReturnValue([existingItem]) + + const useCase = createUseCase() + + const result = await useCase.execute({ + userUuid: '00000000-0000-0000-0000-000000000000', + }) + + expect(result.isFailed()).toBeFalsy() + expect(itemRepository.updateContentSize).toHaveBeenCalledTimes(0) + }) +}) diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.ts new file mode 100644 index 000000000..c59f8c6e2 --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.ts @@ -0,0 +1,65 @@ +import { Result, UseCaseInterface, Uuid } from '@standardnotes/domain-core' +import { Logger } from 'winston' + +import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface' + +import { FixContentSizesDTO } from './FixContentSizesDTO' + +export class FixContentSizes implements UseCaseInterface { + constructor( + private itemRepository: ItemRepositoryInterface, + private logger: Logger, + ) {} + + async execute(dto: FixContentSizesDTO): Promise> { + const userUuidOrError = Uuid.create(dto.userUuid) + if (userUuidOrError.isFailed()) { + return Result.fail(userUuidOrError.getError()) + } + const userUuid = userUuidOrError.getValue() + + const count = await this.itemRepository.countAll({ + userUuid: userUuid.value, + }) + + this.logger.info(`Fixing content sizes for ${count} items`, { + userId: userUuid.value, + codeTag: 'FixContentSizes', + }) + + const pageSize = 100 + let page = 1 + const totalPages = Math.ceil(count / pageSize) + + for (page; page <= totalPages; page++) { + const items = await this.itemRepository.findAll({ + userUuid: userUuid.value, + sortOrder: 'ASC', + sortBy: 'created_at_timestamp', + offset: (page - 1) * pageSize, + limit: pageSize, + }) + + for (const item of items) { + if (item.props.contentSize != item.calculateContentSize()) { + this.logger.info(`Fixing content size for item ${item.id}`, { + userId: userUuid.value, + codeTag: 'FixContentSizes', + itemUuid: item.uuid.value, + oldContentSize: item.props.contentSize, + newContentSize: item.calculateContentSize(), + }) + + await this.itemRepository.updateContentSize(item.id.toString(), item.calculateContentSize()) + } + } + } + + this.logger.info(`Finished fixing content sizes for ${count} items`, { + userId: userUuid.value, + codeTag: 'FixContentSizes', + }) + + return Result.ok() + } +} diff --git a/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizesDTO.ts b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizesDTO.ts new file mode 100644 index 000000000..6c36de75a --- /dev/null +++ b/packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizesDTO.ts @@ -0,0 +1,3 @@ +export interface FixContentSizesDTO { + userUuid: string +}