feat(syncing-server): add procedure to recalculate content sizes (#1027)

packages/syncing-server/bin/content_size.ts

@@ -0,0 +1,50 @@
+import 'reflect-metadata'
+
+import { Logger } from 'winston'
+
+import { ContainerConfigLoader } from '../src/Bootstrap/Container'
+import TYPES from '../src/Bootstrap/Types'
+import { Env } from '../src/Bootstrap/Env'
+import { FixContentSizes } from '../src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes'
+import { Result } from '@standardnotes/domain-core'
+
+const inputArgs = process.argv.slice(2)
+const userUuid = inputArgs[0]
+
+const container = new ContainerConfigLoader('worker')
+void container.load().then((container) => {
+  const env: Env = new Env()
+  env.load()
+
+  const logger: Logger = container.get(TYPES.Sync_Logger)
+
+  logger.info('Starting fixing of content sizes', {
+    userId: userUuid,
+  })
+
+  const fixContentSizes = container.get<FixContentSizes>(TYPES.Sync_FixContentSizes)
+
+  Promise.resolve(fixContentSizes.execute({ userUuid }))
+    .then((result: Result<void>) => {
+      if (result.isFailed()) {
+        logger.error(`Error while fixing content sizes: ${result.getError()}`, {
+          userId: userUuid,
+        })
+
+        process.exit(1)
+      }
+
+      logger.info('Finished fixing of content sizes', {
+        userId: userUuid,
+      })
+
+      process.exit(0)
+    })
+    .catch((error) => {
+      logger.error(`Error while fixing content sizes: ${error.message}`, {
+        userId: userUuid,
+      })
+
+      process.exit(1)
+    })
+})

packages/syncing-server/docker/entrypoint-content-size.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/content_size.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/syncing-server/docker/entrypoint.sh

@@ -16,6 +16,11 @@ case "$COMMAND" in
     exec node docker/entrypoint-statistics.js
     ;;
 
+  'content-size' )
+    EMAIL=$1 && shift 1
+    exec node docker/entrypoint-content-size.js $EMAIL
+    ;;
+
    * )
     echo "[Docker] Unknown command"
     ;;

packages/syncing-server/src/Bootstrap/Container.ts

@@ -167,6 +167,7 @@ import { MetricsStoreInterface } from '../Domain/Metrics/MetricsStoreInterface'
 import { RedisMetricStore } from '../Infra/Redis/RedisMetricStore'
 import { DummyMetricStore } from '../Infra/Dummy/DummyMetricStore'
 import { CheckForTrafficAbuse } from '../Domain/UseCase/Syncing/CheckForTrafficAbuse/CheckForTrafficAbuse'
+import { FixContentSizes } from '../Domain/UseCase/Syncing/FixContentSizes/FixContentSizes'
 
 export class ContainerConfigLoader {
   private readonly DEFAULT_CONTENT_SIZE_TRANSFER_LIMIT = 10_000_000
@@ -955,6 +956,14 @@ export class ContainerConfigLoader {
           container.get<DomainEventPublisherInterface>(TYPES.Sync_DomainEventPublisher),
         ),
       )
+    container
+      .bind<FixContentSizes>(TYPES.Sync_FixContentSizes)
+      .toConstantValue(
+        new FixContentSizes(
+          container.get<ItemRepositoryInterface>(TYPES.Sync_SQLItemRepository),
+          container.get<Logger>(TYPES.Sync_Logger),
+        ),
+      )
 
     // Services
     container

packages/syncing-server/src/Bootstrap/Types.ts

@@ -97,6 +97,7 @@ const TYPES = {
   Sync_TransferSharedVaultItems: Symbol.for('Sync_TransferSharedVaultItems'),
   Sync_DumpItem: Symbol.for('Sync_DumpItem'),
   Sync_CheckForTrafficAbuse: Symbol.for('Sync_CheckForTrafficAbuse'),
+  Sync_FixContentSizes: Symbol.for('Sync_FixContentSizes'),
   // Handlers
   Sync_AccountDeletionRequestedEventHandler: Symbol.for('Sync_AccountDeletionRequestedEventHandler'),
   Sync_AccountDeletionVerificationRequestedEventHandler: Symbol.for(

packages/syncing-server/src/Domain/Item/Item.spec.ts

@@ -249,4 +249,25 @@ describe('Item', () => {
 
     expect(entity.isIdenticalTo(otherEntity)).toBeFalsy()
   })
+
+  it('should calculate content size of the item', () => {
+    const entity = Item.create(
+      {
+        duplicateOf: null,
+        itemsKeyId: 'items-key-id',
+        content: 'content',
+        contentType: ContentType.create(ContentType.TYPES.Note).getValue(),
+        encItemKey: 'enc-item-key',
+        authHash: 'auth-hash',
+        userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
+        deleted: false,
+        updatedWithSession: null,
+        dates: Dates.create(new Date(123), new Date(123)).getValue(),
+        timestamps: Timestamps.create(123, 123).getValue(),
+      },
+      new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
+    ).getValue()
+
+    expect(entity.calculateContentSize()).toEqual(943)
+  })
 })

packages/syncing-server/src/Domain/Item/Item.ts

@@ -16,6 +16,10 @@ export class Item extends Aggregate<ItemProps> {
     return Result.ok<Item>(new Item(props, id))
   }
 
+  calculateContentSize(): number {
+    return Buffer.byteLength(JSON.stringify(this))
+  }
+
   get uuid(): Uuid {
     const uuidOrError = Uuid.create(this._id.toString())
     if (uuidOrError.isFailed()) {

packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.spec.ts

@@ -0,0 +1,92 @@
+import { Logger } from 'winston'
+import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface'
+import { FixContentSizes } from './FixContentSizes'
+import { Uuid, ContentType, Dates, Timestamps, UniqueEntityId } from '@standardnotes/domain-core'
+import { Item } from '../../../Item/Item'
+
+describe('FixContentSizes', () => {
+  let itemRepository: ItemRepositoryInterface
+  let logger: Logger
+
+  const createUseCase = () => new FixContentSizes(itemRepository, logger)
+
+  beforeEach(() => {
+    const existingItem = Item.create(
+      {
+        userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
+        updatedWithSession: null,
+        content: 'foobar',
+        contentType: ContentType.create(ContentType.TYPES.Note).getValue(),
+        encItemKey: null,
+        authHash: null,
+        itemsKeyId: null,
+        duplicateOf: null,
+        deleted: false,
+        dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(),
+        timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(),
+      },
+      new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
+    ).getValue()
+
+    itemRepository = {} as jest.Mocked<ItemRepositoryInterface>
+    itemRepository.findAll = jest.fn().mockReturnValue([existingItem])
+    itemRepository.countAll = jest.fn().mockReturnValue(1)
+    itemRepository.updateContentSize = jest.fn()
+
+    logger = {} as jest.Mocked<Logger>
+    logger.info = jest.fn()
+  })
+
+  it('should fix content sizes', async () => {
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({
+      userUuid: '00000000-0000-0000-0000-000000000000',
+    })
+
+    expect(result.isFailed()).toBeFalsy()
+    expect(itemRepository.updateContentSize).toHaveBeenCalledTimes(1)
+    expect(itemRepository.updateContentSize).toHaveBeenCalledWith('00000000-0000-0000-0000-000000000000', 947)
+  })
+
+  it('should return an error if user uuid is invalid', async () => {
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({
+      userUuid: 'invalid',
+    })
+
+    expect(result.isFailed()).toBeTruthy()
+    expect(result.getError()).toEqual('Given value is not a valid uuid: invalid')
+  })
+
+  it('should do nothing if the content size is correct', async () => {
+    const existingItem = Item.create(
+      {
+        userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
+        updatedWithSession: null,
+        content: 'foobar',
+        contentType: ContentType.create(ContentType.TYPES.Note).getValue(),
+        contentSize: 947,
+        encItemKey: null,
+        authHash: null,
+        itemsKeyId: null,
+        duplicateOf: null,
+        deleted: false,
+        dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(),
+        timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(),
+      },
+      new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
+    ).getValue()
+    itemRepository.findAll = jest.fn().mockReturnValue([existingItem])
+
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({
+      userUuid: '00000000-0000-0000-0000-000000000000',
+    })
+
+    expect(result.isFailed()).toBeFalsy()
+    expect(itemRepository.updateContentSize).toHaveBeenCalledTimes(0)
+  })
+})

packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizes.ts

@@ -0,0 +1,65 @@
+import { Result, UseCaseInterface, Uuid } from '@standardnotes/domain-core'
+import { Logger } from 'winston'
+
+import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface'
+
+import { FixContentSizesDTO } from './FixContentSizesDTO'
+
+export class FixContentSizes implements UseCaseInterface<void> {
+  constructor(
+    private itemRepository: ItemRepositoryInterface,
+    private logger: Logger,
+  ) {}
+
+  async execute(dto: FixContentSizesDTO): Promise<Result<void>> {
+    const userUuidOrError = Uuid.create(dto.userUuid)
+    if (userUuidOrError.isFailed()) {
+      return Result.fail(userUuidOrError.getError())
+    }
+    const userUuid = userUuidOrError.getValue()
+
+    const count = await this.itemRepository.countAll({
+      userUuid: userUuid.value,
+    })
+
+    this.logger.info(`Fixing content sizes for ${count} items`, {
+      userId: userUuid.value,
+      codeTag: 'FixContentSizes',
+    })
+
+    const pageSize = 100
+    let page = 1
+    const totalPages = Math.ceil(count / pageSize)
+
+    for (page; page <= totalPages; page++) {
+      const items = await this.itemRepository.findAll({
+        userUuid: userUuid.value,
+        sortOrder: 'ASC',
+        sortBy: 'created_at_timestamp',
+        offset: (page - 1) * pageSize,
+        limit: pageSize,
+      })
+
+      for (const item of items) {
+        if (item.props.contentSize != item.calculateContentSize()) {
+          this.logger.info(`Fixing content size for item ${item.id}`, {
+            userId: userUuid.value,
+            codeTag: 'FixContentSizes',
+            itemUuid: item.uuid.value,
+            oldContentSize: item.props.contentSize,
+            newContentSize: item.calculateContentSize(),
+          })
+
+          await this.itemRepository.updateContentSize(item.id.toString(), item.calculateContentSize())
+        }
+      }
+    }
+
+    this.logger.info(`Finished fixing content sizes for ${count} items`, {
+      userId: userUuid.value,
+      codeTag: 'FixContentSizes',
+    })
+
+    return Result.ok()
+  }
+}

packages/syncing-server/src/Domain/UseCase/Syncing/FixContentSizes/FixContentSizesDTO.ts

@@ -0,0 +1,3 @@
+export interface FixContentSizesDTO {
+  userUuid: string
+}