beenull/standardnotes-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(syncing-server): return cursor token upon transfer limit breached (#902)

Browse source 
* fix(syncing-server): return cursor token upon transfer limit breached

* fix e2e env vars
This commit is contained in:
Karol Sójko 2023-11-06 15:38:28 +01:00 committed by GitHub
parent 2742075edc
commit 71689c1497
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 50 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/ci.env vendored
View file

@ -26,3 +26,5 @@ MYSQL_ROOT_PASSWORD=changeme123
AUTH_JWT_SECRET=f95259c5e441f5a4646d76422cfb3df4c4488842901aa50b6c51b8be2e0040e9
AUTH_SERVER_ENCRYPTION_SERVER_KEY=1087415dfde3093797f9a7ca93a49e7d7aa1861735eb0d32aae9c303b8c3d060
VALET_TOKEN_SECRET=4b886819ebe1e908077c6cae96311b48a8416bd60cc91c03060e15bdf6b30d1f
SYNCING_SERVER_CONTENT_SIZE_TRANSFER_LIMIT=1000000

1
.github/workflows/e2e-home-server.yml vendored
View file

@ -70,6 +70,7 @@ jobs:
echo "ACCESS_TOKEN_AGE=4" >> packages/home-server/.env
echo "REFRESH_TOKEN_AGE=10" >> packages/home-server/.env
echo "REVISIONS_FREQUENCY=2" >> packages/home-server/.env
echo "CONTENT_SIZE_TRANSFER_LIMIT=1000000" >> packages/home-server/.env
echo "DB_HOST=localhost" >> packages/home-server/.env
echo "DB_PORT=3306" >> packages/home-server/.env
echo "DB_DATABASE=standardnotes" >> packages/home-server/.env

36
packages/syncing-server/src/Domain/Item/ItemTransferCalculator.spec.ts
View file

@ -25,11 +25,14 @@ describe('ItemTransferCalculator', () => {
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50)
expect(result).toEqual([
'00000000-0000-0000-0000-000000000000',
'00000000-0000-0000-0000-000000000001',
'00000000-0000-0000-0000-000000000002',
])
expect(result).toEqual({
uuids: [
'00000000-0000-0000-0000-000000000000',
'00000000-0000-0000-0000-000000000001',
'00000000-0000-0000-0000-000000000002',
],
transferLimitBreachedBeforeEndOfItems: false,
})
})
it('should compute uuids to fetch based on transfer limit - exact limit fit', async () => {
@ -41,7 +44,10 @@ describe('ItemTransferCalculator', () => {
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40)
expect(result).toEqual(['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'])
expect(result).toEqual({
uuids: ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],
transferLimitBreachedBeforeEndOfItems: true,
})
})
it('should compute uuids to fetch based on transfer limit - content size not defined on an item', async () => {
@ -53,11 +59,14 @@ describe('ItemTransferCalculator', () => {
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 50)
expect(result).toEqual([
'00000000-0000-0000-0000-000000000000',
'00000000-0000-0000-0000-000000000001',
'00000000-0000-0000-0000-000000000002',
])
expect(result).toEqual({
uuids: [
'00000000-0000-0000-0000-000000000000',
'00000000-0000-0000-0000-000000000001',
'00000000-0000-0000-0000-000000000002',
],
transferLimitBreachedBeforeEndOfItems: false,
})
})
it('should compute uuids to fetch based on transfer limit - first item over the limit', async () => {
@ -69,7 +78,10 @@ describe('ItemTransferCalculator', () => {
const result = await createCalculator().computeItemUuidsToFetch(itemContentSizeDescriptors, 40)
expect(result).toEqual(['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'])
expect(result).toEqual({
uuids: ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'],
transferLimitBreachedBeforeEndOfItems: true,
})
})
})

11
packages/syncing-server/src/Domain/Item/ItemTransferCalculator.ts
View file

@ -9,16 +9,17 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
async computeItemUuidsToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
): Promise<Array<string>> {
): Promise<{ uuids: Array<string>; transferLimitBreachedBeforeEndOfItems: boolean }> {
const itemUuidsToFetch = []
let totalContentSizeInBytes = 0
let transferLimitBreached = false
for (const itemContentSize of itemContentSizeDescriptors) {
const contentSize = itemContentSize.props.contentSize ?? 0
itemUuidsToFetch.push(itemContentSize.props.uuid.value)
totalContentSizeInBytes += contentSize
const transferLimitBreached = this.isTransferLimitBreached({
transferLimitBreached = this.isTransferLimitBreached({
totalContentSizeInBytes,
bytesTransferLimit,
itemUuidsToFetch,
@ -30,7 +31,11 @@ export class ItemTransferCalculator implements ItemTransferCalculatorInterface {
}
}
return itemUuidsToFetch
return {
uuids: itemUuidsToFetch,
transferLimitBreachedBeforeEndOfItems:
transferLimitBreached && itemUuidsToFetch.length < itemContentSizeDescriptors.length,
}
}
async computeItemUuidBundlesToFetch(

2
packages/syncing-server/src/Domain/Item/ItemTransferCalculatorInterface.ts
View file

@ -4,7 +4,7 @@ export interface ItemTransferCalculatorInterface {
computeItemUuidsToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,
): Promise<Array<string>>
): Promise<{ uuids: Array<string>; transferLimitBreachedBeforeEndOfItems: boolean }>
computeItemUuidBundlesToFetch(
itemContentSizeDescriptors: ItemContentSizeDescriptor[],
bytesTransferLimit: number,

4
packages/syncing-server/src/Domain/UseCase/Syncing/GetItems/GetItems.spec.ts
View file

@ -49,7 +49,9 @@ describe('GetItems', () => {
.mockResolvedValue([ItemContentSizeDescriptor.create('00000000-0000-0000-0000-000000000000', 20).getValue()])
itemTransferCalculator = {} as jest.Mocked<ItemTransferCalculatorInterface>
itemTransferCalculator.computeItemUuidsToFetch = jest.fn().mockResolvedValue(['item-uuid'])
itemTransferCalculator.computeItemUuidsToFetch = jest
.fn()
.mockResolvedValue({ uuids: ['item-uuid'], transferLimitBreachedBeforeEndOfItems: false })
timer = {} as jest.Mocked<TimerInterface>
timer.getTimestampInMicroseconds = jest.fn().mockReturnValue(123)

16
packages/syncing-server/src/Domain/UseCase/Syncing/GetItems/GetItems.ts
View file

@ -60,22 +60,22 @@ export class GetItems implements UseCaseInterface<GetItemsResult> {
}
const itemContentSizeDescriptors = await this.itemRepository.findContentSizeForComputingTransferLimit(itemQuery)
const itemUuidsToFetch = await this.itemTransferCalculator.computeItemUuidsToFetch(
const { uuids, transferLimitBreachedBeforeEndOfItems } = await this.itemTransferCalculator.computeItemUuidsToFetch(
itemContentSizeDescriptors,
this.contentSizeTransferLimit,
)
let items: Array<Item> = []
if (itemUuidsToFetch.length > 0) {
if (uuids.length > 0) {
items = await this.itemRepository.findAll({
uuids: itemUuidsToFetch,
uuids,
sortBy: 'updated_at_timestamp',
sortOrder: 'ASC',
})
}
const totalItemsCount = await this.itemRepository.countAll(itemQuery)
let cursorToken = undefined
if (totalItemsCount > upperBoundLimit) {
const thereAreStillMoreItemsToFetch = await this.stillMoreItemsToFetch(itemQuery, upperBoundLimit)
if (transferLimitBreachedBeforeEndOfItems || thereAreStillMoreItemsToFetch) {
const lastSyncTime = items[items.length - 1].props.timestamps.updatedAt / Time.MicrosecondsInASecond
cursorToken = Buffer.from(`${this.SYNC_TOKEN_VERSION}:${lastSyncTime}`, 'utf-8').toString('base64')
}
@ -87,6 +87,12 @@ export class GetItems implements UseCaseInterface<GetItemsResult> {
})
}
private async stillMoreItemsToFetch(itemQuery: ItemQuery, upperBoundLimit: number): Promise<boolean> {
const totalItemsCount = await this.itemRepository.countAll(itemQuery)
return totalItemsCount > upperBoundLimit
}
private getLastSyncTime(dto: GetItemsDTO): Result<number | null> {
let token = dto.syncToken
if (dto.cursorToken !== undefined && dto.cursorToken !== null) {