fix(revisions): handle transitions with already existing data in secondary

This commit is contained in:
Karol Sójko 2023-09-12 17:42:07 +02:00
parent 74b4312928
commit aa835268ea
No known key found for this signature in database
GPG key ID: C2F813669419D05F
2 changed files with 77 additions and 460 deletions

View file

@ -1,455 +0,0 @@
import { Logger } from 'winston'
import { RevisionRepositoryInterface } from '../../../Revision/RevisionRepositoryInterface'
import { TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser } from './TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser'
import { Revision } from '../../../Revision/Revision'
import { ContentType, Dates, UniqueEntityId, Uuid } from '@standardnotes/domain-core'
import { TimerInterface } from '@standardnotes/time'
describe('TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser', () => {
let primaryRevisionRepository: RevisionRepositoryInterface
let secondaryRevisionRepository: RevisionRepositoryInterface | null
let logger: Logger
let primaryRevision1: Revision
let primaryRevision2: Revision
let secondaryRevision1: Revision
let secondaryRevision2: Revision
let timer: TimerInterface
const createUseCase = () =>
new TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser(
primaryRevisionRepository,
secondaryRevisionRepository,
timer,
logger,
1,
)
beforeEach(() => {
primaryRevision1 = Revision.create(
{
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
).getValue()
primaryRevision2 = Revision.create(
{
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc2d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000001'),
).getValue()
secondaryRevision1 = Revision.create(
{
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
).getValue()
secondaryRevision2 = Revision.create(
{
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc2d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000001'),
).getValue()
primaryRevisionRepository = {} as jest.Mocked<RevisionRepositoryInterface>
primaryRevisionRepository.countByUserUuid = jest.fn().mockResolvedValue(2)
primaryRevisionRepository.findByUserUuid = jest
.fn()
.mockResolvedValueOnce([primaryRevision1])
.mockResolvedValueOnce([primaryRevision2])
.mockResolvedValueOnce([primaryRevision1])
.mockResolvedValueOnce([primaryRevision2])
primaryRevisionRepository.removeByUserUuid = jest.fn().mockResolvedValue(undefined)
secondaryRevisionRepository = {} as jest.Mocked<RevisionRepositoryInterface>
secondaryRevisionRepository.insert = jest.fn().mockResolvedValue(true)
secondaryRevisionRepository.removeByUserUuid = jest.fn().mockResolvedValue(undefined)
secondaryRevisionRepository.countByUserUuid = jest.fn().mockResolvedValueOnce(0).mockResolvedValueOnce(2)
secondaryRevisionRepository.findOneByUuid = jest
.fn()
.mockResolvedValueOnce(secondaryRevision1)
.mockResolvedValueOnce(secondaryRevision2)
logger = {} as jest.Mocked<Logger>
logger.error = jest.fn()
logger.info = jest.fn()
logger.debug = jest.fn()
timer = {} as jest.Mocked<TimerInterface>
timer.sleep = jest.fn()
timer.getTimestampInMicroseconds = jest.fn().mockReturnValue(123)
timer.convertMicrosecondsToTimeStructure = jest.fn().mockReturnValue({
days: 0,
hours: 0,
minutes: 0,
seconds: 0,
milliseconds: 0,
})
})
describe('successfull transition', () => {
it('should transition Revisions from primary to secondary database', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeFalsy()
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledTimes(3)
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledWith(
Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
)
expect(primaryRevisionRepository.findByUserUuid).toHaveBeenCalledTimes(4)
expect(primaryRevisionRepository.findByUserUuid).toHaveBeenNthCalledWith(1, {
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
limit: 1,
offset: 0,
})
expect(primaryRevisionRepository.findByUserUuid).toHaveBeenNthCalledWith(2, {
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
limit: 1,
offset: 1,
})
expect(primaryRevisionRepository.findByUserUuid).toHaveBeenNthCalledWith(3, {
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
limit: 1,
offset: 0,
})
expect(primaryRevisionRepository.findByUserUuid).toHaveBeenNthCalledWith(4, {
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
limit: 1,
offset: 1,
})
expect((secondaryRevisionRepository as RevisionRepositoryInterface).insert).toHaveBeenCalledTimes(2)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).insert).toHaveBeenCalledWith(primaryRevision1)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).insert).toHaveBeenCalledWith(primaryRevision2)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).not.toHaveBeenCalled()
expect(primaryRevisionRepository.removeByUserUuid).toHaveBeenCalledTimes(1)
})
it('should log an error if deleting Revisions from primary database fails', async () => {
primaryRevisionRepository.removeByUserUuid = jest.fn().mockRejectedValue(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeFalsy()
expect(logger.error).toHaveBeenCalledTimes(1)
expect(logger.error).toHaveBeenCalledWith(
'Failed to clean up primary database revisions for user 00000000-0000-0000-0000-000000000000: Errored when deleting revisions for user 00000000-0000-0000-0000-000000000000: error',
)
})
})
describe('failed transition', () => {
it('should remove Revisions from secondary database if integrity check fails', async () => {
const secondaryRevision2WithDifferentContent = Revision.create({
...secondaryRevision2.props,
content: 'different-content',
}).getValue()
;(secondaryRevisionRepository as RevisionRepositoryInterface).findOneByUuid = jest
.fn()
.mockResolvedValueOnce(secondaryRevision1)
.mockResolvedValueOnce(secondaryRevision2WithDifferentContent)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
})
it('should remove Revisions from secondary database if migrating Revisions fails', async () => {
primaryRevisionRepository.findByUserUuid = jest
.fn()
.mockResolvedValueOnce([primaryRevision1])
.mockRejectedValueOnce(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Errored when migrating revisions for user 00000000-0000-0000-0000-000000000000: error',
)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
})
it('should return an error for a specific revision if it errors when saving to secondary database', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).insert = jest
.fn()
.mockResolvedValueOnce(true)
.mockRejectedValueOnce(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '84c0f8e8-544a-4c7e-9adf-26209303bc1d',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Errored when saving revision 00000000-0000-0000-0000-000000000001 to secondary database: error',
)
})
it('should log an error if deleting Revisions from secondary database fails upon migration failure', async () => {
primaryRevisionRepository.findByUserUuid = jest
.fn()
.mockResolvedValueOnce([primaryRevision1])
.mockRejectedValueOnce(new Error('error'))
;(secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid = jest
.fn()
.mockRejectedValue(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(logger.error).toHaveBeenCalledTimes(1)
expect(logger.error).toHaveBeenCalledWith(
'Failed to clean up secondary database revisions for user 00000000-0000-0000-0000-000000000000: Errored when deleting revisions for user 00000000-0000-0000-0000-000000000000: error',
)
})
it('should log an error if deleting Revisions from secondary database fails upon integrity check failure', async () => {
const secondaryRevision2WithDifferentContent = Revision.create({
...secondaryRevision2.props,
content: 'different-content',
}).getValue()
;(secondaryRevisionRepository as RevisionRepositoryInterface).findOneByUuid = jest
.fn()
.mockResolvedValueOnce(secondaryRevision1)
.mockResolvedValueOnce(secondaryRevision2WithDifferentContent)
;(secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid = jest
.fn()
.mockRejectedValue(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(logger.error).toHaveBeenCalledTimes(1)
expect(logger.error).toHaveBeenCalledWith(
'Failed to clean up secondary database revisions for user 00000000-0000-0000-0000-000000000000: Errored when deleting revisions for user 00000000-0000-0000-0000-000000000000: error',
)
})
it('should not perform the transition if secondary Revision repository is not set', async () => {
secondaryRevisionRepository = null
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual('Secondary revision repository is not set')
expect(primaryRevisionRepository.countByUserUuid).not.toHaveBeenCalled()
expect(primaryRevisionRepository.findByUserUuid).not.toHaveBeenCalled()
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
})
it('should not perform the transition if the user uuid is invalid', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: 'invalid-uuid',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual('Given value is not a valid uuid: invalid-uuid')
expect(primaryRevisionRepository.countByUserUuid).not.toHaveBeenCalled()
expect(primaryRevisionRepository.findByUserUuid).not.toHaveBeenCalled()
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
})
it('should fail integrity check if the Revision count is not the same in both databases', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).countByUserUuid = jest
.fn()
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(1)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Total revisions count for user 00000000-0000-0000-0000-000000000000 in primary database (2) does not match total revisions count in secondary database (1)',
)
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledTimes(3)
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledWith(
Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
)
expect((secondaryRevisionRepository as RevisionRepositoryInterface).countByUserUuid).toHaveBeenCalledTimes(2)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
})
it('should fail if one Revision is not found in the secondary database', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).findOneByUuid = jest
.fn()
.mockResolvedValueOnce(secondaryRevision1)
.mockResolvedValueOnce(null)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual('Revision 00000000-0000-0000-0000-000000000001 not found in secondary database')
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledTimes(3)
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledWith(
Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
})
it('should fail if an error is thrown during integrity check between primary and secondary database', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).countByUserUuid = jest
.fn()
.mockReturnValueOnce(0)
.mockRejectedValue(new Error('error'))
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual('Errored when checking integrity between primary and secondary database: error')
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
})
it('should fail if a revisions did not save in the secondary database', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).insert = jest.fn().mockResolvedValue(false)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(result.getError()).toEqual(
'Failed to save revision 00000000-0000-0000-0000-000000000000 to secondary database',
)
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).toHaveBeenCalledTimes(1)
})
})
it('should not migrate revisions if there are no revisions in the primary database', async () => {
primaryRevisionRepository.countByUserUuid = jest.fn().mockResolvedValue(0)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeFalsy()
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledTimes(1)
expect(primaryRevisionRepository.countByUserUuid).toHaveBeenCalledWith(
Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
)
expect(primaryRevisionRepository.findByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).insert).not.toHaveBeenCalled()
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).not.toHaveBeenCalled()
})
it('should not migrate revisions if there are already revisions in the secondary database', async () => {
;(secondaryRevisionRepository as RevisionRepositoryInterface).countByUserUuid = jest.fn().mockResolvedValueOnce(1)
const useCase = createUseCase()
const result = await useCase.execute({
userUuid: '00000000-0000-0000-0000-000000000000',
})
expect(result.isFailed()).toBeTruthy()
expect(primaryRevisionRepository.findByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).insert).not.toHaveBeenCalled()
expect(primaryRevisionRepository.removeByUserUuid).not.toHaveBeenCalled()
expect((secondaryRevisionRepository as RevisionRepositoryInterface).removeByUserUuid).not.toHaveBeenCalled()
})
})

View file

@ -1,9 +1,11 @@
/* istanbul ignore file */
import { Result, UseCaseInterface, Uuid } from '@standardnotes/domain-core'
import { TimerInterface } from '@standardnotes/time'
import { Logger } from 'winston'
import { TransitionRevisionsFromPrimaryToSecondaryDatabaseForUserDTO } from './TransitionRevisionsFromPrimaryToSecondaryDatabaseForUserDTO'
import { RevisionRepositoryInterface } from '../../../Revision/RevisionRepositoryInterface'
import { Revision } from '../../../Revision/Revision'
export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements UseCaseInterface<void> {
constructor(
@ -33,10 +35,21 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
return Result.ok()
}
let newRevisionsInSecondaryCount = 0
if (await this.hasAlreadyDataInSecondaryDatabase(userUuid)) {
return Result.fail(`Revisions for user ${userUuid.value} already exist in secondary database`)
const newRevisions = await this.getNewRevisionsCreatedInSecondaryDatabase(userUuid)
for (const existingRevision of newRevisions.alreadyExistingInPrimary) {
await (this.secondRevisionsRepository as RevisionRepositoryInterface).removeOneByUuid(
Uuid.create(existingRevision.id.toString()).getValue(),
userUuid,
)
}
newRevisionsInSecondaryCount = newRevisions.newRevisionsInSecondary.length
}
await this.allowForSecondaryDatabaseToCatchUp()
const migrationTimeStart = this.timer.getTimestampInMicroseconds()
this.logger.debug(`Transitioning revisions for user ${userUuid.value}`)
@ -55,7 +68,10 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
await this.allowForSecondaryDatabaseToCatchUp()
const integrityCheckResult = await this.checkIntegrityBetweenPrimaryAndSecondaryDatabase(userUuid)
const integrityCheckResult = await this.checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid,
newRevisionsInSecondaryCount,
)
if (integrityCheckResult.isFailed()) {
const cleanupResult = await this.deleteRevisionsForUser(userUuid, this.secondRevisionsRepository)
if (cleanupResult.isFailed()) {
@ -157,13 +173,66 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
return totalRevisionsCountForUserInSecondary > 0
}
private async getNewRevisionsCreatedInSecondaryDatabase(userUuid: Uuid): Promise<{
alreadyExistingInPrimary: Revision[]
newRevisionsInSecondary: Revision[]
}> {
const revisions = await (this.secondRevisionsRepository as RevisionRepositoryInterface).findByUserUuid({
userUuid: userUuid,
})
const alreadyExistingInPrimary: Revision[] = []
const newRevisionsInSecondary: Revision[] = []
for (const revision of revisions) {
const revisionExistsInPrimary = await this.checkIfRevisionExistsInPrimaryDatabase(revision)
if (revisionExistsInPrimary) {
alreadyExistingInPrimary.push(revision)
} else {
newRevisionsInSecondary.push(revision)
}
}
return {
alreadyExistingInPrimary: alreadyExistingInPrimary,
newRevisionsInSecondary: newRevisionsInSecondary,
}
}
private async checkIfRevisionExistsInPrimaryDatabase(revision: Revision): Promise<boolean> {
const revisionInPrimary = await this.primaryRevisionsRepository.findOneByUuid(
Uuid.create(revision.id.toString()).getValue(),
revision.props.userUuid as Uuid,
[],
)
if (revisionInPrimary === null) {
return false
}
if (!revision.isIdenticalTo(revisionInPrimary)) {
this.logger.error(
`Revision ${revision.id.toString()} is not identical in primary and secondary database. Revision in secondary database: ${JSON.stringify(
revision,
)}, revision in primary database: ${JSON.stringify(revisionInPrimary)}`,
)
return false
}
return true
}
private async isAlreadyMigrated(userUuid: Uuid): Promise<boolean> {
const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
return totalRevisionsCountForUserInPrimary === 0
}
private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(userUuid: Uuid): Promise<Result<boolean>> {
private async checkIntegrityBetweenPrimaryAndSecondaryDatabase(
userUuid: Uuid,
newRevisionsInSecondaryCount: number,
): Promise<Result<boolean>> {
try {
const totalRevisionsCountForUserInPrimary = await this.primaryRevisionsRepository.countByUserUuid(userUuid)
@ -206,9 +275,12 @@ export class TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser implements
this.secondRevisionsRepository as RevisionRepositoryInterface
).countByUserUuid(userUuid)
if (totalRevisionsCountForUserInPrimary !== totalRevisionsCountForUserInSecondary) {
if (
totalRevisionsCountForUserInPrimary + newRevisionsInSecondaryCount !==
totalRevisionsCountForUserInSecondary
) {
return Result.fail(
`Total revisions count for user ${userUuid.value} in primary database (${totalRevisionsCountForUserInPrimary}) does not match total revisions count in secondary database (${totalRevisionsCountForUserInSecondary})`,
`Total revisions count for user ${userUuid.value} in primary database (${totalRevisionsCountForUserInPrimary} + ${newRevisionsInSecondaryCount}) does not match total revisions count in secondary database (${totalRevisionsCountForUserInSecondary})`,
)
}