feat: refactor handling revision creation from dump (#854)

* feat: refactor handling revision creation from dump

* fix: dump repository handling
This commit is contained in:
Karol Sójko 2023-09-26 11:47:41 +02:00 committed by GitHub
parent 1bb5980b45
commit ca6dbc0053
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 497 additions and 394 deletions

View file

@ -20,7 +20,7 @@
"release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
"publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
"postversion": "./scripts/push-tags-one-by-one.sh",
"e2e": "yarn workspace @standardnotes/home-server run build && PORT=3123 yarn workspace @standardnotes/home-server start",
"e2e": "yarn build && PORT=3123 yarn workspace @standardnotes/home-server start",
"start": "yarn workspace @standardnotes/home-server run build && yarn workspace @standardnotes/home-server start"
},
"devDependencies": {

View file

@ -18,7 +18,7 @@
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"worker": "yarn node dist/bin/worker.js",
"report": "yarn node dist/bin/report.js",
"setup:env": "cp .env.sample .env",

View file

@ -19,7 +19,7 @@
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --fix --ext .ts",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js",
"cleanup": "yarn node dist/bin/cleanup.js",

View file

@ -20,7 +20,7 @@
"clean": "rm -fr dist",
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"test": "jest spec --coverage"
"test": "jest --coverage --no-cache"
},
"devDependencies": {
"@types/jest": "^29.5.1",

View file

@ -21,7 +21,7 @@
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"test": "jest spec --coverage --passWithNoTests"
"test": "jest --coverage --no-cache --passWithNoTests"
},
"dependencies": {
"uuid": "^9.0.0"

View file

@ -21,7 +21,7 @@
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"test": "jest spec --coverage"
"test": "jest --coverage --no-cache"
},
"dependencies": {
"@aws-sdk/client-sns": "^3.332.0",

View file

@ -20,7 +20,7 @@
"clean": "rm -fr dist",
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"test": "jest spec --coverage --passWithNoTests"
"test": "jest --coverage --no-cache --passWithNoTests"
},
"dependencies": {
"@standardnotes/predicates": "workspace:*",

View file

@ -13,7 +13,7 @@
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"worker": "yarn node dist/bin/worker.js"
},
"author": "Karol Sójko <karol@standardnotes.com>",

View file

@ -22,7 +22,7 @@
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --fix --ext .ts",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js"
},

View file

@ -22,7 +22,7 @@
"start": "tsc -p tsconfig.json --watch",
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"test": "jest spec --coverage --passWithNoTests"
"test": "jest --coverage --no-cache --passWithNoTests"
},
"devDependencies": {
"@types/jest": "^29.5.1",

View file

@ -20,7 +20,7 @@
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js"
},

View file

@ -71,6 +71,7 @@ import { TransitionRequestedEventHandler } from '../Domain/Handler/TransitionReq
import { SharedVaultRemovedEventHandler } from '../Domain/Handler/SharedVaultRemovedEventHandler'
import { TransitionRepositoryInterface } from '../Domain/Transition/TransitionRepositoryInterface'
import { RedisTransitionRepository } from '../Infra/Redis/RedisTransitionRepository'
import { CreateRevisionFromDump } from '../Domain/UseCase/CreateRevisionFromDump/CreateRevisionFromDump'
export class ContainerConfigLoader {
constructor(private mode: 'server' | 'worker' = 'server') {}
@ -330,6 +331,21 @@ export class ContainerConfigLoader {
.toDynamicValue((context: interfaces.Context) => {
return new RevisionMetadataHttpMapper(context.container.get(TYPES.Revisions_GetRequiredRoleToViewRevision))
})
container
.bind<MapperInterface<Revision, string>>(TYPES.Revisions_RevisionItemStringMapper)
.toDynamicValue(() => new RevisionItemStringMapper())
container
.bind<DumpRepositoryInterface>(TYPES.Revisions_DumpRepository)
.toConstantValue(
env.get('S3_AWS_REGION', true)
? new S3DumpRepository(
container.get(TYPES.Revisions_S3_BACKUP_BUCKET_NAME),
container.get(TYPES.Revisions_S3),
container.get(TYPES.Revisions_RevisionItemStringMapper),
)
: new FSDumpRepository(container.get(TYPES.Revisions_RevisionItemStringMapper)),
)
// use cases
container
@ -385,6 +401,14 @@ export class ContainerConfigLoader {
: container.get<RevisionRepositoryInterface>(TYPES.Revisions_SQLRevisionRepository),
),
)
container
.bind<CreateRevisionFromDump>(TYPES.Revisions_CreateRevisionFromDump)
.toConstantValue(
new CreateRevisionFromDump(
container.get<DumpRepositoryInterface>(TYPES.Revisions_DumpRepository),
container.get<RevisionRepositoryResolverInterface>(TYPES.Revisions_RevisionRepositoryResolver),
),
)
// env vars
container.bind(TYPES.Revisions_AUTH_JWT_SECRET).toConstantValue(env.get('AUTH_JWT_SECRET'))
@ -409,31 +433,12 @@ export class ContainerConfigLoader {
)
})
// Map
container
.bind<MapperInterface<Revision, string>>(TYPES.Revisions_RevisionItemStringMapper)
.toDynamicValue(() => new RevisionItemStringMapper())
container
.bind<DumpRepositoryInterface>(TYPES.Revisions_DumpRepository)
.toConstantValue(
env.get('S3_AWS_REGION', true)
? new S3DumpRepository(
container.get(TYPES.Revisions_S3_BACKUP_BUCKET_NAME),
container.get(TYPES.Revisions_S3),
container.get(TYPES.Revisions_RevisionItemStringMapper),
container.get(TYPES.Revisions_Logger),
)
: new FSDumpRepository(container.get(TYPES.Revisions_RevisionItemStringMapper)),
)
// Handlers
container
.bind<ItemDumpedEventHandler>(TYPES.Revisions_ItemDumpedEventHandler)
.toConstantValue(
new ItemDumpedEventHandler(
container.get<DumpRepositoryInterface>(TYPES.Revisions_DumpRepository),
container.get<RevisionRepositoryResolverInterface>(TYPES.Revisions_RevisionRepositoryResolver),
container.get<CreateRevisionFromDump>(TYPES.Revisions_CreateRevisionFromDump),
container.get<winston.Logger>(TYPES.Revisions_Logger),
),
)

View file

@ -49,6 +49,7 @@ const TYPES = {
'Revisions_TransitionRevisionsFromPrimaryToSecondaryDatabaseForUser',
),
Revisions_RemoveRevisionsFromSharedVault: Symbol.for('Revisions_RemoveRevisionsFromSharedVault'),
Revisions_CreateRevisionFromDump: Symbol.for('Revisions_CreateRevisionFromDump'),
// Controller
Revisions_ControllerContainer: Symbol.for('Revisions_ControllerContainer'),
Revisions_RevisionsController: Symbol.for('Revisions_RevisionsController'),

View file

@ -1,6 +1,8 @@
import { Result } from '@standardnotes/domain-core'
import { Revision } from '../Revision/Revision'
export interface DumpRepositoryInterface {
getRevisionFromDumpPath(path: string): Promise<Revision | null>
getRevisionFromDumpPath(path: string): Promise<Result<Revision>>
removeDump(path: string): Promise<void>
}

View file

@ -1,79 +0,0 @@
import { ItemDumpedEvent } from '@standardnotes/domain-events'
import { Logger } from 'winston'
import { Uuid, ContentType, Dates } from '@standardnotes/domain-core'
import { DumpRepositoryInterface } from '../Dump/DumpRepositoryInterface'
import { Revision } from '../Revision/Revision'
import { RevisionRepositoryInterface } from '../Revision/RevisionRepositoryInterface'
import { ItemDumpedEventHandler } from './ItemDumpedEventHandler'
import { RevisionRepositoryResolverInterface } from '../Revision/RevisionRepositoryResolverInterface'
describe('ItemDumpedEventHandler', () => {
let dumpRepository: DumpRepositoryInterface
let revisionRepository: RevisionRepositoryInterface
let revisionRepositoryResolver: RevisionRepositoryResolverInterface
let revision: Revision
let event: ItemDumpedEvent
let logger: Logger
const createHandler = () => new ItemDumpedEventHandler(dumpRepository, revisionRepositoryResolver, logger)
beforeEach(() => {
revision = Revision.create({
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
}).getValue()
dumpRepository = {} as jest.Mocked<DumpRepositoryInterface>
dumpRepository.getRevisionFromDumpPath = jest.fn().mockReturnValue(revision)
dumpRepository.removeDump = jest.fn()
revisionRepository = {} as jest.Mocked<RevisionRepositoryInterface>
revisionRepository.insert = jest.fn()
revisionRepositoryResolver = {} as jest.Mocked<RevisionRepositoryResolverInterface>
revisionRepositoryResolver.resolve = jest.fn().mockReturnValue(revisionRepository)
event = {} as jest.Mocked<ItemDumpedEvent>
event.payload = {
fileDumpPath: 'foobar',
roleNames: ['CORE_USER'],
}
logger = {} as jest.Mocked<Logger>
logger.debug = jest.fn()
logger.error = jest.fn()
})
it('should save a revision from file dump', async () => {
await createHandler().handle(event)
expect(revisionRepository.insert).toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
it('should do nothing if role names are not valid', async () => {
event.payload.roleNames = ['INVALID_ROLE_NAME']
await createHandler().handle(event)
expect(revisionRepository.insert).not.toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
it('should not save a revision if it could not be created from dump', async () => {
dumpRepository.getRevisionFromDumpPath = jest.fn().mockReturnValue(null)
await createHandler().handle(event)
expect(revisionRepository.insert).not.toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
})

View file

@ -1,44 +1,22 @@
import { DomainEventHandlerInterface, ItemDumpedEvent } from '@standardnotes/domain-events'
import { DumpRepositoryInterface } from '../Dump/DumpRepositoryInterface'
import { RevisionRepositoryResolverInterface } from '../Revision/RevisionRepositoryResolverInterface'
import { RoleNameCollection } from '@standardnotes/domain-core'
import { Logger } from 'winston'
import { CreateRevisionFromDump } from '../UseCase/CreateRevisionFromDump/CreateRevisionFromDump'
export class ItemDumpedEventHandler implements DomainEventHandlerInterface {
constructor(
private dumpRepository: DumpRepositoryInterface,
private revisionRepositoryResolver: RevisionRepositoryResolverInterface,
private createRevisionFromDump: CreateRevisionFromDump,
private logger: Logger,
) {}
async handle(event: ItemDumpedEvent): Promise<void> {
const revision = await this.dumpRepository.getRevisionFromDumpPath(event.payload.fileDumpPath)
if (revision === null) {
this.logger.error(`Revision not found for dump path ${event.payload.fileDumpPath}`)
const result = await this.createRevisionFromDump.execute({
filePath: event.payload.fileDumpPath,
roleNames: event.payload.roleNames,
})
await this.dumpRepository.removeDump(event.payload.fileDumpPath)
return
if (result.isFailed()) {
this.logger.error(`Item dumped event handler failed: ${result.getError()}`)
}
const roleNamesOrError = RoleNameCollection.create(event.payload.roleNames)
if (roleNamesOrError.isFailed()) {
this.logger.error(`Invalid role names ${event.payload.roleNames}`)
await this.dumpRepository.removeDump(event.payload.fileDumpPath)
return
}
const roleNames = roleNamesOrError.getValue()
const revisionRepository = this.revisionRepositoryResolver.resolve(roleNames)
const successfullyInserted = await revisionRepository.insert(revision)
if (!successfullyInserted) {
this.logger.error(`Could not insert revision ${revision.id.toString()}`)
}
await this.dumpRepository.removeDump(event.payload.fileDumpPath)
}
}

View file

@ -0,0 +1,99 @@
import { Uuid, ContentType, Dates, Result } from '@standardnotes/domain-core'
import { DumpRepositoryInterface } from '../../Dump/DumpRepositoryInterface'
import { Revision } from '../../Revision/Revision'
import { RevisionRepositoryInterface } from '../../Revision/RevisionRepositoryInterface'
import { RevisionRepositoryResolverInterface } from '../../Revision/RevisionRepositoryResolverInterface'
import { CreateRevisionFromDump } from './CreateRevisionFromDump'
describe('CreateRevisionFromDump', () => {
let revisionRepository: RevisionRepositoryInterface
let revision: Revision
let dumpRepository: DumpRepositoryInterface
let revisionRepositoryResolver: RevisionRepositoryResolverInterface
const createUseCase = () => new CreateRevisionFromDump(dumpRepository, revisionRepositoryResolver)
beforeEach(() => {
revision = Revision.create({
itemUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
userUuid: Uuid.create('84c0f8e8-544a-4c7e-9adf-26209303bc1d').getValue(),
content: 'test',
contentType: ContentType.create('Note').getValue(),
itemsKeyId: 'test',
encItemKey: 'test',
authHash: 'test',
creationDate: new Date(1),
dates: Dates.create(new Date(1), new Date(2)).getValue(),
}).getValue()
dumpRepository = {} as jest.Mocked<DumpRepositoryInterface>
dumpRepository.getRevisionFromDumpPath = jest.fn().mockReturnValue(Result.ok(revision))
dumpRepository.removeDump = jest.fn()
revisionRepository = {} as jest.Mocked<RevisionRepositoryInterface>
revisionRepository.insert = jest.fn().mockReturnValue(true)
revisionRepositoryResolver = {} as jest.Mocked<RevisionRepositoryResolverInterface>
revisionRepositoryResolver.resolve = jest.fn().mockReturnValue(revisionRepository)
})
it('should create a revision from file dump', async () => {
const result = await createUseCase().execute({
filePath: 'foobar',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBeFalsy()
expect(revisionRepository.insert).toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
it('should fail if file path is empty', async () => {
const result = await createUseCase().execute({
filePath: '',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBeTruthy()
expect(revisionRepository.insert).not.toHaveBeenCalled()
expect(dumpRepository.removeDump).not.toHaveBeenCalled()
})
it('should fail if role name is invalid', async () => {
const result = await createUseCase().execute({
filePath: 'foobar',
roleNames: ['INVALID_ROLE_NAME'],
})
expect(result.isFailed()).toBeTruthy()
expect(revisionRepository.insert).not.toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
it('should fail if revision cannot be found', async () => {
dumpRepository.getRevisionFromDumpPath = jest.fn().mockReturnValue(Result.fail('Oops'))
const result = await createUseCase().execute({
filePath: 'foobar',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBeTruthy()
expect(revisionRepository.insert).not.toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
it('should fail if revision cannot be inserted', async () => {
revisionRepository.insert = jest.fn().mockReturnValue(false)
const result = await createUseCase().execute({
filePath: 'foobar',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBeTruthy()
expect(revisionRepository.insert).toHaveBeenCalled()
expect(dumpRepository.removeDump).toHaveBeenCalled()
})
})

View file

@ -0,0 +1,47 @@
import { Result, RoleNameCollection, UseCaseInterface, Validator } from '@standardnotes/domain-core'
import { DumpRepositoryInterface } from '../../Dump/DumpRepositoryInterface'
import { RevisionRepositoryResolverInterface } from '../../Revision/RevisionRepositoryResolverInterface'
import { CreateRevisionFromDumpDTO } from './CreateRevisionFromDumpDTO'
export class CreateRevisionFromDump implements UseCaseInterface<void> {
constructor(
private dumpRepository: DumpRepositoryInterface,
private revisionRepositoryResolver: RevisionRepositoryResolverInterface,
) {}
async execute(dto: CreateRevisionFromDumpDTO): Promise<Result<void>> {
const filePathValidationResult = Validator.isNotEmptyString(dto.filePath)
if (filePathValidationResult.isFailed()) {
return Result.fail(`Could not create revision from dump: ${filePathValidationResult.getError()}`)
}
const revisionOrError = await this.dumpRepository.getRevisionFromDumpPath(dto.filePath)
if (revisionOrError.isFailed()) {
await this.dumpRepository.removeDump(dto.filePath)
return Result.fail(`Could not create revision from dump: ${revisionOrError.getError()}`)
}
const revision = revisionOrError.getValue()
const roleNamesOrError = RoleNameCollection.create(dto.roleNames)
if (roleNamesOrError.isFailed()) {
await this.dumpRepository.removeDump(dto.filePath)
return Result.fail(`Could not create revision from dump: ${roleNamesOrError.getError()}`)
}
const roleNames = roleNamesOrError.getValue()
const revisionRepository = this.revisionRepositoryResolver.resolve(roleNames)
const successfullyInserted = await revisionRepository.insert(revision)
if (!successfullyInserted) {
await this.dumpRepository.removeDump(dto.filePath)
return Result.fail(`Could not insert revision from dump: ${revision.id.toString()}`)
}
await this.dumpRepository.removeDump(dto.filePath)
return Result.ok()
}
}

View file

@ -0,0 +1,4 @@
export interface CreateRevisionFromDumpDTO {
filePath: string
roleNames: string[]
}

View file

@ -1,4 +1,4 @@
import { MapperInterface } from '@standardnotes/domain-core'
import { MapperInterface, Result } from '@standardnotes/domain-core'
import { promises } from 'fs'
import { DumpRepositoryInterface } from '../../Domain/Dump/DumpRepositoryInterface'
@ -7,12 +7,16 @@ import { Revision } from '../../Domain/Revision/Revision'
export class FSDumpRepository implements DumpRepositoryInterface {
constructor(private revisionStringItemMapper: MapperInterface<Revision, string>) {}
async getRevisionFromDumpPath(path: string): Promise<Revision | null> {
const contents = (await promises.readFile(path)).toString()
async getRevisionFromDumpPath(path: string): Promise<Result<Revision>> {
try {
const contents = (await promises.readFile(path)).toString()
const revision = this.revisionStringItemMapper.toDomain(contents)
const revision = this.revisionStringItemMapper.toDomain(contents)
return revision
return Result.ok(revision)
} catch (error) {
return Result.fail(`Failed to read dump file: ${(error as Error).message}`)
}
}
async removeDump(path: string): Promise<void> {

View file

@ -1,6 +1,5 @@
import { DeleteObjectCommand, GetObjectCommand, S3Client } from '@aws-sdk/client-s3'
import { MapperInterface } from '@standardnotes/domain-core'
import { Logger } from 'winston'
import { MapperInterface, Result } from '@standardnotes/domain-core'
import { DumpRepositoryInterface } from '../../Domain/Dump/DumpRepositoryInterface'
import { Revision } from '../../Domain/Revision/Revision'
@ -10,26 +9,27 @@ export class S3DumpRepository implements DumpRepositoryInterface {
private dumpBucketName: string,
private s3Client: S3Client,
private revisionStringItemMapper: MapperInterface<Revision, string>,
private logger: Logger,
) {}
async getRevisionFromDumpPath(path: string): Promise<Revision | null> {
const s3Object = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.dumpBucketName,
Key: path,
}),
)
async getRevisionFromDumpPath(path: string): Promise<Result<Revision>> {
try {
const s3Object = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.dumpBucketName,
Key: path,
}),
)
if (s3Object.Body === undefined) {
this.logger.warn(`Could not find revision dump at path: ${path}`)
if (s3Object.Body === undefined) {
return Result.fail(`Could not find revision dump at path: ${path}`)
}
return null
const revision = this.revisionStringItemMapper.toDomain(await s3Object.Body.transformToString())
return Result.ok(revision)
} catch (error) {
return Result.fail(`Failed to read dump file: ${(error as Error).message}`)
}
const revision = this.revisionStringItemMapper.toDomain(await s3Object.Body.transformToString())
return revision
}
async removeDump(path: string): Promise<void> {

View file

@ -16,7 +16,7 @@
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"worker": "yarn node dist/bin/worker.js",
"verify:jobs": "yarn node dist/bin/verify.js",
"setup:env": "cp .env.sample .env",

View file

@ -22,7 +22,7 @@
"start": "tsc -p tsconfig.json --watch",
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"test": "jest spec --coverage"
"test": "jest --coverage --no-cache"
},
"dependencies": {
"jsonwebtoken": "^9.0.0",

View file

@ -20,7 +20,7 @@
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js",
"content-size": "yarn node dist/bin/content.js",

View file

@ -175,6 +175,7 @@ import { TransferSharedVault } from '../Domain/UseCase/SharedVaults/TransferShar
import { TransitionRepositoryInterface } from '../Domain/Transition/TransitionRepositoryInterface'
import { RedisTransitionRepository } from '../Infra/Redis/RedisTransitionRepository'
import { TransferSharedVaultItems } from '../Domain/UseCase/SharedVaults/TransferSharedVaultItems/TransferSharedVaultItems'
import { DumpItem } from '../Domain/UseCase/Syncing/DumpItem/DumpItem'
export class ContainerConfigLoader {
private readonly DEFAULT_CONTENT_SIZE_TRANSFER_LIMIT = 10_000_000
@ -590,6 +591,24 @@ export class ContainerConfigLoader {
]),
)
container
.bind<ItemBackupServiceInterface>(TYPES.Sync_ItemBackupService)
.toConstantValue(
env.get('S3_AWS_REGION', true)
? new S3ItemBackupService(
container.get(TYPES.Sync_S3_BACKUP_BUCKET_NAME),
container.get(TYPES.Sync_ItemBackupMapper),
container.get(TYPES.Sync_ItemHttpMapper),
container.get(TYPES.Sync_Logger),
container.get(TYPES.Sync_S3),
)
: new FSItemBackupService(
container.get(TYPES.Sync_FILE_UPLOAD_PATH),
container.get(TYPES.Sync_ItemBackupMapper),
container.get(TYPES.Sync_Logger),
),
)
// use cases
container
.bind<GetItems>(TYPES.Sync_GetItems)
@ -932,6 +951,16 @@ export class ContainerConfigLoader {
container.get<DeleteSharedVault>(TYPES.Sync_DeleteSharedVault),
),
)
container
.bind<DumpItem>(TYPES.Sync_DumpItem)
.toConstantValue(
new DumpItem(
container.get<ItemRepositoryResolverInterface>(TYPES.Sync_ItemRepositoryResolver),
container.get<ItemBackupServiceInterface>(TYPES.Sync_ItemBackupService),
container.get<DomainEventFactoryInterface>(TYPES.Sync_DomainEventFactory),
container.get<DomainEventPublisherInterface>(TYPES.Sync_DomainEventPublisher),
),
)
// Services
container
@ -959,24 +988,6 @@ export class ContainerConfigLoader {
)
})
container
.bind<ItemBackupServiceInterface>(TYPES.Sync_ItemBackupService)
.toConstantValue(
env.get('S3_AWS_REGION', true)
? new S3ItemBackupService(
container.get(TYPES.Sync_S3_BACKUP_BUCKET_NAME),
container.get(TYPES.Sync_ItemBackupMapper),
container.get(TYPES.Sync_ItemHttpMapper),
container.get(TYPES.Sync_Logger),
container.get(TYPES.Sync_S3),
)
: new FSItemBackupService(
container.get(TYPES.Sync_FILE_UPLOAD_PATH),
container.get(TYPES.Sync_ItemBackupMapper),
container.get(TYPES.Sync_Logger),
),
)
// Handlers
container
.bind<DuplicateItemSyncedEventHandler>(TYPES.Sync_DuplicateItemSyncedEventHandler)
@ -1002,10 +1013,8 @@ export class ContainerConfigLoader {
.bind<ItemRevisionCreationRequestedEventHandler>(TYPES.Sync_ItemRevisionCreationRequestedEventHandler)
.toConstantValue(
new ItemRevisionCreationRequestedEventHandler(
container.get<ItemRepositoryResolverInterface>(TYPES.Sync_ItemRepositoryResolver),
container.get<ItemBackupServiceInterface>(TYPES.Sync_ItemBackupService),
container.get<DomainEventFactoryInterface>(TYPES.Sync_DomainEventFactory),
container.get<DomainEventPublisherInterface>(TYPES.Sync_DomainEventPublisher),
container.get<DumpItem>(TYPES.Sync_DumpItem),
container.get<Logger>(TYPES.Sync_Logger),
),
)
container

View file

@ -92,6 +92,7 @@ const TYPES = {
Sync_RemoveUserFromSharedVaults: Symbol.for('Sync_RemoveUserFromSharedVaults'),
Sync_TransferSharedVault: Symbol.for('Sync_TransferSharedVault'),
Sync_TransferSharedVaultItems: Symbol.for('Sync_TransferSharedVaultItems'),
Sync_DumpItem: Symbol.for('Sync_DumpItem'),
// Handlers
Sync_AccountDeletionRequestedEventHandler: Symbol.for('Sync_AccountDeletionRequestedEventHandler'),
Sync_DuplicateItemSyncedEventHandler: Symbol.for('Sync_DuplicateItemSyncedEventHandler'),

View file

@ -1,125 +0,0 @@
import 'reflect-metadata'
import {
DomainEventPublisherInterface,
DomainEventService,
ItemRevisionCreationRequestedEvent,
} from '@standardnotes/domain-events'
import { Item } from '../Item/Item'
import { ItemRepositoryInterface } from '../Item/ItemRepositoryInterface'
import { ItemRevisionCreationRequestedEventHandler } from './ItemRevisionCreationRequestedEventHandler'
import { ItemBackupServiceInterface } from '../Item/ItemBackupServiceInterface'
import { DomainEventFactoryInterface } from '../Event/DomainEventFactoryInterface'
import { Uuid, ContentType, Dates, Timestamps, UniqueEntityId } from '@standardnotes/domain-core'
import { ItemRepositoryResolverInterface } from '../Item/ItemRepositoryResolverInterface'
describe('ItemRevisionCreationRequestedEventHandler', () => {
let itemRepositoryResolver: ItemRepositoryResolverInterface
let itemRepository: ItemRepositoryInterface
let event: ItemRevisionCreationRequestedEvent
let item: Item
let itemBackupService: ItemBackupServiceInterface
let domainEventFactory: DomainEventFactoryInterface
let domainEventPublisher: DomainEventPublisherInterface
const createHandler = () =>
new ItemRevisionCreationRequestedEventHandler(
itemRepositoryResolver,
itemBackupService,
domainEventFactory,
domainEventPublisher,
)
beforeEach(() => {
item = Item.create(
{
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
updatedWithSession: null,
content: 'foobar1',
contentType: ContentType.create(ContentType.TYPES.Note).getValue(),
encItemKey: null,
authHash: null,
itemsKeyId: null,
duplicateOf: null,
deleted: false,
dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(),
timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
).getValue()
itemRepository = {} as jest.Mocked<ItemRepositoryInterface>
itemRepository.findByUuid = jest.fn().mockReturnValue(item)
itemRepositoryResolver = {} as jest.Mocked<ItemRepositoryResolverInterface>
itemRepositoryResolver.resolve = jest.fn().mockReturnValue(itemRepository)
event = {} as jest.Mocked<ItemRevisionCreationRequestedEvent>
event.createdAt = new Date(1)
event.payload = {
itemUuid: '00000000-0000-0000-0000-000000000000',
roleNames: ['CORE_USER'],
}
event.meta = {
correlation: {
userIdentifier: '1-2-3',
userIdentifierType: 'uuid',
},
origin: DomainEventService.SyncingServer,
}
itemBackupService = {} as jest.Mocked<ItemBackupServiceInterface>
itemBackupService.dump = jest.fn().mockReturnValue('foo://bar')
domainEventFactory = {} as jest.Mocked<DomainEventFactoryInterface>
domainEventFactory.createItemDumpedEvent = jest.fn()
domainEventPublisher = {} as jest.Mocked<DomainEventPublisherInterface>
domainEventPublisher.publish = jest.fn()
})
it('should create a revision for an item', async () => {
await createHandler().handle(event)
expect(domainEventPublisher.publish).toHaveBeenCalled()
expect(domainEventFactory.createItemDumpedEvent).toHaveBeenCalled()
})
it('should do nothing if roles names are not valid', async () => {
event.payload.roleNames = ['INVALID_ROLE_NAME']
await createHandler().handle(event)
expect(domainEventPublisher.publish).not.toHaveBeenCalled()
expect(domainEventFactory.createItemDumpedEvent).not.toHaveBeenCalled()
})
it('should not create a revision for an item that does not exist', async () => {
itemRepository.findByUuid = jest.fn().mockReturnValue(null)
itemRepositoryResolver.resolve = jest.fn().mockReturnValue(itemRepository)
await createHandler().handle(event)
expect(domainEventPublisher.publish).not.toHaveBeenCalled()
})
it('should not create a revision for an item if the dump was not created', async () => {
itemBackupService.dump = jest.fn().mockReturnValue('')
await createHandler().handle(event)
expect(domainEventPublisher.publish).not.toHaveBeenCalled()
expect(domainEventFactory.createItemDumpedEvent).not.toHaveBeenCalled()
})
it('should not create a revision if the item uuid is invalid', async () => {
event.payload.itemUuid = 'invalid-uuid'
await createHandler().handle(event)
expect(domainEventPublisher.publish).not.toHaveBeenCalled()
expect(domainEventFactory.createItemDumpedEvent).not.toHaveBeenCalled()
})
})

View file

@ -1,59 +1,22 @@
import {
ItemRevisionCreationRequestedEvent,
DomainEventHandlerInterface,
DomainEventPublisherInterface,
} from '@standardnotes/domain-events'
import { RoleNameCollection, Uuid } from '@standardnotes/domain-core'
import { ItemRevisionCreationRequestedEvent, DomainEventHandlerInterface } from '@standardnotes/domain-events'
import { DomainEventFactoryInterface } from '../Event/DomainEventFactoryInterface'
import { ItemBackupServiceInterface } from '../Item/ItemBackupServiceInterface'
import { ItemRepositoryInterface } from '../Item/ItemRepositoryInterface'
import { ItemRepositoryResolverInterface } from '../Item/ItemRepositoryResolverInterface'
import { DumpItem } from '../UseCase/Syncing/DumpItem/DumpItem'
import { Logger } from 'winston'
export class ItemRevisionCreationRequestedEventHandler implements DomainEventHandlerInterface {
constructor(
private itemRepositoryResolver: ItemRepositoryResolverInterface,
private itemBackupService: ItemBackupServiceInterface,
private domainEventFactory: DomainEventFactoryInterface,
private domainEventPublisher: DomainEventPublisherInterface,
private dumpItem: DumpItem,
private logger: Logger,
) {}
async handle(event: ItemRevisionCreationRequestedEvent): Promise<void> {
const roleNamesOrError = RoleNameCollection.create(event.payload.roleNames)
if (roleNamesOrError.isFailed()) {
return
}
const roleNames = roleNamesOrError.getValue()
const result = await this.dumpItem.execute({
itemUuid: event.payload.itemUuid,
roleNames: event.payload.roleNames,
})
const itemRepository = this.itemRepositoryResolver.resolve(roleNames)
await this.createItemDump(event, itemRepository)
}
private async createItemDump(
event: ItemRevisionCreationRequestedEvent,
itemRepository: ItemRepositoryInterface,
): Promise<void> {
const itemUuidOrError = Uuid.create(event.payload.itemUuid)
if (itemUuidOrError.isFailed()) {
return
}
const itemUuid = itemUuidOrError.getValue()
const item = await itemRepository.findByUuid(itemUuid)
if (item === null) {
return
}
const fileDumpPath = await this.itemBackupService.dump(item)
if (fileDumpPath) {
await this.domainEventPublisher.publish(
this.domainEventFactory.createItemDumpedEvent({
fileDumpPath,
userUuid: event.meta.correlation.userIdentifier,
roleNames: event.payload.roleNames,
}),
)
if (result.isFailed()) {
this.logger.error(`Item revision requested handler failed: ${result.getError()}`)
}
}
}

View file

@ -1,7 +1,9 @@
import { KeyParamsData } from '@standardnotes/responses'
import { Result } from '@standardnotes/domain-core'
import { Item } from './Item'
export interface ItemBackupServiceInterface {
backup(items: Array<Item>, authParams: KeyParamsData, contentSizeLimit?: number): Promise<string[]>
dump(item: Item): Promise<string>
dump(item: Item): Promise<Result<string>>
}

View file

@ -0,0 +1,117 @@
import { DomainEventInterface, DomainEventPublisherInterface } from '@standardnotes/domain-events'
import { DomainEventFactoryInterface } from '../../../Event/DomainEventFactoryInterface'
import { ItemBackupServiceInterface } from '../../../Item/ItemBackupServiceInterface'
import { ItemRepositoryResolverInterface } from '../../../Item/ItemRepositoryResolverInterface'
import { DumpItem } from './DumpItem'
import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface'
import { Item } from '../../../Item/Item'
import { Uuid, ContentType, Dates, Timestamps, UniqueEntityId, Result } from '@standardnotes/domain-core'
describe('DumpItem', () => {
let itemRepositoryResolver: ItemRepositoryResolverInterface
let itemRepository: ItemRepositoryInterface
let item: Item
let itemBackupService: ItemBackupServiceInterface
let domainEventFactory: DomainEventFactoryInterface
let domainEventPublisher: DomainEventPublisherInterface
const createUseCase = () =>
new DumpItem(itemRepositoryResolver, itemBackupService, domainEventFactory, domainEventPublisher)
beforeEach(() => {
item = Item.create(
{
userUuid: Uuid.create('00000000-0000-0000-0000-000000000000').getValue(),
updatedWithSession: null,
content: 'foobar',
contentType: ContentType.create(ContentType.TYPES.Note).getValue(),
encItemKey: null,
authHash: null,
itemsKeyId: null,
duplicateOf: null,
deleted: false,
dates: Dates.create(new Date(1616164633241311), new Date(1616164633241311)).getValue(),
timestamps: Timestamps.create(1616164633241311, 1616164633241311).getValue(),
},
new UniqueEntityId('00000000-0000-0000-0000-000000000000'),
).getValue()
itemRepository = {} as jest.Mocked<ItemRepositoryInterface>
itemRepository.findByUuid = jest.fn().mockResolvedValue(item)
itemRepositoryResolver = {} as jest.Mocked<ItemRepositoryResolverInterface>
itemRepositoryResolver.resolve = jest.fn().mockReturnValue(itemRepository)
itemBackupService = {} as jest.Mocked<ItemBackupServiceInterface>
itemBackupService.dump = jest.fn().mockResolvedValue(Result.ok('dump-path'))
domainEventFactory = {} as jest.Mocked<DomainEventFactoryInterface>
domainEventFactory.createItemDumpedEvent = jest.fn().mockReturnValue({} as jest.Mocked<DomainEventInterface>)
domainEventPublisher = {} as jest.Mocked<DomainEventPublisherInterface>
domainEventPublisher.publish = jest.fn()
})
it('should dump item', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
itemUuid: '00000000-0000-0000-0000-000000000000',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBe(false)
expect(itemBackupService.dump).toHaveBeenCalled()
expect(domainEventPublisher.publish).toHaveBeenCalled()
})
it('should fail if item cannot be found', async () => {
itemRepository.findByUuid = jest.fn().mockResolvedValue(null)
const useCase = createUseCase()
const result = await useCase.execute({
itemUuid: '00000000-0000-0000-0000-000000000000',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBe(true)
})
it('should fail if item cannot be dumped', async () => {
itemBackupService.dump = jest.fn().mockResolvedValue(Result.fail('error'))
const useCase = createUseCase()
const result = await useCase.execute({
itemUuid: '00000000-0000-0000-0000-000000000000',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBe(true)
})
it('should fail if item uuid is invalid', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
itemUuid: 'invalid-uuid',
roleNames: ['CORE_USER'],
})
expect(result.isFailed()).toBe(true)
})
it('should fail if role names are invalid', async () => {
const useCase = createUseCase()
const result = await useCase.execute({
itemUuid: '00000000-0000-0000-0000-000000000000',
roleNames: ['invalid-role'],
})
expect(result.isFailed()).toBe(true)
})
})

View file

@ -0,0 +1,63 @@
import { Result, RoleNameCollection, UseCaseInterface, Uuid } from '@standardnotes/domain-core'
import { DumpItemDTO } from './DumpItemDTO'
import { DomainEventPublisherInterface } from '@standardnotes/domain-events'
import { DomainEventFactoryInterface } from '../../../Event/DomainEventFactoryInterface'
import { ItemBackupServiceInterface } from '../../../Item/ItemBackupServiceInterface'
import { ItemRepositoryInterface } from '../../../Item/ItemRepositoryInterface'
import { ItemRepositoryResolverInterface } from '../../../Item/ItemRepositoryResolverInterface'
export class DumpItem implements UseCaseInterface<void> {
constructor(
private itemRepositoryResolver: ItemRepositoryResolverInterface,
private itemBackupService: ItemBackupServiceInterface,
private domainEventFactory: DomainEventFactoryInterface,
private domainEventPublisher: DomainEventPublisherInterface,
) {}
async execute(dto: DumpItemDTO): Promise<Result<void>> {
const itemUuidOrError = Uuid.create(dto.itemUuid)
if (itemUuidOrError.isFailed()) {
return Result.fail(`Failed to dump item: ${itemUuidOrError.getError()}`)
}
const itemUuid = itemUuidOrError.getValue()
const itemRepositoryOrError = this.getItemRepository(dto.roleNames)
if (itemRepositoryOrError.isFailed()) {
return Result.fail(`Failed to dump item: ${itemRepositoryOrError.getError()}`)
}
const itemRepository = itemRepositoryOrError.getValue()
const item = await itemRepository.findByUuid(itemUuid)
if (item === null) {
return Result.fail('Failed to dump item: Item not found')
}
const fileDumpPathOrError = await this.itemBackupService.dump(item)
if (fileDumpPathOrError.isFailed()) {
return Result.fail(`Failed to dump item: ${fileDumpPathOrError.getError()}`)
}
const fileDumpPath = fileDumpPathOrError.getValue()
await this.domainEventPublisher.publish(
this.domainEventFactory.createItemDumpedEvent({
fileDumpPath,
userUuid: item.props.userUuid.value,
roleNames: dto.roleNames,
}),
)
return Result.ok()
}
private getItemRepository(stringRoleNames: string[]): Result<ItemRepositoryInterface> {
const roleNamesOrError = RoleNameCollection.create(stringRoleNames)
if (roleNamesOrError.isFailed()) {
return Result.fail(`Could not obtain item repository based on role names: ${roleNamesOrError.getError()}`)
}
const roleNames = roleNamesOrError.getValue()
const itemRepository = this.itemRepositoryResolver.resolve(roleNames)
return Result.ok(itemRepository)
}
}

View file

@ -0,0 +1,4 @@
export interface DumpItemDTO {
itemUuid: string
roleNames: string[]
}

View file

@ -1,5 +1,5 @@
import { KeyParamsData } from '@standardnotes/responses'
import { MapperInterface } from '@standardnotes/domain-core'
import { MapperInterface, Result } from '@standardnotes/domain-core'
import { promises } from 'fs'
import * as uuid from 'uuid'
import { Logger } from 'winston'
@ -20,25 +20,29 @@ export class FSItemBackupService implements ItemBackupServiceInterface {
throw new Error('Method not implemented.')
}
async dump(item: Item): Promise<string> {
const contents = JSON.stringify({
item: this.mapper.toProjection(item),
})
async dump(item: Item): Promise<Result<string>> {
try {
const contents = JSON.stringify({
item: this.mapper.toProjection(item),
})
const path = `${this.fileUploadPath}/dumps/${uuid.v4()}`
const path = `${this.fileUploadPath}/dumps/${uuid.v4()}`
this.logger.debug(`Dumping item ${item.id.toString()} to ${path}`)
this.logger.debug(`Dumping item ${item.id.toString()} to ${path}`)
await promises.mkdir(dirname(path), { recursive: true })
await promises.mkdir(dirname(path), { recursive: true })
await promises.writeFile(path, contents)
await promises.writeFile(path, contents)
const fileCreated = (await promises.stat(path)).isFile()
const fileCreated = (await promises.stat(path)).isFile()
if (!fileCreated) {
throw new Error(`Could not create dump file ${path}`)
if (!fileCreated) {
return Result.fail(`Could not create dump file ${path}`)
}
return Result.ok(path)
} catch (error) {
return Result.fail(`Could not dump item: ${(error as Error).message}`)
}
return path
}
}

View file

@ -5,7 +5,7 @@ import { Logger } from 'winston'
import { Item } from '../../Domain/Item/Item'
import { ItemBackupServiceInterface } from '../../Domain/Item/ItemBackupServiceInterface'
import { MapperInterface } from '@standardnotes/domain-core'
import { MapperInterface, Result } from '@standardnotes/domain-core'
import { ItemBackupRepresentation } from '../../Mapping/Backup/ItemBackupRepresentation'
import { ItemHttpRepresentation } from '../../Mapping/Http/ItemHttpRepresentation'
@ -18,25 +18,29 @@ export class S3ItemBackupService implements ItemBackupServiceInterface {
private s3Client?: S3Client,
) {}
async dump(item: Item): Promise<string> {
if (!this.s3BackupBucketName || this.s3Client === undefined) {
this.logger.warn('S3 backup not configured')
async dump(item: Item): Promise<Result<string>> {
try {
if (!this.s3BackupBucketName || this.s3Client === undefined) {
this.logger.warn('S3 backup not configured')
return ''
}
return Result.fail('S3 backup not configured')
}
const s3Key = uuid.v4()
await this.s3Client.send(
new PutObjectCommand({
Bucket: this.s3BackupBucketName,
Key: s3Key,
Body: JSON.stringify({
item: this.backupMapper.toProjection(item),
const s3Key = uuid.v4()
await this.s3Client.send(
new PutObjectCommand({
Bucket: this.s3BackupBucketName,
Key: s3Key,
Body: JSON.stringify({
item: this.backupMapper.toProjection(item),
}),
}),
}),
)
)
return s3Key
return Result.ok(s3Key)
} catch (error) {
return Result.fail(`Could not dump item: ${(error as Error).message}`)
}
}
async backup(items: Item[], authParams: KeyParamsData, contentSizeLimit?: number): Promise<string[]> {

View file

@ -20,7 +20,7 @@
"clean": "rm -fr dist",
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"test": "jest spec --coverage"
"test": "jest --coverage --no-cache"
},
"dependencies": {
"dayjs": "^1.11.6",

View file

@ -16,7 +16,7 @@
"build": "tsc --build",
"lint": "eslint . --ext .ts",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"test": "jest --coverage --no-cache --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js",
"typeorm": "typeorm-ts-node-commonjs"