feat: script to mass delete accounts from CSV source (#913)

packages/auth/bin/delete_accounts.ts

@@ -0,0 +1,43 @@
+import 'reflect-metadata'
+
+import { Logger } from 'winston'
+
+import { ContainerConfigLoader } from '../src/Bootstrap/Container'
+import TYPES from '../src/Bootstrap/Types'
+import { Env } from '../src/Bootstrap/Env'
+import { DeleteAccountsFromCSVFile } from '../src/Domain/UseCase/DeleteAccountsFromCSVFile/DeleteAccountsFromCSVFile'
+
+const inputArgs = process.argv.slice(2)
+const fileName = inputArgs[0]
+const mode = inputArgs[1]
+
+const deleteAccounts = async (deleteAccountsFromCSVFile: DeleteAccountsFromCSVFile): Promise<void> => {
+  await deleteAccountsFromCSVFile.execute({
+    fileName,
+    dryRun: mode !== 'delete',
+  })
+}
+
+const container = new ContainerConfigLoader('worker')
+void container.load().then((container) => {
+  const env: Env = new Env()
+  env.load()
+
+  const logger: Logger = container.get(TYPES.Auth_Logger)
+
+  logger.info('Starting mass accounts deletion from CSV file')
+
+  const deleteAccountsFromCSVFile = container.get<DeleteAccountsFromCSVFile>(TYPES.Auth_DeleteAccountsFromCSVFile)
+
+  Promise.resolve(deleteAccounts(deleteAccountsFromCSVFile))
+    .then(() => {
+      logger.info('Accounts deleted.')
+
+      process.exit(0)
+    })
+    .catch((error) => {
+      logger.error(`Could not delete accounts: ${error.message}`)
+
+      process.exit(1)
+    })
+})

packages/auth/docker/entrypoint-delete-accounts.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/delete-accounts.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint.sh

@@ -40,6 +40,13 @@ case "$COMMAND" in
     node docker/entrypoint-user-email-backup.js $EMAIL
     ;;
 
+  'delete-accounts' )
+    echo "[Docker] Starting Accounts Deleting from CSV..."
+    FILE_NAME=$1 && shift 1
+    MODE=$1 && shift 1
+    node docker/entrypoint-delete-accounts.js $FILE_NAME $MODE
+    ;;
+
    * )
     echo "[Docker] Unknown command"
     ;;

packages/auth/package.json

@@ -32,6 +32,7 @@
     "migrate": "yarn build && yarn typeorm migration:run -d dist/src/Bootstrap/DataSource.js"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.445.0",
     "@aws-sdk/client-sns": "^3.427.0",
     "@aws-sdk/client-sqs": "^3.427.0",
     "@cbor-extract/cbor-extract-linux-arm64": "^2.1.1",

packages/auth/src/Bootstrap/Container.ts

@@ -2,6 +2,7 @@ import * as winston from 'winston'
 import Redis from 'ioredis'
 import { SNSClient, SNSClientConfig } from '@aws-sdk/client-sns'
 import { SQSClient, SQSClientConfig } from '@aws-sdk/client-sqs'
+import { S3Client } from '@aws-sdk/client-s3'
 import { Container } from 'inversify'
 import {
   DomainEventHandlerInterface,
@@ -276,6 +277,9 @@ import { UserInvitedToSharedVaultEventHandler } from '../Domain/Handler/UserInvi
 import { TriggerPostSettingUpdateActions } from '../Domain/UseCase/TriggerPostSettingUpdateActions/TriggerPostSettingUpdateActions'
 import { TriggerEmailBackupForUser } from '../Domain/UseCase/TriggerEmailBackupForUser/TriggerEmailBackupForUser'
 import { TriggerEmailBackupForAllUsers } from '../Domain/UseCase/TriggerEmailBackupForAllUsers/TriggerEmailBackupForAllUsers'
+import { CSVFileReaderInterface } from '../Domain/CSV/CSVFileReaderInterface'
+import { S3CsvFileReader } from '../Infra/S3/S3CsvFileReader'
+import { DeleteAccountsFromCSVFile } from '../Domain/UseCase/DeleteAccountsFromCSVFile/DeleteAccountsFromCSVFile'
 
 export class ContainerConfigLoader {
   constructor(private mode: 'server' | 'worker' = 'server') {}
@@ -370,6 +374,19 @@ export class ContainerConfigLoader {
       }
       const sqsClient = new SQSClient(sqsConfig)
       container.bind<SQSClient>(TYPES.Auth_SQS).toConstantValue(sqsClient)
+
+      container.bind<S3Client>(TYPES.Auth_S3).toConstantValue(
+        new S3Client({
+          apiVersion: 'latest',
+          region: env.get('S3_AWS_REGION', true),
+        }),
+      )
+
+      container
+        .bind<CSVFileReaderInterface>(TYPES.Auth_CSVFileReader)
+        .toConstantValue(
+          new S3CsvFileReader(env.get('S3_AUTH_SCRIPTS_DATA_BUCKET', true), container.get<S3Client>(TYPES.Auth_S3)),
+        )
     }
 
     container.bind(TYPES.Auth_SNS_TOPIC_ARN).toConstantValue(env.get('SNS_TOPIC_ARN', true))
@@ -1251,6 +1268,17 @@ export class ContainerConfigLoader {
           container.get<TriggerEmailBackupForUser>(TYPES.Auth_TriggerEmailBackupForUser),
         ),
       )
+    if (!isConfiguredForHomeServer) {
+      container
+        .bind<DeleteAccountsFromCSVFile>(TYPES.Auth_DeleteAccountsFromCSVFile)
+        .toConstantValue(
+          new DeleteAccountsFromCSVFile(
+            container.get<CSVFileReaderInterface>(TYPES.Auth_CSVFileReader),
+            container.get<DeleteAccount>(TYPES.Auth_DeleteAccount),
+            container.get<winston.Logger>(TYPES.Auth_Logger),
+          ),
+        )
+    }
 
     // Controller
     container

packages/auth/src/Bootstrap/Types.ts

@@ -3,6 +3,7 @@ const TYPES = {
   Auth_Redis: Symbol.for('Auth_Redis'),
   Auth_SNS: Symbol.for('Auth_SNS'),
   Auth_SQS: Symbol.for('Auth_SQS'),
+  Auth_S3: Symbol.for('Auth_S3'),
   // Mapping
   Auth_SessionTracePersistenceMapper: Symbol.for('Auth_SessionTracePersistenceMapper'),
   Auth_AuthenticatorChallengePersistenceMapper: Symbol.for('Auth_AuthenticatorChallengePersistenceMapper'),
@@ -167,6 +168,7 @@ const TYPES = {
   Auth_TriggerPostSettingUpdateActions: Symbol.for('Auth_TriggerPostSettingUpdateActions'),
   Auth_TriggerEmailBackupForUser: Symbol.for('Auth_TriggerEmailBackupForUser'),
   Auth_TriggerEmailBackupForAllUsers: Symbol.for('Auth_TriggerEmailBackupForAllUsers'),
+  Auth_DeleteAccountsFromCSVFile: Symbol.for('Auth_DeleteAccountsFromCSVFile'),
   // Handlers
   Auth_AccountDeletionRequestedEventHandler: Symbol.for('Auth_AccountDeletionRequestedEventHandler'),
   Auth_SubscriptionPurchasedEventHandler: Symbol.for('Auth_SubscriptionPurchasedEventHandler'),
@@ -251,6 +253,7 @@ const TYPES = {
   Auth_BaseOfflineController: Symbol.for('Auth_BaseOfflineController'),
   Auth_BaseListedController: Symbol.for('Auth_BaseListedController'),
   Auth_BaseFeaturesController: Symbol.for('Auth_BaseFeaturesController'),
+  Auth_CSVFileReader: Symbol.for('Auth_CSVFileReader'),
 }
 
 export default TYPES

packages/auth/src/Domain/CSV/CSVFileReaderInterface.ts

@@ -0,0 +1,5 @@
+import { Result } from '@standardnotes/domain-core'
+
+export interface CSVFileReaderInterface {
+  getValues(fileName: string): Promise<Result<string[]>>
+}

packages/auth/src/Domain/UseCase/DeleteAccountsFromCSVFile/DeleteAccountsFromCSVFile.spec.ts

@@ -0,0 +1,72 @@
+import { Logger } from 'winston'
+import { Result } from '@standardnotes/domain-core'
+
+import { CSVFileReaderInterface } from '../../CSV/CSVFileReaderInterface'
+import { DeleteAccount } from '../DeleteAccount/DeleteAccount'
+import { DeleteAccountsFromCSVFile } from './DeleteAccountsFromCSVFile'
+
+describe('DeleteAccountsFromCSVFile', () => {
+  let csvFileReader: CSVFileReaderInterface
+  let deleteAccount: DeleteAccount
+  let logger: Logger
+
+  const createUseCase = () => new DeleteAccountsFromCSVFile(csvFileReader, deleteAccount, logger)
+
+  beforeEach(() => {
+    csvFileReader = {} as jest.Mocked<CSVFileReaderInterface>
+    csvFileReader.getValues = jest.fn().mockResolvedValue(Result.ok(['email1']))
+
+    deleteAccount = {} as jest.Mocked<DeleteAccount>
+    deleteAccount.execute = jest.fn().mockResolvedValue(Result.ok(''))
+
+    logger = {} as jest.Mocked<Logger>
+    logger.info = jest.fn()
+  })
+
+  it('should delete accounts', async () => {
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({ fileName: 'test.csv', dryRun: false })
+
+    expect(result.isFailed()).toBeFalsy()
+  })
+
+  it('should return error if csv file is invalid', async () => {
+    csvFileReader.getValues = jest.fn().mockResolvedValue(Result.fail('Oops'))
+
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({ fileName: 'test.csv', dryRun: false })
+
+    expect(result.isFailed()).toBeTruthy()
+  })
+
+  it('should return error if csv file is empty', async () => {
+    csvFileReader.getValues = jest.fn().mockResolvedValue(Result.ok([]))
+
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({ fileName: 'test.csv', dryRun: false })
+
+    expect(result.isFailed()).toBeTruthy()
+  })
+
+  it('should do nothing on a dry run', async () => {
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({ fileName: 'test.csv', dryRun: true })
+
+    expect(deleteAccount.execute).not.toHaveBeenCalled()
+    expect(result.isFailed()).toBeFalsy()
+  })
+
+  it('should return error if delete account fails', async () => {
+    deleteAccount.execute = jest.fn().mockResolvedValue(Result.fail('Oops'))
+
+    const useCase = createUseCase()
+
+    const result = await useCase.execute({ fileName: 'test.csv', dryRun: false })
+
+    expect(result.isFailed()).toBeTruthy()
+  })
+})

packages/auth/src/Domain/UseCase/DeleteAccountsFromCSVFile/DeleteAccountsFromCSVFile.ts

@@ -0,0 +1,47 @@
+import { Result, UseCaseInterface } from '@standardnotes/domain-core'
+import { Logger } from 'winston'
+
+import { DeleteAccount } from '../DeleteAccount/DeleteAccount'
+import { CSVFileReaderInterface } from '../../CSV/CSVFileReaderInterface'
+import { DeleteAccountsFromCSVFileDTO } from './DeleteAccountsFromCSVFileDTO'
+
+export class DeleteAccountsFromCSVFile implements UseCaseInterface<void> {
+  constructor(
+    private csvFileReader: CSVFileReaderInterface,
+    private deleteAccount: DeleteAccount,
+    private logger: Logger,
+  ) {}
+
+  async execute(dto: DeleteAccountsFromCSVFileDTO): Promise<Result<void>> {
+    const emailsOrError = await this.csvFileReader.getValues(dto.fileName)
+    if (emailsOrError.isFailed()) {
+      return Result.fail(emailsOrError.getError())
+    }
+    const emails = emailsOrError.getValue()
+
+    if (emails.length === 0) {
+      return Result.fail(`No emails found in CSV file ${dto.fileName}`)
+    }
+
+    if (dto.dryRun) {
+      const firstTenEmails = emails.slice(0, 10)
+      this.logger.info(
+        `Dry run mode enabled. Would delete ${emails.length} accounts. First 10 emails: ${firstTenEmails}`,
+      )
+
+      return Result.ok()
+    }
+
+    for (const email of emails) {
+      const deleteAccountOrError = await this.deleteAccount.execute({
+        username: email,
+      })
+
+      if (deleteAccountOrError.isFailed()) {
+        return Result.fail(deleteAccountOrError.getError())
+      }
+    }
+
+    return Result.ok()
+  }
+}

packages/auth/src/Domain/UseCase/DeleteAccountsFromCSVFile/DeleteAccountsFromCSVFileDTO.ts

@@ -0,0 +1,4 @@
+export interface DeleteAccountsFromCSVFileDTO {
+  fileName: string
+  dryRun: boolean
+}

packages/auth/src/Infra/S3/S3CsvFileReader.ts

@@ -0,0 +1,32 @@
+import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3'
+import { Result } from '@standardnotes/domain-core'
+
+import { CSVFileReaderInterface } from '../../Domain/CSV/CSVFileReaderInterface'
+
+export class S3CsvFileReader implements CSVFileReaderInterface {
+  constructor(
+    private s3BucketName: string,
+    private s3Client: S3Client,
+  ) {}
+
+  async getValues(fileName: string): Promise<Result<string[]>> {
+    if (this.s3BucketName.length === 0) {
+      return Result.fail('S3 bucket name is not set')
+    }
+
+    const response = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: this.s3BucketName,
+        Key: fileName,
+      }),
+    )
+
+    if (response.Body === undefined) {
+      return Result.fail(`Could not find CSV file at path: ${fileName}`)
+    }
+
+    const csvContent = await response.Body.transformToString()
+
+    return Result.ok(csvContent.split('\n').filter((line) => line.length > 0))
+  }
+}