feat(revisions): add sqlite driver (#575)

.pnp.cjs

@@ -4500,6 +4500,7 @@ const RAW_RUNTIME_STATE =
           ["@standardnotes/responses", "npm:1.13.9"],\
           ["@standardnotes/security", "workspace:packages/security"],\
           ["@standardnotes/time", "workspace:packages/time"],\
+          ["@types/better-sqlite3", "npm:7.6.4"],\
           ["@types/cors", "npm:2.8.12"],\
           ["@types/dotenv", "npm:8.2.0"],\
           ["@types/express", "npm:4.17.14"],\
@@ -4507,6 +4508,7 @@ const RAW_RUNTIME_STATE =
           ["@types/jest", "npm:29.1.1"],\
           ["@types/newrelic", "npm:9.13.0"],\
           ["@typescript-eslint/eslint-plugin", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:5.48.2"],\
+          ["better-sqlite3", "npm:8.3.0"],\
           ["cors", "npm:2.8.5"],\
           ["dotenv", "npm:16.0.1"],\
           ["eslint", "npm:8.32.0"],\
@@ -4713,7 +4715,7 @@ const RAW_RUNTIME_STATE =
           ["prettyjson", "npm:1.2.5"],\
           ["reflect-metadata", "npm:0.1.13"],\
           ["ts-jest", "virtual:fd909b174d079e30b336c4ce72c38a88c1e447767b1a8dd7655e07719a1e31b97807f0931368724fc78897ff15e6a6d00b83316c0f76d11f85111f342e08bb79#npm:29.0.3"],\
-          ["typeorm", "virtual:67ad3a1ca34e24ce4821cc48979e98af0c3e5dd7aabc7ad0b5d22d1d977d6f943f81c9f141a420105ebdc61ef777e508a96c7946081decd98f8c30543d468b33#npm:0.3.10"],\
+          ["typeorm", "virtual:365b8c88cdf194291829ee28b79556e2328175d26a621363e703848100bea0042e9500db2a1206c9bbc3a4a76a1d169639ef774b2ea3a1a98584a9936b58c6be#npm:0.3.10"],\
           ["typescript", "patch:typescript@npm%3A4.8.4#optional!builtin<compat/typescript>::version=4.8.4&hash=701156"],\
           ["ua-parser-js", "npm:1.0.32"],\
           ["uuid", "npm:9.0.0"],\
@@ -14762,100 +14764,6 @@ const RAW_RUNTIME_STATE =
           ["@google-cloud/spanner", null],\
           ["@sap/hana-client", null],\
           ["@sqltools/formatter", "npm:1.2.5"],\
-          ["@types/better-sqlite3", null],\
-          ["@types/google-cloud__spanner", null],\
-          ["@types/hdb-pool", null],\
-          ["@types/ioredis", null],\
-          ["@types/mongodb", null],\
-          ["@types/mssql", null],\
-          ["@types/mysql2", null],\
-          ["@types/oracledb", null],\
-          ["@types/pg", null],\
-          ["@types/pg-native", null],\
-          ["@types/pg-query-stream", null],\
-          ["@types/redis", null],\
-          ["@types/sap__hana-client", null],\
-          ["@types/sql.js", null],\
-          ["@types/sqlite3", null],\
-          ["@types/ts-node", null],\
-          ["@types/typeorm-aurora-data-api-driver", null],\
-          ["app-root-path", "npm:3.1.0"],\
-          ["better-sqlite3", null],\
-          ["buffer", "npm:6.0.3"],\
-          ["chalk", "npm:4.1.2"],\
-          ["cli-highlight", "npm:2.1.11"],\
-          ["date-fns", "npm:2.29.3"],\
-          ["debug", "virtual:b86a9fb34323a98c6519528ed55faa0d9b44ca8879307c0b29aa384bde47ff59a7d0c9051b31246f14521dfb71ba3c5d6d0b35c29fffc17bf875aa6ad977d9e8#npm:4.3.4"],\
-          ["dotenv", "npm:16.0.3"],\
-          ["glob", "npm:7.2.3"],\
-          ["hdb-pool", null],\
-          ["ioredis", null],\
-          ["js-yaml", "npm:4.1.0"],\
-          ["mkdirp", "npm:1.0.4"],\
-          ["mongodb", null],\
-          ["mssql", null],\
-          ["mysql2", "npm:3.0.1"],\
-          ["oracledb", null],\
-          ["pg", null],\
-          ["pg-native", null],\
-          ["pg-query-stream", null],\
-          ["redis", null],\
-          ["reflect-metadata", "npm:0.1.13"],\
-          ["sha.js", "npm:2.4.11"],\
-          ["sql.js", null],\
-          ["sqlite3", null],\
-          ["ts-node", null],\
-          ["tslib", "npm:2.4.0"],\
-          ["typeorm-aurora-data-api-driver", null],\
-          ["uuid", "npm:8.3.2"],\
-          ["xml2js", "npm:0.4.23"],\
-          ["yargs", "npm:17.5.1"]\
-        ],\
-        "packagePeers": [\
-          "@google-cloud/spanner",\
-          "@sap/hana-client",\
-          "@types/better-sqlite3",\
-          "@types/google-cloud__spanner",\
-          "@types/hdb-pool",\
-          "@types/ioredis",\
-          "@types/mongodb",\
-          "@types/mssql",\
-          "@types/mysql2",\
-          "@types/oracledb",\
-          "@types/pg-native",\
-          "@types/pg-query-stream",\
-          "@types/pg",\
-          "@types/redis",\
-          "@types/sap__hana-client",\
-          "@types/sql.js",\
-          "@types/sqlite3",\
-          "@types/ts-node",\
-          "@types/typeorm-aurora-data-api-driver",\
-          "better-sqlite3",\
-          "hdb-pool",\
-          "ioredis",\
-          "mongodb",\
-          "mssql",\
-          "mysql2",\
-          "oracledb",\
-          "pg-native",\
-          "pg-query-stream",\
-          "pg",\
-          "redis",\
-          "sql.js",\
-          "sqlite3",\
-          "ts-node",\
-          "typeorm-aurora-data-api-driver"\
-        ],\
-        "linkType": "HARD"\
-      }],\
-      ["virtual:67ad3a1ca34e24ce4821cc48979e98af0c3e5dd7aabc7ad0b5d22d1d977d6f943f81c9f141a420105ebdc61ef777e508a96c7946081decd98f8c30543d468b33#npm:0.3.10", {\
-        "packageLocation": "./.yarn/__virtual__/typeorm-virtual-f86b034570/0/cache/typeorm-npm-0.3.10-4667857f33-749e1a6777.zip/node_modules/typeorm/",\
-        "packageDependencies": [\
-          ["typeorm", "virtual:67ad3a1ca34e24ce4821cc48979e98af0c3e5dd7aabc7ad0b5d22d1d977d6f943f81c9f141a420105ebdc61ef777e508a96c7946081decd98f8c30543d468b33#npm:0.3.10"],\
-          ["@google-cloud/spanner", null],\
-          ["@sap/hana-client", null],\
-          ["@sqltools/formatter", "npm:1.2.5"],\
           ["@types/better-sqlite3", "npm:7.6.4"],\
           ["@types/google-cloud__spanner", null],\
           ["@types/hdb-pool", null],\

.yarn/unplugged/better-sqlite3-npm-8.3.0-d1ef3f5776/node_modules/better-sqlite3/build/Makefile

@@ -342,7 +342,7 @@ endif
 
 quiet_cmd_regen_makefile = ACTION Regenerating $@
 cmd_regen_makefile = cd $(srcdir); /Users/karolsojko/workspace/server/.yarn/unplugged/node-gyp-npm-9.0.0-0eccfca4d1/node_modules/node-gyp/gyp/gyp_main.py -fmake --ignore-environment "-Dlibrary=shared_library" "-Dvisibility=default" "-Dnode_root_dir=/Users/karolsojko/Library/Caches/node-gyp/18.15.0" "-Dnode_gyp_dir=/Users/karolsojko/workspace/server/.yarn/unplugged/node-gyp-npm-9.0.0-0eccfca4d1/node_modules/node-gyp" "-Dnode_lib_file=/Users/karolsojko/Library/Caches/node-gyp/18.15.0/<(target_arch)/node.lib" "-Dmodule_root_dir=/Users/karolsojko/workspace/server/.yarn/unplugged/better-sqlite3-npm-8.3.0-d1ef3f5776/node_modules/better-sqlite3" "-Dnode_engine=v8" "--depth=." "-Goutput_dir=." "--generator-output=build" -I/Users/karolsojko/workspace/server/.yarn/unplugged/better-sqlite3-npm-8.3.0-d1ef3f5776/node_modules/better-sqlite3/build/config.gypi -I/Users/karolsojko/workspace/server/.yarn/unplugged/node-gyp-npm-9.0.0-0eccfca4d1/node_modules/node-gyp/addon.gypi -I/Users/karolsojko/Library/Caches/node-gyp/18.15.0/include/node/common.gypi "--toplevel-dir=." binding.gyp
-Makefile: $(srcdir)/deps/defines.gypi $(srcdir)/../../../../../../../Library/Caches/node-gyp/18.15.0/include/node/common.gypi $(srcdir)/deps/sqlite3.gyp $(srcdir)/binding.gyp $(srcdir)/../../../node-gyp-npm-9.0.0-0eccfca4d1/node_modules/node-gyp/addon.gypi $(srcdir)/deps/common.gypi $(srcdir)/build/config.gypi
+Makefile: $(srcdir)/build/config.gypi $(srcdir)/binding.gyp $(srcdir)/../../../../../../../Library/Caches/node-gyp/18.15.0/include/node/common.gypi $(srcdir)/deps/defines.gypi $(srcdir)/../../../node-gyp-npm-9.0.0-0eccfca4d1/node_modules/node-gyp/addon.gypi $(srcdir)/deps/common.gypi $(srcdir)/deps/sqlite3.gyp
 	$(call do_cmd,regen_makefile)
 
 # "all" is a concatenation of the "all" targets from all the included

packages/revisions/migrations/sqlite/1682678053275-initial_boilerplate.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class initialBoilerplate1682678053275 implements MigrationInterface {
+  name = 'initialBoilerplate1682678053275'
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      'CREATE TABLE "revisions" ("uuid" varchar PRIMARY KEY NOT NULL, "item_uuid" varchar(36) NOT NULL, "user_uuid" varchar(36), "content" text, "content_type" varchar(255), "items_key_id" varchar(255), "enc_item_key" text, "auth_hash" varchar(255), "creation_date" date, "created_at" datetime(6), "updated_at" datetime(6))',
+    )
+    await queryRunner.query('CREATE INDEX "item_uuid" ON "revisions" ("item_uuid") ')
+    await queryRunner.query('CREATE INDEX "user_uuid" ON "revisions" ("user_uuid") ')
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('DROP INDEX "user_uuid"')
+    await queryRunner.query('DROP INDEX "item_uuid"')
+    await queryRunner.query('DROP TABLE "revisions"')
+  }
+}

packages/revisions/package.json

@@ -35,6 +35,7 @@
     "@standardnotes/responses": "^1.13.9",
     "@standardnotes/security": "workspace:^",
     "@standardnotes/time": "workspace:^",
+    "better-sqlite3": "^8.3.0",
     "cors": "2.8.5",
     "dotenv": "^16.0.1",
     "express": "^4.18.2",
@@ -47,6 +48,7 @@
     "winston": "^3.8.1"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.4",
     "@types/cors": "^2.8.9",
     "@types/dotenv": "^8.2.0",
     "@types/express": "^4.17.14",

packages/revisions/src/Bootstrap/CommonContainerConfigLoader.ts

@@ -2,10 +2,11 @@ import { MapperInterface } from '@standardnotes/domain-core'
 import { Container, interfaces } from 'inversify'
 import { Repository } from 'typeorm'
 import * as winston from 'winston'
+
 import { Revision } from '../Domain/Revision/Revision'
 import { RevisionMetadata } from '../Domain/Revision/RevisionMetadata'
 import { RevisionRepositoryInterface } from '../Domain/Revision/RevisionRepositoryInterface'
-import { MySQLRevisionRepository } from '../Infra/MySQL/MySQLRevisionRepository'
+import { TypeORMRevisionRepository } from '../Infra/TypeORM/TypeORMRevisionRepository'
 import { TypeORMRevision } from '../Infra/TypeORM/TypeORMRevision'
 import { RevisionMetadataPersistenceMapper } from '../Mapping/RevisionMetadataPersistenceMapper'
 import { RevisionPersistenceMapper } from '../Mapping/RevisionPersistenceMapper'
@@ -67,7 +68,7 @@ export class CommonContainerConfigLoader {
     container
       .bind<RevisionRepositoryInterface>(TYPES.RevisionRepository)
       .toDynamicValue((context: interfaces.Context) => {
-        return new MySQLRevisionRepository(
+        return new TypeORMRevisionRepository(
           context.container.get(TYPES.ORMRevisionRepository),
           context.container.get(TYPES.RevisionMetadataPersistenceMapper),
           context.container.get(TYPES.RevisionPersistenceMapper),

packages/revisions/src/Bootstrap/DataSource.ts

@@ -1,4 +1,6 @@
 import { DataSource, LoggerOptions } from 'typeorm'
+import { BetterSqlite3ConnectionOptions } from 'typeorm/driver/better-sqlite3/BetterSqlite3ConnectionOptions'
+import { MysqlConnectionOptions } from 'typeorm/driver/mysql/MysqlConnectionOptions'
 
 import { TypeORMRevision } from '../Infra/TypeORM/TypeORMRevision'
 
@@ -7,6 +9,8 @@ import { Env } from './Env'
 const env: Env = new Env()
 env.load()
 
+const isConfiguredForMySQL = env.get('DB_TYPE') === 'mysql'
+
 const maxQueryExecutionTime = env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
   ? +env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
   : 45_000
@@ -34,22 +38,32 @@ const replicationConfig = {
   restoreNodeTimeout: 5,
 }
 
-const dataSource = new DataSource({
+const commonDataSourceOptions = {
+  maxQueryExecutionTime,
+  entities: [TypeORMRevision],
+  migrations: [`dist/migrations/${isConfiguredForMySQL ? 'mysql' : 'sqlite'}/*.js`],
+  migrationsRun: true,
+  logging: <LoggerOptions>env.get('DB_DEBUG_LEVEL'),
+}
+
+const mySQLDataSourceOptions: MysqlConnectionOptions = {
+  ...commonDataSourceOptions,
   type: 'mysql',
   charset: 'utf8mb4',
   supportBigNumbers: true,
   bigNumberStrings: false,
-  maxQueryExecutionTime,
   replication: inReplicaMode ? replicationConfig : undefined,
   host: inReplicaMode ? undefined : env.get('DB_HOST'),
   port: inReplicaMode ? undefined : parseInt(env.get('DB_PORT')),
   username: inReplicaMode ? undefined : env.get('DB_USERNAME'),
   password: inReplicaMode ? undefined : env.get('DB_PASSWORD'),
   database: inReplicaMode ? undefined : env.get('DB_DATABASE'),
-  entities: [TypeORMRevision],
-  migrations: [env.get('DB_MIGRATIONS_PATH', true) ?? 'dist/migrations/*.js'],
-  migrationsRun: true,
-  logging: <LoggerOptions>env.get('DB_DEBUG_LEVEL'),
-})
+}
+
+const sqliteDataSourceOptions: BetterSqlite3ConnectionOptions = {
+  ...commonDataSourceOptions,
+  type: 'better-sqlite3',
+  database: `data/${env.get('DB_DATABASE')}.sqlite`,
+}
 
-export const AppDataSource = dataSource
+export const AppDataSource = new DataSource(isConfiguredForMySQL ? mySQLDataSourceOptions : sqliteDataSourceOptions)

packages/revisions/src/Infra/TypeORM/TypeORMRevision.ts

@@ -22,7 +22,7 @@ export class TypeORMRevision {
   declare userUuid: string | null
 
   @Column({
-    type: 'mediumtext',
+    type: 'text',
     nullable: true,
   })
   declare content: string | null

packages/revisions/src/Infra/MySQL/MySQLRevisionRepository.ts → packages/revisions/src/Infra/TypeORM/TypeORMRevisionRepository.ts

@@ -5,9 +5,9 @@ import { Logger } from 'winston'
 import { Revision } from '../../Domain/Revision/Revision'
 import { RevisionMetadata } from '../../Domain/Revision/RevisionMetadata'
 import { RevisionRepositoryInterface } from '../../Domain/Revision/RevisionRepositoryInterface'
-import { TypeORMRevision } from '../TypeORM/TypeORMRevision'
+import { TypeORMRevision } from './TypeORMRevision'
 
-export class MySQLRevisionRepository implements RevisionRepositoryInterface {
+export class TypeORMRevisionRepository implements RevisionRepositoryInterface {
   constructor(
     private ormRepository: Repository<TypeORMRevision>,
     private revisionMetadataMapper: MapperInterface<RevisionMetadata, TypeORMRevision>,
@@ -97,7 +97,7 @@ export class MySQLRevisionRepository implements RevisionRepositoryInterface {
     const simplifiedRevisions = await queryBuilder.getRawMany()
 
     this.logger.debug(
-      `Found ${simplifiedRevisions.length} revisions MySQL entries for item ${itemUuid.value}`,
+      `Found ${simplifiedRevisions.length} revisions entries for item ${itemUuid.value}`,
       simplifiedRevisions,
     )
 

yarn.lock

@@ -3604,6 +3604,7 @@ __metadata:
     "@standardnotes/responses": "npm:^1.13.9"
     "@standardnotes/security": "workspace:^"
     "@standardnotes/time": "workspace:^"
+    "@types/better-sqlite3": "npm:^7.6.4"
     "@types/cors": "npm:^2.8.9"
     "@types/dotenv": "npm:^8.2.0"
     "@types/express": "npm:^4.17.14"
@@ -3611,6 +3612,7 @@ __metadata:
     "@types/jest": "npm:^29.1.1"
     "@types/newrelic": "npm:^9.13.0"
     "@typescript-eslint/eslint-plugin": "npm:^5.48.2"
+    better-sqlite3: "npm:^8.3.0"
     cors: "npm:2.8.5"
     dotenv: "npm:^16.0.1"
     eslint: "npm:^8.32.0"