From c9289204c419a8cd54ce705bd20a3a4ee43c249d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Mon, 6 Feb 2023 11:20:11 +0100 Subject: [PATCH] revisions setup --- .dockerignore | 2 + docker/docker-entrypoint.sh | 52 +++++++++++++++++++ docker/supervisord.conf | 18 ++++++- package.json | 6 ++- .../migrations/1669113322388-init.ts | 16 +++++- .../1669636497932-remove-date-indexes.ts | 17 +++++- 6 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..3f6fff7b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +dist +coverage \ No newline at end of file diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 33acca2ea..c418a3cc2 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -22,6 +22,10 @@ else export FILES_SERVER_PORT=$EXPOSED_FILES_SERVER_PORT fi +if [ -z "$REVISIONS_SERVER_PORT" ]; then + export REVISIONS_SERVER_PORT=3105 +fi + ###### # DB # ###### @@ -294,6 +298,54 @@ fi printenv | grep FILES_SERVER_ | sed 's/FILES_SERVER_//g' > /opt/server/packages/files/.env +############# +# REVISIONS # +############# + +if [ -z "$REVISIONS_SERVER_LOG_LEVEL" ]; then + export REVISIONS_SERVER_LOG_LEVEL="info" +fi + +export REVISIONS_SERVER_NODE_ENV="production" +export REVISIONS_SERVER_VERSION="local" + +export REVISIONS_SERVER_REDIS_EVENTS_CHANNEL="revisions-events" + +export REVISIONS_SERVER_NEW_RELIC_ENABLED=false + +if [ -z "$REVISIONS_SERVER_SNS_TOPIC_ARN" ]; then + export REVISIONS_SERVER_SNS_TOPIC_ARN="arn:aws:sns:us-east-1:000000000000:revisions-server-local-topic" +fi +if [ -z "$REVISIONS_SERVER_SNS_ENDPOINT" ]; then + export REVISIONS_SERVER_SNS_ENDPOINT="http://localstack:4566" +fi +if [ -z "$REVISIONS_SERVER_SNS_SECRET_ACCESS_KEY" ]; then + export REVISIONS_SERVER_SNS_SECRET_ACCESS_KEY="x" +fi +if [ -z "$REVISIONS_SERVER_SNS_ACCESS_KEY_ID" ]; then + export REVISIONS_SERVER_SNS_ACCESS_KEY_ID="x" +fi +if [ -z "$REVISIONS_SERVER_SNS_AWS_REGION" ]; then + export REVISIONS_SERVER_SNS_AWS_REGION="us-east-1" +fi +if [ -z "$REVISIONS_SERVER_SQS_QUEUE_URL" ]; then + export REVISIONS_SERVER_SQS_QUEUE_URL="http://localstack:4566/000000000000/revisions-server-local-queue" +fi +if [ -z "$REVISIONS_SERVER_SQS_AWS_REGION" ]; then + export REVISIONS_SERVER_SQS_AWS_REGION="us-east-1" +fi +if [ -z "$REVISIONS_SERVER_SQS_ACCESS_KEY_ID" ]; then + export REVISIONS_SERVER_SQS_ACCESS_KEY_ID="x" +fi +if [ -z "$REVISIONS_SERVER_SQS_SECRET_ACCESS_KEY" ]; then + export REVISIONS_SERVER_SQS_SECRET_ACCESS_KEY="x" +fi +if [ -z "$REVISIONS_SERVER_SQS_ENDPOINT" ]; then + export REVISIONS_SERVER_SQS_ENDPOINT="http://localstack:4566" +fi + +printenv | grep REVISIONS_SERVER_ | sed 's/REVISIONS_SERVER_//g' > /opt/server/packages/revisions/.env + ############### # API GATEWAY # ############### diff --git a/docker/supervisord.conf b/docker/supervisord.conf index a3e03f25c..24480bf22 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -56,4 +56,20 @@ command=yarn start:files-worker autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/files-worker.log -stderr_logfile=/var/lib/server/logs/files-worker.err \ No newline at end of file +stderr_logfile=/var/lib/server/logs/files-worker.err + +[program:revisions] +directory=/opt/server +command=yarn start:revisions +autostart=true +autorestart=true +stdout_logfile=/var/lib/server/logs/revisions.log +stderr_logfile=/var/lib/server/logs/revisions.err + +[program:revisions-worker] +directory=/opt/server +command=yarn start:revisions-worker +autostart=true +autorestart=true +stdout_logfile=/var/lib/server/logs/revisions-worker.log +stderr_logfile=/var/lib/server/logs/revisions-worker.err \ No newline at end of file diff --git a/package.json b/package.json index 7c1ba7086..70a6b385a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT", "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT", "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT", + "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT", "start:auth": "yarn wait-for:db && yarn workspace @standardnotes/auth-server start", "start:auth-worker": "yarn wait-for:db && yarn workspace @standardnotes/auth-server worker", "start:scheduler": "yarn workspace @standardnotes/scheduler-server worker", @@ -35,11 +36,12 @@ "start:syncing-server-worker": "yarn wait-for:db && yarn workspace @standardnotes/syncing-server worker", "start:files": "yarn wait-for:db && yarn workspace @standardnotes/files-server start", "start:files-worker": "yarn wait-for:db && yarn workspace @standardnotes/files-server worker", - "start:api-gateway": "yarn wait-for:auth && yarn wait-for:syncing-server && yarn wait-for:files && yarn workspace @standardnotes/api-gateway start", + "start:api-gateway": "yarn wait-for:auth && yarn wait-for:syncing-server && yarn wait-for:files && yarn wait-for:revisions && yarn workspace @standardnotes/api-gateway start", "start:websockets": "yarn workspace @standardnotes/websockets-server start", "start:workspace": "yarn workspace @standardnotes/workspace-server start", "start:analytics": "yarn workspace @standardnotes/analytics worker", - "start:revisions": "yarn workspace @standardnotes/revisions-server start", + "start:revisions": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server start", + "start:revisions-worker": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server worker", "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"", "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose", "postversion": "./scripts/push-tags-one-by-one.sh", diff --git a/packages/revisions/migrations/1669113322388-init.ts b/packages/revisions/migrations/1669113322388-init.ts index 16538506d..3b5a66c83 100644 --- a/packages/revisions/migrations/1669113322388-init.ts +++ b/packages/revisions/migrations/1669113322388-init.ts @@ -4,8 +4,10 @@ export class init1669113322388 implements MigrationInterface { name = 'init1669113322388' public async up(queryRunner: QueryRunner): Promise { + await this.syncSchemaBetweenLegacyRevisions(queryRunner) + await queryRunner.query( - 'CREATE TABLE `revisions` (`uuid` varchar(36) NOT NULL, `item_uuid` varchar(36) NOT NULL, `user_uuid` varchar(36) NOT NULL, `content` mediumtext NULL, `content_type` varchar(255) NULL, `items_key_id` varchar(255) NULL, `enc_item_key` text NULL, `auth_hash` varchar(255) NULL, `creation_date` date NULL, `created_at` datetime(6) NULL, `updated_at` datetime(6) NULL, INDEX `item_uuid` (`item_uuid`), INDEX `user_uuid` (`user_uuid`), INDEX `creation_date` (`creation_date`), INDEX `created_at` (`created_at`), PRIMARY KEY (`uuid`)) ENGINE=InnoDB', + 'CREATE TABLE IF NOT EXISTS `revisions` (`uuid` varchar(36) NOT NULL, `item_uuid` varchar(36) NOT NULL, `user_uuid` varchar(36) NOT NULL, `content` mediumtext NULL, `content_type` varchar(255) NULL, `items_key_id` varchar(255) NULL, `enc_item_key` text NULL, `auth_hash` varchar(255) NULL, `creation_date` date NULL, `created_at` datetime(6) NULL, `updated_at` datetime(6) NULL, INDEX `item_uuid` (`item_uuid`), INDEX `user_uuid` (`user_uuid`), INDEX `creation_date` (`creation_date`), INDEX `created_at` (`created_at`), PRIMARY KEY (`uuid`)) ENGINE=InnoDB', ) } @@ -16,4 +18,16 @@ export class init1669113322388 implements MigrationInterface { await queryRunner.query('DROP INDEX `item_uuid` ON `revisions`') await queryRunner.query('DROP TABLE `revisions`') } + + private async syncSchemaBetweenLegacyRevisions(queryRunner: QueryRunner): Promise { + const revisionsTableExistsQueryResult = await queryRunner.manager.query( + 'SELECT COUNT(*) as count FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = "revisions"', + ) + const revisionsTableExists = revisionsTableExistsQueryResult[0].count === 1 + if (!revisionsTableExists) { + return + } + + await queryRunner.query('ALTER TABLE `revisions` ADD COLUMN `user_uuid` varchar(36) NULL') + } } diff --git a/packages/revisions/migrations/1669636497932-remove-date-indexes.ts b/packages/revisions/migrations/1669636497932-remove-date-indexes.ts index ac1586cbe..9e644078c 100644 --- a/packages/revisions/migrations/1669636497932-remove-date-indexes.ts +++ b/packages/revisions/migrations/1669636497932-remove-date-indexes.ts @@ -4,8 +4,21 @@ export class removeDateIndexes1669636497932 implements MigrationInterface { name = 'removeDateIndexes1669636497932' public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query('DROP INDEX `created_at` ON `revisions`') - await queryRunner.query('DROP INDEX `creation_date` ON `revisions`') + const indexRevisionsOnCreatedAt = await queryRunner.manager.query( + 'SHOW INDEX FROM `revisions` where `key_name` = "created_at"', + ) + const indexRevisionsOnCreatedAtExist = indexRevisionsOnCreatedAt && indexRevisionsOnCreatedAt.length > 0 + if (indexRevisionsOnCreatedAtExist) { + await queryRunner.query('DROP INDEX `created_at` ON `revisions`') + } + + const indexRevisionsOnCreationDate = await queryRunner.manager.query( + 'SHOW INDEX FROM `revisions` where `key_name` = "creation_date"', + ) + const indexRevisionsOnCreationDateAtExist = indexRevisionsOnCreationDate && indexRevisionsOnCreationDate.length > 0 + if (indexRevisionsOnCreationDateAtExist) { + await queryRunner.query('DROP INDEX `creation_date` ON `revisions`') + } } public async down(queryRunner: QueryRunner): Promise {