feat: optimize memory usage (#444)

* fix(syncing-server): optimize memory usge

* fix: optimize remaining self hostable services

.github/workflows/pr.yml

@@ -94,3 +94,81 @@ jobs:
     name: E2E
     uses: standardnotes/server/.github/workflows/common-e2e.yml@main
     secrets: inherit
+
+  legacy_e2e:
+    needs: build
+    name: Legacy E2E
+    strategy:
+      matrix:
+        application:
+          - { "service_name": "api-gateway", "workspace_name": "@standardnotes/api-gateway", "e2e_tag_parameter_name": "api_gateway_image_tag", "package_path": "packages/api-gateway" }
+          - { "service_name": "auth", "workspace_name": "@standardnotes/auth-server", "e2e_tag_parameter_name": "auth_image_tag", "package_path": "packages/auth" }
+          - { "service_name": "files", "workspace_name": "@standardnotes/files-server", "e2e_tag_parameter_name": "files_image_tag", "package_path": "packages/files" }
+          - { "service_name": "revisions", "workspace_name": "@standardnotes/revisions-server", "e2e_tag_parameter_name": "revisions_image_tag", "package_path": "packages/revisions"}
+          - { "service_name": "syncing-server-js",  "workspace_name": "@standardnotes/syncing-server", "e2e_tag_parameter_name": "syncing_server_js_image_tag", "package_path": "packages/syncing-server" }
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Create Bundle Dir
+      id: bundle-dir
+      run: echo "temp_dir=$(mktemp -d -t ${{ matrix.application.service_name }}-${{ github.sha }}-XXXXXXX)" >> $GITHUB_OUTPUT
+
+    - name: Cache build
+      id: cache-build
+      uses: actions/cache@v3
+      with:
+        path: |
+          packages/**/dist
+          ${{ needs.legacy_e2e.outputs.temp_dir }}
+        key: ${{ runner.os }}-build-${{ github.sha }}
+
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        registry-url: 'https://registry.npmjs.org'
+        node-version-file: '.nvmrc'
+
+    - name: Build
+      if: steps.cache-build.outputs.cache-hit != 'true'
+      run: yarn build
+
+    - name: Bundle
+      run: yarn workspace ${{ matrix.application.workspace_name }} bundle --no-compress --output-directory ${{ steps.bundle-dir.outputs.temp_dir }}
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@master
+      with:
+        platforms: all
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@master
+
+    - name: Publish Docker image for E2E testing
+      uses: docker/build-push-action@v3
+      with:
+        builder: ${{ steps.buildx.outputs.name }}
+        context: ${{ steps.bundle-dir.outputs.temp_dir }}
+        file: ${{ steps.bundle-dir.outputs.temp_dir }}/${{ matrix.application.package_path }}/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: standardnotes/${{ matrix.application.service_name }}:${{ github.sha }}
+
+    - name: Run E2E test suite
+      uses: convictional/trigger-workflow-and-wait@master
+      with:
+        owner: standardnotes
+        repo: self-hosted
+        github_token: ${{ secrets.CI_PAT_TOKEN }}
+        workflow_file_name: testing-with-updating-client-and-server.yml
+        wait_interval: 30
+        client_payload: '{"${{ matrix.application.e2e_tag_parameter_name }}": "${{ github.sha }}"}'
+        propagate_failure: true
+        trigger_workflow: true
+        wait_workflow: true

packages/api-gateway/Dockerfile

@@ -8,10 +8,10 @@ ENV NODE_ENV production
 
 RUN corepack enable
 
-WORKDIR /workspace
-
 COPY ./ /workspace
 
+WORKDIR /workspace/packages/api-gateway
+
 ENTRYPOINT [ "/workspace/packages/api-gateway/docker/entrypoint.sh" ]
 
 CMD [ "start-web" ]

packages/api-gateway/docker/entrypoint.sh

@@ -4,16 +4,9 @@ set -e
 COMMAND=$1 && shift 1
 
 case "$COMMAND" in
-  'start-local' )
-    echo "Building the project..."
-    yarn workspace @standardnotes/api-gateway build
-    echo "Starting Web..."
-    yarn workspace @standardnotes/api-gateway start
-    ;;
-
   'start-web' )
     echo "Starting Web..."
-    yarn workspace @standardnotes/api-gateway start
+    node docker/entrypoint-server.js
     ;;
 
    * )

packages/api-gateway/package.json

@@ -16,9 +16,7 @@
     "build": "tsc --build",
     "lint": "eslint . --ext .ts",
     "setup:env": "cp .env.sample .env",
-    "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
-    "supervisor:start": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT && node dist/bin/server.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },
   "dependencies": {

packages/api-gateway/supervisor/supervisor-server.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $SYNCING_SERVER_PORT
-node supervisor/entrypoint-server.js
+node docker/entrypoint-server.js

packages/auth/Dockerfile

@@ -8,10 +8,10 @@ ENV NODE_ENV production
 
 RUN corepack enable
 
-WORKDIR /workspace
-
 COPY ./ /workspace
 
+WORKDIR /workspace/packages/auth
+
 ENTRYPOINT [ "/workspace/packages/auth/docker/entrypoint.sh" ]
 
 CMD [ "start-web" ]

packages/auth/docker/entrypoint-backup.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/backup.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint-cleanup.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/cleanup.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint-content.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/content.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint-stats.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/stats.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint-user-email-backup.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/user_email_backup.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/auth/docker/entrypoint.sh

@@ -4,65 +4,60 @@ set -e
 COMMAND=$1 && shift 1
 
 case "$COMMAND" in
-  'start-local' )
-    echo "[Docker] Starting Web..."
-    yarn workspace @standardnotes/auth-server start:local
-    ;;
-
   'start-web' )
     echo "[Docker] Starting Web..."
-    yarn workspace @standardnotes/auth-server start
+    node docker/entrypoint-server.js
     ;;
 
   'start-worker' )
     echo "[Docker] Starting Worker..."
-    yarn workspace @standardnotes/auth-server worker
+    node docker/entrypoint-worker.js
     ;;
 
   'cleanup' )
     echo "[Docker] Starting Cleanup..."
-    yarn workspace @standardnotes/auth-server cleanup
+    node docker/entrypoint-cleanup.js
     ;;
 
   'stats' )
     echo "[Docker] Starting Persisting Stats..."
-    yarn workspace @standardnotes/auth-server stats
+    node docker/entrypoint-stats.js
     ;;
 
   'email-daily-backup' )
     echo "[Docker] Starting Email Daily Backup..."
-    yarn workspace @standardnotes/auth-server daily-backup:email
+    node docker/entrypoint-backup.js email daily
     ;;
 
   'email-weekly-backup' )
     echo "[Docker] Starting Email Weekly Backup..."
-    yarn workspace @standardnotes/auth-server weekly-backup:email
+    node docker/entrypoint-backup.js email weekly
     ;;
 
   'email-backup' )
     echo "[Docker] Starting Email Backup For Single User..."
     EMAIL=$1 && shift 1
-    yarn workspace @standardnotes/auth-server user-email-backup $EMAIL
+    node docker/entrypoint-user-email-backup.js $EMAIL
     ;;
 
   'dropbox-daily-backup' )
     echo "[Docker] Starting Dropbox Daily Backup..."
-    yarn workspace @standardnotes/auth-server daily-backup:dropbox
+    node docker/entrypoint-backup.js dropbox daily
     ;;
 
   'google-drive-daily-backup' )
     echo "[Docker] Starting Google Drive Daily Backup..."
-    yarn workspace @standardnotes/auth-server daily-backup:google_drive
+    node docker/entrypoint-backup.js google_drive daily
     ;;
 
   'one-drive-daily-backup' )
     echo "[Docker] Starting One Drive Daily Backup..."
-    yarn workspace @standardnotes/auth-server daily-backup:one_drive
+    node docker/entrypoint-backup.js one_drive daily
     ;;
 
   'content-recalculation' )
     echo "[Docker] Starting Content Size Recalculation..."
-    yarn workspace @standardnotes/auth-server content-recalculation
+    node docker/entrypoint-content.js
     ;;
 
    * )

packages/auth/package.json

@@ -17,12 +17,8 @@
     "lint": "eslint . --ext .ts",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
-    "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
-    "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
-    "supervisor:start": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT && node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
-    "supervisor:worker": "docker/wait-for.sh localhost $AUTH_SERVER_PORT && node dist/bin/worker.js",
     "cleanup": "yarn node dist/bin/cleanup.js",
     "stats": "yarn node dist/bin/stats.js",
     "daily-backup:email": "yarn node dist/bin/backup.js email daily",

packages/auth/supervisor/supervisor-server.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $SYNCING_SERVER_PORT
-node supervisor/entrypoint-server.js
+node docker/entrypoint-server.js

packages/auth/supervisor/supervisor-worker.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $AUTH_SERVER_PORT
-node supervisor/entrypoint-worker.js
+node docker/entrypoint-worker.js

packages/files/Dockerfile

@@ -8,10 +8,10 @@ ENV NODE_ENV production
 
 RUN corepack enable
 
-WORKDIR /workspace
-
 COPY ./ /workspace
 
+WORKDIR /workspace/packages/files
+
 ENTRYPOINT [ "/workspace/packages/files/docker/entrypoint.sh" ]
 
 CMD [ "start-web" ]

packages/files/docker/entrypoint.sh

@@ -4,19 +4,14 @@ set -e
 COMMAND=$1 && shift 1
 
 case "$COMMAND" in
-  'start-local')
-    echo "Starting Web in Local Mode..."
-    yarn workspace @standardnotes/files-server start:local
-    ;;
-
   'start-web' )
     echo "Starting Web..."
-    yarn workspace @standardnotes/files-server start
+    node docker/entrypoint-server.js
     ;;
 
   'start-worker' )
     echo "Starting Worker..."
-    yarn workspace @standardnotes/files-server worker
+    node docker/entrypoint-worker.js
     ;;
 
    * )

packages/files/package.json

@@ -20,12 +20,8 @@
     "lint": "eslint . --ext .ts",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
-    "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
-    "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
-    "supervisor:start": "docker/wait-for.sh localhost $AUTH_SERVER_PORT && node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
-    "supervisor:worker": "docker/wait-for.sh localhost $FILES_SERVER_PORT && node dist/bin/worker.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },
   "dependencies": {

packages/files/supervisor/supervisor-server.sh

@@ -4,4 +4,4 @@ set -euo pipefail
 
 sh supervisor/wait-for.sh $DB_HOST $DB_PORT
 sh supervisor/wait-for.sh $REDIS_HOST $REDIS_PORT
-node supervisor/entrypoint-server.js
+node docker/entrypoint-server.js

packages/files/supervisor/supervisor-worker.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $SYNCING_SERVER_PORT
-node supervisor/entrypoint-worker.js
+node docker/entrypoint-worker.js

packages/revisions/Dockerfile

@@ -8,10 +8,10 @@ ENV NODE_ENV production
 
 RUN corepack enable
 
-WORKDIR /workspace
-
 COPY ./ /workspace
 
+WORKDIR /workspace/packages/revisions
+
 ENTRYPOINT [ "/workspace/packages/revisions/docker/entrypoint.sh" ]
 
 CMD [ "start-web" ]

packages/revisions/docker/entrypoint.sh

@@ -6,12 +6,12 @@ COMMAND=$1 && shift 1
 case "$COMMAND" in
   'start-web' )
     echo "Starting Web..."
-    yarn workspace @standardnotes/revisions-server start
+    node docker/entrypoint-server.js
     ;;
 
   'start-worker' )
     echo "Starting Worker..."
-    yarn workspace @standardnotes/revisions-server worker
+    node docker/entrypoint-worker.js
     ;;
 
    * )

packages/revisions/package.json

@@ -19,11 +19,7 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
-    "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
-    "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
-    "supervisor:start": "docker/wait-for.sh localhost $FILES_SERVER_PORT && node dist/bin/server.js",
-    "supervisor:worker": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT && node dist/bin/worker.js",
     "worker": "yarn node dist/bin/worker.js"
   },
   "dependencies": {

packages/revisions/supervisor/supervisor-server.sh

@@ -4,4 +4,4 @@ set -euo pipefail
 
 sh supervisor/wait-for.sh $DB_HOST $DB_PORT
 sh supervisor/wait-for.sh $REDIS_HOST $REDIS_PORT
-node supervisor/entrypoint-server.js
+node docker/entrypoint-server.js

packages/revisions/supervisor/supervisor-worker.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $SYNCING_SERVER_PORT
-node supervisor/entrypoint-worker.js
+node docker/entrypoint-worker.js

packages/syncing-server/Dockerfile

@@ -8,10 +8,10 @@ ENV NODE_ENV production
 
 RUN corepack enable
 
-WORKDIR /workspace
-
 COPY ./ /workspace
 
+WORKDIR /workspace/packages/syncing-server
+
 ENTRYPOINT [ "/workspace/packages/syncing-server/docker/entrypoint.sh" ]
 
 CMD [ "start-web" ]

packages/syncing-server/docker/entrypoint-content.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const path = require('path')
+
+const pnp = require(path.normalize(path.resolve(__dirname, '../../..', '.pnp.cjs'))).setup()
+
+const index = require(path.normalize(path.resolve(__dirname, '../dist/bin/content.js')))
+
+Object.defineProperty(exports, '__esModule', { value: true })
+
+exports.default = index

packages/syncing-server/docker/entrypoint.sh

@@ -4,25 +4,20 @@ set -e
 COMMAND=$1 && shift 1
 
 case "$COMMAND" in
-  'start-local')
-    echo "[Docker] Starting Web in Local Mode..."
-    yarn workspace @standardnotes/syncing-server start:local
-    ;;
-
   'start-web' )
     echo "[Docker] Starting Web..."
-    yarn workspace @standardnotes/syncing-server start
+    node docker/entrypoint-server.js
     ;;
 
   'start-worker' )
     echo "[Docker] Starting Worker..."
-    yarn workspace @standardnotes/syncing-server worker
+    node docker/entrypoint-worker.js
     ;;
 
   'content-size-recalculate' )
     echo "[Docker] Starting Content Size Recalculation..."
     USER_UUID=$1 && shift 1
-    yarn workspace @standardnotes/syncing-server content-size $USER_UUID
+    node docker/entrypoint-content.js $USER_UUID
     ;;
 
    * )

packages/syncing-server/package.json

@@ -19,14 +19,8 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
-    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
-    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
-    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
-    "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
-    "supervisor:start": "docker/wait-for.sh $DB_HOST $DB_PORT && docker/wait-for.sh $REDIS_HOST $REDIS_PORT && node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
-    "supervisor:worker": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT && node dist/bin/worker.js",
     "content-size": "yarn node dist/bin/content.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },

packages/syncing-server/supervisor/supervisor-server.sh

@@ -4,4 +4,4 @@ set -euo pipefail
 
 sh supervisor/wait-for.sh $DB_HOST $DB_PORT
 sh supervisor/wait-for.sh $REDIS_HOST $REDIS_PORT
-node supervisor/entrypoint-server.js
+node docker/entrypoint-server.js

packages/syncing-server/supervisor/supervisor-worker.sh

@@ -3,4 +3,4 @@
 set -euo pipefail
 
 sh supervisor/wait-for.sh localhost $SYNCING_SERVER_PORT
-node supervisor/entrypoint-worker.js
+node docker/entrypoint-worker.js