fix: optimize memory usage

Dockerfile

@@ -26,4 +26,17 @@ RUN yarn install --immutable
 
 RUN CI=true yarn build
 
+RUN mkdir -p \
+  /opt/bundled/syncing-server \
+  /opt/bundled/auth \
+  /opt/bundled/files \
+  /opt/bundled/revisions \
+  /opt/bundled/api-gateway
+
+RUN yarn workspace @standardnotes/syncing-server bundle --no-compress --output-directory /opt/bundled/syncing-server
+RUN yarn workspace @standardnotes/auth-server bundle --no-compress --output-directory /opt/bundled/auth
+RUN yarn workspace @standardnotes/files-server bundle --no-compress --output-directory /opt/bundled/files
+RUN yarn workspace @standardnotes/revisions-server bundle --no-compress --output-directory /opt/bundled/revisions
+RUN yarn workspace @standardnotes/api-gateway bundle --no-compress --output-directory /opt/bundled/api-gateway
+
 ENTRYPOINT ["docker-entrypoint.sh"]

docker/docker-entrypoint.sh

@@ -178,7 +178,7 @@ if [ -z "$AUTH_SERVER_U2F_REQUIRE_USER_VERIFICATION" ]; then
   export AUTH_SERVER_U2F_REQUIRE_USER_VERIFICATION=false
 fi
 
-printenv | grep AUTH_SERVER_ | sed 's/AUTH_SERVER_//g' > /opt/server/packages/auth/.env
+printenv | grep AUTH_SERVER_ | sed 's/AUTH_SERVER_//g' > /opt/bundled/auth/packages/auth/.env
 
 ##################
 # SYNCING SERVER #
@@ -239,7 +239,7 @@ if [ -z "$SYNCING_SERVER_FILE_UPLOAD_PATH" ]; then
   export SYNCING_SERVER_FILE_UPLOAD_PATH="data/uploads"
 fi
 
-printenv | grep SYNCING_SERVER_ | sed 's/SYNCING_SERVER_//g' > /opt/server/packages/syncing-server/.env
+printenv | grep SYNCING_SERVER_ | sed 's/SYNCING_SERVER_//g' > /opt/bundled/syncing-server/packages/syncing-server/.env
 
 
 ################
@@ -289,7 +289,7 @@ if [ -z "$FILES_SERVER_SQS_ENDPOINT" ]; then
   export FILES_SERVER_SQS_ENDPOINT="http://localstack:4566"
 fi
 
-printenv | grep FILES_SERVER_ | sed 's/FILES_SERVER_//g' > /opt/server/packages/files/.env
+printenv | grep FILES_SERVER_ | sed 's/FILES_SERVER_//g' > /opt/bundled/files/packages/files/.env
 
 #############
 # REVISIONS #
@@ -335,7 +335,7 @@ if [ -z "$REVISIONS_SERVER_SQS_ENDPOINT" ]; then
   export REVISIONS_SERVER_SQS_ENDPOINT="http://localstack:4566"
 fi
 
-printenv | grep REVISIONS_SERVER_ | sed 's/REVISIONS_SERVER_//g' > /opt/server/packages/revisions/.env
+printenv | grep REVISIONS_SERVER_ | sed 's/REVISIONS_SERVER_//g' > /opt/bundled/revisions/packages/revisions/.env
 
 ###############
 # API GATEWAY #
@@ -361,7 +361,7 @@ else
   export API_GATEWAY_FILES_SERVER_URL=$PUBLIC_FILES_SERVER_URL
 fi
 
-printenv | grep API_GATEWAY_ | sed 's/API_GATEWAY_//g' > /opt/server/packages/api-gateway/.env
+printenv | grep API_GATEWAY_ | sed 's/API_GATEWAY_//g' > /opt/bundled/api-gateway/packages/api-gateway/.env
 
 # Run supervisor
 

docker/supervisord.conf

@@ -3,72 +3,72 @@ nodaemon=true
 logfile=/tmp/supervisord.log
 
 [program:syncing-server]
-directory=/opt/server
-command=yarn supervisor:start:syncing-server
+directory=/opt/bundled/syncing-server
+command=yarn workspace @standardnotes/syncing-server supervisor:start
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/syncing-server.log
 stderr_logfile=/var/lib/server/logs/syncing-server.err
 
 [program:syncing-server-worker]
-directory=/opt/server
-command=yarn supervisor:start:syncing-server-worker
+directory=/opt/bundled/syncing-server
+command=yarn workspace @standardnotes/syncing-server supervisor:worker
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/syncing-server-worker.log
 stderr_logfile=/var/lib/server/logs/syncing-server-worker.err
 
 [program:auth]
-directory=/opt/server
-command=yarn supervisor:start:auth
+directory=/opt/bundled/auth
+command=yarn workspace @standardnotes/auth-server supervisor:start
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/auth.log
 stderr_logfile=/var/lib/server/logs/auth.err
 
 [program:auth-worker]
-directory=/opt/server
-command=yarn supervisor:start:auth-worker
+directory=/opt/bundled/auth
+command=yarn workspace @standardnotes/auth-server supervisor:worker
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/auth-worker.log
 stderr_logfile=/var/lib/server/logs/auth-worker.err
 
 [program:files]
-directory=/opt/server
-command=yarn supervisor:start:files
+directory=/opt/bundled/files
+command=yarn workspace @standardnotes/files-server supervisor:start
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/files.log
 stderr_logfile=/var/lib/server/logs/files.err
 
 [program:files-worker]
-directory=/opt/server
-command=yarn supervisor:start:files-worker
+directory=/opt/bundled/files
+command=yarn workspace @standardnotes/files-server supervisor:worker
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/files-worker.log
 stderr_logfile=/var/lib/server/logs/files-worker.err
 
 [program:revisions]
-directory=/opt/server
-command=yarn supervisor:start:revisions
+directory=/opt/bundled/revisions
+command=yarn workspace @standardnotes/revisions-server supervisor:start
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/revisions.log
 stderr_logfile=/var/lib/server/logs/revisions.err
 
 [program:revisions-worker]
-directory=/opt/server
-command=yarn supervisor:start:revisions-worker
+directory=/opt/bundled/revisions
+command=yarn workspace @standardnotes/revisions-server supervisor:worker
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/revisions-worker.log
 stderr_logfile=/var/lib/server/logs/revisions-worker.err
 
 [program:api-gateway]
-directory=/opt/server
-command=yarn supervisor:start:api-gateway
+directory=/opt/bundled/api-gateway
+command=yarn workspace @standardnotes/api-gateway supervisor:start
 autostart=true
 autorestart=true
 stdout_logfile=/var/lib/server/logs/api-gateway.log

package.json

@@ -12,47 +12,8 @@
   },
   "scripts": {
     "lint": "yarn workspaces foreach -p -j 10 --verbose run lint",
-    "lint:auth": "yarn workspace @standardnotes/auth-server lint",
-    "lint:scheduler": "yarn workspace @standardnotes/scheduler-server lint",
-    "lint:syncing-server-js": "yarn workspace @standardnotes/syncing-server lint",
-    "lint:files": "yarn workspace @standardnotes/files-server lint",
-    "lint:api-gateway": "yarn workspace @standardnotes/api-gateway lint",
-    "lint:event-store": "yarn workspace @standardnotes/event-store lint",
-    "lint:websockets": "yarn workspace @standardnotes/websockets-server lint",
-    "lint:workspace": "yarn workspace @standardnotes/workspace-server lint",
-    "lint:analytics": "yarn workspace @standardnotes/analytics lint",
-    "lint:revisions": "yarn workspace @standardnotes/revisions-server lint",
     "clean": "yarn workspaces foreach -p --verbose run clean",
     "setup:env": "cp .env.sample .env && yarn workspaces foreach -p --verbose run setup:env",
-    "start:auth": "yarn workspace @standardnotes/auth-server start",
-    "start:auth-worker": "yarn workspace @standardnotes/auth-server worker",
-    "start:scheduler": "yarn workspace @standardnotes/scheduler-server worker",
-    "start:syncing-server": "yarn workspace @standardnotes/syncing-server start",
-    "start:syncing-server-worker": "yarn workspace @standardnotes/syncing-server worker",
-    "start:files": "yarn workspace @standardnotes/files-server start",
-    "start:files-worker": "yarn workspace @standardnotes/files-server worker",
-    "start:api-gateway": "yarn workspace @standardnotes/api-gateway start",
-    "start:websockets": "yarn workspace @standardnotes/websockets-server start",
-    "start:workspace": "yarn workspace @standardnotes/workspace-server start",
-    "start:analytics": "yarn workspace @standardnotes/analytics worker",
-    "start:revisions": "yarn workspace @standardnotes/revisions-server start",
-    "start:revisions-worker": "yarn workspace @standardnotes/revisions-server worker",
-    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
-    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
-    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
-    "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
-    "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
-    "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
-    "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
-    "supervisor:start:syncing-server": "yarn wait-for:infra && yarn start:syncing-server",
-    "supervisor:start:syncing-server-worker": "yarn wait-for:syncing-server && yarn start:syncing-server-worker",
-    "supervisor:start:auth": "yarn wait-for:syncing-server && yarn start:auth",
-    "supervisor:start:auth-worker": "yarn wait-for:auth && yarn start:auth-worker",
-    "supervisor:start:files": "yarn wait-for:auth && yarn start:files",
-    "supervisor:start:files-worker": "yarn wait-for:files && yarn start:files-worker",
-    "supervisor:start:revisions": "yarn wait-for:files && yarn start:revisions",
-    "supervisor:start:revisions-worker": "yarn wait-for:revisions && yarn start:revisions-worker",
-    "supervisor:start:api-gateway": "yarn wait-for:revisions && yarn start:api-gateway",
     "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
     "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
     "postversion": "./scripts/push-tags-one-by-one.sh",

packages/api-gateway/docker/wait-for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+WAIT_FOR_HOST="$1"
+shift
+WAIT_FOR_PORT="$1"
+shift
+
+while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do
+  echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start"
+  sleep 10
+done
+
+echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup."

packages/api-gateway/package.json

@@ -16,7 +16,13 @@
     "build": "tsc --build",
     "lint": "eslint . --ext .ts",
     "setup:env": "cp .env.sample .env",
+    "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
+    "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
+    "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
+    "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
+    "wait-for:services": "yarn wait-for:syncing-server && yarn wait-for:auth && yarn wait-for:files && yarn wait-for:revisions",
     "start": "yarn node dist/bin/server.js",
+    "supervisor:start": "yarn wait-for:services && yarn node dist/bin/server.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },
   "dependencies": {

packages/auth/docker/wait-for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+WAIT_FOR_HOST="$1"
+shift
+WAIT_FOR_PORT="$1"
+shift
+
+while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do
+  echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start"
+  sleep 10
+done
+
+echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup."

packages/auth/package.json

@@ -17,8 +17,14 @@
     "lint": "eslint . --ext .ts",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
+    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
+    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
+    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
+    "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
+    "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
+    "supervisor:worker": "yarn wait-for:auth && yarn node dist/bin/worker.js",
     "cleanup": "yarn node dist/bin/cleanup.js",
     "stats": "yarn node dist/bin/stats.js",
     "daily-backup:email": "yarn node dist/bin/backup.js email daily",

packages/files/docker/wait-for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+WAIT_FOR_HOST="$1"
+shift
+WAIT_FOR_PORT="$1"
+shift
+
+while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do
+  echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start"
+  sleep 10
+done
+
+echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup."

packages/files/package.json

@@ -20,8 +20,14 @@
     "lint": "eslint . --ext .ts",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
+    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
+    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
+    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
+    "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
+    "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
+    "supervisor:worker": "yarn wait-for:files && yarn node dist/bin/worker.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },
   "dependencies": {

packages/revisions/docker/wait-for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+WAIT_FOR_HOST="$1"
+shift
+WAIT_FOR_PORT="$1"
+shift
+
+while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do
+  echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start"
+  sleep 10
+done
+
+echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup."

packages/revisions/package.json

@@ -19,7 +19,13 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
+    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
+    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
+    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
+    "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
+    "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js",
+    "supervisor:worker": "yarn wait-for:revisions && yarn node dist/bin/worker.js",
     "worker": "yarn node dist/bin/worker.js"
   },
   "dependencies": {

packages/syncing-server/docker/wait-for.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+WAIT_FOR_HOST="$1"
+shift
+WAIT_FOR_PORT="$1"
+shift
+
+while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do
+  echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start"
+  sleep 10
+done
+
+echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup."

packages/syncing-server/package.json

@@ -19,8 +19,14 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "pretest": "yarn lint && yarn build",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
+    "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
+    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
+    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
+    "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
     "start": "yarn node dist/bin/server.js",
+    "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js",
     "worker": "yarn node dist/bin/worker.js",
+    "supervisor:worker": "yarn wait-for:syncing-server && yarn node dist/bin/worker.js",
     "content-size": "yarn node dist/bin/content.js",
     "revisions-ownership": "yarn node dist/bin/revisions.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"