From e96fd6d69e1252842b5c91b1bedefa36e5d4a232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Thu, 9 Feb 2023 10:23:10 +0100 Subject: [PATCH] fix: optimize memory usage --- Dockerfile | 13 ++++++++ docker/docker-entrypoint.sh | 10 +++--- docker/supervisord.conf | 36 ++++++++++---------- package.json | 39 ---------------------- packages/api-gateway/docker/wait-for.sh | 13 ++++++++ packages/api-gateway/package.json | 6 ++++ packages/auth/docker/wait-for.sh | 13 ++++++++ packages/auth/package.json | 6 ++++ packages/files/docker/wait-for.sh | 13 ++++++++ packages/files/package.json | 6 ++++ packages/revisions/docker/wait-for.sh | 13 ++++++++ packages/revisions/package.json | 6 ++++ packages/syncing-server/docker/wait-for.sh | 13 ++++++++ packages/syncing-server/package.json | 6 ++++ 14 files changed, 131 insertions(+), 62 deletions(-) create mode 100755 packages/api-gateway/docker/wait-for.sh create mode 100755 packages/auth/docker/wait-for.sh create mode 100755 packages/files/docker/wait-for.sh create mode 100755 packages/revisions/docker/wait-for.sh create mode 100755 packages/syncing-server/docker/wait-for.sh diff --git a/Dockerfile b/Dockerfile index aa0df5d91..4d09f34e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,4 +26,17 @@ RUN yarn install --immutable RUN CI=true yarn build +RUN mkdir -p \ + /opt/bundled/syncing-server \ + /opt/bundled/auth \ + /opt/bundled/files \ + /opt/bundled/revisions \ + /opt/bundled/api-gateway + +RUN yarn workspace @standardnotes/syncing-server bundle --no-compress --output-directory /opt/bundled/syncing-server +RUN yarn workspace @standardnotes/auth-server bundle --no-compress --output-directory /opt/bundled/auth +RUN yarn workspace @standardnotes/files-server bundle --no-compress --output-directory /opt/bundled/files +RUN yarn workspace @standardnotes/revisions-server bundle --no-compress --output-directory /opt/bundled/revisions +RUN yarn workspace @standardnotes/api-gateway bundle --no-compress --output-directory /opt/bundled/api-gateway + ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 9ff66c8f5..2f71b2ab2 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -178,7 +178,7 @@ if [ -z "$AUTH_SERVER_U2F_REQUIRE_USER_VERIFICATION" ]; then export AUTH_SERVER_U2F_REQUIRE_USER_VERIFICATION=false fi -printenv | grep AUTH_SERVER_ | sed 's/AUTH_SERVER_//g' > /opt/server/packages/auth/.env +printenv | grep AUTH_SERVER_ | sed 's/AUTH_SERVER_//g' > /opt/bundled/auth/packages/auth/.env ################## # SYNCING SERVER # @@ -239,7 +239,7 @@ if [ -z "$SYNCING_SERVER_FILE_UPLOAD_PATH" ]; then export SYNCING_SERVER_FILE_UPLOAD_PATH="data/uploads" fi -printenv | grep SYNCING_SERVER_ | sed 's/SYNCING_SERVER_//g' > /opt/server/packages/syncing-server/.env +printenv | grep SYNCING_SERVER_ | sed 's/SYNCING_SERVER_//g' > /opt/bundled/syncing-server/packages/syncing-server/.env ################ @@ -289,7 +289,7 @@ if [ -z "$FILES_SERVER_SQS_ENDPOINT" ]; then export FILES_SERVER_SQS_ENDPOINT="http://localstack:4566" fi -printenv | grep FILES_SERVER_ | sed 's/FILES_SERVER_//g' > /opt/server/packages/files/.env +printenv | grep FILES_SERVER_ | sed 's/FILES_SERVER_//g' > /opt/bundled/files/packages/files/.env ############# # REVISIONS # @@ -335,7 +335,7 @@ if [ -z "$REVISIONS_SERVER_SQS_ENDPOINT" ]; then export REVISIONS_SERVER_SQS_ENDPOINT="http://localstack:4566" fi -printenv | grep REVISIONS_SERVER_ | sed 's/REVISIONS_SERVER_//g' > /opt/server/packages/revisions/.env +printenv | grep REVISIONS_SERVER_ | sed 's/REVISIONS_SERVER_//g' > /opt/bundled/revisions/packages/revisions/.env ############### # API GATEWAY # @@ -361,7 +361,7 @@ else export API_GATEWAY_FILES_SERVER_URL=$PUBLIC_FILES_SERVER_URL fi -printenv | grep API_GATEWAY_ | sed 's/API_GATEWAY_//g' > /opt/server/packages/api-gateway/.env +printenv | grep API_GATEWAY_ | sed 's/API_GATEWAY_//g' > /opt/bundled/api-gateway/packages/api-gateway/.env # Run supervisor diff --git a/docker/supervisord.conf b/docker/supervisord.conf index b5508c64a..f29e7fe9a 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -3,72 +3,72 @@ nodaemon=true logfile=/tmp/supervisord.log [program:syncing-server] -directory=/opt/server -command=yarn supervisor:start:syncing-server +directory=/opt/bundled/syncing-server +command=yarn workspace @standardnotes/syncing-server supervisor:start autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/syncing-server.log stderr_logfile=/var/lib/server/logs/syncing-server.err [program:syncing-server-worker] -directory=/opt/server -command=yarn supervisor:start:syncing-server-worker +directory=/opt/bundled/syncing-server +command=yarn workspace @standardnotes/syncing-server supervisor:worker autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/syncing-server-worker.log stderr_logfile=/var/lib/server/logs/syncing-server-worker.err [program:auth] -directory=/opt/server -command=yarn supervisor:start:auth +directory=/opt/bundled/auth +command=yarn workspace @standardnotes/auth-server supervisor:start autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/auth.log stderr_logfile=/var/lib/server/logs/auth.err [program:auth-worker] -directory=/opt/server -command=yarn supervisor:start:auth-worker +directory=/opt/bundled/auth +command=yarn workspace @standardnotes/auth-server supervisor:worker autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/auth-worker.log stderr_logfile=/var/lib/server/logs/auth-worker.err [program:files] -directory=/opt/server -command=yarn supervisor:start:files +directory=/opt/bundled/files +command=yarn workspace @standardnotes/files-server supervisor:start autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/files.log stderr_logfile=/var/lib/server/logs/files.err [program:files-worker] -directory=/opt/server -command=yarn supervisor:start:files-worker +directory=/opt/bundled/files +command=yarn workspace @standardnotes/files-server supervisor:worker autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/files-worker.log stderr_logfile=/var/lib/server/logs/files-worker.err [program:revisions] -directory=/opt/server -command=yarn supervisor:start:revisions +directory=/opt/bundled/revisions +command=yarn workspace @standardnotes/revisions-server supervisor:start autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/revisions.log stderr_logfile=/var/lib/server/logs/revisions.err [program:revisions-worker] -directory=/opt/server -command=yarn supervisor:start:revisions-worker +directory=/opt/bundled/revisions +command=yarn workspace @standardnotes/revisions-server supervisor:worker autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/revisions-worker.log stderr_logfile=/var/lib/server/logs/revisions-worker.err [program:api-gateway] -directory=/opt/server -command=yarn supervisor:start:api-gateway +directory=/opt/bundled/api-gateway +command=yarn workspace @standardnotes/api-gateway supervisor:start autostart=true autorestart=true stdout_logfile=/var/lib/server/logs/api-gateway.log diff --git a/package.json b/package.json index 2d1f61d6b..b175abea4 100644 --- a/package.json +++ b/package.json @@ -12,47 +12,8 @@ }, "scripts": { "lint": "yarn workspaces foreach -p -j 10 --verbose run lint", - "lint:auth": "yarn workspace @standardnotes/auth-server lint", - "lint:scheduler": "yarn workspace @standardnotes/scheduler-server lint", - "lint:syncing-server-js": "yarn workspace @standardnotes/syncing-server lint", - "lint:files": "yarn workspace @standardnotes/files-server lint", - "lint:api-gateway": "yarn workspace @standardnotes/api-gateway lint", - "lint:event-store": "yarn workspace @standardnotes/event-store lint", - "lint:websockets": "yarn workspace @standardnotes/websockets-server lint", - "lint:workspace": "yarn workspace @standardnotes/workspace-server lint", - "lint:analytics": "yarn workspace @standardnotes/analytics lint", - "lint:revisions": "yarn workspace @standardnotes/revisions-server lint", "clean": "yarn workspaces foreach -p --verbose run clean", "setup:env": "cp .env.sample .env && yarn workspaces foreach -p --verbose run setup:env", - "start:auth": "yarn workspace @standardnotes/auth-server start", - "start:auth-worker": "yarn workspace @standardnotes/auth-server worker", - "start:scheduler": "yarn workspace @standardnotes/scheduler-server worker", - "start:syncing-server": "yarn workspace @standardnotes/syncing-server start", - "start:syncing-server-worker": "yarn workspace @standardnotes/syncing-server worker", - "start:files": "yarn workspace @standardnotes/files-server start", - "start:files-worker": "yarn workspace @standardnotes/files-server worker", - "start:api-gateway": "yarn workspace @standardnotes/api-gateway start", - "start:websockets": "yarn workspace @standardnotes/websockets-server start", - "start:workspace": "yarn workspace @standardnotes/workspace-server start", - "start:analytics": "yarn workspace @standardnotes/analytics worker", - "start:revisions": "yarn workspace @standardnotes/revisions-server start", - "start:revisions-worker": "yarn workspace @standardnotes/revisions-server worker", - "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", - "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", - "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", - "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT", - "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT", - "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT", - "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT", - "supervisor:start:syncing-server": "yarn wait-for:infra && yarn start:syncing-server", - "supervisor:start:syncing-server-worker": "yarn wait-for:syncing-server && yarn start:syncing-server-worker", - "supervisor:start:auth": "yarn wait-for:syncing-server && yarn start:auth", - "supervisor:start:auth-worker": "yarn wait-for:auth && yarn start:auth-worker", - "supervisor:start:files": "yarn wait-for:auth && yarn start:files", - "supervisor:start:files-worker": "yarn wait-for:files && yarn start:files-worker", - "supervisor:start:revisions": "yarn wait-for:files && yarn start:revisions", - "supervisor:start:revisions-worker": "yarn wait-for:revisions && yarn start:revisions-worker", - "supervisor:start:api-gateway": "yarn wait-for:revisions && yarn start:api-gateway", "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"", "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose", "postversion": "./scripts/push-tags-one-by-one.sh", diff --git a/packages/api-gateway/docker/wait-for.sh b/packages/api-gateway/docker/wait-for.sh new file mode 100755 index 000000000..90438f0d2 --- /dev/null +++ b/packages/api-gateway/docker/wait-for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +WAIT_FOR_HOST="$1" +shift +WAIT_FOR_PORT="$1" +shift + +while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do + echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start" + sleep 10 +done + +echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup." \ No newline at end of file diff --git a/packages/api-gateway/package.json b/packages/api-gateway/package.json index 1f39bd402..25bbaf3b3 100644 --- a/packages/api-gateway/package.json +++ b/packages/api-gateway/package.json @@ -16,7 +16,13 @@ "build": "tsc --build", "lint": "eslint . --ext .ts", "setup:env": "cp .env.sample .env", + "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT", + "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT", + "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT", + "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT", + "wait-for:services": "yarn wait-for:syncing-server && yarn wait-for:auth && yarn wait-for:files && yarn wait-for:revisions", "start": "yarn node dist/bin/server.js", + "supervisor:start": "yarn wait-for:services && yarn node dist/bin/server.js", "upgrade:snjs": "yarn ncu -u '@standardnotes/*'" }, "dependencies": { diff --git a/packages/auth/docker/wait-for.sh b/packages/auth/docker/wait-for.sh new file mode 100755 index 000000000..90438f0d2 --- /dev/null +++ b/packages/auth/docker/wait-for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +WAIT_FOR_HOST="$1" +shift +WAIT_FOR_PORT="$1" +shift + +while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do + echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start" + sleep 10 +done + +echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup." \ No newline at end of file diff --git a/packages/auth/package.json b/packages/auth/package.json index bd97af34f..1002300de 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -17,8 +17,14 @@ "lint": "eslint . --ext .ts", "pretest": "yarn lint && yarn build", "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%", + "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", + "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", + "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", + "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT", "start": "yarn node dist/bin/server.js", + "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js", "worker": "yarn node dist/bin/worker.js", + "supervisor:worker": "yarn wait-for:auth && yarn node dist/bin/worker.js", "cleanup": "yarn node dist/bin/cleanup.js", "stats": "yarn node dist/bin/stats.js", "daily-backup:email": "yarn node dist/bin/backup.js email daily", diff --git a/packages/files/docker/wait-for.sh b/packages/files/docker/wait-for.sh new file mode 100755 index 000000000..90438f0d2 --- /dev/null +++ b/packages/files/docker/wait-for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +WAIT_FOR_HOST="$1" +shift +WAIT_FOR_PORT="$1" +shift + +while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do + echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start" + sleep 10 +done + +echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup." \ No newline at end of file diff --git a/packages/files/package.json b/packages/files/package.json index e2a0cf8f7..9eca1135d 100644 --- a/packages/files/package.json +++ b/packages/files/package.json @@ -20,8 +20,14 @@ "lint": "eslint . --ext .ts", "pretest": "yarn lint && yarn build", "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%", + "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", + "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", + "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", + "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT", "start": "yarn node dist/bin/server.js", + "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js", "worker": "yarn node dist/bin/worker.js", + "supervisor:worker": "yarn wait-for:files && yarn node dist/bin/worker.js", "upgrade:snjs": "yarn ncu -u '@standardnotes/*'" }, "dependencies": { diff --git a/packages/revisions/docker/wait-for.sh b/packages/revisions/docker/wait-for.sh new file mode 100755 index 000000000..90438f0d2 --- /dev/null +++ b/packages/revisions/docker/wait-for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +WAIT_FOR_HOST="$1" +shift +WAIT_FOR_PORT="$1" +shift + +while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do + echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start" + sleep 10 +done + +echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup." \ No newline at end of file diff --git a/packages/revisions/package.json b/packages/revisions/package.json index 6aac631a3..02c7b76fe 100644 --- a/packages/revisions/package.json +++ b/packages/revisions/package.json @@ -19,7 +19,13 @@ "lint:fix": "eslint . --ext .ts --fix", "pretest": "yarn lint && yarn build", "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%", + "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", + "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", + "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", + "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT", "start": "yarn node dist/bin/server.js", + "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js", + "supervisor:worker": "yarn wait-for:revisions && yarn node dist/bin/worker.js", "worker": "yarn node dist/bin/worker.js" }, "dependencies": { diff --git a/packages/syncing-server/docker/wait-for.sh b/packages/syncing-server/docker/wait-for.sh new file mode 100755 index 000000000..90438f0d2 --- /dev/null +++ b/packages/syncing-server/docker/wait-for.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +WAIT_FOR_HOST="$1" +shift +WAIT_FOR_PORT="$1" +shift + +while ! nc -vz $WAIT_FOR_HOST $WAIT_FOR_PORT; do + echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is unavailable yet - waiting for it to start" + sleep 10 +done + +echo "$WAIT_FOR_HOST:$WAIT_FOR_PORT is up. Proceeding to startup." \ No newline at end of file diff --git a/packages/syncing-server/package.json b/packages/syncing-server/package.json index 2d12a3c97..86c52054b 100644 --- a/packages/syncing-server/package.json +++ b/packages/syncing-server/package.json @@ -19,8 +19,14 @@ "lint:fix": "eslint . --ext .ts --fix", "pretest": "yarn lint && yarn build", "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%", + "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", + "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", + "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", + "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT", "start": "yarn node dist/bin/server.js", + "supervisor:start": "yarn wait-for:infra && yarn node dist/bin/server.js", "worker": "yarn node dist/bin/worker.js", + "supervisor:worker": "yarn wait-for:syncing-server && yarn node dist/bin/worker.js", "content-size": "yarn node dist/bin/content.js", "revisions-ownership": "yarn node dist/bin/revisions.js", "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"