From 5aa561dcb20eedf94ed8d5bed2902bd653e3f42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Mon, 6 Feb 2023 13:35:58 +0100 Subject: [PATCH] add localstack --- .dockerignore | 13 ++- .env.sample | 2 +- Dockerfile | 15 ++- docker-compose.yml | 18 +++ docker/docker-entrypoint.sh | 5 +- docker/localstack_bootstrap.sh | 196 +++++++++++++++++++++++++++++++++ package.json | 18 +-- 7 files changed, 248 insertions(+), 19 deletions(-) create mode 100755 docker/localstack_bootstrap.sh diff --git a/.dockerignore b/.dockerignore index 3f6fff7b2..75e5b59ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,13 @@ dist -coverage \ No newline at end of file +coverage + +.yarn/* +!.yarn/cache +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/unplugged +!.yarn/sdks +!.yarn/versions + +data/* \ No newline at end of file diff --git a/.env.sample b/.env.sample index a83361a50..3495c2b7e 100644 --- a/.env.sample +++ b/.env.sample @@ -26,4 +26,4 @@ DB_MIGRATIONS_PATH=dist/migrations/*.js ######### REDIS_PORT=6379 -REDIS_URL=redis://cache \ No newline at end of file +REDIS_HOST=cache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cfc48f8a0..d5394f596 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,12 @@ FROM node:18.13.0-alpine -RUN apk add --update \ - bash \ - curl \ - py3-pip \ - openssl \ - && rm -rf /var/cache/apk/* - ENV NODE_ENV production -RUN corepack enable +RUN apk add --update --no-cache \ + openssl \ + curl \ + bash \ + py3-pip RUN pip install --no-cache-dir --upgrade supervisor @@ -23,6 +20,8 @@ COPY . /opt/server WORKDIR /opt/server +RUN corepack enable + RUN yarn install --immutable RUN yarn build diff --git a/docker-compose.yml b/docker-compose.yml index 204a71b98..07ef589e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,30 @@ services: server: build: . env_file: .env + container_name: server-self-hosted ports: - ${EXPOSED_PORT}:3000 volumes: - ./logs:/var/lib/server/logs + localstack: + image: localstack/localstack:1.3 + container_name: localstack-self-hosted + expose: + - 4566 + restart: unless-stopped + environment: + - SERVICES=sns,sqs + - DOCKER_HOST=unix:///var/run/docker.sock + - HOSTNAME_EXTERNAL=localstack + - LS_LOG=warn + volumes: + - ./docker/localstack_bootstrap.sh:/etc/localstack/init/ready.d/localstack_bootstrap.sh + - /var/run/docker.sock:/var/run/docker.sock + db: image: mysql:8 + container_name: db-self-hosted environment: MYSQL_DATABASE: '${DB_DATABASE}' MYSQL_USER: '${DB_USERNAME}' @@ -24,6 +41,7 @@ services: cache: image: redis:6.0-alpine + container_name: cache-self-hosted volumes: - ./data/redis/:/data expose: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index bd6c2fa4e..57acf04cf 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -59,8 +59,11 @@ fi if [ -z "$REDIS_PORT" ]; then export REDIS_PORT=6379 fi -if [ -z "$REDIS_URL" ]; then + +if [ -z "$REDIS_HOST" ]; then export REDIS_URL="redis://cache" +else + export REDIS_URL="redis://$REDIS_HOST" fi ########## diff --git a/docker/localstack_bootstrap.sh b/docker/localstack_bootstrap.sh new file mode 100755 index 000000000..74d8f25cf --- /dev/null +++ b/docker/localstack_bootstrap.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "configuring sns/sqs" +echo "===================" +LOCALSTACK_HOST=localhost +AWS_REGION=us-east-1 +LOCALSTACK_DUMMY_ID=000000000000 + +get_all_queues() { + awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sqs list-queues +} + +create_queue() { + local QUEUE_NAME_TO_CREATE=$1 + awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sqs create-queue --queue-name ${QUEUE_NAME_TO_CREATE} +} + +get_all_topics() { + awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns list-topics +} + +create_topic() { + local TOPIC_NAME_TO_CREATE=$1 + awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns create-topic --name ${TOPIC_NAME_TO_CREATE} +} + +link_queue_and_topic() { + local TOPIC_ARN_TO_LINK=$1 + local QUEUE_ARN_TO_LINK=$2 + awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns subscribe --topic-arn ${TOPIC_ARN_TO_LINK} --protocol sqs --notification-endpoint ${QUEUE_ARN_TO_LINK} +} + +get_queue_arn_from_name() { + local QUEUE_NAME=$1 + echo "arn:aws:sns:${AWS_REGION}:${LOCALSTACK_DUMMY_ID}:$QUEUE_NAME" +} + +get_topic_arn_from_name() { + local TOPIC_NAME=$1 + echo "arn:aws:sns:${AWS_REGION}:${LOCALSTACK_DUMMY_ID}:$TOPIC_NAME" +} + +PAYMENTS_TOPIC_NAME="payments-local-topic" + +echo "creating topic $PAYMENTS_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${PAYMENTS_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +PAYMENTS_TOPIC_ARN=$(get_topic_arn_from_name $PAYMENTS_TOPIC_NAME) + +SYNCING_SERVER_TOPIC_NAME="syncing-server-local-topic" + +echo "creating topic $SYNCING_SERVER_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${SYNCING_SERVER_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +SYNCING_SERVER_TOPIC_ARN=$(get_topic_arn_from_name $SYNCING_SERVER_TOPIC_NAME) + +AUTH_TOPIC_NAME="auth-local-topic" + +echo "creating topic $AUTH_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${AUTH_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +AUTH_TOPIC_ARN=$(get_topic_arn_from_name $AUTH_TOPIC_NAME) + +FILES_TOPIC_NAME="files-local-topic" + +echo "creating topic $FILES_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${FILES_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +FILES_TOPIC_ARN=$(get_topic_arn_from_name $FILES_TOPIC_NAME) + +ANALYTICS_TOPIC_NAME="analytics-local-topic" + +echo "creating topic $ANALYTICS_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${ANALYTICS_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +ANALYTICS_TOPIC_ARN=$(get_topic_arn_from_name $ANALYTICS_TOPIC_NAME) + +REVISIONS_TOPIC_NAME="revisions-server-local-topic" + +echo "creating topic $REVISIONS_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${REVISIONS_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +REVISIONS_TOPIC_ARN=$(get_topic_arn_from_name $REVISIONS_TOPIC_NAME) + +SCHEDULER_TOPIC_NAME="scheduler-local-topic" + +echo "creating topic $SCHEDULER_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${SCHEDULER_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +SCHEDULER_TOPIC_ARN=$(get_topic_arn_from_name $SCHEDULER_TOPIC_NAME) + +WORKSPACE_TOPIC_NAME="workspace-local-topic" + +echo "creating topic $WORKSPACE_TOPIC_NAME" +TOPIC_CREATED_RESULT=$(create_topic ${WORKSPACE_TOPIC_NAME}) +echo "created topic: $TOPIC_CREATED_RESULT" +WORKSPACE_TOPIC_ARN=$(get_topic_arn_from_name $WORKSPACE_TOPIC_NAME) + +QUEUE_NAME="analytics-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +ANALYTICS_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "linking topic $PAYMENTS_TOPIC_ARN to queue $ANALYTICS_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $PAYMENTS_TOPIC_ARN $ANALYTICS_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +QUEUE_NAME="auth-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +AUTH_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "linking topic $PAYMENTS_TOPIC_ARN to queue $AUTH_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $PAYMENTS_TOPIC_ARN $AUTH_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" +echo "linking topic $AUTH_TOPIC_ARN to queue $AUTH_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $AUTH_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" +echo "linking topic $FILES_TOPIC_ARN to queue $AUTH_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $FILES_TOPIC_ARN $AUTH_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +QUEUE_NAME="files-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +FILES_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "linking topic $AUTH_TOPIC_ARN to queue $FILES_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $FILES_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +QUEUE_NAME="syncing-server-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +SYNCING_SERVER_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" +echo "linking topic $AUTH_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +QUEUE_NAME="revisions-server-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +REVISIONS_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $REVISIONS_QUEUE_ARN" +LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $REVISIONS_QUEUE_ARN) +echo "linking done:" +echo "$LINKING_RESULT" + +QUEUE_NAME="scheduler-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +SCHEDULER_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +QUEUE_NAME="workspace-local-queue" + +echo "creating queue $QUEUE_NAME" +QUEUE_URL=$(create_queue ${QUEUE_NAME}) +echo "created queue: $QUEUE_URL" +WORKSPACE_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME) + +echo "all topics are:" +echo "$(get_all_topics)" + +echo "all queues are:" +echo "$(get_all_queues)" diff --git a/package.json b/package.json index 70a6b385a..0b12f4f61 100644 --- a/package.json +++ b/package.json @@ -24,24 +24,26 @@ "lint:revisions": "yarn workspace @standardnotes/revisions-server lint", "clean": "yarn workspaces foreach -p --verbose run clean", "setup:env": "cp .env.sample .env && yarn workspaces foreach -p --verbose run setup:env", + "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT", "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT", + "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache", "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT", "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT", "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT", "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT", - "start:auth": "yarn wait-for:db && yarn workspace @standardnotes/auth-server start", - "start:auth-worker": "yarn wait-for:db && yarn workspace @standardnotes/auth-server worker", + "start:auth": "yarn wait-for:infra && yarn workspace @standardnotes/auth-server start", + "start:auth-worker": "yarn wait-for:infra && yarn workspace @standardnotes/auth-server worker", "start:scheduler": "yarn workspace @standardnotes/scheduler-server worker", - "start:syncing-server": "yarn wait-for:db && yarn workspace @standardnotes/syncing-server start", - "start:syncing-server-worker": "yarn wait-for:db && yarn workspace @standardnotes/syncing-server worker", - "start:files": "yarn wait-for:db && yarn workspace @standardnotes/files-server start", - "start:files-worker": "yarn wait-for:db && yarn workspace @standardnotes/files-server worker", + "start:syncing-server": "yarn wait-for:infra && yarn workspace @standardnotes/syncing-server start", + "start:syncing-server-worker": "yarn wait-for:infra && yarn workspace @standardnotes/syncing-server worker", + "start:files": "yarn wait-for:infra && yarn workspace @standardnotes/files-server start", + "start:files-worker": "yarn wait-for:infra && yarn workspace @standardnotes/files-server worker", "start:api-gateway": "yarn wait-for:auth && yarn wait-for:syncing-server && yarn wait-for:files && yarn wait-for:revisions && yarn workspace @standardnotes/api-gateway start", "start:websockets": "yarn workspace @standardnotes/websockets-server start", "start:workspace": "yarn workspace @standardnotes/workspace-server start", "start:analytics": "yarn workspace @standardnotes/analytics worker", - "start:revisions": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server start", - "start:revisions-worker": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server worker", + "start:revisions": "yarn wait-for:infra && yarn workspace @standardnotes/revisions-server start", + "start:revisions-worker": "yarn wait-for:infra && yarn workspace @standardnotes/revisions-server worker", "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"", "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose", "postversion": "./scripts/push-tags-one-by-one.sh",