add localstack

.dockerignore

@@ -1,2 +1,13 @@
 dist
-coverage
+coverage
+
+.yarn/*
+!.yarn/cache
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/unplugged
+!.yarn/sdks
+!.yarn/versions
+
+data/*

.env.sample

@@ -26,4 +26,4 @@ DB_MIGRATIONS_PATH=dist/migrations/*.js
 #########
 
 REDIS_PORT=6379
-REDIS_URL=redis://cache
+REDIS_HOST=cache

Dockerfile

@@ -1,15 +1,12 @@
 FROM node:18.13.0-alpine
 
-RUN apk add --update \
-  bash \
-  curl \
-  py3-pip \
-  openssl \
-  && rm -rf /var/cache/apk/*
-
 ENV NODE_ENV production
 
-RUN corepack enable
+RUN apk add --update --no-cache \
+  openssl \
+  curl \
+  bash \
+  py3-pip
 
 RUN pip install --no-cache-dir --upgrade supervisor
 
@@ -23,6 +20,8 @@ COPY . /opt/server
 
 WORKDIR /opt/server
 
+RUN corepack enable
+
 RUN yarn install --immutable
 
 RUN yarn build

docker-compose.yml

@@ -2,13 +2,30 @@ services:
   server:
     build: .
     env_file: .env
+    container_name: server-self-hosted
     ports:
       - ${EXPOSED_PORT}:3000
     volumes:
       - ./logs:/var/lib/server/logs
 
+  localstack:
+    image: localstack/localstack:1.3
+    container_name: localstack-self-hosted
+    expose:
+      - 4566
+    restart: unless-stopped
+    environment:
+      - SERVICES=sns,sqs
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOSTNAME_EXTERNAL=localstack
+      - LS_LOG=warn
+    volumes:
+      - ./docker/localstack_bootstrap.sh:/etc/localstack/init/ready.d/localstack_bootstrap.sh
+      - /var/run/docker.sock:/var/run/docker.sock
+
   db:
     image: mysql:8
+    container_name: db-self-hosted
     environment:
       MYSQL_DATABASE: '${DB_DATABASE}'
       MYSQL_USER: '${DB_USERNAME}'
@@ -24,6 +41,7 @@ services:
 
   cache:
     image: redis:6.0-alpine
+    container_name: cache-self-hosted
     volumes:
       - ./data/redis/:/data
     expose:

docker/docker-entrypoint.sh

@@ -59,8 +59,11 @@ fi
 if [ -z "$REDIS_PORT" ]; then
   export REDIS_PORT=6379
 fi
-if [ -z "$REDIS_URL" ]; then
+
+if [ -z "$REDIS_HOST" ]; then
   export REDIS_URL="redis://cache"
+else
+  export REDIS_URL="redis://$REDIS_HOST"
 fi
 
 ##########

docker/localstack_bootstrap.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "configuring sns/sqs"
+echo "==================="
+LOCALSTACK_HOST=localhost
+AWS_REGION=us-east-1
+LOCALSTACK_DUMMY_ID=000000000000
+
+get_all_queues() {
+  awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sqs list-queues
+}
+
+create_queue() {
+  local QUEUE_NAME_TO_CREATE=$1
+  awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sqs create-queue --queue-name ${QUEUE_NAME_TO_CREATE}
+}
+
+get_all_topics() {
+  awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns list-topics
+}
+
+create_topic() {
+  local TOPIC_NAME_TO_CREATE=$1
+  awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns create-topic --name ${TOPIC_NAME_TO_CREATE}
+}
+
+link_queue_and_topic() {
+  local TOPIC_ARN_TO_LINK=$1
+  local QUEUE_ARN_TO_LINK=$2
+  awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 sns subscribe --topic-arn ${TOPIC_ARN_TO_LINK} --protocol sqs --notification-endpoint ${QUEUE_ARN_TO_LINK}
+}
+
+get_queue_arn_from_name() {
+  local QUEUE_NAME=$1
+  echo "arn:aws:sns:${AWS_REGION}:${LOCALSTACK_DUMMY_ID}:$QUEUE_NAME"
+}
+
+get_topic_arn_from_name() {
+  local TOPIC_NAME=$1
+  echo "arn:aws:sns:${AWS_REGION}:${LOCALSTACK_DUMMY_ID}:$TOPIC_NAME"
+}
+
+PAYMENTS_TOPIC_NAME="payments-local-topic"
+
+echo "creating topic $PAYMENTS_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${PAYMENTS_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+PAYMENTS_TOPIC_ARN=$(get_topic_arn_from_name $PAYMENTS_TOPIC_NAME)
+
+SYNCING_SERVER_TOPIC_NAME="syncing-server-local-topic"
+
+echo "creating topic $SYNCING_SERVER_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${SYNCING_SERVER_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+SYNCING_SERVER_TOPIC_ARN=$(get_topic_arn_from_name $SYNCING_SERVER_TOPIC_NAME)
+
+AUTH_TOPIC_NAME="auth-local-topic"
+
+echo "creating topic $AUTH_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${AUTH_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+AUTH_TOPIC_ARN=$(get_topic_arn_from_name $AUTH_TOPIC_NAME)
+
+FILES_TOPIC_NAME="files-local-topic"
+
+echo "creating topic $FILES_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${FILES_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+FILES_TOPIC_ARN=$(get_topic_arn_from_name $FILES_TOPIC_NAME)
+
+ANALYTICS_TOPIC_NAME="analytics-local-topic"
+
+echo "creating topic $ANALYTICS_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${ANALYTICS_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+ANALYTICS_TOPIC_ARN=$(get_topic_arn_from_name $ANALYTICS_TOPIC_NAME)
+
+REVISIONS_TOPIC_NAME="revisions-server-local-topic"
+
+echo "creating topic $REVISIONS_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${REVISIONS_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+REVISIONS_TOPIC_ARN=$(get_topic_arn_from_name $REVISIONS_TOPIC_NAME)
+
+SCHEDULER_TOPIC_NAME="scheduler-local-topic"
+
+echo "creating topic $SCHEDULER_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${SCHEDULER_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+SCHEDULER_TOPIC_ARN=$(get_topic_arn_from_name $SCHEDULER_TOPIC_NAME)
+
+WORKSPACE_TOPIC_NAME="workspace-local-topic"
+
+echo "creating topic $WORKSPACE_TOPIC_NAME"
+TOPIC_CREATED_RESULT=$(create_topic ${WORKSPACE_TOPIC_NAME})
+echo "created topic: $TOPIC_CREATED_RESULT"
+WORKSPACE_TOPIC_ARN=$(get_topic_arn_from_name $WORKSPACE_TOPIC_NAME)
+
+QUEUE_NAME="analytics-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+ANALYTICS_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "linking topic $PAYMENTS_TOPIC_ARN to queue $ANALYTICS_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $PAYMENTS_TOPIC_ARN $ANALYTICS_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+QUEUE_NAME="auth-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+AUTH_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "linking topic $PAYMENTS_TOPIC_ARN to queue $AUTH_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $PAYMENTS_TOPIC_ARN $AUTH_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+echo "linking topic $AUTH_TOPIC_ARN to queue $AUTH_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $AUTH_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+echo "linking topic $FILES_TOPIC_ARN to queue $AUTH_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $FILES_TOPIC_ARN $AUTH_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+QUEUE_NAME="files-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+FILES_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "linking topic $AUTH_TOPIC_ARN to queue $FILES_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $FILES_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+QUEUE_NAME="syncing-server-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+SYNCING_SERVER_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+echo "linking topic $AUTH_TOPIC_ARN to queue $SYNCING_SERVER_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $AUTH_TOPIC_ARN $SYNCING_SERVER_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+QUEUE_NAME="revisions-server-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+REVISIONS_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "linking topic $SYNCING_SERVER_TOPIC_ARN to queue $REVISIONS_QUEUE_ARN"
+LINKING_RESULT=$(link_queue_and_topic $SYNCING_SERVER_TOPIC_ARN $REVISIONS_QUEUE_ARN)
+echo "linking done:"
+echo "$LINKING_RESULT"
+
+QUEUE_NAME="scheduler-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+SCHEDULER_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+QUEUE_NAME="workspace-local-queue"
+
+echo "creating queue $QUEUE_NAME"
+QUEUE_URL=$(create_queue ${QUEUE_NAME})
+echo "created queue: $QUEUE_URL"
+WORKSPACE_QUEUE_ARN=$(get_queue_arn_from_name $QUEUE_NAME)
+
+echo "all topics are:"
+echo "$(get_all_topics)"
+
+echo "all queues are:"
+echo "$(get_all_queues)"

package.json

@@ -24,24 +24,26 @@
     "lint:revisions": "yarn workspace @standardnotes/revisions-server lint",
     "clean": "yarn workspaces foreach -p --verbose run clean",
     "setup:env": "cp .env.sample .env && yarn workspaces foreach -p --verbose run setup:env",
+    "wait-for:cache": "docker/wait-for.sh $REDIS_HOST $REDIS_PORT",
     "wait-for:db": "docker/wait-for.sh $DB_HOST $DB_PORT",
+    "wait-for:infra": "yarn wait-for:db && yarn wait-for:cache",
     "wait-for:auth": "docker/wait-for.sh localhost $AUTH_SERVER_PORT",
     "wait-for:syncing-server": "docker/wait-for.sh localhost $SYNCING_SERVER_PORT",
     "wait-for:files": "docker/wait-for.sh localhost $FILES_SERVER_PORT",
     "wait-for:revisions": "docker/wait-for.sh localhost $REVISIONS_SERVER_PORT",
-    "start:auth": "yarn wait-for:db && yarn workspace @standardnotes/auth-server start",
-    "start:auth-worker": "yarn wait-for:db && yarn workspace @standardnotes/auth-server worker",
+    "start:auth": "yarn wait-for:infra && yarn workspace @standardnotes/auth-server start",
+    "start:auth-worker": "yarn wait-for:infra && yarn workspace @standardnotes/auth-server worker",
     "start:scheduler": "yarn workspace @standardnotes/scheduler-server worker",
-    "start:syncing-server": "yarn wait-for:db && yarn workspace @standardnotes/syncing-server start",
-    "start:syncing-server-worker": "yarn wait-for:db && yarn workspace @standardnotes/syncing-server worker",
-    "start:files": "yarn wait-for:db && yarn workspace @standardnotes/files-server start",
-    "start:files-worker": "yarn wait-for:db && yarn workspace @standardnotes/files-server worker",
+    "start:syncing-server": "yarn wait-for:infra && yarn workspace @standardnotes/syncing-server start",
+    "start:syncing-server-worker": "yarn wait-for:infra && yarn workspace @standardnotes/syncing-server worker",
+    "start:files": "yarn wait-for:infra && yarn workspace @standardnotes/files-server start",
+    "start:files-worker": "yarn wait-for:infra && yarn workspace @standardnotes/files-server worker",
     "start:api-gateway": "yarn wait-for:auth && yarn wait-for:syncing-server && yarn wait-for:files && yarn wait-for:revisions && yarn workspace @standardnotes/api-gateway start",
     "start:websockets": "yarn workspace @standardnotes/websockets-server start",
     "start:workspace": "yarn workspace @standardnotes/workspace-server start",
     "start:analytics": "yarn workspace @standardnotes/analytics worker",
-    "start:revisions": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server start",
-    "start:revisions-worker": "yarn wait-for:db && yarn workspace @standardnotes/revisions-server worker",
+    "start:revisions": "yarn wait-for:infra && yarn workspace @standardnotes/revisions-server start",
+    "start:revisions-worker": "yarn wait-for:infra && yarn workspace @standardnotes/revisions-server worker",
     "release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
     "publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
     "postversion": "./scripts/push-tags-one-by-one.sh",