feat: add workspace microservice

This commit is contained in:
Karol Sójko 2022-10-06 11:54:22 +02:00
parent 7064bd4c4c
commit 44a9ade3fc
No known key found for this signature in database
GPG key ID: A50543BF560BDEB0
37 changed files with 1132 additions and 107 deletions

206
.github/workflows/workspace.release.yml vendored Normal file
View file

@ -0,0 +1,206 @@
name: Workspace Server
concurrency:
group: workspace
cancel-in-progress: true
on:
push:
tags:
- '*standardnotes/workspace-server*'
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Node
uses: actions/setup-node@v3
with:
registry-url: 'https://registry.npmjs.org'
node-version-file: '.nvmrc'
- name: Build
run: yarn build
- name: Lint
run: yarn lint:workspace
- name: Test
run: yarn test:workspace
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Publish Docker image for E2E testing
run: |
yarn docker build @standardnotes/workspace-server -t standardnotes/workspace:${{ github.sha }}
docker push standardnotes/workspace:${{ github.sha }}
- name: Run E2E test suite
uses: convictional/trigger-workflow-and-wait@v1.6.3
with:
owner: standardnotes
repo: e2e
github_token: ${{ secrets.CI_PAT_TOKEN }}
workflow_file_name: testing-with-stable-client.yml
wait_interval: 30
client_payload: '{"workspace_image_tag": "${{ github.sha }}"}'
propagate_failure: true
trigger_workflow: true
wait_workflow: true
publish-aws-ecr:
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Node
uses: actions/setup-node@v3
with:
registry-url: 'https://registry.npmjs.org'
node-version-file: '.nvmrc'
- name: Build locally
run: yarn build
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, tag, and push image to Amazon ECR
id: build-image
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: workspace
IMAGE_TAG: ${{ github.sha }}
run: |
yarn docker build @standardnotes/workspace-server -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest
docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
publish-docker-hub:
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Node
uses: actions/setup-node@v3
with:
registry-url: 'https://registry.npmjs.org'
node-version-file: '.nvmrc'
- name: Build locally
run: yarn build
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Publish Docker image as stable
run: |
yarn docker build @standardnotes/workspace-server -t standardnotes/workspace:latest
docker push standardnotes/workspace:latest
deploy-web:
needs: publish-aws-ecr
runs-on: ubuntu-latest
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Download task definition
run: |
aws ecs describe-task-definition --task-definition workspace-prod --query taskDefinition > task-definition.json
- name: Fill in the new version in the Amazon ECS task definition
run: |
jq '(.containerDefinitions[] | select(.name=="workspace-prod") | .environment[] | select(.name=="VERSION")).value = "${{ github.sha }}"' task-definition.json > tmp.json && mv tmp.json task-definition.json
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def-prod
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: task-definition.json
container-name: workspace-prod
image: ${{ secrets.AWS_ECR_REGISTRY }}/workspace:${{ github.sha }}
- name: Deploy Amazon ECS task definition
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
with:
task-definition: ${{ steps.task-def-prod.outputs.task-definition }}
service: workspace-prod
cluster: prod
wait-for-service-stability: true
deploy-worker:
needs: publish-aws-ecr
runs-on: ubuntu-latest
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Download task definition
run: |
aws ecs describe-task-definition --task-definition workspace-worker-prod --query taskDefinition > task-definition.json
- name: Fill in the new version in the Amazon ECS task definition
run: |
jq '(.containerDefinitions[] | select(.name=="workspace-worker-prod") | .environment[] | select(.name=="VERSION")).value = "${{ github.sha }}"' task-definition.json > tmp.json && mv tmp.json task-definition.json
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def-prod
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: task-definition.json
container-name: workspace-worker-prod
image: ${{ secrets.AWS_ECR_REGISTRY }}/workspace:${{ github.sha }}
- name: Deploy Amazon ECS task definition
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
with:
task-definition: ${{ steps.task-def-prod.outputs.task-definition }}
service: workspace-worker-prod
cluster: prod
wait-for-service-stability: true
newrelic:
needs: [ deploy-web, deploy-worker ]
runs-on: ubuntu-latest
steps:
- name: Create New Relic deployment marker for Web
uses: newrelic/deployment-marker-action@v1
with:
accountId: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
apiKey: ${{ secrets.NEW_RELIC_API_KEY }}
applicationId: ${{ secrets.NEW_RELIC_APPLICATION_ID_WORKSPACE_WEB_PROD }}
revision: "${{ github.sha }}"
description: "Automated Deployment via Github Actions"
user: "${{ github.actor }}"
- name: Create New Relic deployment marker for Worker
uses: newrelic/deployment-marker-action@v1
with:
accountId: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
apiKey: ${{ secrets.NEW_RELIC_API_KEY }}
applicationId: ${{ secrets.NEW_RELIC_APPLICATION_ID_WORKSPACE_WORKER_PROD }}
revision: "${{ github.sha }}"
description: "Automated Deployment via Github Actions"
user: "${{ github.actor }}"

44
.pnp.cjs generated
View file

@ -79,6 +79,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
{\
"name": "@standardnotes/time",\
"reference": "workspace:packages/time"\
},\
{\
"name": "@standardnotes/workspace-server",\
"reference": "workspace:packages/workspace"\
}\
],\
"enableTopLevelFallback": true,\
@ -99,7 +103,8 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
["@standardnotes/settings", ["workspace:packages/settings"]],\
["@standardnotes/sncrypto-node", ["workspace:packages/sncrypto-node"]],\
["@standardnotes/syncing-server", ["workspace:packages/syncing-server"]],\
["@standardnotes/time", ["workspace:packages/time"]]\
["@standardnotes/time", ["workspace:packages/time"]],\
["@standardnotes/workspace-server", ["workspace:packages/workspace"]]\
],\
"fallbackPool": [\
],\
@ -3099,6 +3104,43 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
"linkType": "HARD"\
}]\
]],\
["@standardnotes/workspace-server", [\
["workspace:packages/workspace", {\
"packageLocation": "./packages/workspace/",\
"packageDependencies": [\
["@standardnotes/workspace-server", "workspace:packages/workspace"],\
["@newrelic/winston-enricher", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:4.0.0"],\
["@sentry/node", "npm:7.5.0"],\
["@standardnotes/common", "workspace:packages/common"],\
["@standardnotes/domain-events", "workspace:packages/domain-events"],\
["@standardnotes/domain-events-infra", "workspace:packages/domain-events-infra"],\
["@standardnotes/security", "workspace:packages/security"],\
["@types/cors", "npm:2.8.12"],\
["@types/express", "npm:4.17.13"],\
["@types/ioredis", "npm:4.28.10"],\
["@types/jest", "npm:28.1.4"],\
["@types/newrelic", "npm:7.0.3"],\
["@typescript-eslint/eslint-plugin", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:5.30.5"],\
["aws-sdk", "npm:2.1168.0"],\
["cors", "npm:2.8.5"],\
["dotenv", "npm:16.0.1"],\
["eslint", "npm:8.19.0"],\
["eslint-plugin-prettier", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:4.2.1"],\
["express", "npm:4.18.1"],\
["inversify", "npm:6.0.1"],\
["inversify-express-utils", "npm:6.4.3"],\
["ioredis", "npm:5.2.0"],\
["jest", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:28.1.2"],\
["mysql2", "npm:2.3.3"],\
["newrelic", "npm:9.0.0"],\
["reflect-metadata", "npm:0.1.13"],\
["ts-jest", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:28.0.5"],\
["typeorm", "virtual:31b5a94a105c89c9294c3d524a7f8929fe63ee5a2efadf21951ca4c0cfd2ecf02e8f4ef5a066bbda091f1e3a56e57c6749069a080618c96b22e51131a330fc4a#npm:0.3.7"],\
["winston", "npm:3.8.1"]\
],\
"linkType": "SOFT"\
}]\
]],\
["@szmarczak/http-timer", [\
["npm:5.0.1", {\
"packageLocation": "./.yarn/cache/@szmarczak-http-timer-npm-5.0.1-52261e5986-fc9cb993e8.zip/node_modules/@szmarczak/http-timer/",\

View file

@ -18,12 +18,14 @@
"lint:files": "yarn workspace @standardnotes/files-server lint",
"lint:api-gateway": "yarn workspace @standardnotes/api-gateway lint",
"lint:event-store": "yarn workspace @standardnotes/event-store lint",
"lint:workspace": "yarn workspace @standardnotes/workspace-server lint",
"test": "yarn workspaces foreach -p -j 10 --verbose run test",
"test:auth": "yarn workspace @standardnotes/auth-server test",
"test:scheduler": "yarn workspace @standardnotes/scheduler-server test",
"test:syncing-server": "yarn workspace @standardnotes/syncing-server test",
"test:files": "yarn workspace @standardnotes/files-server test",
"test:event-store": "yarn workspace @standardnotes/event-store test",
"test:workspace": "yarn workspace @standardnotes/workspace-server test",
"clean": "yarn workspaces foreach -p --verbose run clean",
"setup:env": "cp .env.sample .env && yarn workspaces foreach -p --verbose run setup:env",
"build": "yarn workspaces foreach -pt -j 10 --verbose run build",
@ -32,6 +34,7 @@
"build:syncing-server": "yarn workspace @standardnotes/syncing-server build",
"build:files": "yarn workspace @standardnotes/files-server build",
"build:api-gateway": "yarn workspace @standardnotes/api-gateway build",
"build:workspace": "yarn workspace @standardnotes/workspace-server build",
"start:auth": "yarn workspace @standardnotes/auth-server start",
"start:auth-worker": "yarn workspace @standardnotes/auth-server worker",
"start:scheduler": "yarn workspace @standardnotes/scheduler-server worker",
@ -40,6 +43,7 @@
"start:files": "yarn workspace @standardnotes/files-server start",
"start:files-worker": "yarn workspace @standardnotes/files-server worker",
"start:api-gateway": "yarn workspace @standardnotes/api-gateway start",
"start:workspace": "yarn workspace @standardnotes/workspace-server start",
"release": "lerna version --conventional-graduate --conventional-commits --yes -m \"chore(release): publish new version\"",
"publish": "lerna publish from-git --yes --no-verify-access --loglevel verbose",
"postversion": "./scripts/push-tags-one-by-one.sh",

View file

@ -0,0 +1,17 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class removeGroups1665047863774 implements MigrationInterface {
name = 'removeGroups1665047863774'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query('ALTER TABLE `group_users` DROP FOREIGN KEY `FK_9d1bcb8c649eb05d7a2eb62114e`')
await queryRunner.query('ALTER TABLE `group_users` DROP FOREIGN KEY `FK_b97989611efde2c54b074127920`')
await queryRunner.query('DROP INDEX `index_group_users_on_group_and_user` ON `group_users`')
await queryRunner.query('DROP TABLE `group_users`')
await queryRunner.query('DROP TABLE `groups`')
}
public async down(): Promise<void> {
return
}
}

View file

@ -1,7 +1,5 @@
import { DataSource, LoggerOptions } from 'typeorm'
import { AnalyticsEntity } from '../Domain/Analytics/AnalyticsEntity'
import { Group } from '../Domain/Group/Group'
import { GroupUser } from '../Domain/Group/GroupUser'
import { Permission } from '../Domain/Permission/Permission'
import { Role } from '../Domain/Role/Role'
import { RevokedSession } from '../Domain/Session/RevokedSession'
@ -59,8 +57,6 @@ export const AppDataSource = new DataSource({
SharedSubscriptionInvitation,
SubscriptionSetting,
AnalyticsEntity,
Group,
GroupUser,
],
migrations: [env.get('DB_MIGRATIONS_PATH', true) ?? 'dist/migrations/*.js'],
migrationsRun: true,

View file

@ -1,23 +0,0 @@
import { Column, Entity, OneToMany, PrimaryGeneratedColumn } from 'typeorm'
import { User } from '../User/User'
import { GroupType } from './GroupType'
import { GroupUser } from './GroupUser'
@Entity({ name: 'groups' })
export class Group {
@PrimaryGeneratedColumn('uuid')
declare uuid: string
@Column({
length: 64,
})
declare type: GroupType
@OneToMany(
/* istanbul ignore next */
() => GroupUser,
/* istanbul ignore next */
(groupUser) => groupUser.group,
)
declare users: Promise<User[]>
}

View file

@ -1,4 +0,0 @@
export enum GroupType {
Team = 'team',
Private = 'private',
}

View file

@ -1,64 +0,0 @@
import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'
import { User } from '../User/User'
import { Group } from './Group'
import { GroupAccessLevel } from './GroupAccessLevel'
@Entity({ name: 'group_users' })
@Index('index_group_users_on_group_and_user', ['userUuid', 'groupUuid'], { unique: true })
export class GroupUser {
@PrimaryGeneratedColumn('uuid')
declare uuid: string
@Column({
name: 'access_level',
length: 64,
})
declare accessLevel: GroupAccessLevel
@Column({
name: 'user_uuid',
length: 36,
})
declare userUuid: string
@Column({
name: 'group_uuid',
length: 36,
})
declare groupUuid: string
@Column({
name: 'encrypted_group_key',
length: 255,
type: 'varchar',
})
declare encryptedGroupKey: string
@ManyToOne(
/* istanbul ignore next */
() => User,
/* istanbul ignore next */
(user) => user.groups,
/* istanbul ignore next */
{ onDelete: 'CASCADE' },
)
@JoinColumn(
/* istanbul ignore next */
{ name: 'user_uuid' },
)
declare user: Promise<User>
@ManyToOne(
/* istanbul ignore next */
() => Group,
/* istanbul ignore next */
(group) => group.users,
/* istanbul ignore next */
{ onDelete: 'CASCADE' },
)
@JoinColumn(
/* istanbul ignore next */
{ name: 'group_uuid' },
)
declare group: Promise<Group>
}

View file

@ -5,8 +5,6 @@ import { Setting } from '../Setting/Setting'
import { UserSubscription } from '../Subscription/UserSubscription'
import { AnalyticsEntity } from '../Analytics/AnalyticsEntity'
import { ProtocolVersion } from '@standardnotes/common'
import { Group } from '../Group/Group'
import { GroupUser } from '../Group/GroupUser'
@Entity({ name: 'users' })
export class User {
@ -194,14 +192,6 @@ export class User {
)
declare analyticsEntity: Promise<AnalyticsEntity>
@OneToMany(
/* istanbul ignore next */
() => GroupUser,
/* istanbul ignore next */
(groupUser) => groupUser.user,
)
declare groups: Promise<Group[]>
supportsSessions(): boolean {
return parseInt(this.version) >= parseInt(ProtocolVersion.V004)
}

View file

@ -0,0 +1,34 @@
LOG_LEVEL=debug
NODE_ENV=development
VERSION=development
AUTH_JWT_SECRET=auth_jwt_secret
PORT=3000
DB_HOST=127.0.0.1
DB_REPLICA_HOST=127.0.0.1
DB_PORT=3306
DB_USERNAME=workspace
DB_PASSWORD=changeme123
DB_DATABASE=workspace
DB_DEBUG_LEVEL=all # "all" | "query" | "schema" | "error" | "warn" | "info" | "log" | "migration"
DB_MIGRATIONS_PATH=dist/migrations/*.js
REDIS_URL=redis://cache
SNS_TOPIC_ARN=
SNS_AWS_REGION=
SQS_QUEUE_URL=
SQS_AWS_REGION=
REDIS_EVENTS_CHANNEL=events
# (Optional) New Relic Setup
NEW_RELIC_ENABLED=false
NEW_RELIC_APP_NAME=Workspace
NEW_RELIC_LICENSE_KEY=
NEW_RELIC_NO_CONFIG_FILE=true
NEW_RELIC_DISTRIBUTED_TRACING_ENABLED=false
NEW_RELIC_LOG_ENABLED=false
NEW_RELIC_LOG_LEVEL=info

View file

@ -0,0 +1,3 @@
dist
test-setup.ts
data

View file

@ -0,0 +1,6 @@
{
"extends": "../../.eslintrc",
"parserOptions": {
"project": "./linter.tsconfig.json"
}
}

View file

@ -0,0 +1,27 @@
FROM node:16.15.1-alpine AS builder
# Install dependencies for building native libraries
RUN apk add --update git openssh-client python3 alpine-sdk
WORKDIR /workspace
# docker-build plugin copies everything needed for `yarn install` to `manifests` folder.
COPY manifests ./
RUN yarn install --immutable
FROM node:16.15.1-alpine
RUN apk add --update curl
WORKDIR /workspace
# Copy the installed dependencies from the previous stage.
COPY --from=builder /workspace ./
# docker-build plugin runs `yarn pack` in all workspace dependencies and copies them to `packs` folder.
COPY packs ./
ENTRYPOINT [ "/workspace/packages/workspace/docker/entrypoint.sh" ]
CMD [ "start-web" ]

View file

@ -0,0 +1,69 @@
import 'reflect-metadata'
import 'newrelic'
import * as Sentry from '@sentry/node'
import '../src/Infra/InversifyExpressUtils/InversifyExpressHealthCheckController'
import * as cors from 'cors'
import { urlencoded, json, Request, Response, NextFunction, RequestHandler, ErrorRequestHandler } from 'express'
import * as winston from 'winston'
import { InversifyExpressServer } from 'inversify-express-utils'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
import TYPES from '../src/Bootstrap/Types'
import { Env } from '../src/Bootstrap/Env'
const container = new ContainerConfigLoader()
void container.load().then((container) => {
const env: Env = new Env()
env.load()
const server = new InversifyExpressServer(container)
server.setConfig((app) => {
app.use((_request: Request, response: Response, next: NextFunction) => {
response.setHeader('X-Auth-Version', container.get(TYPES.VERSION))
next()
})
app.use(json())
app.use(urlencoded({ extended: true }))
app.use(cors())
if (env.get('SENTRY_DSN', true)) {
Sentry.init({
dsn: env.get('SENTRY_DSN'),
integrations: [new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true })],
tracesSampleRate: 0,
})
app.use(Sentry.Handlers.requestHandler() as RequestHandler)
}
})
const logger: winston.Logger = container.get(TYPES.Logger)
server.setErrorConfig((app) => {
if (env.get('SENTRY_DSN', true)) {
app.use(Sentry.Handlers.errorHandler() as ErrorRequestHandler)
}
app.use((error: Record<string, unknown>, _request: Request, response: Response, _next: NextFunction) => {
logger.error(error.stack)
response.status(500).send({
error: {
message:
"Unfortunately, we couldn't handle your request. Please try again or contact our support if the error persists.",
},
})
})
})
const serverInstance = server.build()
serverInstance.listen(env.get('PORT'))
logger.info(`Server started on port ${process.env.PORT}`)
})

View file

@ -0,0 +1,25 @@
import 'reflect-metadata'
import 'newrelic'
import { Logger } from 'winston'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
import TYPES from '../src/Bootstrap/Types'
import { Env } from '../src/Bootstrap/Env'
import { DomainEventSubscriberFactoryInterface } from '@standardnotes/domain-events'
const container = new ContainerConfigLoader()
void container.load().then((container) => {
const env: Env = new Env()
env.load()
const logger: Logger = container.get(TYPES.Logger)
logger.info('Starting worker...')
const subscriberFactory: DomainEventSubscriberFactoryInterface = container.get(TYPES.DomainEventSubscriberFactory)
subscriberFactory.create().start()
setInterval(() => logger.info('Alive and kicking!'), 20 * 60 * 1000)
})

View file

@ -0,0 +1,22 @@
#!/bin/sh
set -e
COMMAND=$1 && shift 1
case "$COMMAND" in
'start-web' )
echo "Starting Web..."
yarn workspace @standardnotes/workspace-server start
;;
'start-worker' )
echo "Starting Worker..."
yarn workspace @standardnotes/workspace-server worker
;;
* )
echo "Unknown command"
;;
esac
exec "$@"

View file

@ -0,0 +1,13 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires
const base = require('../../jest.config')
module.exports = {
...base,
globals: {
'ts-jest': {
tsconfig: 'tsconfig.json',
},
},
coveragePathIgnorePatterns: ['/Bootstrap/', '/InversifyExpressUtils/'],
setupFilesAfterEnv: ['./test-setup.ts'],
}

View file

@ -0,0 +1,4 @@
{
"extends": "./tsconfig.json",
"exclude": ["dist", "test-setup.ts"]
}

View file

@ -0,0 +1,20 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class initial1665049971623 implements MigrationInterface {
name = 'initial1665049971623'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
'CREATE TABLE `workspaces` (`uuid` varchar(36) NOT NULL, `type` varchar(64) NOT NULL, `name` varchar(255) NULL, `key_rotation_index` int NOT NULL DEFAULT 0, PRIMARY KEY (`uuid`)) ENGINE=InnoDB',
)
await queryRunner.query(
'CREATE TABLE `workspace_users` (`uuid` varchar(36) NOT NULL, `access_level` varchar(64) NOT NULL, `user_uuid` varchar(36) NOT NULL, `workspace_uuid` varchar(36) NOT NULL, `encrypted_workspace_key` varchar(255) NULL, `public_key` varchar(255) NOT NULL, `private_key` varchar(255) NOT NULL, `status` varchar(64) NOT NULL, `key_rotation_index` int NOT NULL DEFAULT 0, UNIQUE INDEX `index_workspace_users_on_workspace_and_user` (`user_uuid`, `workspace_uuid`), PRIMARY KEY (`uuid`)) ENGINE=InnoDB',
)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query('DROP INDEX `index_workspace_users_on_workspace_and_user` ON `workspace_users`')
await queryRunner.query('DROP TABLE `workspace_users`')
await queryRunner.query('DROP TABLE `workspaces`')
}
}

View file

@ -0,0 +1,57 @@
{
"name": "@standardnotes/workspace-server",
"version": "1.0.0",
"engines": {
"node": ">=16.0.0 <17.0.0"
},
"private": true,
"description": "Workspace Server",
"main": "dist/src/index.js",
"typings": "dist/src/index.d.ts",
"author": "Karol Sójko <karol@standardnotes.com>",
"license": "AGPL-3.0-or-later",
"scripts": {
"clean": "rm -fr dist",
"setup:env": "cp .env.sample .env",
"prebuild": "yarn clean",
"build": "tsc --rootDir ./",
"lint": "eslint . --ext .ts",
"pretest": "yarn lint && yarn build",
"test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
"start": "yarn node dist/bin/server.js",
"worker": "yarn node dist/bin/worker.js",
"typeorm": "typeorm-ts-node-commonjs"
},
"dependencies": {
"@newrelic/winston-enricher": "^4.0.0",
"@sentry/node": "^7.3.0",
"@standardnotes/common": "workspace:*",
"@standardnotes/domain-events": "workspace:*",
"@standardnotes/domain-events-infra": "workspace:*",
"@standardnotes/security": "workspace:*",
"aws-sdk": "^2.1159.0",
"cors": "2.8.5",
"dotenv": "^16.0.1",
"express": "^4.18.1",
"inversify": "^6.0.1",
"inversify-express-utils": "^6.4.3",
"ioredis": "^5.2.0",
"mysql2": "^2.3.3",
"newrelic": "^9.0.0",
"reflect-metadata": "0.1.13",
"typeorm": "^0.3.6",
"winston": "^3.8.1"
},
"devDependencies": {
"@types/cors": "^2.8.9",
"@types/express": "^4.17.11",
"@types/ioredis": "^4.28.10",
"@types/jest": "^28.1.4",
"@types/newrelic": "^7.0.3",
"@typescript-eslint/eslint-plugin": "^5.29.0",
"eslint": "^8.14.0",
"eslint-plugin-prettier": "^4.0.0",
"jest": "^28.1.2",
"ts-jest": "^28.0.5"
}
}

View file

@ -0,0 +1,154 @@
import * as winston from 'winston'
import Redis from 'ioredis'
import * as AWS from 'aws-sdk'
import { Container } from 'inversify'
import {
DomainEventHandlerInterface,
DomainEventMessageHandlerInterface,
DomainEventSubscriberFactoryInterface,
} from '@standardnotes/domain-events'
import { Env } from './Env'
import TYPES from './Types'
import { AppDataSource } from './DataSource'
import {
RedisDomainEventPublisher,
RedisDomainEventSubscriberFactory,
RedisEventMessageHandler,
SNSDomainEventPublisher,
SQSDomainEventSubscriberFactory,
SQSEventMessageHandler,
SQSNewRelicEventMessageHandler,
} from '@standardnotes/domain-events-infra'
import { ApiGatewayAuthMiddleware } from '../Controller/ApiGatewayAuthMiddleware'
import { CrossServiceTokenData, TokenDecoder, TokenDecoderInterface } from '@standardnotes/security'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const newrelicFormatter = require('@newrelic/winston-enricher')
export class ContainerConfigLoader {
async load(): Promise<Container> {
const env: Env = new Env()
env.load()
const container = new Container()
await AppDataSource.initialize()
const redisUrl = env.get('REDIS_URL')
const isRedisInClusterMode = redisUrl.indexOf(',') > 0
let redis
if (isRedisInClusterMode) {
redis = new Redis.Cluster(redisUrl.split(','))
} else {
redis = new Redis(redisUrl)
}
container.bind(TYPES.Redis).toConstantValue(redis)
const newrelicWinstonFormatter = newrelicFormatter(winston)
const winstonFormatters = [winston.format.splat(), winston.format.json()]
if (env.get('NEW_RELIC_ENABLED', true) === 'true') {
winstonFormatters.push(newrelicWinstonFormatter())
}
const logger = winston.createLogger({
level: env.get('LOG_LEVEL') || 'info',
format: winston.format.combine(...winstonFormatters),
transports: [new winston.transports.Console({ level: env.get('LOG_LEVEL') || 'info' })],
})
container.bind<winston.Logger>(TYPES.Logger).toConstantValue(logger)
if (env.get('SNS_AWS_REGION', true)) {
container.bind<AWS.SNS>(TYPES.SNS).toConstantValue(
new AWS.SNS({
apiVersion: 'latest',
region: env.get('SNS_AWS_REGION', true),
}),
)
}
if (env.get('SQS_QUEUE_URL', true)) {
const sqsConfig: AWS.SQS.Types.ClientConfiguration = {
apiVersion: 'latest',
region: env.get('SQS_AWS_REGION', true),
}
if (env.get('SQS_ACCESS_KEY_ID', true) && env.get('SQS_SECRET_ACCESS_KEY', true)) {
sqsConfig.credentials = {
accessKeyId: env.get('SQS_ACCESS_KEY_ID', true),
secretAccessKey: env.get('SQS_SECRET_ACCESS_KEY', true),
}
}
container.bind<AWS.SQS>(TYPES.SQS).toConstantValue(new AWS.SQS(sqsConfig))
}
// Controller
// Repositories
// ORM
// Middleware
container.bind<ApiGatewayAuthMiddleware>(TYPES.ApiGatewayAuthMiddleware).to(ApiGatewayAuthMiddleware)
// env vars
container.bind(TYPES.AUTH_JWT_SECRET).toConstantValue(env.get('AUTH_JWT_SECRET'))
container.bind(TYPES.REDIS_URL).toConstantValue(env.get('REDIS_URL'))
container.bind(TYPES.SNS_TOPIC_ARN).toConstantValue(env.get('SNS_TOPIC_ARN', true))
container.bind(TYPES.SNS_AWS_REGION).toConstantValue(env.get('SNS_AWS_REGION', true))
container.bind(TYPES.SQS_QUEUE_URL).toConstantValue(env.get('SQS_QUEUE_URL', true))
container.bind(TYPES.REDIS_EVENTS_CHANNEL).toConstantValue(env.get('REDIS_EVENTS_CHANNEL'))
container.bind(TYPES.NEW_RELIC_ENABLED).toConstantValue(env.get('NEW_RELIC_ENABLED', true))
container.bind(TYPES.VERSION).toConstantValue(env.get('VERSION'))
// use cases
// Handlers
// Services
container
.bind<TokenDecoderInterface<CrossServiceTokenData>>(TYPES.CrossServiceTokenDecoder)
.toConstantValue(new TokenDecoder<CrossServiceTokenData>(container.get(TYPES.AUTH_JWT_SECRET)))
if (env.get('SNS_TOPIC_ARN', true)) {
container
.bind<SNSDomainEventPublisher>(TYPES.DomainEventPublisher)
.toConstantValue(new SNSDomainEventPublisher(container.get(TYPES.SNS), container.get(TYPES.SNS_TOPIC_ARN)))
} else {
container
.bind<RedisDomainEventPublisher>(TYPES.DomainEventPublisher)
.toConstantValue(
new RedisDomainEventPublisher(container.get(TYPES.Redis), container.get(TYPES.REDIS_EVENTS_CHANNEL)),
)
}
const eventHandlers: Map<string, DomainEventHandlerInterface> = new Map([])
if (env.get('SQS_QUEUE_URL', true)) {
container
.bind<DomainEventMessageHandlerInterface>(TYPES.DomainEventMessageHandler)
.toConstantValue(
env.get('NEW_RELIC_ENABLED', true) === 'true'
? new SQSNewRelicEventMessageHandler(eventHandlers, container.get(TYPES.Logger))
: new SQSEventMessageHandler(eventHandlers, container.get(TYPES.Logger)),
)
container
.bind<DomainEventSubscriberFactoryInterface>(TYPES.DomainEventSubscriberFactory)
.toConstantValue(
new SQSDomainEventSubscriberFactory(
container.get(TYPES.SQS),
container.get(TYPES.SQS_QUEUE_URL),
container.get(TYPES.DomainEventMessageHandler),
),
)
} else {
container
.bind<DomainEventMessageHandlerInterface>(TYPES.DomainEventMessageHandler)
.toConstantValue(new RedisEventMessageHandler(eventHandlers, container.get(TYPES.Logger)))
container
.bind<DomainEventSubscriberFactoryInterface>(TYPES.DomainEventSubscriberFactory)
.toConstantValue(
new RedisDomainEventSubscriberFactory(
container.get(TYPES.Redis),
container.get(TYPES.DomainEventMessageHandler),
container.get(TYPES.REDIS_EVENTS_CHANNEL),
),
)
}
return container
}
}

View file

@ -0,0 +1,41 @@
import { DataSource, LoggerOptions } from 'typeorm'
import { Workspace } from '../Domain/Workspace/Workspace'
import { WorkspaceUser } from '../Domain/Workspace/WorkspaceUser'
import { Env } from './Env'
const env: Env = new Env()
env.load()
const maxQueryExecutionTime = env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
? +env.get('DB_MAX_QUERY_EXECUTION_TIME', true)
: 45_000
export const AppDataSource = new DataSource({
type: 'mysql',
supportBigNumbers: true,
bigNumberStrings: false,
maxQueryExecutionTime,
replication: {
master: {
host: env.get('DB_HOST'),
port: parseInt(env.get('DB_PORT')),
username: env.get('DB_USERNAME'),
password: env.get('DB_PASSWORD'),
database: env.get('DB_DATABASE'),
},
slaves: [
{
host: env.get('DB_REPLICA_HOST'),
port: parseInt(env.get('DB_PORT')),
username: env.get('DB_USERNAME'),
password: env.get('DB_PASSWORD'),
database: env.get('DB_DATABASE'),
},
],
removeNodeErrorCount: 10,
},
entities: [Workspace, WorkspaceUser],
migrations: [env.get('DB_MIGRATIONS_PATH', true) ?? 'dist/migrations/*.js'],
migrationsRun: true,
logging: <LoggerOptions>env.get('DB_DEBUG_LEVEL'),
})

View file

@ -0,0 +1,24 @@
import { config, DotenvParseOutput } from 'dotenv'
import { injectable } from 'inversify'
@injectable()
export class Env {
private env?: DotenvParseOutput
public load(): void {
const output = config()
this.env = <DotenvParseOutput>output.parsed
}
public get(key: string, optional = false): string {
if (!this.env) {
this.load()
}
if (!process.env[key] && !optional) {
throw new Error(`Environment variable ${key} not set`)
}
return <string>process.env[key]
}
}

View file

@ -0,0 +1,30 @@
const TYPES = {
Logger: Symbol.for('Logger'),
Redis: Symbol.for('Redis'),
SNS: Symbol.for('SNS'),
SQS: Symbol.for('SQS'),
// Controller
// Repositories
// ORM
// Middleware
ApiGatewayAuthMiddleware: Symbol.for('ApiGatewayAuthMiddleware'),
// env vars
AUTH_JWT_SECRET: Symbol.for('AUTH_JWT_SECRET'),
REDIS_URL: Symbol.for('REDIS_URL'),
SNS_TOPIC_ARN: Symbol.for('SNS_TOPIC_ARN'),
SNS_AWS_REGION: Symbol.for('SNS_AWS_REGION'),
SQS_QUEUE_URL: Symbol.for('SQS_QUEUE_URL'),
SQS_AWS_REGION: Symbol.for('SQS_AWS_REGION'),
REDIS_EVENTS_CHANNEL: Symbol.for('REDIS_EVENTS_CHANNEL'),
NEW_RELIC_ENABLED: Symbol.for('NEW_RELIC_ENABLED'),
VERSION: Symbol.for('VERSION'),
// use cases
// Handlers
// Services
CrossServiceTokenDecoder: Symbol.for('CrossServiceTokenDecoder'),
DomainEventPublisher: Symbol.for('DomainEventPublisher'),
DomainEventSubscriberFactory: Symbol.for('DomainEventSubscriberFactory'),
DomainEventMessageHandler: Symbol.for('DomainEventMessageHandler'),
}
export default TYPES

View file

@ -0,0 +1,99 @@
import 'reflect-metadata'
import { ApiGatewayAuthMiddleware } from './ApiGatewayAuthMiddleware'
import { NextFunction, Request, Response } from 'express'
import { Logger } from 'winston'
import { CrossServiceTokenData, TokenDecoderInterface } from '@standardnotes/security'
import { RoleName } from '@standardnotes/common'
describe('ApiGatewayAuthMiddleware', () => {
let tokenDecoder: TokenDecoderInterface<CrossServiceTokenData>
let request: Request
let response: Response
let next: NextFunction
const logger = {
debug: jest.fn(),
} as unknown as jest.Mocked<Logger>
const createMiddleware = () => new ApiGatewayAuthMiddleware(tokenDecoder, logger)
beforeEach(() => {
tokenDecoder = {} as jest.Mocked<TokenDecoderInterface<CrossServiceTokenData>>
tokenDecoder.decodeToken = jest.fn().mockReturnValue({
user: {
uuid: '1-2-3',
email: 'test@test.te',
},
roles: [
{
uuid: 'a-b-c',
name: RoleName.CoreUser,
},
],
})
request = {
headers: {},
} as jest.Mocked<Request>
response = {
locals: {},
} as jest.Mocked<Response>
response.status = jest.fn().mockReturnThis()
response.send = jest.fn()
next = jest.fn()
})
it('should authorize user', async () => {
request.headers['x-auth-token'] = 'auth-jwt-token'
await createMiddleware().handler(request, response, next)
expect(response.locals.user).toEqual({
uuid: '1-2-3',
email: 'test@test.te',
})
expect(response.locals.roles).toEqual([
{
uuid: 'a-b-c',
name: RoleName.CoreUser,
},
])
expect(next).toHaveBeenCalled()
})
it('should not authorize if request is missing auth jwt token in headers', async () => {
await createMiddleware().handler(request, response, next)
expect(response.status).toHaveBeenCalledWith(401)
expect(next).not.toHaveBeenCalled()
})
it('should not authorize if auth jwt token is malformed', async () => {
request.headers['x-auth-token'] = 'auth-jwt-token'
tokenDecoder.decodeToken = jest.fn().mockReturnValue(undefined)
await createMiddleware().handler(request, response, next)
expect(response.status).toHaveBeenCalledWith(401)
expect(next).not.toHaveBeenCalled()
})
it('should pass the error to next middleware if one occurres', async () => {
request.headers['x-auth-token'] = 'auth-jwt-token'
const error = new Error('Ooops')
tokenDecoder.decodeToken = jest.fn().mockImplementation(() => {
throw error
})
await createMiddleware().handler(request, response, next)
expect(response.status).not.toHaveBeenCalled()
expect(next).toHaveBeenCalledWith(error)
})
})

View file

@ -0,0 +1,59 @@
import { CrossServiceTokenData, TokenDecoderInterface } from '@standardnotes/security'
import { NextFunction, Request, Response } from 'express'
import { inject, injectable } from 'inversify'
import { BaseMiddleware } from 'inversify-express-utils'
import { Logger } from 'winston'
import TYPES from '../Bootstrap/Types'
@injectable()
export class ApiGatewayAuthMiddleware extends BaseMiddleware {
constructor(
@inject(TYPES.CrossServiceTokenDecoder) private tokenDecoder: TokenDecoderInterface<CrossServiceTokenData>,
@inject(TYPES.Logger) private logger: Logger,
) {
super()
}
async handler(request: Request, response: Response, next: NextFunction): Promise<void> {
try {
if (!request.headers['x-auth-token']) {
this.logger.debug('ApiGatewayAuthMiddleware missing x-auth-token header.')
response.status(401).send({
error: {
tag: 'invalid-auth',
message: 'Invalid login credentials.',
},
})
return
}
const token: CrossServiceTokenData | undefined = this.tokenDecoder.decodeToken(
request.headers['x-auth-token'] as string,
)
if (token === undefined) {
this.logger.debug('ApiGatewayAuthMiddleware authentication failure.')
response.status(401).send({
error: {
tag: 'invalid-auth',
message: 'Invalid login credentials.',
},
})
return
}
response.locals.user = token.user
response.locals.roles = token.roles
response.locals.session = token.session
response.locals.readOnlyAccess = token.session?.readonly_access ?? false
return next()
} catch (error) {
return next(error)
}
}
}

View file

@ -0,0 +1,26 @@
import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'
import { WorkspaceType } from './WorkspaceType'
@Entity({ name: 'workspaces' })
export class Workspace {
@PrimaryGeneratedColumn('uuid')
declare uuid: string
@Column({
length: 64,
})
declare type: WorkspaceType
@Column({
length: 255,
nullable: true,
type: 'varchar',
})
declare name: string | null
@Column({
name: 'key_rotation_index',
default: 0,
})
declare keyRotationIndex: number
}

View file

@ -1,4 +1,4 @@
export enum GroupAccessLevel {
export enum WorkspaceAccessLevel {
Owner = 'owner',
Admin = 'admin',
ReadOnly = 'read-only',

View file

@ -0,0 +1,5 @@
export enum WorkspaceType {
Root = 'root',
Team = 'team',
Private = 'private',
}

View file

@ -0,0 +1,62 @@
import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'
import { WorkspaceAccessLevel } from './WorkspaceAccessLevel'
import { WorkspaceUserStatus } from './WorkspaceUserStatus'
@Entity({ name: 'workspace_users' })
@Index('index_workspace_users_on_workspace_and_user', ['userUuid', 'workspaceUuid'], { unique: true })
export class WorkspaceUser {
@PrimaryGeneratedColumn('uuid')
declare uuid: string
@Column({
name: 'access_level',
length: 64,
})
declare accessLevel: WorkspaceAccessLevel
@Column({
name: 'user_uuid',
length: 36,
})
declare userUuid: string
@Column({
name: 'workspace_uuid',
length: 36,
})
declare workspaceUuid: string
@Column({
name: 'encrypted_workspace_key',
length: 255,
type: 'varchar',
nullable: true,
})
declare encryptedWorkspaceKey: string | null
@Column({
name: 'public_key',
length: 255,
type: 'varchar',
})
declare publicKey: string
@Column({
name: 'private_key',
length: 255,
type: 'varchar',
})
declare privateKey: string
@Column({
name: 'status',
length: 64,
})
declare status: WorkspaceUserStatus
@Column({
name: 'key_rotation_index',
default: 0,
})
declare keyRotationIndex: number
}

View file

@ -0,0 +1,4 @@
export enum WorkspaceUserStatus {
Active = 'active',
PendingKeyshare = 'pending-keyshare',
}

View file

@ -0,0 +1,9 @@
import { controller, httpGet } from 'inversify-express-utils'
@controller('/healthcheck')
export class InversifyExpressHealthCheckController {
@httpGet('/')
public async get(): Promise<string> {
return 'OK'
}
}

View file

View file

@ -0,0 +1,13 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"composite": true,
"outDir": "./dist",
},
"include": [
"src/**/*",
"bin/**/*",
"migrations/**/*",
],
"references": []
}

17
packages/workspace/wait-for.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/sh
set -e
host="$1"
shift
port="$1"
shift
cmd="$@"
while ! nc -vz $host $port; do
>&2 echo "$host:$port is unavailable yet - waiting for it to start"
sleep 10
done
>&2 echo "$host:$port is up - executing command"
exec $cmd

View file

@ -63,6 +63,9 @@
},
{
"path": "./packages/time"
},
{
"path": "./packages/workspace"
}
]
}

View file

@ -2339,6 +2339,41 @@ __metadata:
languageName: node
linkType: hard
"@standardnotes/workspace-server@workspace:packages/workspace":
version: 0.0.0-use.local
resolution: "@standardnotes/workspace-server@workspace:packages/workspace"
dependencies:
"@newrelic/winston-enricher": ^4.0.0
"@sentry/node": ^7.3.0
"@standardnotes/common": "workspace:*"
"@standardnotes/domain-events": "workspace:*"
"@standardnotes/domain-events-infra": "workspace:*"
"@standardnotes/security": "workspace:*"
"@types/cors": ^2.8.9
"@types/express": ^4.17.11
"@types/ioredis": ^4.28.10
"@types/jest": ^28.1.4
"@types/newrelic": ^7.0.3
"@typescript-eslint/eslint-plugin": ^5.29.0
aws-sdk: ^2.1159.0
cors: 2.8.5
dotenv: ^16.0.1
eslint: ^8.14.0
eslint-plugin-prettier: ^4.0.0
express: ^4.18.1
inversify: ^6.0.1
inversify-express-utils: ^6.4.3
ioredis: ^5.2.0
jest: ^28.1.2
mysql2: ^2.3.3
newrelic: ^9.0.0
reflect-metadata: 0.1.13
ts-jest: ^28.0.5
typeorm: ^0.3.6
winston: ^3.8.1
languageName: unknown
linkType: soft
"@szmarczak/http-timer@npm:^5.0.1":
version: 5.0.1
resolution: "@szmarczak/http-timer@npm:5.0.1"