feat: create server-preload script to run migrations upon server start

Dockerfile

@@ -24,6 +24,7 @@ RUN npm run build
 WORKDIR /dashboard
 COPY ./packages/dashboard /dashboard
 RUN npm run build
+RUN npm run bundle:preload
 
 FROM node:${NODE_VERSION}-buster-slim as app
 
@@ -38,6 +39,8 @@ COPY --from=builder /api/dist /api/dist
 
 WORKDIR /dashboard
 COPY --from=builder /dashboard/next.config.mjs ./
+COPY --from=builder /dashboard/migrations ./migrations
+COPY --from=builder /dashboard/server-preload.js ./
 COPY --from=builder /dashboard/public ./public
 COPY --from=builder /dashboard/package.json ./package.json
 COPY --from=builder --chown=node:node /dashboard/.next/standalone ./

Dockerfile.dev

@@ -21,4 +21,7 @@ RUN npm install
 COPY ./packages/system-api /api
 COPY ./packages/dashboard /dashboard
 
+WORKDIR /dashboard
+RUN npm run bundle:preload
+
 WORKDIR /

docker-compose.rc.yml

@@ -95,7 +95,7 @@ services:
 
   dashboard:
     image: meienberger/runtipi:rc-${TIPI_VERSION}
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     container_name: dashboard
     networks:
       - tipi_main_network

docker-compose.test.yml

@@ -99,7 +99,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     restart: unless-stopped
     container_name: dashboard
     networks:

docker-compose.yml

@@ -95,7 +95,7 @@ services:
 
   dashboard:
     image: meienberger/runtipi:${TIPI_VERSION}
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     restart: unless-stopped
     container_name: dashboard
     networks:

package.json

@@ -9,7 +9,7 @@
     "start:dev": "./scripts/start-dev.sh",
     "start:rc": "docker-compose -f docker-compose.rc.yml --env-file .env up --build",
     "start:prod": "docker-compose -f docker-compose.test.yml --env-file .env up --build",
-    "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres",
+    "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres:14",
     "version": "echo $npm_package_version",
     "release:rc": "./scripts/deploy/release-rc.sh",
     "test:build": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t meienberger/runtipi:test .",

packages/dashboard/.env.test

@@ -3,5 +3,3 @@ POSTGRES_DBNAME=postgres
 POSTGRES_USERNAME=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_PORT=5433
-
-DATABASE_URL="postgresql://postgres:postgres@localhost:5433/postgres"

packages/dashboard/.gitignore

@@ -14,6 +14,8 @@
 
 # production
 /build
+/dist
+server-preload.js
 
 # misc
 .DS_Store
@@ -32,4 +34,4 @@ yarn-error.log*
 .vercel
 
 # typescript
-*.tsbuildinfo
+*.tsbuildinfo

packages/dashboard/esbuild.js

@@ -0,0 +1,15 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable import/no-extraneous-dependencies */
+const esbuild = require('esbuild');
+
+/* Bundle server preload */
+esbuild.build({
+  entryPoints: ['./server-preload.ts'],
+  bundle: true,
+  allowOverwrite: true,
+  external: ['pg-native'],
+  platform: 'node',
+  target: 'node18',
+  outfile: 'server-preload.js',
+  logLevel: 'info',
+});

packages/dashboard/package.json

@@ -3,22 +3,24 @@
   "version": "0.8.1",
   "private": true,
   "scripts": {
-    "migrate:postgres:test": "dotenv -e .env.test -- npx prisma migrate dev --name postgres-init",
-    "test": "npm run migrate:postgres:test && jest --colors",
+    "prisma:pull": "prisma db pull",
+    "test": "dotenv -e .env.test -- ts-node ./run-migration.ts && jest --colors",
     "test:client": "jest --colors --selectProjects client --",
     "test:server": "jest --colors --selectProjects server --",
     "postinstall": "prisma generate",
-    "dev": "next dev",
+    "dev": "node --require ./server-preload.js ./node_modules/.bin/next dev",
     "build": "next build",
-    "start": "next start",
+    "start": "NODE_ENV=production node --require ./server-preload.js server.js",
     "lint": "next lint",
     "lint:fix": "next lint --fix",
-    "gen": "graphql-codegen --config codegen.yml"
+    "gen": "graphql-codegen --config codegen.yml",
+    "bundle:preload": "node esbuild.js"
   },
   "dependencies": {
     "@apollo/client": "^3.6.8",
     "@hookform/resolvers": "^2.9.10",
     "@prisma/client": "^4.8.0",
+    "@runtipi/postgres-migrations": "^5.3.0",
     "@tabler/core": "1.0.0-beta16",
     "@tabler/icons": "^1.109.0",
     "@tanstack/react-query": "^4.20.4",
@@ -34,6 +36,7 @@
     "isomorphic-fetch": "^3.0.0",
     "jsonwebtoken": "^9.0.0",
     "next": "13.1.1",
+    "pg": "^8.7.3",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.38.0",
@@ -71,6 +74,7 @@
     "@types/jest": "^29.2.4",
     "@types/jsonwebtoken": "^9.0.0",
     "@types/node": "18.11.18",
+    "@types/pg": "^8.6.5",
     "@types/react": "18.0.8",
     "@types/react-dom": "18.0.3",
     "@types/semver": "^7.3.12",
@@ -80,6 +84,7 @@
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
     "dotenv-cli": "^6.0.0",
+    "esbuild": "^0.16.6",
     "eslint": "8.30.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^17.0.0",

packages/dashboard/run-migration.ts

@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+import path from 'path';
+import pg from 'pg';
+import { migrate } from '@runtipi/postgres-migrations';
+
+export const runPostgresMigrations = async () => {
+  console.log('Starting database migration');
+
+  const { POSTGRES_HOST, POSTGRES_DBNAME, POSTGRES_USERNAME, POSTGRES_PASSWORD, POSTGRES_PORT = 5432 } = process.env;
+
+  console.log('Connecting to database', POSTGRES_DBNAME, 'on', POSTGRES_HOST, 'as', POSTGRES_USERNAME, 'on port', POSTGRES_PORT);
+
+  const client = new pg.Client({
+    user: POSTGRES_USERNAME,
+    host: POSTGRES_HOST,
+    database: POSTGRES_DBNAME,
+    password: POSTGRES_PASSWORD,
+    port: Number(POSTGRES_PORT),
+  });
+  await client.connect();
+
+  console.log('Client connected');
+
+  try {
+    const { rows } = await client.query('SELECT * FROM migrations');
+    // if rows contains a migration with name 'Initial1657299198975' (legacy typeorm) delete table migrations. As all migrations are idempotent we can safely delete the table and start over.
+    if (rows.find((row) => row.name === 'Initial1657299198975')) {
+      console.log('Found legacy migration. Deleting table migrations');
+      await client.query('DROP TABLE migrations');
+    }
+  } catch (e) {
+    console.log('Migrations table not found, creating it');
+  }
+
+  console.log('Running migrations');
+  try {
+    await migrate({ client }, path.join(__dirname, 'migrations'), { skipCreateMigrationTable: true });
+  } catch (e) {
+    console.log('Error running migrations. Dropping table migrations and trying again');
+    await client.query('DROP TABLE migrations');
+    await migrate({ client }, path.join(__dirname, 'migrations'), { skipCreateMigrationTable: true });
+  }
+
+  console.log('Migration complete');
+  await client.end();
+};
+
+runPostgresMigrations();

packages/dashboard/server-preload.ts

@@ -0,0 +1,21 @@
+import { runPostgresMigrations } from './run-migration';
+import { EventDispatcher, EventTypes } from './src/server/core/EventDispatcher';
+import { getConfig, setConfig } from './src/server/core/TipiConfig';
+
+const main = async () => {
+  // Run database migrations
+  await runPostgresMigrations();
+
+  // Update app store repository
+  await EventDispatcher.dispatchEventAsync(EventTypes.CLONE_REPO, [getConfig().appsRepoUrl]);
+  await EventDispatcher.dispatchEventAsync(EventTypes.UPDATE_REPO, [getConfig().appsRepoUrl]);
+
+  // startJobs();
+  setConfig('status', 'RUNNING');
+
+  // Start apps
+  // appsService.startAllApps();
+  console.info(`Config: ${JSON.stringify(getConfig(), null, 2)}`);
+};
+
+main();

packages/dashboard/src/server/core/TipiConfig/TipiConfig.ts

@@ -10,7 +10,6 @@ enum AppSupportedArchitecturesEnum {
   AMD64 = 'amd64',
 }
 
-const { serverRuntimeConfig } = nextConfig();
 const {
   NODE_ENV,
   JWT_SECRET,
@@ -27,7 +26,8 @@ const {
   POSTGRES_USERNAME,
   POSTGRES_PASSWORD,
   POSTGRES_PORT = 5432,
-} = serverRuntimeConfig;
+} = nextConfig()?.serverRuntimeConfig || process.env;
+// Use process.env if nextConfig is not available (e.g. in in server-preload.ts)
 
 const configSchema = z.object({
   NODE_ENV: z.union([z.literal('development'), z.literal('production'), z.literal('test')]),

packages/dashboard/src/server/db/client.ts

@@ -11,11 +11,10 @@ declare global {
 export const prisma =
   global.prisma ||
   new PrismaClient({
-    log: getConfig().NODE_ENV === 'development' ? ['query', 'error', 'warn'] : ['error'],
+    log: getConfig().NODE_ENV === 'development' ? ['query', 'warn', 'error'] : ['error'],
     datasources: {
       db: {
-        url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}`,
-        // url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}`,
+        url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}?connect_timeout=300`,
       },
     },
   });

packages/dashboard/tsconfig.json

@@ -12,7 +12,7 @@
     "forceConsistentCasingInFileNames": true,
     "noEmit": true,
     "esModuleInterop": true,
-    "module": "esnext",
+    "module": "CommonJS",
     "moduleResolution": "node",
     "resolveJsonModule": true,
     "isolatedModules": true,

pnpm-lock.yaml

@@ -27,6 +27,7 @@ importers:
       '@graphql-codegen/typescript-react-apollo': ^3.2.16
       '@hookform/resolvers': ^2.9.10
       '@prisma/client': ^4.8.0
+      '@runtipi/postgres-migrations': ^5.3.0
       '@tabler/core': 1.0.0-beta16
       '@tabler/icons': ^1.109.0
       '@tanstack/react-query': ^4.20.4
@@ -43,6 +44,7 @@ importers:
       '@types/jest': ^29.2.4
       '@types/jsonwebtoken': ^9.0.0
       '@types/node': 18.11.18
+      '@types/pg': ^8.6.5
       '@types/react': 18.0.8
       '@types/react-dom': 18.0.3
       '@types/semver': ^7.3.12
@@ -54,6 +56,7 @@ importers:
       argon2: ^0.29.1
       clsx: ^1.1.1
       dotenv-cli: ^6.0.0
+      esbuild: ^0.16.6
       eslint: 8.30.0
       eslint-config-airbnb: ^19.0.4
       eslint-config-airbnb-typescript: ^17.0.0
@@ -73,6 +76,7 @@ importers:
       msw: ^0.49.2
       next: 13.1.1
       next-router-mock: ^0.8.0
+      pg: ^8.7.3
       prisma: ^4.8.0
       react: 18.2.0
       react-dom: 18.2.0
@@ -102,6 +106,7 @@ importers:
       '@apollo/client': 3.6.8_o264z5epwuajru7y4dsijkqr44
       '@hookform/resolvers': 2.9.10_react-hook-form@7.40.0
       '@prisma/client': 4.8.0_prisma@4.8.0
+      '@runtipi/postgres-migrations': 5.3.0
       '@tabler/core': 1.0.0-beta16_biqbaboplfbrettd7655fr4n2y
       '@tabler/icons': 1.116.1_biqbaboplfbrettd7655fr4n2y
       '@tanstack/react-query': 4.20.4_biqbaboplfbrettd7655fr4n2y
@@ -117,6 +122,7 @@ importers:
       isomorphic-fetch: 3.0.0
       jsonwebtoken: 9.0.0
       next: 13.1.1_7nrowiyds4jpk2wpzkb7237oey
+      pg: 8.7.3
       react: 18.2.0
       react-dom: 18.2.0_react@18.2.0
       react-hook-form: 7.40.0_react@18.2.0
@@ -153,6 +159,7 @@ importers:
       '@types/jest': 29.2.4
       '@types/jsonwebtoken': 9.0.0
       '@types/node': 18.11.18
+      '@types/pg': 8.6.5
       '@types/react': 18.0.8
       '@types/react-dom': 18.0.3
       '@types/semver': 7.3.12
@@ -162,6 +169,7 @@ importers:
       '@typescript-eslint/eslint-plugin': 5.47.1_txmweb6yn7coi7nfrp22gpyqmy
       '@typescript-eslint/parser': 5.47.1_lzzuuodtsqwxnvqeq4g4likcqa
       dotenv-cli: 6.0.0
+      esbuild: 0.16.8
       eslint: 8.30.0
       eslint-config-airbnb: 19.0.4_j3uyvjk2vb2gkfzhvqukeu5rlq
       eslint-config-airbnb-typescript: 17.0.0_qipeoi3mvzxgzndpeo4r6kwevy
@@ -176,7 +184,7 @@ importers:
       msw: 0.49.2_typescript@4.9.4
       next-router-mock: 0.8.0_next@13.1.1+react@18.2.0
       prisma: 4.8.0
-      ts-jest: 29.0.3_fckurgq3dkz2wyxkuqw26iqkyi
+      ts-jest: 29.0.3_iyz3vhhlowkpp2xbqliblzwv3y
       ts-node: 10.9.1_awa2wsr5thmg3i7jqycphctjfq
       typescript: 4.9.4
       whatwg-fetch: 3.6.2
@@ -3150,6 +3158,17 @@ packages:
       '@redis/client': 1.3.1
     dev: false
 
+  /@runtipi/postgres-migrations/5.3.0:
+    resolution: {integrity: sha512-pTAx/8j843L4n9f4TOCRh6eGFQD827jY64EVy5luHZNOfaiX1KI6SaWpzMfNPdAwy1od0k5FZrDJjpyHXC0ppg==}
+    engines: {node: '>10.17.0'}
+    hasBin: true
+    dependencies:
+      pg: 8.7.3
+      sql-template-strings: 2.2.2
+    transitivePeerDependencies:
+      - pg-native
+    dev: false
+
   /@rushstack/eslint-patch/1.2.0:
     resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==}
     dev: true
@@ -3691,7 +3710,7 @@ packages:
   /@types/pg/8.6.5:
     resolution: {integrity: sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==}
     dependencies:
-      '@types/node': 17.0.31
+      '@types/node': 18.11.18
       pg-protocol: 1.5.0
       pg-types: 2.2.0
     dev: true
@@ -7554,7 +7573,7 @@ packages:
     dev: true
 
   /has-flag/3.0.0:
-    resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
+    resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
     engines: {node: '>=4'}
 
   /has-flag/4.0.0:
@@ -7708,7 +7727,7 @@ packages:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
 
   /ignore-by-default/1.0.1:
-    resolution: {integrity: sha1-SMptcvbGo68Aqa1K5odr44ieKwk=}
+    resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}
     dev: true
 
   /ignore/5.2.0:
@@ -10506,7 +10525,7 @@ packages:
     dev: true
 
   /nopt/1.0.10:
-    resolution: {integrity: sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=}
+    resolution: {integrity: sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==}
     hasBin: true
     dependencies:
       abbrev: 1.1.1
@@ -12006,6 +12025,11 @@ packages:
     resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
     dev: true
 
+  /sql-template-strings/2.2.2:
+    resolution: {integrity: sha512-UXhXR2869FQaD+GMly8jAMCRZ94nU5KcrFetZfWEMd+LVVG6y0ExgHAhatEcKZ/wk8YcKPdi+hiD2wm75lq3/Q==}
+    engines: {node: '>=4.0.0'}
+    dev: false
+
   /stack-trace/0.0.10:
     resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
     dev: false
@@ -12500,7 +12524,7 @@ packages:
       yargs-parser: 20.2.9
     dev: true
 
-  /ts-jest/29.0.3_fckurgq3dkz2wyxkuqw26iqkyi:
+  /ts-jest/29.0.3_iyz3vhhlowkpp2xbqliblzwv3y:
     resolution: {integrity: sha512-Ibygvmuyq1qp/z3yTh9QTwVVAbFdDy/+4BtIQR2sp6baF2SJU/8CKK/hhnGIDY2L90Az2jIqTwZPnN2p+BweiQ==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     hasBin: true
@@ -12523,6 +12547,7 @@ packages:
     dependencies:
       '@babel/core': 7.17.10
       bs-logger: 0.2.6
+      esbuild: 0.16.8
       fast-json-stable-stringify: 2.1.0
       jest: 29.3.1_zfha7dvnw4nti6zkbsmhmn6xo4
       jest-util: 29.3.1