refactor(server): migrate to esbuild to have a smaller docker image size

Migrated the server build to esbuild in order to have one bundle for the whole app including the
used modules

Dockerfile

@@ -1,5 +1,12 @@
-FROM node:18 AS builder
+FROM node:18-alpine3.16 AS builder
 
+# Required for argon2
+RUN apk --no-cache add g++
+RUN apk --no-cache add make
+RUN apk --no-cache add python3
+
+# Required for sharp
+RUN apk --no-cache add vips-dev=8.12.2-r5
 RUN npm install node-gyp -g
 
 WORKDIR /api
@@ -18,24 +25,13 @@ WORKDIR /dashboard
 COPY ./packages/dashboard /dashboard
 RUN npm run build
 
-
-FROM alpine:3.16.0 as app
+FROM node:18-alpine3.16 as app
 
 WORKDIR /
 
-# # Install dependencies
-RUN apk --no-cache add nodejs npm
-RUN apk --no-cache add g++
-RUN apk --no-cache add make
-RUN apk --no-cache add python3
-
-RUN npm install node-gyp -g
-
 WORKDIR /api
-COPY ./packages/system-api/package*.json /api/
-RUN npm install --omit=dev
-
-COPY --from=builder /api/dist /api/dist
+COPY ./packages/system-api/package.json /api/
+COPY --from=builder --chown=node:node /api/dist /api/dist
 
 WORKDIR /dashboard
 COPY --from=builder /dashboard/next.config.js ./
@@ -44,4 +40,8 @@ COPY --from=builder /dashboard/package.json ./package.json
 COPY --from=builder --chown=node:node /dashboard/.next/standalone ./
 COPY --from=builder --chown=node:node /dashboard/.next/static ./.next/static
 
+RUN mkdir -p /app/logs
+
+USER node
+
 WORKDIR /

docker-compose.dev.yml

@@ -24,7 +24,7 @@ services:
     restart: unless-stopped
     stop_grace_period: 1m
     volumes:
-      - ./data/postgres:/var/lib/postgresql/data
+      - pgdata:/var/lib/postgresql/data
     ports:
       - 5432:5432
     environment:
@@ -135,3 +135,6 @@ networks:
       driver: default
       config:
         - subnet: 10.21.21.0/24
+
+volumes:
+  pgdata:

docker-compose.test.yml

@@ -0,0 +1,151 @@
+version: "3.7"
+
+services:
+  reverse-proxy:
+    container_name: reverse-proxy
+    image: traefik:v2.8
+    restart: unless-stopped
+    ports:
+      - ${NGINX_PORT-80}:80
+      - ${NGINX_PORT_SSL-443}:443
+    command: --providers.docker
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${PWD}/traefik:/root/.config
+      - ${PWD}/traefik/shared:/shared
+    networks:
+      - tipi_main_network
+
+  tipi-db:
+    container_name: tipi-db
+    image: postgres:14
+    restart: unless-stopped
+    stop_grace_period: 1m
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USER: tipi
+      POSTGRES_DB: tipi
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d tipi -U tipi"]
+      interval: 5s
+      timeout: 10s
+      retries: 120
+    networks:
+      - tipi_main_network
+
+  tipi-redis:
+    container_name: tipi-redis
+    image: redis:alpine
+    restart: unless-stopped
+    volumes:
+      - ./data/redis:/data
+    networks:
+      - tipi_main_network
+
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: /bin/sh -c "cd /api && npm run start"
+    restart: unless-stopped
+    container_name: api
+    depends_on:
+      tipi-db:
+        condition: service_healthy
+    volumes:
+      - ${PWD}/repos:/runtipi/repos:ro
+      - ${PWD}/apps:/runtipi/apps
+      - ${PWD}/state:/runtipi/state
+      - ${PWD}/logs:/app/logs
+      - ${STORAGE_PATH}:/app/storage
+      - ${PWD}/.env:/runtipi/.env:ro
+    environment:
+      INTERNAL_IP: ${INTERNAL_IP}
+      TIPI_VERSION: ${TIPI_VERSION}
+      JWT_SECRET: ${JWT_SECRET}
+      NGINX_PORT: ${NGINX_PORT}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USERNAME: tipi
+      POSTGRES_DBNAME: tipi
+      POSTGRES_HOST: tipi-db
+      NODE_ENV: production
+      APPS_REPO_ID: ${APPS_REPO_ID}
+      APPS_REPO_URL: ${APPS_REPO_URL}
+      DOMAIN: ${DOMAIN}
+      ARCHITECTURE: ${ARCHITECTURE}
+    networks:
+      - tipi_main_network
+    labels:
+      traefik.enable: true
+      # Web
+      traefik.http.routers.api.rule: PathPrefix(`/api`)
+      traefik.http.routers.api.service: api
+      traefik.http.routers.api.entrypoints: web
+      traefik.http.routers.api.middlewares: api-stripprefix
+      traefik.http.services.api.loadbalancer.server.port: 3001
+      # Websecure
+      traefik.http.routers.api-secure.rule: (Host(`${DOMAIN}`) && PathPrefix(`/api`))
+      traefik.http.routers.api-secure.entrypoints: websecure
+      traefik.http.routers.api-secure.service: api-secure
+      traefik.http.routers.api-secure.tls.certresolver: myresolver
+      traefik.http.routers.api-secure.middlewares: api-stripprefix
+      traefik.http.services.api-secure.loadbalancer.server.port: 3001
+      # Middlewares
+      traefik.http.middlewares.api-stripprefix.stripprefix.prefixes: /api
+
+  dashboard:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: /bin/sh -c "cd /dashboard && node server.js"
+    restart: unless-stopped
+    container_name: dashboard
+    networks:
+      - tipi_main_network
+    depends_on:
+      api:
+        condition: service_started
+    environment:
+      NODE_ENV: production
+    labels:
+      traefik.enable: true
+      traefik.http.routers.dashboard-redirect.rule: PathPrefix("/")
+      traefik.http.routers.dashboard-redirect.entrypoints: web
+      traefik.http.routers.dashboard-redirect.middlewares: redirect-middleware
+      traefik.http.routers.dashboard-redirect.service: dashboard
+      traefik.http.services.dashboard-redirect.loadbalancer.server.port: 3000
+
+      traefik.http.routers.dashboard-redirect-secure.rule: Host(`${DOMAIN}`) && PathPrefix(`/`)
+      traefik.http.routers.dashboard-redirect-secure.entrypoints: websecure
+      traefik.http.routers.dashboard-redirect-secure.middlewares: redirect-middleware
+      traefik.http.routers.dashboard-redirect-secure.service: dashboard
+      traefik.http.routers.dashboard-redirect-secure.tls.certresolver: myresolver
+      traefik.http.services.dashboard-redirect-secure.loadbalancer.server.port: 3000
+
+      # Web
+      traefik.http.routers.dashboard.rule: PathPrefix("/dashboard")
+      traefik.http.routers.dashboard.service: dashboard
+      traefik.http.routers.dashboard.entrypoints: web
+      traefik.http.services.dashboard.loadbalancer.server.port: 3000
+      # Websecure
+      traefik.http.routers.dashboard-secure.rule: Host(`${DOMAIN}`) && PathPrefix(`/dashboard`)
+      traefik.http.routers.dashboard-secure.service: dashboard-secure
+      traefik.http.routers.dashboard-secure.entrypoints: websecure
+      traefik.http.routers.dashboard-secure.tls.certresolver: myresolver
+      traefik.http.services.dashboard-secure.loadbalancer.server.port: 3000
+      # Middlewares
+      traefik.http.middlewares.redirect-middleware.redirectregex.regex: .*
+      traefik.http.middlewares.redirect-middleware.redirectregex.replacement: /dashboard
+
+networks:
+  tipi_main_network:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.21.21.0/24
+
+volumes:
+  pgdata:

package.json

@@ -9,10 +9,11 @@
     "act:docker": "act --container-architecture linux/amd64 --secret-file github.secrets -j build-images",
     "start:dev": "./scripts/start-dev.sh",
     "start:rc": "docker-compose -f docker-compose.rc.yml --env-file .env up --build",
-    "start:prod": "docker-compose --env-file .env up --build",
+    "start:prod": "docker-compose -f docker-compose.test.yml --env-file .env up --build",
     "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres",
     "version": "echo $npm_package_version",
-    "release:rc": "./scripts/deploy/release-rc.sh"
+    "release:rc": "./scripts/deploy/release-rc.sh",
+    "test:build": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t meienberger/runtipi:test ."
   },
   "devDependencies": {
     "@commitlint/cli": "^17.0.3",

packages/dashboard/package.json

@@ -31,6 +31,7 @@
     "remark-mdx": "^2.1.1",
     "sass": "^1.55.0",
     "semver": "^7.3.7",
+    "sharp": "0.30.7",
     "swr": "^1.3.0",
     "tslib": "^2.4.0",
     "validator": "^13.7.0",

packages/dashboard/src/pages/_document.tsx

@@ -9,13 +9,14 @@ export default function MyDocument() {
       <Head>
         <link rel="preconnect" href="https://fonts.googleapis.com" />
         <link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin="true" />
+        <link rel="preconnect" href="https://cdn.jsdelivr.net" />
         <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&display=swap" rel="stylesheet" />
         <link rel="apple-touch-icon" sizes="180x180" href={getUrl('apple-touch-icon.png')} />
         <link rel="icon" type="image/png" sizes="32x32" href={getUrl('favicon-32x32.png')} />
         <link rel="icon" type="image/png" sizes="16x16" href={getUrl('favicon-16x16.png')} />
         <link rel="manifest" href={getUrl('site.webmanifest')} />
         <link rel="mask-icon" href={getUrl('safari-pinned-tab.svg')} color="#5bbad5" />
-        <script src="https://cdn.jsdelivr.net/npm/@tabler/core@latest/dist/js/tabler.min.js" defer />
+        <script src="https://cdn.jsdelivr.net/npm/@tabler/core@latest/dist/js/tabler.min.js" async />
         <meta name="msapplication-TileColor" content="#da532c" />
         <meta name="theme-color" content="#ffffff" />
       </Head>

packages/system-api/.eslintrc.cjs → packages/system-api/.eslintrc.js

@@ -18,6 +18,7 @@ module.exports = {
     'import/prefer-default-export': 0,
     'no-underscore-dangle': 0,
     '@typescript-eslint/ban-ts-comment': 0,
+    'import/no-extraneous-dependencies': ['error', { devDependencies: ['**/*.test.ts', '**/*.spec.ts', '**/*.factory.ts', 'esbuild.js'] }],
   },
   globals: {
     NodeJS: true,

packages/system-api/.swcrc

@@ -1,17 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/swcrc",
-  "jsc": {
-    "parser": {
-      "syntax": "typescript",
-      "tsx": false,
-      "decorators": true,
-      "dynamicImport": true
-    },
-    "target": "es2022"
-  },
-  "module": {
-    "type": "es6"
-  },
-  "minify": true,
-  "isModule": true
-}

packages/system-api/esbuild.js

@@ -0,0 +1,85 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const esbuild = require('esbuild');
+const path = require('path');
+
+const commandArgs = process.argv.slice(2);
+
+const nativeNodeModulesPlugin = () => ({
+  name: 'native-node-modules',
+  setup(build) {
+    // If a ".node" file is imported within a module in the "file" namespace, resolve
+    // it to an absolute path and put it into the "node-file" virtual namespace.
+    build.onResolve({ filter: /\.node$/, namespace: 'file' }, (args) => {
+      const resolvedId = require.resolve(args.path, {
+        paths: [args.resolveDir],
+      });
+      if (resolvedId.endsWith('.node')) {
+        return {
+          path: resolvedId,
+          namespace: 'node-file',
+        };
+      }
+      return {
+        path: resolvedId,
+      };
+    });
+
+    // Files in the "node-file" virtual namespace call "require()" on the
+    // path from esbuild of the ".node" file in the output directory.
+    build.onLoad({ filter: /.*/, namespace: 'node-file' }, (args) => ({
+      contents: `
+              import path from ${JSON.stringify(args.path)}
+              try { module.exports = require(path) }
+              catch {}
+            `,
+      resolveDir: path.dirname(args.path),
+    }));
+
+    // If a ".node" file is imported within a module in the "node-file" namespace, put
+    // it in the "file" namespace where esbuild's default loading behavior will handle
+    // it. It is already an absolute path since we resolved it to one above.
+    build.onResolve({ filter: /\.node$/, namespace: 'node-file' }, (args) => ({
+      path: args.path,
+      namespace: 'file',
+    }));
+
+    // Tell esbuild's default loading behavior to use the "file" loader for
+    // these ".node" files.
+    const opts = build.initialOptions;
+    opts.loader = opts.loader || {};
+    opts.loader['.node'] = 'file';
+  },
+});
+
+/* Bundle server */
+esbuild.build({
+  entryPoints: ['./src/server.ts'],
+  bundle: true,
+  platform: 'node',
+  target: 'node18',
+  external: ['pg-native'],
+  sourcemap: commandArgs.includes('--sourcemap'),
+  watch: commandArgs.includes('--watch'),
+  outfile: 'dist/server.bundle.js',
+  plugins: [nativeNodeModulesPlugin()],
+  logLevel: 'info',
+  minifySyntax: true,
+  minifyWhitespace: true,
+});
+
+const glob = require('glob');
+
+/* Migrations */
+const migrationFiles = glob.sync('./src/config/migrations/*.ts');
+
+esbuild.buildSync({
+  entryPoints: migrationFiles,
+  platform: 'node',
+  target: 'node18',
+  minify: false,
+  outdir: 'dist/config/migrations',
+  logLevel: 'info',
+  format: 'cjs',
+  minifySyntax: true,
+  minifyWhitespace: true,
+});

packages/system-api/package.json

@@ -2,8 +2,7 @@
   "name": "system-api",
   "version": "0.7.4",
   "description": "",
-  "exports": "./dist/server.js",
-  "type": "module",
+  "exports": "./dist/server.bundle.js",
   "engines": {
     "node": ">=14.16"
   },
@@ -13,20 +12,18 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "test": "jest --colors",
     "test:watch": "jest --watch",
-    "build": "rm -rf dist && swc ./src -d dist",
-    "build:watch": "swc ./src -d dist --watch",
-    "start:dev": "NODE_ENV=development && nodemon --experimental-specifier-resolution=node --trace-deprecation --trace-warnings --watch dist dist/server.js",
+    "build": "rm -rf dist && node esbuild.js",
+    "build:watch": "node esbuild.js --sourcemap --watch",
+    "start:dev": "NODE_ENV=development && nodemon --watch dist dist/server.bundle.js",
     "dev": "concurrently \"npm run build:watch\" \"npm run start:dev\"",
-    "start": "NODE_ENV=production && node --experimental-specifier-resolution=node dist/server.js",
-    "typeorm": "node --experimental-specifier-resolution=node --loader ts-node/esm ./node_modules/typeorm/cli.js -d ormconfig.ts",
-    "migration:generate": "npm run typeorm migration:generate ./src/config/migrations/$npm_config_name",
-    "migration:create": "node --experimental-specifier-resolution=node --loader ts-node/esm ./node_modules/typeorm/cli.js migration:create ./src/config/migrations/$npm_config_name"
+    "start": "NODE_ENV=production && node dist/server.bundle.js",
+    "start:test": "NODE_ENV=test && node dist/server.bundle.js",
+    "typeorm": "typeorm-ts-node-commonjs -d ./ormconfig.ts",
+    "migration:generate": "npm run typeorm migration:generate"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@apollo/utils.keyvadapter": "^1.1.2",
-    "@keyv/redis": "^2.5.3",
     "apollo-server-core": "^3.10.0",
     "apollo-server-express": "^3.9.0",
     "argon2": "^0.29.1",
@@ -40,15 +37,13 @@
     "graphql-type-json": "^0.3.2",
     "http": "0.0.1-security",
     "jsonwebtoken": "^8.5.1",
-    "keyv": "^4.5.2",
-    "node-cache": "^5.1.2",
     "node-cron": "^3.0.1",
     "pg": "^8.7.3",
     "redis": "^4.3.1",
     "reflect-metadata": "^0.1.13",
     "semver": "^7.3.7",
     "type-graphql": "^1.1.1",
-    "typeorm": "^0.3.6",
+    "typeorm": "^0.3.11",
     "uuid": "^9.0.0",
     "validator": "^13.7.0",
     "winston": "^3.7.2",
@@ -56,8 +51,6 @@
   },
   "devDependencies": {
     "@faker-js/faker": "^7.3.0",
-    "@swc/cli": "^0.1.57",
-    "@swc/core": "^1.2.210",
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.13",
     "@types/fs-extra": "^9.0.13",
@@ -72,18 +65,20 @@
     "@typescript-eslint/eslint-plugin": "^5.18.0",
     "@typescript-eslint/parser": "^5.22.0",
     "concurrently": "^7.1.0",
+    "esbuild": "^0.16.6",
     "eslint": "^8.13.0",
     "eslint-config-airbnb-typescript": "^17.0.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-prettier": "^4.0.0",
+    "glob": "^8.0.3",
     "graphql-import-node": "^0.0.5",
     "jest": "^28.1.0",
     "nodemon": "^2.0.15",
     "prettier": "2.6.2",
     "rimraf": "^3.0.2",
     "ts-jest": "^28.0.2",
-    "ts-node": "^10.8.2",
+    "ts-node": "^10.9.1",
     "typescript": "4.6.4"
   }
 }

packages/system-api/src/config/logger/logger.ts

@@ -5,11 +5,6 @@ import { getConfig } from '../../core/config/TipiConfig';
 
 const { align, printf, timestamp, combine, colorize } = format;
 
-// Create the logs directory if it does not exist
-if (!fs.existsSync(getConfig().logs.LOGS_FOLDER)) {
-  fs.mkdirSync(getConfig().logs.LOGS_FOLDER);
-}
-
 /**
  * Production logger format
  */
@@ -27,24 +22,29 @@ const combinedLogFormatDev = combine(
   printf((info) => `${info.level}: ${info.message}`),
 );
 
-const Logger = createLogger({
-  level: 'info',
-  format: combinedLogFormat,
-  transports: [
-    //
-    // - Write to all logs with level `info` and below to `app.log`
-    // - Write all logs error (and below) to `error.log`.
-    //
-    new transports.File({
-      filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_ERROR),
-      level: 'error',
-    }),
-    new transports.File({
-      filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_APP),
-    }),
-  ],
-  exceptionHandlers: [new transports.File({ filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_ERROR) })],
-});
+const productionLogger = () => {
+  if (!fs.existsSync(getConfig().logs.LOGS_FOLDER)) {
+    fs.mkdirSync(getConfig().logs.LOGS_FOLDER);
+  }
+  return createLogger({
+    level: 'info',
+    format: combinedLogFormat,
+    transports: [
+      //
+      // - Write to all logs with level `info` and below to `app.log`
+      // - Write all logs error (and below) to `error.log`.
+      //
+      new transports.File({
+        filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_ERROR),
+        level: 'error',
+      }),
+      new transports.File({
+        filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_APP),
+      }),
+    ],
+    exceptionHandlers: [new transports.File({ filename: path.join(getConfig().logs.LOGS_FOLDER, getConfig().logs.LOGS_ERROR) })],
+  });
+};
 
 //
 // If we're not in production then log to the `console
@@ -59,4 +59,4 @@ const LoggerDev = createLogger({
   ],
 });
 
-export default process.env.NODE_ENV === 'production' ? Logger : LoggerDev;
+export default process.env.NODE_ENV === 'production' ? productionLogger() : LoggerDev;

packages/system-api/src/core/config/TipiConfig.ts

@@ -99,7 +99,7 @@ class Config {
 
     const parsed = configSchema.parse({
       ...this.config,
-      ...fileConfig,
+      ...(fileConfig as object),
     });
 
     this.config = parsed;

packages/system-api/src/modules/system/__tests__/system.service.test.ts

@@ -5,7 +5,6 @@ import { faker } from '@faker-js/faker';
 import SystemService from '../system.service';
 import TipiCache from '../../../config/TipiCache';
 import { setConfig } from '../../../core/config/TipiConfig';
-import logger from '../../../config/logger/logger';
 import EventDispatcher from '../../../core/config/EventDispatcher';
 
 jest.mock('fs-extra');

packages/system-api/src/server.ts

@@ -5,8 +5,6 @@ import { ApolloServer } from 'apollo-server-express';
 import { createServer } from 'http';
 import { ZodError } from 'zod';
 import cors, { CorsOptions } from 'cors';
-import Keyv from 'keyv';
-import { KeyvAdapter } from '@apollo/utils.keyvadapter';
 import { createSchema } from './schema';
 import { ApolloLogs } from './config/logger/apollo.logger';
 import logger from './config/logger/logger';
@@ -69,7 +67,7 @@ const main = async () => {
       schema,
       context: ({ req, res }): MyContext => ({ req, res }),
       plugins,
-      cache: new KeyvAdapter(new Keyv(`redis://${getConfig().REDIS_HOST}:6379`)),
+      cache: 'bounded',
     });
 
     await apolloServer.start();

packages/system-api/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "es2018",
-    "module": "es2022",
+    "module": "CommonJS",
     "lib": ["es2021", "ESNext.AsyncIterable"],
     "allowJs": true,
     "skipLibCheck": true,
@@ -19,6 +19,6 @@
     "allowSyntheticDefaultImports": true,
     "outDir": "./dist"
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.cjs", "jest.config.cjs"],
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.js", "jest.config.js"],
   "exclude": ["node_modules"]
 }