ci: run e2e tests on digital ocean droplet

.github/workflows/e2e.yml

@@ -0,0 +1,170 @@
+name: E2E Tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  build:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push images
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: meienberger/runtipi:e2e
+          cache-from: type=registry,ref=meienberger/runtipi:buildcache
+          cache-to: type=registry,ref=meienberger/runtipi:buildcache,mode=max
+
+  deploy:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    needs: [build]
+    outputs:
+      droplet_id: ${{ steps.create-droplet.outputs.droplet_id }}
+      droplet_ip: ${{ steps.get-droplet-ip.outputs.droplet_ip }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_KEY }}
+          known_hosts: unnecessary
+          name: id_rsa
+
+      - name: Get sha of last commit
+        id: get-sha
+        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Create new Droplet
+        id: create-droplet
+        run: |
+          droplet_id=$(doctl compute droplet create runtipi-${{ steps.get-sha.outputs.sha }} \
+          --image ubuntu-20-04-x64 \
+          --size s-1vcpu-1gb \
+          --format ID \
+          --no-header \
+          --ssh-keys ${{ secrets.SSH_KEY_FINGERPRINT }})
+          echo "droplet_id=$droplet_id" >> $GITHUB_OUTPUT
+
+      - name: Wait for Droplet to become active
+        run: |
+          while ! doctl compute droplet get ${{ steps.create-droplet.outputs.droplet_id }} --format Status --no-header | grep -q "active"; do sleep 5; done
+
+      - name: Get Droplet IP address
+        id: get-droplet-ip
+        run: |
+          droplet_ip=$(doctl compute droplet get ${{ steps.create-droplet.outputs.droplet_id }} --format PublicIPv4 --no-header)
+          echo "droplet_ip=$droplet_ip" >> $GITHUB_OUTPUT
+
+      - name: Wait for SSH to be ready on Droplet
+        run: |
+          while ! ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa root@${{ steps.get-droplet-ip.outputs.droplet_ip }} "echo 'SSH is ready'"; do sleep 5; done
+
+      - name: Wait 1 minute for Droplet to be ready
+        run: sleep 60
+
+      - name: Deploy app to Droplet
+        uses: fifsky/ssh-action@master
+        with:
+          command: |
+            echo 'Cloning repo on branch ${{ github.head_ref }}'
+            git clone --single-branch --branch ${{ github.head_ref }} https://github.com/${{ github.repository }}
+            echo 'Waiting for dpkg lock to be released'
+            cd runtipi
+            echo 'Checking out branch ${{ github.head_ref }}'
+            git checkout ${{ github.head_ref }}
+            sudo ./scripts/start-e2e.sh latest
+            echo 'App deployed'
+          host: ${{ steps.get-droplet-ip.outputs.droplet_ip }}
+          user: root
+          key: ${{ secrets.SSH_KEY }}
+
+  e2e:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    needs: [deploy]
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: pnpm/action-setup@v2.2.4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Create .env.e2e file with Droplet IP
+        run: echo "SERVER_IP=${{ needs.deploy.outputs.droplet_ip }}" > .env.e2e
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: npm run test:e2e
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+  teardown:
+    runs-on: ubuntu-latest
+    if: always()
+    needs: [e2e, deploy]
+    steps:
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Delete Droplet
+        run: doctl compute droplet delete ${{ needs.deploy.outputs.droplet_id }} --force

.github/workflows/playwright.yml

@@ -1,59 +0,0 @@
-name: Playwright Tests
-on:
-  push:
-    branches: [develop, master, test/e2e-playwright]
-  pull_request:
-    branches: [develop, master]
-
-jobs:
-  e2e:
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-
-      - uses: pnpm/action-setup@v2.2.2
-        name: Install pnpm
-        id: pnpm-install
-        with:
-          version: 8
-          run_install: false
-
-      - name: Install fswatch
-        run: sudo apt-get update && sudo apt-get install fswatch
-
-      - name: Get pnpm store directory
-        id: pnpm-cache
-        run: |
-          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
-
-      - uses: actions/cache@v3
-        name: Setup pnpm cache
-        with:
-          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
-          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-store-
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
-
-      - name: Build and run the app
-        run: npm run start:e2e
-
-      - name: Run Playwright tests
-        run: npm run test:e2e
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30

docker-compose.e2e.yml

@@ -50,9 +50,7 @@ services:
       - tipi_main_network
 
   dashboard:
-    build:
+    image: meienberger/runtipi:${DOCKER_TAG}
-      context: .
-      dockerfile: Dockerfile
     restart: unless-stopped
     container_name: dashboard
     networks:

e2e/0002-login.spec.ts

@@ -1,6 +1,12 @@
 import { test, expect } from '@playwright/test';
-import { loginUser } from './fixtures/fixtures';
+import { registerUser } from './fixtures/fixtures';
 import { testUser } from './helpers/constants';
+import { clearDatabase } from './helpers/db';
+
+test.beforeEach(async ({ page }) => {
+  await clearDatabase();
+  await registerUser(page);
+});
 
 test('user can login and is redirected to the dashboard', async ({ page }) => {
   await page.goto('/login');
@@ -13,7 +19,6 @@ test('user can login and is redirected to the dashboard', async ({ page }) => {
 });
 
 test('user can logout', async ({ page }) => {
-  await loginUser(page);
   await page.getByTestId('logout-button').click();
 
   await expect(page.getByText('Login to your account')).toBeVisible();

e2e/0003-apps.spec.ts

@@ -1,8 +1,10 @@
 import { test, expect } from '@playwright/test';
-import { loginUser } from './fixtures/fixtures';
+import { registerUser } from './fixtures/fixtures';
+import { clearDatabase } from './helpers/db';
 
 test.beforeEach(async ({ page, isMobile }) => {
-  await loginUser(page);
+  await clearDatabase();
+  await registerUser(page);
 
   // Go to hello world app
   if (isMobile) {

e2e/fixtures/fixtures.ts

@@ -1,6 +1,18 @@
 import { expect, Page } from '@playwright/test';
 import { testUser } from '../helpers/constants';
 
+export const registerUser = async (page: Page) => {
+  await page.goto('/register');
+  await page.getByPlaceholder('you@example.com').click();
+  await page.getByPlaceholder('you@example.com').fill(testUser.email);
+
+  await page.getByPlaceholder('Enter your password', { exact: true }).fill(testUser.password);
+  await page.getByPlaceholder('Confirm your password').fill(testUser.password);
+
+  await page.getByRole('button', { name: 'Register' }).click();
+  await expect(page).toHaveTitle(/Dashboard/);
+};
+
 export const loginUser = async (page: Page) => {
   await page.goto('/login');
 

e2e/helpers/db.ts

@@ -1,13 +1,12 @@
 import pg from 'pg';
-import { getConfig } from '../../src/server/core/TipiConfig';
 
 export const clearDatabase = async () => {
   const pgClient = new pg.Client({
-    user: getConfig().postgresUsername,
+    user: 'tipi',
-    host: '127.0.0.1',
+    host: process.env.SERVER_IP,
-    database: getConfig().postgresDatabase,
+    database: 'tipi',
-    password: getConfig().postgresPassword,
+    password: 'postgres',
-    port: getConfig().postgresPort,
+    port: 5432,
   });
 
   await pgClient.connect();

e2e/helpers/global-setup.ts

@@ -5,7 +5,6 @@ import { clearDatabase } from './db';
  */
 async function globalSetup() {
   await clearDatabase();
-  console.log('Global setup...');
 }
 
 export default globalSetup;

package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "copy:migrations": "mkdir -p dist/migrations && cp -r ./src/server/migrations dist",
     "test": "dotenv -e .env.test -- jest --colors",
-    "test:e2e": "NODE_ENV=test dotenv -e .env -- playwright test",
+    "test:e2e": "NODE_ENV=test dotenv -e .env -e .env.e2e -- playwright test",
     "test:e2e:ui": "NODE_ENV=test dotenv -e .env -- playwright test --ui",
     "test:client": "jest --colors --selectProjects client --",
     "test:server": "jest --colors --selectProjects server --",
@@ -60,7 +60,6 @@
     "next": "13.4.4",
     "next-intl": "^2.14.2",
     "node-cron": "^3.0.1",
-    "node-fetch-commonjs": "^3.2.4",
     "pg": "^8.11.0",
     "qrcode.react": "^3.1.0",
     "react": "18.2.0",
@@ -70,6 +69,7 @@
     "react-markdown": "^8.0.7",
     "react-select": "^5.7.3",
     "react-tooltip": "^5.13.1",
+    "redaxios": "^0.5.1",
     "redis": "^4.6.6",
     "remark-breaks": "^3.0.3",
     "remark-gfm": "^3.0.1",
@@ -90,7 +90,6 @@
     "@faker-js/faker": "^8.0.1",
     "@testing-library/dom": "^9.3.0",
     "@playwright/test": "^1.32.3",
-    "@testing-library/dom": "^9.0.1",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",

playwright.config.ts

@@ -26,7 +26,7 @@ export default defineConfig({
 
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */
-    baseURL: 'http://localhost:3000',
+    baseURL: `http://${process.env.SERVER_IP}`,
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
@@ -42,21 +42,21 @@ export default defineConfig({
       use: { ...devices['Desktop Chrome'] },
     },
 
-    {
+    // {
-      name: 'firefox',
+    //   name: 'firefox',
-      use: { ...devices['Desktop Firefox'] },
+    //   use: { ...devices['Desktop Firefox'] },
-    },
+    // },
 
-    {
+    // {
-      name: 'webkit',
+    //   name: 'webkit',
-      use: { ...devices['Desktop Safari'] },
+    //   use: { ...devices['Desktop Safari'] },
-    },
+    // },
 
     /* Test against mobile viewports. */
-    {
+    // {
-      name: 'Mobile Chrome',
+    //   name: 'Mobile Chrome',
-      use: { ...devices['Pixel 5'] },
+    //   use: { ...devices['Pixel 5'] },
-    },
+    // },
     // {
     //   name: 'Mobile Safari',
     //   use: { ...devices['iPhone 12'] },
@@ -74,9 +74,9 @@ export default defineConfig({
   ],
 
   /* Run your local dev server before starting the tests */
-  webServer: {
+  // webServer: {
-    command: 'npm run start:e2e',
+  //   command: 'npm run start:e2e',
-    url: 'http://127.0.0.1:3000',
+  //   url: 'http://174.138.79.40',
-    reuseExistingServer: true,
+  //   reuseExistingServer: true,
-  },
+  // },
 });

pnpm-lock.yaml

@@ -91,9 +91,6 @@ dependencies:
   node-cron:
     specifier: ^3.0.1
     version: 3.0.2
-  node-fetch-commonjs:
-    specifier: ^3.2.4
-    version: 3.2.4
   pg:
     specifier: ^8.11.0
     version: 8.11.0
@@ -121,6 +118,9 @@ dependencies:
   react-tooltip:
     specifier: ^5.13.1
     version: 5.13.1(react-dom@18.2.0)(react@18.2.0)
+  redaxios:
+    specifier: ^0.5.1
+    version: 0.5.1
   redis:
     specifier: ^4.6.6
     version: 4.6.6
@@ -175,7 +175,7 @@ devDependencies:
     specifier: ^1.32.3
     version: 1.32.3
   '@testing-library/dom':
-    specifier: ^9.0.1
+    specifier: ^9.3.0
     version: 9.3.0
   '@testing-library/jest-dom':
     specifier: ^5.16.5
@@ -1618,7 +1618,7 @@ packages:
     engines: {node: '>=14'}
     hasBin: true
     dependencies:
-      '@types/node': 20.2.1
+      '@types/node': 20.2.5
       playwright-core: 1.32.3
     optionalDependencies:
       fsevents: 2.3.2
@@ -5100,14 +5100,6 @@ packages:
     resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==}
     dev: false
 
-  /fetch-blob@3.2.0:
-    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
-    engines: {node: ^12.20 || >= 14.13}
-    dependencies:
-      node-domexception: 1.0.0
-      web-streams-polyfill: 3.2.1
-    dev: false
-
   /figures@3.2.0:
     resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==}
     engines: {node: '>=8'}
@@ -5194,13 +5186,6 @@ packages:
       mime-types: 2.1.35
     dev: true
 
-  /formdata-polyfill@4.0.10:
-    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
-    engines: {node: '>=12.20.0'}
-    dependencies:
-      fetch-blob: 3.2.0
-    dev: false
-
   /formidable@2.1.2:
     resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==}
     dependencies:
@@ -7415,19 +7400,6 @@ packages:
       uuid: 8.3.2
     dev: false
 
-  /node-domexception@1.0.0:
-    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
-    engines: {node: '>=10.5.0'}
-    dev: false
-
-  /node-fetch-commonjs@3.2.4:
-    resolution: {integrity: sha512-bZW7+ldcuuMPLTJk8DufhT6qHDRdljYD0jqBjmrYfcInaYcReX5kK42SQsu/jvtit/tER28yYjnk63PEEmNPtg==}
-    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
-    dependencies:
-      formdata-polyfill: 4.0.10
-      web-streams-polyfill: 3.2.1
-    dev: false
-
   /node-fetch@2.6.9:
     resolution: {integrity: sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==}
     engines: {node: 4.x || >=6.0.0}
@@ -8209,6 +8181,10 @@ packages:
     dependencies:
       picomatch: 2.3.1
 
+  /redaxios@0.5.1:
+    resolution: {integrity: sha512-FSD2AmfdbkYwl7KDExYQlVvIrFz6Yd83pGfaGjBzM9F6rpq8g652Q4Yq5QD4c+nf4g2AgeElv1y+8ajUPiOYMg==}
+    dev: false
+
   /redent@3.0.0:
     resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==}
     engines: {node: '>=8'}
@@ -9436,11 +9412,6 @@ packages:
       - supports-color
     dev: false
 
-  /web-streams-polyfill@3.2.1:
-    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
-    engines: {node: '>= 8'}
-    dev: false
-
   /webidl-conversions@3.0.1:
     resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
 

scripts/common.sh

@@ -95,6 +95,7 @@ function kill_watcher() {
             kill -9 $watcher_pid
         fi
     fi
+    # pkill -f "watcher.sh"
 }
 
 
@@ -140,6 +141,7 @@ function generate_env_file() {
   local postgres_port=$(get_json_field "$json_file" postgres_port)
   local redis_host=$(get_json_field "$json_file" redis_host)
   local demo_mode=$(get_json_field "$json_file" demo_mode)
+  local docker_tag=$(get_json_field "$json_file" docker_tag)
   local root_folder=$(get_json_field "$json_file" root_folder | sed 's/\//\\\//g')
   local apps_repository=$(get_json_field "$json_file" apps_repository | sed 's/\//\\\//g')
   local storage_path=$(get_json_field "$json_file" storage_path | sed 's/\//\\\//g')
@@ -164,7 +166,7 @@ function generate_env_file() {
     if [[ "$(get_json_field "${STATE_FOLDER}/settings.json" appsRepoUrl)" != "null" ]]; then
       apps_repository_temp=$(get_json_field "${STATE_FOLDER}/settings.json" appsRepoUrl)
       apps_repository="$(echo "${apps_repository_temp}" | sed 's/\//\\\//g')"
-      repo_id="$("${ROOT_FOLDER}"/scripts/git.sh get_hash "${apps_repository}")"
+      repo_id="$("${ROOT_FOLDER}"/scripts/git.sh get_hash "${apps_repository_temp}")"
     fi
 
     # If port is set in settings.json, use it
@@ -233,6 +235,7 @@ function generate_env_file() {
       sed "${sed_args[@]}" "s/<postgres_host>/${postgres_host}/g" "${template}"
       sed "${sed_args[@]}" "s/<redis_host>/${redis_host}/g" "${template}"
       sed "${sed_args[@]}" "s/<demo_mode>/${demo_mode}/g" "${template}"
+      sed "${sed_args[@]}" "s/<docker_tag>/${docker_tag}/g" "${template}"
   done
 
   mv -f "$env_file" "$ROOT_FOLDER/.env"

scripts/configure.sh

@@ -8,12 +8,10 @@ function install_generic() {
   local os="${2}"
 
   if [[ "${os}" == "debian" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "${dependency}"
-    sudo apt-get install -y "${dependency}"
     return 0
   elif [[ "${os}" == "ubuntu" || "${os}" == "pop" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "${dependency}"
-    sudo apt-get install -y "${dependency}"
     return 0
   elif [[ "${os}" == "centos" ]]; then
     sudo yum install -y --allowerasing "${dependency}"
@@ -31,27 +29,23 @@ function install_generic() {
 
 function install_docker() {
   local os="${1}"
-  echo "Installing docker for os ${os}" >/dev/tty
+  echo "Installing docker for os ${os}"
 
   if [[ "${os}" == "debian" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg lsb-release
-    sudo apt-get upgrade
-    sudo apt-get install -y ca-certificates curl gnupg lsb-release
     sudo mkdir -p /etc/apt/keyrings
     curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
     echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get update -y
-    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
     return 0
   elif [[ "${os}" == "ubuntu" || "${os}" == "pop" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg lsb-release
-    sudo apt-get upgrade
-    sudo apt-get install -y ca-certificates curl gnupg lsb-release
     sudo mkdir -p /etc/apt/keyrings
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
     echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get update -y
-    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
     return 0
   elif [[ "${os}" == "centos" ]]; then
     sudo yum install -y yum-utils
@@ -82,12 +76,10 @@ function update_docker() {
   echo "Updating Docker for os ${os}" >/dev/tty
 
   if [[ "${os}" == "debian" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
-    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
     return 0
   elif [[ "${os}" == "ubuntu" || "${os}" == "pop" ]]; then
-    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
-    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
     return 0
   elif [[ "${os}" == "centos" ]]; then
     sudo yum install -y --allowerasing docker-ce docker-ce-cli containerd.io docker-compose-plugin
@@ -103,7 +95,13 @@ function update_docker() {
   fi
 }
 
+echo "Updating system"
+sudo DEBIAN_FRONTEND=noninteractive apt-get update -y
+echo "Upgrading system"
+sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
+
 if ! command -v docker >/dev/null; then
+  echo "Installing docker"
   install_docker "${OS}"
   docker_result=$?
 

scripts/start-e2e.sh

@@ -6,6 +6,8 @@ if [[ "${TRACE-0}" == "1" ]]; then
     set -o xtrace
 fi
 
+export DEBIAN_FRONTEND=noninteractive
+
 source "${BASH_SOURCE%/*}/common.sh"
 
 clean_logs
@@ -15,6 +17,26 @@ clean_logs
 ### --------------------------------
 ROOT_FOLDER="${PWD}"
 STATE_FOLDER="${ROOT_FOLDER}/state"
+## Comes from first argument
+DOCKER_TAG="${1}"
+echo "Starting e2e tests with tag meienberger/runtipi:${DOCKER_TAG}"
+
+### --------------------------------
+### Pre-configuration
+### --------------------------------
+sudo "${ROOT_FOLDER}/scripts/configure.sh"
+mkdir -p "${ROOT_FOLDER}/state"
+STATE_FOLDER="${ROOT_FOLDER}/state"
+
+if [[ ! -f "${STATE_FOLDER}/seed" ]]; then
+  echo "Generating seed..."
+  mkdir -p "${STATE_FOLDER}"
+  touch "${STATE_FOLDER}/seed"
+
+  if ! tr </dev/urandom -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 >"${STATE_FOLDER}/seed"; then
+    echo "Created seed file..."
+  fi
+fi
 
 ### --------------------------------
 ### Apps repository configuration
@@ -26,7 +48,7 @@ env_variables_json=$(cat <<EOF
   "dns_ip": "9.9.9.9",
   "domain": "tipi.localhost",
   "root_folder": "${ROOT_FOLDER}",
-  "nginx_port": 3000,
+  "nginx_port": 80,
   "nginx_port_ssl": 443,
   "jwt_secret": "secret",
   "postgres_password": "postgres",
@@ -40,7 +62,8 @@ env_variables_json=$(cat <<EOF
   "demo_mode": false,
   "apps_repository": "${apps_repository}",
   "storage_path": "${ROOT_FOLDER}",
-  "repo_id": "$("${ROOT_FOLDER}"/scripts/git.sh get_hash ${apps_repository})"
+  "repo_id": "$("${ROOT_FOLDER}"/scripts/git.sh get_hash ${apps_repository})",
+  "docker_tag": "${DOCKER_TAG}"
 }
 EOF
 )
@@ -48,27 +71,32 @@ EOF
 ### --------------------------------
 ### Watcher and system-info
 ### --------------------------------
-mkdir -p "${ROOT_FOLDER}/state"
 
+echo "creating events file"
 if [[ ! -f "${ROOT_FOLDER}/state/events" ]]; then
     touch "${ROOT_FOLDER}/state/events"
 fi
 
+echo "creating system-info file"
 if [[ ! -f "${ROOT_FOLDER}/state/system-info.json" ]]; then
     echo "{}" >"${ROOT_FOLDER}/state/system-info.json"
 fi
 
 chmod -R a+rwx "${ROOT_FOLDER}/state/events"
 chmod -R a+rwx "${ROOT_FOLDER}/state/system-info.json"
+echo "kill previous watcher"
 kill_watcher
-"${ROOT_FOLDER}/scripts/watcher.sh" &
+echo "starting watcher"
+nohup "${ROOT_FOLDER}/scripts/watcher.sh" > /dev/null 2>&1 &
 
 ### --------------------------------
 ### env file generation
 ### --------------------------------
+echo "Generating env file..."
 generate_env_file "${env_variables_json}"
 
 ### --------------------------------
 ### Start the project
 ### --------------------------------
+echo "Starting docker-compose..."
 docker compose -f docker-compose.e2e.yml up -d --build

scripts/start.sh

@@ -22,7 +22,6 @@ clean_logs
 "${ROOT_FOLDER}/scripts/configure.sh"
 
 STATE_FOLDER="${ROOT_FOLDER}/state"
-# Create seed file with cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1
 if [[ ! -f "${STATE_FOLDER}/seed" ]]; then
   echo "Generating seed..."
   if ! tr </dev/urandom -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 >"${STATE_FOLDER}/seed"; then

src/server/index.ts

@@ -50,7 +50,9 @@ nextApp.prepare().then(async () => {
     EventDispatcher.clear();
 
     // Run database migrations
+    if (getConfig().NODE_ENV !== 'development') {
       await runPostgresMigrations();
+    }
     setConfig('status', 'RUNNING');
 
     // Clone and update apps repo

src/server/services/system/system.service.test.ts

@@ -1,17 +1,19 @@
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
 import fs from 'fs-extra';
 import semver from 'semver';
 import { faker } from '@faker-js/faker';
-import fetch from 'node-fetch-commonjs';
 import { EventDispatcher } from '../../core/EventDispatcher';
 import { setConfig } from '../../core/TipiConfig';
 import TipiCache from '../../core/TipiCache';
 import { SystemServiceClass } from '.';
 
 jest.mock('redis');
-jest.mock('node-fetch-commonjs');
 
 const SystemService = new SystemServiceClass();
 
+const server = setupServer();
+
 beforeEach(async () => {
   jest.mock('fs-extra');
   jest.resetModules();
@@ -63,19 +65,28 @@ describe('Test: systemInfo', () => {
 });
 
 describe('Test: getVersion', () => {
+  beforeAll(() => {
+    server.listen();
+  });
+
   beforeEach(() => {
+    server.resetHandlers();
     TipiCache.del('latestVersion');
   });
 
   afterAll(() => {
+    server.close();
     jest.restoreAllMocks();
   });
 
   it('It should return version with body', async () => {
     // Arrange
     const body = faker.lorem.words(10);
-    // @ts-expect-error Mocking fetch
+    server.use(
-    fetch.mockImplementationOnce(() => Promise.resolve({ json: () => Promise.resolve({ name: `v${faker.string.numeric(1)}.${faker.string.numeric(1)}.${faker.string.numeric()}`, body }) }));
+      rest.get('https://api.github.com/repos/meienberger/runtipi/releases/latest', (_, res, ctx) => {
+        return res(ctx.json({ name: `v${faker.string.numeric(1)}.${faker.string.numeric(1)}.${faker.string.numeric()}`, body }));
+      }),
+    );
 
     // Act
     const version = await SystemService.getVersion();
@@ -88,8 +99,11 @@ describe('Test: getVersion', () => {
   });
 
   it('Should return undefined for latest if request fails', async () => {
-    // @ts-expect-error Mocking fetch
+    server.use(
-    fetch.mockImplementationOnce(() => Promise.reject(new Error('API is down')));
+      rest.get('https://api.github.com/repos/meienberger/runtipi/releases/latest', (_, res, ctx) => {
+        return res(ctx.status(500));
+      }),
+    );
 
     const version = await SystemService.getVersion();
 
@@ -100,8 +114,11 @@ describe('Test: getVersion', () => {
 
   it('Should return cached version', async () => {
     // Arrange
-    // @ts-expect-error Mocking fetch
+    server.use(
-    fetch.mockImplementationOnce(() => Promise.resolve({ json: () => Promise.resolve({ name: `v${faker.string.numeric(1)}.${faker.string.numeric(1)}.${faker.string.numeric()}` }) }));
+      rest.get('https://api.github.com/repos/meienberger/runtipi/releases/latest', (_, res, ctx) => {
+        return res(ctx.json({ name: `v${faker.string.numeric(1)}.${faker.string.numeric(1)}.${faker.string.numeric()}` }));
+      }),
+    );
 
     // Act
     const version = await SystemService.getVersion();
@@ -156,8 +173,11 @@ describe('Test: update', () => {
   it('Should throw an error if latest version is not set', async () => {
     // Arrange
     TipiCache.del('latestVersion');
-    // @ts-expect-error Mocking fetch
+    server.use(
-    fetch.mockImplementationOnce(() => Promise.resolve({ json: () => Promise.resolve({ name: null }) }));
+      rest.get('https://api.github.com/repos/meienberger/runtipi/releases/latest', (_, res, ctx) => {
+        return res(ctx.json({ name: null }));
+      }),
+    );
     setConfig('version', '0.0.1');
 
     // Act & Assert

src/server/services/system/system.service.ts

@@ -1,6 +1,6 @@
 import semver from 'semver';
 import { z } from 'zod';
-import fetch from 'node-fetch-commonjs';
+import axios from 'redaxios';
 import { TranslatedError } from '@/server/utils/errors';
 import { readJsonFile } from '../../common/fs.helpers';
 import { EventDispatcher } from '../../core/EventDispatcher';
@@ -48,11 +48,10 @@ export class SystemServiceClass {
       let body = await this.cache.get('latestVersionBody');
 
       if (!version) {
-        const data = await fetch('https://api.github.com/repos/meienberger/runtipi/releases/latest');
+        const { data } = await axios.get<{ name: string; body: string }>('https://api.github.com/repos/meienberger/runtipi/releases/latest');
-        const release = (await data.json()) as { name: string; body: string };
 
-        version = release.name.replace('v', '');
+        version = data.name.replace('v', '');
-        body = release.body;
+        body = data.body;
 
         await this.cache.set('latestVersion', version?.replace('v', '') || '', 60 * 60);
         await this.cache.set('latestVersionBody', body || '', 60 * 60);

templates/env-sample

@@ -21,3 +21,4 @@ POSTGRES_PASSWORD=<postgres_password>
 POSTGRES_PORT=<postgres_port>
 REDIS_HOST=<redis_host>
 DEMO_MODE=<demo_mode>
+DOCKER_TAG=<docker_tag>

tests/server/jest.setup.ts

@@ -1,6 +1,12 @@
+import { fromPartial } from '@total-typescript/shoehorn';
 import { EventDispatcher } from '../../src/server/core/EventDispatcher';
 
 global.fetch = jest.fn();
+// Mock global location
+global.location = fromPartial({
+  hostname: 'localhost',
+});
+
 console.error = jest.fn();
 
 // Mock Logger