WIP: File based db migration

packages/dashboard/.dockerignore

@@ -1,2 +1,5 @@
 node_modules/
 .next/
+dist/
+sessions/
+logs/

packages/system-api/.dockerignore

@@ -1,2 +1,4 @@
 node_modules/
-dist/
+dist/
+sessions/
+logs/

packages/system-api/.swcrc

@@ -11,5 +11,6 @@
   "module": {
     "type": "es6"
   },
+  "minify": true,
   "isModule": true
 }

packages/system-api/package.json

@@ -13,7 +13,7 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "test": "jest --colors",
     "test:watch": "jest --watch",
-    "build": "rm -rf dist && swc ./src -d dist",
+    "build": "rm -rf dist && swc ./src --ignore **/*.test.* -d dist",
     "build:watch": "swc ./src -d dist --watch",
     "start:dev": "NODE_ENV=development && nodemon --experimental-specifier-resolution=node --trace-deprecation --trace-warnings --watch dist dist/server.js",
     "dev": "concurrently \"npm run build:watch\" \"npm run start:dev\"",

packages/system-api/src/config/apps.ts

@@ -1,21 +0,0 @@
-export const appNames = [
-  'nextcloud',
-  'syncthing',
-  'freshrss',
-  'anonaddy',
-  'filebrowser',
-  'wg-easy',
-  'jackett',
-  'sonarr',
-  'radarr',
-  'transmission',
-  'jellyfin',
-  'pihole',
-  'tailscale',
-  'n8n',
-  'invidious',
-  'joplin',
-  'homarr',
-  'code-server',
-  'calibre-web',
-] as const;

packages/system-api/src/core/updates/__tests__/v040.test.ts

@@ -0,0 +1,105 @@
+import fs from 'fs';
+import { DataSource } from 'typeorm';
+import App from '../../../modules/apps/app.entity';
+import { AppInfo, AppStatusEnum } from '../../../modules/apps/apps.types';
+import { createApp } from '../../../modules/apps/__tests__/apps.factory';
+import Update, { UpdateStatusEnum } from '../../../modules/system/update.entity';
+import { setupConnection, teardownConnection } from '../../../test/connection';
+import { updateV040 } from '../v040';
+
+jest.mock('fs');
+
+let db: DataSource | null = null;
+const TEST_SUITE = 'updatev040';
+
+beforeAll(async () => {
+  db = await setupConnection(TEST_SUITE);
+});
+
+beforeEach(async () => {
+  jest.resetModules();
+  jest.resetAllMocks();
+  await App.clear();
+  await Update.clear();
+});
+
+afterAll(async () => {
+  await db?.destroy();
+  await teardownConnection(TEST_SUITE);
+});
+
+const createState = (apps: string[]) => {
+  return JSON.stringify({ installed: apps.join(' ') });
+};
+
+describe('No state/apps.json', () => {
+  it('Should do nothing and create the update with status SUCCES', async () => {
+    await updateV040();
+
+    const update = await Update.findOne({ where: { name: 'v040' } });
+
+    expect(update).toBeDefined();
+    expect(update!.status).toBe(UpdateStatusEnum.SUCCESS);
+
+    const apps = await App.find();
+
+    expect(apps).toHaveLength(0);
+  });
+});
+
+describe('State/apps.json exists with no installed app', () => {
+  beforeEach(async () => {
+    const { MockFiles } = await createApp();
+    MockFiles['/tipi/state/apps.json'] = createState([]);
+    // @ts-ignore
+    fs.__createMockFiles(MockFiles);
+  });
+
+  it('Should do nothing and create the update with status SUCCES', async () => {
+    await updateV040();
+    const update = await Update.findOne({ where: { name: 'v040' } });
+
+    expect(update).toBeDefined();
+    expect(update?.status).toBe(UpdateStatusEnum.SUCCESS);
+
+    const apps = await App.find();
+    expect(apps).toHaveLength(0);
+  });
+
+  it('Should delete state file after update', async () => {
+    await updateV040();
+    expect(fs.existsSync('/tipi/state/apps.json')).toBe(false);
+  });
+});
+
+describe('State/apps.json exists with one installed app', () => {
+  let app1: AppInfo | null = null;
+  beforeEach(async () => {
+    const { MockFiles, appInfo } = await createApp();
+    app1 = appInfo;
+    MockFiles['/tipi/state/apps.json'] = createState([appInfo.id]);
+    MockFiles[`/tipi/app-data/${appInfo.id}`] = '';
+    MockFiles[`/tipi/app-data/${appInfo.id}/app.env`] = 'TEST=test\nAPP_PORT=3000\nTEST_FIELD=test';
+    // @ts-ignore
+    fs.__createMockFiles(MockFiles);
+  });
+
+  it('Should create a new app and update', async () => {
+    await updateV040();
+
+    const app = await App.findOne({ where: { id: app1?.id } });
+    const update = await Update.findOne({ where: { name: 'v040' } });
+
+    expect(app).toBeDefined();
+    expect(app?.status).toBe(AppStatusEnum.STOPPED);
+    expect(update).toBeDefined();
+    expect(update?.status).toBe('SUCCESS');
+  });
+
+  it("Should correctly pick up app's variables from existing .env file", async () => {
+    await updateV040();
+    const app = await App.findOne({ where: { id: app1?.id } });
+
+    expect(app?.config).toStrictEqual({ TEST_FIELD: 'test' });
+  });
+});

packages/system-api/src/core/updates/run.ts

@@ -0,0 +1,6 @@
+import { updateV040 } from './v040';
+
+export const runUpdates = async (): Promise<void> => {
+  // v040: Update to 0.4.0
+  await updateV040();
+};

packages/system-api/src/core/updates/v040.ts

@@ -0,0 +1,63 @@
+import logger from '../../config/logger/logger';
+import App from '../../modules/apps/app.entity';
+import { AppInfo, AppStatusEnum } from '../../modules/apps/apps.types';
+import { deleteFolder, fileExists, readFile, readJsonFile } from '../../modules/fs/fs.helpers';
+import Update, { UpdateStatusEnum } from '../../modules/system/update.entity';
+
+type AppsState = { installed: string };
+
+const UPDATE_NAME = 'v040';
+
+export const updateV040 = async (): Promise<void> => {
+  try {
+    const update = await Update.findOne({ where: { name: UPDATE_NAME } });
+
+    if (update) {
+      logger.info(`Update ${UPDATE_NAME} already applied`);
+      return;
+    }
+
+    if (fileExists('/state/apps.json')) {
+      const state: AppsState = await readJsonFile('/state/apps.json');
+      const installed: string[] = state.installed.split(' ').filter(Boolean);
+
+      for (const appId of installed) {
+        const app = await App.findOne({ where: { id: appId } });
+
+        if (!app) {
+          const envFile = readFile(`/app-data/${appId}/app.env`).toString();
+          const envVars = envFile.split('\n');
+          const envVarsMap = new Map<string, string>();
+
+          envVars.forEach((envVar) => {
+            const [key, value] = envVar.split('=');
+            envVarsMap.set(key, value);
+          });
+
+          const form: Record<string, string> = {};
+
+          const configFile: AppInfo = readJsonFile(`/apps/${appId}/config.json`);
+          configFile.form_fields?.forEach((field) => {
+            const envVar = field.env_variable;
+            const envVarValue = envVarsMap.get(envVar);
+
+            if (envVarValue) {
+              form[field.env_variable] = envVarValue;
+            }
+          });
+
+          await App.create({ id: appId, status: AppStatusEnum.STOPPED, config: form }).save();
+        } else {
+          logger.info('App already migrated');
+        }
+      }
+    }
+
+    deleteFolder('/state/apps.json');
+    await Update.create({ name: UPDATE_NAME, status: UpdateStatusEnum.SUCCESS }).save();
+  } catch (error) {
+    logger.error(error);
+    console.error(error);
+    await Update.create({ name: UPDATE_NAME, status: UpdateStatusEnum.FAILED }).save();
+  }
+};

packages/system-api/src/modules/apps/apps.helpers.ts

@@ -1,5 +1,4 @@
 import portUsed from 'tcp-port-used';
-import p from 'p-iteration';
 import { fileExists, readdirSync, readFile, readJsonFile, runScript, writeFile } from '../fs/fs.helpers';
 import InternalIp from 'internal-ip';
 import config from '../../config';
@@ -10,12 +9,12 @@ export const checkAppRequirements = async (appName: string) => {
   const configFile: AppInfo = readJsonFile(`/apps/${appName}/config.json`);
 
   if (configFile?.requirements?.ports) {
-    await p.forEachSeries(configFile?.requirements.ports, async (port: number) => {
+    for (const port of configFile.requirements.ports) {
       const ip = await InternalIp.v4();
       const used = await portUsed.check(port, ip);
 
       if (used) valid = false;
-    });
+    }
   }
 
   return valid;

packages/system-api/src/modules/apps/apps.service.ts

@@ -2,6 +2,23 @@ import { createFolder, readFile, readJsonFile } from '../fs/fs.helpers';
 import { checkAppRequirements, checkEnvFile, generateEnvFile, getAvailableApps, runAppScript } from './apps.helpers';
 import { AppInfo, AppStatusEnum, ListAppsResonse } from './apps.types';
 import App from './app.entity';
+import datasource from '../../config/datasource';
+
+const startAllApps = async (): Promise<void> => {
+  const apps = await App.find({ where: { status: AppStatusEnum.RUNNING } });
+
+  await Promise.all(
+    apps.map(async (app) => {
+      // Regenerate env file
+      generateEnvFile(app.id, app.config);
+      checkEnvFile(app.id);
+
+      await App.update({ id: app.id }, { status: AppStatusEnum.STARTING });
+      await runAppScript(['start', app.id]);
+      await App.update({ id: app.id }, { status: AppStatusEnum.RUNNING });
+    }),
+  );
+};
 
 const startApp = async (appName: string): Promise<App> => {
   let app = await App.findOne({ where: { id: appName } });
@@ -18,7 +35,7 @@ const startApp = async (appName: string): Promise<App> => {
   await App.update({ id: appName }, { status: AppStatusEnum.STARTING });
   // Run script
   await runAppScript(['start', appName]);
-  const result = await App.update({ id: appName }, { status: AppStatusEnum.RUNNING });
+  const result = await datasource.createQueryBuilder().update(App).set({ status: AppStatusEnum.RUNNING }).where('id = :id', { id: appName }).returning('*').execute();
 
   return result.raw[0];
 };
@@ -47,7 +64,7 @@ const installApp = async (id: string, form: Record<string, string>): Promise<App
     await runAppScript(['install', id]);
   }
 
-  const result = await App.update({ id }, { status: AppStatusEnum.RUNNING });
+  const result = await datasource.createQueryBuilder().update(App).set({ status: AppStatusEnum.RUNNING }).where('id = :id', { id }).returning('*').execute();
 
   return result.raw[0];
 };
@@ -78,7 +95,7 @@ const updateAppConfig = async (id: string, form: Record<string, string>): Promis
   }
 
   generateEnvFile(id, form);
-  const result = await App.update({ id }, { config: form });
+  const result = await datasource.createQueryBuilder().update(App).set({ config: form }).where('id = :id', { id }).returning('*').execute();
 
   return result.raw[0];
 };
@@ -93,8 +110,7 @@ const stopApp = async (id: string): Promise<App> => {
   // Run script
   await App.update({ id }, { status: AppStatusEnum.STOPPING });
   await runAppScript(['stop', id]);
-  const result = await App.update({ id }, { status: AppStatusEnum.STOPPED });
-
+  const result = await datasource.createQueryBuilder().update(App).set({ status: AppStatusEnum.STOPPED }).where('id = :id', { id }).returning('*').execute();
   return result.raw[0];
 };
 
@@ -126,4 +142,4 @@ const getApp = async (id: string): Promise<App> => {
   return app;
 };
 
-export default { installApp, startApp, listApps, getApp, updateAppConfig, stopApp, uninstallApp };
+export default { installApp, startApp, listApps, getApp, updateAppConfig, stopApp, uninstallApp, startAllApps };

packages/system-api/src/modules/system/update.entity.ts

@@ -0,0 +1,24 @@
+import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from 'typeorm';
+
+export enum UpdateStatusEnum {
+  FAILED = 'FAILED',
+  SUCCESS = 'SUCCESS',
+}
+
+@Entity()
+export default class Update extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column({ type: 'varchar', unique: true, nullable: false })
+  name!: string;
+
+  @Column({ type: 'enum', enum: UpdateStatusEnum, nullable: false })
+  status!: UpdateStatusEnum;
+
+  @CreateDateColumn()
+  createdAt!: Date;
+
+  @UpdateDateColumn()
+  updatedAt!: Date;
+}

packages/system-api/src/server.ts

@@ -12,12 +12,15 @@ import { MyContext } from './types';
 import { __prod__ } from './config/constants/constants';
 import cors from 'cors';
 import datasource from './config/datasource';
+import appsService from './modules/apps/apps.service';
+import { runUpdates } from './core/updates/run';
 
 const main = async () => {
   try {
     const app = express();
     const port = 3001;
 
+    app.set('proxy', 1);
     app.use(
       cors({
         credentials: true,
@@ -43,9 +46,7 @@ const main = async () => {
     }
 
     const schema = await createSchema();
-
     const httpServer = createServer(app);
-
     const plugins = [ApolloLogs];
 
     if (!__prod__) {
@@ -57,10 +58,17 @@ const main = async () => {
       context: ({ req, res }): MyContext => ({ req, res }),
       plugins,
     });
+
     await apolloServer.start();
     apolloServer.applyMiddleware({ app });
 
+    // Run migrations
+    // await runUpdates();
+
     httpServer.listen(port, () => {
+      // Start apps
+      appsService.startAllApps();
+
       logger.info(`Server running on port ${port}`);
     });
   } catch (error) {

packages/system-api/src/test/connection.ts

@@ -2,6 +2,7 @@ import { DataSource } from 'typeorm';
 import App from '../modules/apps/app.entity';
 import User from '../modules/auth/user.entity';
 import pg from 'pg';
+import Update from '../modules/system/update.entity';
 
 const pgClient = new pg.Client({
   user: 'postgres',
@@ -28,7 +29,7 @@ export const setupConnection = async (testsuite: string): Promise<DataSource> =>
     dropSchema: true,
     logging: false,
     synchronize: true,
-    entities: [App, User],
+    entities: [App, User, Update],
   });
 
   return AppDataSource.initialize();