feat(api): copy app files locally instead of reading from repo

This commit is contained in:
Nicolas Meienberger 2022-08-09 20:44:07 +02:00
parent 2334cff67f
commit cb38cc9c90
18 changed files with 130 additions and 51 deletions

2
.gitignore vendored
View file

@ -9,6 +9,8 @@ traefik/ssl/*
!app-data/.gitkeep
repos/*
!repos/.gitkeep
apps/*
!apps/.gitkeep
scripts/pacapt

0
.husky/pre-push Normal file → Executable file
View file

0
apps/.gitkeep Normal file
View file

View file

@ -14,4 +14,7 @@ module.exports = {
'max-len': [1, { code: 200 }],
'import/extensions': ['error', 'ignorePackages', { js: 'never', jsx: 'never', ts: 'never', tsx: 'never' }],
},
globals: {
JSX: true,
},
};

View file

@ -8,7 +8,8 @@ const fs: {
rmSync: typeof rmSync;
readdirSync: typeof readdirSync;
copyFileSync: typeof copyFileSync;
} = jest.genMockFromModule('fs');
copySync: typeof copyFileSync;
} = jest.genMockFromModule('fs-extra');
let mockFiles = Object.create(null);
@ -74,6 +75,16 @@ const copyFileSync = (source: string, destination: string) => {
mockFiles[destination] = mockFiles[source];
};
const copySync = (source: string, destination: string) => {
mockFiles[destination] = mockFiles[source];
if (mockFiles[source] instanceof Array) {
mockFiles[source].forEach((file: string) => {
mockFiles[destination + '/' + file] = mockFiles[source + '/' + file];
});
}
};
fs.readdirSync = readdirSync;
fs.existsSync = existsSync;
fs.readFileSync = readFileSync;
@ -81,6 +92,7 @@ fs.writeFileSync = writeFileSync;
fs.mkdirSync = mkdirSync;
fs.rmSync = rmSync;
fs.copyFileSync = copyFileSync;
fs.copySync = copySync;
fs.__createMockFiles = createMockFiles;
module.exports = fs;

View file

@ -36,6 +36,7 @@
"dotenv": "^16.0.0",
"express": "^4.17.3",
"express-session": "^1.17.3",
"fs-extra": "^10.1.0",
"graphql": "^15.3.0",
"graphql-type-json": "^0.3.2",
"http": "0.0.1-security",
@ -64,6 +65,7 @@
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/express-session": "^1.17.4",
"@types/fs-extra": "^9.0.13",
"@types/jest": "^27.5.0",
"@types/jsonwebtoken": "^8.5.8",
"@types/mock-fs": "^4.13.1",

View file

@ -1,4 +1,4 @@
import fs from 'fs';
import fs from 'fs-extra';
import path from 'path';
import { createLogger, format, transports } from 'winston';
import config from '..';

View file

@ -1,4 +1,4 @@
import fs from 'fs';
import fs from 'fs-extra';
import { DataSource } from 'typeorm';
import logger from '../../../config/logger/logger';
import App from '../../../modules/apps/app.entity';
@ -60,7 +60,7 @@ describe('No state/apps.json', () => {
describe('State/apps.json exists with no installed app', () => {
beforeEach(async () => {
const { MockFiles } = await createApp();
const { MockFiles } = await createApp({});
MockFiles['/tipi/state/apps.json'] = createState([]);
// @ts-ignore
fs.__createMockFiles(MockFiles);
@ -86,7 +86,7 @@ describe('State/apps.json exists with no installed app', () => {
describe('State/apps.json exists with one installed app', () => {
let app1: AppInfo | null = null;
beforeEach(async () => {
const { MockFiles, appInfo } = await createApp();
const { MockFiles, appInfo } = await createApp({});
app1 = appInfo;
MockFiles['/tipi/state/apps.json'] = createState([appInfo.id]);
MockFiles[`/tipi/app-data/${appInfo.id}`] = '';
@ -115,7 +115,7 @@ describe('State/apps.json exists with one installed app', () => {
});
it('Should not try to migrate app if it already exists', async () => {
const { MockFiles, appInfo } = await createApp(true);
const { MockFiles, appInfo } = await createApp({ installed: true });
app1 = appInfo;
MockFiles['/tipi/state/apps.json'] = createState([appInfo.id]);
MockFiles[`/tipi/app-data/${appInfo.id}`] = '';

View file

@ -39,8 +39,8 @@ export const updateV040 = async (): Promise<void> => {
const form: Record<string, string> = {};
const configFile: AppInfo = readJsonFile(`/repos/${config.APPS_REPO_ID}/apps/${appId}/config.json`);
configFile.form_fields?.forEach((field) => {
const configFile: AppInfo | null = readJsonFile(`/repos/${config.APPS_REPO_ID}/apps/${appId}/config.json`);
configFile?.form_fields?.forEach((field) => {
const envVar = field.env_variable;
const envVarValue = envVarsMap.get(envVar);

View file

@ -3,7 +3,16 @@ import { AppCategoriesEnum, AppInfo, AppStatusEnum, FieldTypes } from '../apps.t
import config from '../../../config';
import App from '../app.entity';
const createApp = async (installed = false, status = AppStatusEnum.RUNNING, requiredPort?: number) => {
interface IProps {
installed?: boolean;
status?: AppStatusEnum;
requiredPort?: number;
randomField?: boolean;
}
const createApp = async (props: IProps) => {
const { installed = false, status = AppStatusEnum.RUNNING, requiredPort, randomField = false } = props;
const categories = Object.values(AppCategoriesEnum);
const appInfo: AppInfo = {
@ -47,6 +56,8 @@ const createApp = async (installed = false, status = AppStatusEnum.RUNNING, requ
MockFiles[`${config.ROOT_FOLDER}/app-data/${appInfo.id}`] = '';
MockFiles[`${config.ROOT_FOLDER}/app-data/${appInfo.id}/app.env`] = 'TEST=test\nAPP_PORT=3000\nTEST_FIELD=test';
MockFiles[`${config.ROOT_FOLDER}/apps/${appInfo.id}/config.json`] = JSON.stringify(appInfo);
MockFiles[`${config.ROOT_FOLDER}/apps/${appInfo.id}/metadata/description.md`] = 'md desc';
}
return { appInfo, MockFiles };

View file

@ -1,6 +1,6 @@
import { DataSource } from 'typeorm';
import { setupConnection, teardownConnection } from '../../../test/connection';
import fs from 'fs';
import fs from 'fs-extra';
import { gcall } from '../../../test/gcall';
import App from '../app.entity';
import { getAppQuery, InstalledAppsQuery, listAppInfosQuery } from '../../../test/queries';
@ -43,7 +43,7 @@ describe('ListAppsInfos', () => {
let app1: AppInfo;
beforeEach(async () => {
const { MockFiles, appInfo } = await createApp();
const { MockFiles, appInfo } = await createApp({});
app1 = appInfo;
// @ts-ignore
fs.__createMockFiles(MockFiles);
@ -69,8 +69,8 @@ describe('GetApp', () => {
let app2: AppInfo;
beforeEach(async () => {
const app1create = await createApp();
const app2create = await createApp(true);
const app1create = await createApp({});
const app2create = await createApp({ installed: true });
app1 = app1create.appInfo;
app2 = app2create.appInfo;
// @ts-ignore
@ -109,7 +109,7 @@ describe('InstalledApps', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(app1create.MockFiles);
@ -153,7 +153,7 @@ describe('InstallApp', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp();
const app1create = await createApp({});
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(app1create.MockFiles);
@ -219,7 +219,7 @@ describe('InstallApp', () => {
});
it('Should throw an error if the requirements are not met', async () => {
const { appInfo, MockFiles } = await createApp(false, undefined, 400);
const { appInfo, MockFiles } = await createApp({ requiredPort: 400 });
// @ts-ignore
fs.__createMockFiles(MockFiles);

View file

@ -1,5 +1,5 @@
import AppsService from '../apps.service';
import fs from 'fs';
import fs from 'fs-extra';
import config from '../../../config';
import childProcess from 'child_process';
import { AppInfo, AppStatusEnum } from '../apps.types';
@ -8,7 +8,7 @@ import { createApp } from './apps.factory';
import { setupConnection, teardownConnection } from '../../../test/connection';
import { DataSource } from 'typeorm';
jest.mock('fs');
jest.mock('fs-extra');
jest.mock('child_process');
let db: DataSource | null = null;
@ -34,7 +34,7 @@ describe('Install app', () => {
let app1: AppInfo;
beforeEach(async () => {
const { MockFiles, appInfo } = await createApp();
const { MockFiles, appInfo } = await createApp({});
app1 = appInfo;
// @ts-ignore
fs.__createMockFiles(MockFiles);
@ -96,13 +96,20 @@ describe('Install app', () => {
it('Should throw if required form fields are missing', async () => {
await expect(AppsService.installApp(app1.id, {})).rejects.toThrowError('Variable TEST_FIELD is required');
});
it('Correctly generates a random value if the field has a "random" type', async () => {
// const { appInfo } = await createApp({ randomField: true });
// await AppsService.installApp(appInfo.id, { TEST_FIELD: 'test' });
// const envFile = fs.readFileSync(`${config.ROOT_FOLDER}/app-data/${appInfo.id}/app.env`).toString();
// expect(envFile.trim()).toBe(`TEST=test\nAPP_PORT=${appInfo.port}\nTEST_FIELD=${appInfo.randomValue}`);
});
});
describe('Uninstall app', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(Object.assign(app1create.MockFiles));
@ -154,7 +161,7 @@ describe('Start app', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(Object.assign(app1create.MockFiles));
@ -207,7 +214,7 @@ describe('Stop app', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(Object.assign(app1create.MockFiles));
@ -230,7 +237,7 @@ describe('Update app config', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(Object.assign(app1create.MockFiles));
@ -257,7 +264,7 @@ describe('Get app config', () => {
let app1: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app1create = await createApp({ installed: true });
app1 = app1create.appInfo;
// @ts-ignore
fs.__createMockFiles(Object.assign(app1create.MockFiles));
@ -287,8 +294,8 @@ describe('List apps', () => {
let app2: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app2create = await createApp();
const app1create = await createApp({ installed: true });
const app2create = await createApp({});
app1 = app1create.appInfo;
app2 = app2create.appInfo;
// @ts-ignore
@ -314,8 +321,8 @@ describe('Start all apps', () => {
let app2: AppInfo;
beforeEach(async () => {
const app1create = await createApp(true);
const app2create = await createApp(true);
const app1create = await createApp({ installed: true });
const app2create = await createApp({ installed: true });
app1 = app1create.appInfo;
app2 = app2create.appInfo;
// @ts-ignore
@ -336,7 +343,7 @@ describe('Start all apps', () => {
it('Should not start app which has not status RUNNING', async () => {
const spy = jest.spyOn(childProcess, 'execFile');
await createApp(true, AppStatusEnum.STOPPED);
await createApp({ installed: true, status: AppStatusEnum.STOPPED });
await AppsService.startAllApps();
const apps = await App.find();

View file

@ -32,7 +32,7 @@ class App extends BaseEntity {
config!: Record<string, string>;
@Field(() => Number, { nullable: true })
@Column({ type: 'integer', default: 0, nullable: false })
@Column({ type: 'integer', default: 1, nullable: false })
version!: number;
@Field(() => Date)

View file

@ -9,7 +9,7 @@ import logger from '../../config/logger/logger';
export const checkAppRequirements = async (appName: string) => {
let valid = true;
const configFile: AppInfo = readJsonFile(`/repos/${config.APPS_REPO_ID}/apps/${appName}/config.json`);
const configFile: AppInfo | null = readJsonFile(`/apps/${appName}/config.json`);
if (!configFile) {
throw new Error(`App ${appName} not found`);
@ -41,10 +41,10 @@ export const getEnvMap = (appName: string): Map<string, string> => {
};
export const checkEnvFile = (appName: string) => {
const configFile: AppInfo = readJsonFile(`/repos/${config.APPS_REPO_ID}/apps/${appName}/config.json`);
const configFile: AppInfo | null = readJsonFile(`/apps/${appName}/config.json`);
const envMap = getEnvMap(appName);
configFile.form_fields?.forEach((field) => {
configFile?.form_fields?.forEach((field) => {
const envVar = field.env_variable;
const envVarValue = envMap.get(envVar);
@ -82,7 +82,12 @@ const getEntropy = (name: string, length: number) => {
};
export const generateEnvFile = (appName: string, form: Record<string, string>) => {
const configFile: AppInfo = readJsonFile(`/repos/${config.APPS_REPO_ID}/apps/${appName}/config.json`);
const configFile: AppInfo | null = readJsonFile(`/apps/${appName}/config.json`);
if (!configFile) {
throw new Error(`App ${appName} not found`);
}
const baseEnvFile = readFile('/.env').toString();
let envFile = `${baseEnvFile}\nAPP_PORT=${configFile.port}\n`;
const envMap = getEnvMap(appName);

View file

@ -1,4 +1,4 @@
import { createFolder, readFile, readJsonFile } from '../fs/fs.helpers';
import { createFolder, ensureAppFolder, readFile, readJsonFile } from '../fs/fs.helpers';
import { checkAppRequirements, checkEnvFile, generateEnvFile, getAvailableApps, runAppScript } from './apps.helpers';
import { AppInfo, AppStatusEnum, ListAppsResonse } from './apps.types';
import App from './app.entity';
@ -57,6 +57,8 @@ const startApp = async (appName: string): Promise<App> => {
};
const installApp = async (id: string, form: Record<string, string>): Promise<App> => {
ensureAppFolder(id);
let app = await App.findOne({ where: { id } });
if (app) {
@ -74,7 +76,8 @@ const installApp = async (id: string, form: Record<string, string>): Promise<App
// Create env file
generateEnvFile(id, form);
app = await App.create({ id, status: AppStatusEnum.INSTALLING, config: form }).save();
const appInfo: AppInfo | null = await readJsonFile(`/apps/${id}/config.json`);
app = await App.create({ id, status: AppStatusEnum.INSTALLING, config: form, version: Number(appInfo?.tipi_version || 0) }).save();
// Run script
try {

View file

@ -1,4 +1,4 @@
import fs from 'fs';
import fs from 'fs-extra';
import childProcess from 'child_process';
import config from '../../config';
@ -35,11 +35,16 @@ export const createFolder = (path: string) => {
};
export const deleteFolder = (path: string) => fs.rmSync(getAbsolutePath(path), { recursive: true });
export const copyFile = (source: string, destination: string) => fs.copyFileSync(getAbsolutePath(source), getAbsolutePath(destination));
export const runScript = (path: string, args: string[], callback?: any) => childProcess.execFile(getAbsolutePath(path), args, {}, callback);
export const getSeed = () => {
const seed = readFile('/state/seed');
return seed.toString();
};
export const ensureAppFolder = (appName: string) => {
if (!fileExists(`/apps/${appName}`)) {
// Copy from apps repo
fs.copySync(getAbsolutePath(`/repos/${config.APPS_REPO_ID}/apps/${appName}`), getAbsolutePath(`/apps/${appName}`));
}
};

17
pnpm-lock.yaml generated
View file

@ -143,6 +143,7 @@ importers:
'@types/cors': ^2.8.12
'@types/express': ^4.17.13
'@types/express-session': ^1.17.4
'@types/fs-extra': ^9.0.13
'@types/jest': ^27.5.0
'@types/jsonwebtoken': ^8.5.8
'@types/mock-fs': ^4.13.1
@ -170,6 +171,7 @@ importers:
eslint-plugin-prettier: ^4.0.0
express: ^4.17.3
express-session: ^1.17.3
fs-extra: ^10.1.0
graphql: ^15.3.0
graphql-import-node: ^0.0.5
graphql-type-json: ^0.3.2
@ -208,6 +210,7 @@ importers:
dotenv: 16.0.0
express: 4.18.1
express-session: 1.17.3
fs-extra: 10.1.0
graphql: 15.8.0
graphql-type-json: 0.3.2_graphql@15.8.0
http: 0.0.1-security
@ -235,6 +238,7 @@ importers:
'@types/cors': 2.8.12
'@types/express': 4.17.13
'@types/express-session': 1.17.4
'@types/fs-extra': 9.0.13
'@types/jest': 27.5.0
'@types/jsonwebtoken': 8.5.8
'@types/mock-fs': 4.13.1
@ -3748,6 +3752,12 @@ packages:
'@types/qs': 6.9.7
'@types/serve-static': 1.13.10
/@types/fs-extra/9.0.13:
resolution: {integrity: sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==}
dependencies:
'@types/node': 17.0.31
dev: true
/@types/glob/7.2.0:
resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==}
dependencies:
@ -6385,7 +6395,7 @@ packages:
eslint-import-resolver-webpack:
optional: true
dependencies:
'@typescript-eslint/parser': 5.22.0_uhoeudlwl7kc47h4kncsfowede
'@typescript-eslint/parser': 5.22.0_hcfsmds2fshutdssjqluwm76uu
debug: 3.2.7
eslint-import-resolver-node: 0.3.6
find-up: 2.1.0
@ -7133,7 +7143,6 @@ packages:
graceful-fs: 4.2.10
jsonfile: 6.1.0
universalify: 2.0.0
dev: true
/fs-extra/8.1.0:
resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==}
@ -8801,7 +8810,6 @@ packages:
universalify: 2.0.0
optionalDependencies:
graceful-fs: 4.2.10
dev: true
/jsonify/0.0.0:
resolution: {integrity: sha512-trvBk1ki43VZptdBI5rIlG4YOzyeH/WefQt5rj1grasPn4iiZWKet8nkgc4GlsAylaztn0qZfUYOiTsASJFdNA==}
@ -11989,7 +11997,7 @@ packages:
'@types/jest': 27.5.0
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
jest: 28.1.0_@types+node@17.0.31
jest: 28.1.0_qxft4nzwxz7jey57xog52j3doy
jest-util: 28.1.0
json5: 2.2.1
lodash.memoize: 4.1.2
@ -12398,7 +12406,6 @@ packages:
/universalify/2.0.0:
resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==}
engines: {node: '>= 10.0.0'}
dev: true
/unixify/1.0.0:
resolution: {integrity: sha512-6bc58dPYhCMHHuwxldQxO3RRNZ4eCogZ/st++0+fcC1nr0jiGUtAdBJ2qzmLQWSxbtz42pWt4QQMiZ9HvZf5cg==}

View file

@ -70,7 +70,20 @@ else
exit 1
fi
app_dir="${ROOT_FOLDER}/repos/${repo_id}/apps/${app}"
if [[ -z "${root_folder_host}" ]]; then
echo "Error: Root folder not provided"
exit 1
fi
app_dir="${ROOT_FOLDER}/apps/${app}"
if [[ ! -d "${app_dir}" ]]; then
# copy from repo
echo "Copying app from repo"
mkdir -p "${app_dir}"
cp -r "${ROOT_FOLDER}/repos/${repo_id}/apps/${app}"/* "${app_dir}"
fi
app_data_dir="${ROOT_FOLDER}/app-data/${app}"
if [[ -z "${app}" ]] || [[ ! -d "${app_dir}" ]]; then
@ -78,10 +91,6 @@ else
exit 1
fi
if [[ -z "${root_folder_host}" ]]; then
echo "Error: Root folder not provided"
exit 1
fi
fi
if [ -z ${3+x} ]; then
@ -133,8 +142,8 @@ if [[ "$command" = "install" ]]; then
compose "${app}" pull
# Copy default data dir to app data dir if it exists
if [[ -d "${ROOT_FOLDER}/repos/${repo_id}/${app}/data" ]]; then
cp -r "${ROOT_FOLDER}/repos/${repo_id}/${app}/data" "${app_data_dir}/data"
if [[ -d "${app_dir}/data" ]]; then
cp -r "${app_dir}/data" "${app_data_dir}/data"
fi
# Remove all .gitkeep files from app data dir
@ -158,6 +167,10 @@ if [[ "$command" = "uninstall" ]]; then
rm -rf "${app_data_dir}"
fi
if [[ -d "${app_dir}" ]]; then
rm -rf "${app_dir}"
fi
echo "Successfully uninstalled app ${app}"
exit
fi
@ -166,6 +179,15 @@ fi
if [[ "$command" = "update" ]]; then
compose "${app}" up --detach
compose "${app}" down --rmi all --remove-orphans
# Remove app
if [[ -d "${app_dir}" ]]; then
rm -rf "${app_dir}"
fi
# Copy app from repo
cp -r "${ROOT_FOLDER}/repos/${repo_id}/apps/${app}" "${app_dir}"
compose "${app}" pull
compose "${app}" up --detach
exit