wip: external repo for apps [skip ci]

.gitignore

@@ -7,6 +7,8 @@ data/postgres
 traefik/ssl/*
 !traefik/ssl/.gitkeep
 !app-data/.gitkeep
+repos/*
+!repos/.gitkeep
 
 scripts/pacapt
 

Dockerfile

@@ -24,7 +24,7 @@ FROM alpine:3.16.0 as app
 WORKDIR /
 
 # Install dependencies
-RUN apk --no-cache add docker-compose nodejs npm bash g++ make
+RUN apk --no-cache add docker-compose nodejs npm bash g++ make git
 
 RUN npm install node-gyp -g
 

Dockerfile.dev

@@ -3,7 +3,7 @@ FROM alpine:3.16.0 as app
 WORKDIR /
 
 # Install docker
-RUN apk --no-cache add docker-compose nodejs npm bash g++ make
+RUN apk --no-cache add docker-compose nodejs npm bash g++ make git
 
 RUN npm install node-gyp -g
 

docker-compose.dev.yml

@@ -36,6 +36,7 @@ services:
     volumes:
       ## Docker sock
       - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${PWD}/repos:/repos
       - ${PWD}:/tipi
       - ${PWD}/packages/system-api/src:/api/src
       # - /api/node_modules
@@ -49,6 +50,7 @@ services:
       POSTGRES_USERNAME: tipi
       POSTGRES_DBNAME: tipi
       POSTGRES_HOST: tipi-db
+      APPS_REPOSITORY: ${APPS_REPOSITORY}
     networks:
       - tipi_main_network
 

docker-compose.rc.yml

@@ -49,6 +49,7 @@ services:
       ## Docker sock
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ${PWD}:/tipi
+      - ${PWD}/repos:/repos
     environment:
       INTERNAL_IP: ${INTERNAL_IP}
       TIPI_VERSION: ${TIPI_VERSION}
@@ -60,6 +61,7 @@ services:
       POSTGRES_DBNAME: tipi
       POSTGRES_HOST: tipi-db
       NODE_ENV: production
+      APPS_REPOSITORY: ${APPS_REPOSITORY}
     dns:
       - ${DNS_IP}
     networks:

docker-compose.yml

@@ -50,6 +50,7 @@ services:
       ## Docker sock
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ${PWD}:/tipi
+      - ${PWD}/repos:/repos
     environment:
       INTERNAL_IP: ${INTERNAL_IP}
       TIPI_VERSION: ${TIPI_VERSION}
@@ -61,6 +62,7 @@ services:
       POSTGRES_DBNAME: tipi
       POSTGRES_HOST: tipi-db
       NODE_ENV: production
+      APPS_REPOSITORY: ${APPS_REPOSITORY}
     dns:
       - ${DNS_IP}
     networks:

packages/dashboard/src/components/AppLogo/AppLogo.tsx

@@ -1,13 +1,19 @@
 import React from 'react';
+import { useSytemStore } from '../../state/systemStore';
+
+const AppLogo: React.FC<{ id: string; size?: number; className?: string; alt?: string }> = ({ id, size = 80, className = '', alt = '' }) => {
+  const { internalIp } = useSytemStore();
+  const logoUrl = `http://${internalIp}:3001/apps/${id}/metadata/logo.jpg`;
+
+  console.log(logoUrl);
 
-const AppLogo: React.FC<{ src: string; size?: number; className?: string; alt?: string }> = ({ src, size = 80, className = '', alt = '' }) => {
   return (
     <div aria-label={alt} className={`drop-shadow ${className}`} style={{ width: size, height: size }}>
       <svg width={size} height={size} viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
         <mask id="mask0" maskUnits="userSpaceOnUse" x="0" y="0" width="200" height="200">
           <path fillRule="evenodd" clipRule="evenodd" d="M0 100C0 0 0 0 100 0S200 0 200 100 200 200 100 200 0 200 0 100" fill="white" />
         </mask>
-        <image href={src} mask="url(#mask0)" width="200" height="200" />
+        <image href={logoUrl} mask="url(#mask0)" width="200" height="200" />
       </svg>
     </div>
   );

packages/dashboard/src/components/AppTile/index.tsx

@@ -16,7 +16,7 @@ const AppTile: React.FC<{ app: AppTileInfo; status: AppStatusEnum }> = ({ app, s
     <Link href={`/apps/${app.id}`} passHref>
       <SlideFade in className="flex flex-1" offsetY="20px">
         <Box bg={bg} className="flex flex-1 border-2 drop-shadow-sm rounded-lg p-3 items-center cursor-pointer group hover:drop-shadow-md transition-all">
-          <AppLogo alt={`${app.name} logo`} className="mr-3 group-hover:scale-105 transition-all" src={app.image} size={100} />
+          <AppLogo alt={`${app.name} logo`} className="mr-3 group-hover:scale-105 transition-all" id={app.id} size={100} />
           <div className="mr-3 flex-1">
             <h3 className="font-bold text-xl">{app.name}</h3>
             <span>{limitText(app.short_desc, 50)}</span>

packages/dashboard/src/modules/AppStore/components/AppStoreTile.tsx

@@ -17,7 +17,7 @@ const AppStoreTile: React.FC<{ app: App }> = ({ app }) => {
   return (
     <Link href={`/app-store/${app.id}`} passHref>
       <div key={app.id} className="p-2 rounded-md app-store-tile flex items-center group">
-        <AppLogo src={app.image} className="group-hover:scale-105 transition-all" />
+        <AppLogo id={app.id} className="group-hover:scale-105 transition-all" />
         <div className="ml-2">
           <div className="font-bold">{limitText(app.name, 20)}</div>
           <div className="text-sm mb-1">{limitText(app.short_desc, 45)}</div>

packages/dashboard/src/modules/Apps/containers/AppDetails.tsx

@@ -116,7 +116,7 @@ const AppDetails: React.FC<IProps> = ({ app, info }) => {
     <SlideFade in className="flex flex-1" offsetY="20px">
       <div className="flex flex-1  p-4 mt-3 rounded-lg flex-col">
         <Flex className="flex-col md:flex-row">
-          <AppLogo src={info.image} size={180} className="self-center sm:self-auto" alt={info.name} />
+          <AppLogo id={info.id} size={180} className="self-center sm:self-auto" alt={info.name} />
           <VStack align="flex-start" justify="space-between" className="ml-0 md:ml-4">
             <div className="mt-3 items-center self-center flex flex-col sm:items-start sm:self-start md:mt-0">
               <h1 className="font-bold text-2xl">{info.name}</h1>

packages/system-api/src/config/config.ts

@@ -12,6 +12,7 @@ interface IConfig {
   CLIENT_URLS: string[];
   VERSION: string;
   ROOT_FOLDER_HOST: string;
+  APPS_REPOSITORY: string;
 }
 
 if (process.env.NODE_ENV !== 'production') {
@@ -30,6 +31,7 @@ const {
   TIPI_VERSION = '',
   ROOT_FOLDER_HOST = '',
   NGINX_PORT = '80',
+  APPS_REPOSITORY = '',
 } = process.env;
 
 const config: IConfig = {
@@ -44,6 +46,7 @@ const config: IConfig = {
   CLIENT_URLS: ['http://localhost:3000', `http://${INTERNAL_IP}`, `http://${INTERNAL_IP}:${NGINX_PORT}`, `http://${INTERNAL_IP}:3000`],
   VERSION: TIPI_VERSION,
   ROOT_FOLDER_HOST,
+  APPS_REPOSITORY,
 };
 
 export default config;

packages/system-api/src/helpers/repo-helpers.ts

@@ -0,0 +1,42 @@
+import config from '../config';
+import { runScript } from '../modules/fs/fs.helpers';
+
+export const getRepoId = (repo: string): Promise<string> => {
+  return new Promise((resolve, reject) => {
+    runScript('/scripts/git.sh', [...['get_hash', repo], config.ROOT_FOLDER_HOST], (err: string, stdout: string) => {
+      if (err) {
+        reject(err);
+      }
+
+      resolve(stdout.trim());
+    });
+  });
+};
+
+export const updateRepo = (repo: string): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    runScript('/scripts/git.sh', [...['update', repo], config.ROOT_FOLDER_HOST], (err: string, stdout: string) => {
+      if (err) {
+        reject(err);
+      }
+
+      console.info('Update result', stdout);
+
+      resolve();
+    });
+  });
+};
+
+export const cloneRepo = (repo: string): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    runScript('/scripts/git.sh', [...['clone', repo], config.ROOT_FOLDER_HOST], (err: string, stdout: string) => {
+      if (err) {
+        reject(err);
+      }
+
+      console.info('Clone result', stdout);
+
+      resolve();
+    });
+  });
+};

packages/system-api/src/modules/apps/app.entity.ts

@@ -40,7 +40,7 @@ class App extends BaseEntity {
   updatedAt!: Date;
 
   @Field(() => AppInfo)
-  info(): AppInfo {
+  info(): Promise<AppInfo> {
     return getAppInfo(this.id);
   }
 }

packages/system-api/src/modules/apps/apps.helpers.ts

@@ -1,9 +1,11 @@
 import portUsed from 'tcp-port-used';
-import { fileExists, readdirSync, readFile, readJsonFile, runScript, writeFile } from '../fs/fs.helpers';
+import { fileExists, getSeed, readdirSync, readFile, readJsonFile, runScript, writeFile } from '../fs/fs.helpers';
 import InternalIp from 'internal-ip';
 import crypto from 'crypto';
 import config from '../../config';
 import { AppInfo } from './apps.types';
+import { getRepoId } from '../../helpers/repo-helpers';
+import logger from '../../config/logger/logger';
 
 export const checkAppRequirements = async (appName: string) => {
   let valid = true;
@@ -60,13 +62,18 @@ export const checkAppExists = (appName: string) => {
   }
 };
 
-export const runAppScript = (params: string[]): Promise<void> => {
+export const runAppScript = async (params: string[]): Promise<void> => {
+  const repoId = await getRepoId(config.APPS_REPOSITORY);
+
   return new Promise((resolve, reject) => {
-    runScript('/scripts/app.sh', [...params, config.ROOT_FOLDER_HOST], (err: string) => {
+    runScript('/scripts/app.sh', [...params, config.ROOT_FOLDER_HOST, repoId], (err: string, stdout: string) => {
       if (err) {
+        logger.error(err);
         reject(err);
       }
 
+      logger.info(stdout);
+
       resolve();
     });
   });
@@ -74,7 +81,7 @@ export const runAppScript = (params: string[]): Promise<void> => {
 
 const getEntropy = (name: string, length: number) => {
   const hash = crypto.createHash('sha256');
-  hash.update(name);
+  hash.update(name + getSeed());
   return hash.digest('hex').substring(0, length);
 };
 
@@ -107,13 +114,14 @@ export const generateEnvFile = (appName: string, form: Record<string, string>) =
   writeFile(`/app-data/${appName}/app.env`, envFile);
 };
 
-export const getAvailableApps = (): string[] => {
+export const getAvailableApps = async (): Promise<string[]> => {
   const apps: string[] = [];
 
-  const appsDir = readdirSync('/apps');
+  const repoId = await getRepoId(config.APPS_REPOSITORY);
+  const appsDir = readdirSync(`/repos/${repoId}/apps`);
 
   appsDir.forEach((app) => {
-    if (fileExists(`/apps/${app}/config.json`)) {
+    if (fileExists(`/repos/${repoId}/apps/${app}/config.json`)) {
       const configFile: AppInfo = readJsonFile(`/apps/${app}/config.json`);
 
       if (configFile.available) {
@@ -125,13 +133,21 @@ export const getAvailableApps = (): string[] => {
   return apps;
 };
 
-export const getAppInfo = (id: string): AppInfo => {
+export const getAppInfo = async (id: string): Promise<AppInfo> => {
   try {
-    const configFile: AppInfo = readJsonFile(`/apps/${id}/config.json`);
-    configFile.description = readFile(`/apps/${id}/metadata/description.md`);
+    const repoId = await getRepoId(config.APPS_REPOSITORY);
 
-    return configFile;
+    if (fileExists(`/repos/${repoId}`)) {
+      const configFile: AppInfo = readJsonFile(`/repos/${repoId}/apps/${id}/config.json`);
+      configFile.description = readFile(`/repos/${repoId}/apps/${id}/metadata/description.md`);
+
+      if (configFile.available) {
+        return configFile;
+      }
+    }
+
+    throw new Error('No repository found');
   } catch (e) {
-    throw new Error(`App ${id} not found`);
+    throw new Error(`Error loading app ${id}`);
   }
 };

packages/system-api/src/modules/apps/apps.service.ts

@@ -91,7 +91,9 @@ const installApp = async (id: string, form: Record<string, string>): Promise<App
 };
 
 const listApps = async (): Promise<ListAppsResonse> => {
-  const apps: AppInfo[] = getAvailableApps()
+  const folders: string[] = await getAvailableApps();
+
+  const apps: AppInfo[] = folders
     .map((app) => {
       try {
         return readJsonFile(`/apps/${app}/config.json`);

packages/system-api/src/modules/fs/fs.helpers.ts

@@ -38,3 +38,8 @@ export const deleteFolder = (path: string) => fs.rmSync(getAbsolutePath(path), {
 export const copyFile = (source: string, destination: string) => fs.copyFileSync(getAbsolutePath(source), getAbsolutePath(destination));
 
 export const runScript = (path: string, args: string[], callback?: any) => childProcess.execFile(getAbsolutePath(path), args, {}, callback);
+
+export const getSeed = () => {
+  const seed = readFile('/state/seed');
+  return seed.toString();
+};

packages/system-api/src/server.ts

@@ -15,6 +15,7 @@ import datasource from './config/datasource';
 import appsService from './modules/apps/apps.service';
 import { runUpdates } from './core/updates/run';
 import recover from './core/updates/recover-migrations';
+import { cloneRepo, getRepoId, updateRepo } from './helpers/repo-helpers';
 
 let corsOptions = __prod__
   ? {
@@ -38,6 +39,9 @@ const main = async () => {
     const app = express();
     const port = 3001;
 
+    const repoId = await getRepoId(config.APPS_REPOSITORY);
+
+    app.use(express.static(`/repos/${repoId}`));
     app.use(cors(corsOptions));
     app.use(getSessionMiddleware());
 
@@ -72,7 +76,9 @@ const main = async () => {
     // Run migrations
     await runUpdates();
 
-    httpServer.listen(port, () => {
+    httpServer.listen(port, async () => {
+      await cloneRepo(config.APPS_REPOSITORY);
+      await updateRepo(config.APPS_REPOSITORY);
       // Start apps
       appsService.startAllApps();
       console.info(`Server running on port ${port} 🚀 Production => ${__prod__}`);

scripts/app.sh

@@ -61,8 +61,14 @@ if [ -z ${2+x} ]; then
 else
   app="$2"
   root_folder_host="${3:-$ROOT_FOLDER}"
+  repo_id="${4}"
 
-  app_dir="${ROOT_FOLDER}/apps/${app}"
+  if [[ -z "${root_folder_host}" ]]; then
+    echo "Error: Repo id not provided"
+    exit 1
+  fi
+
+  app_dir="/repos/${repo_id}/apps/${app}"
   app_data_dir="${ROOT_FOLDER}/app-data/${app}"
 
   if [[ -z "${app}" ]] || [[ ! -d "${app_dir}" ]]; then
@@ -101,8 +107,7 @@ compose() {
     app_compose_file="${app_dir}/docker-compose.arm.yml"
   fi
 
-  local common_compose_file="${ROOT_FOLDER}/apps/docker-compose.common.yml"
-  local app_dir="${ROOT_FOLDER}/apps/${app}"
+  local common_compose_file="/repos/${repo_id}/apps/docker-compose.common.yml"
 
   # Vars to use in compose file
   export APP_DATA_DIR="${root_folder_host}/app-data/${app}"
@@ -126,8 +131,8 @@ if [[ "$command" = "install" ]]; then
   compose "${app}" pull
 
   # Copy default data dir to app data dir if it exists
-  if [[ -d "${ROOT_FOLDER}/apps/${app}/data" ]]; then
-    cp -r "${ROOT_FOLDER}/apps/${app}/data" "${app_data_dir}/data"
+  if [[ -d "/repos/${repo_id}/${app}/data" ]]; then
+    cp -r "/repos/${repo_id}/${app}/data" "${app_data_dir}/data"
   fi
 
   # Remove all .gitkeep files from app data dir

scripts/git.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# use greadlink instead of readlink on osx
+if [[ "$(uname)" == "Darwin" ]]; then
+    rdlk=greadlink
+else
+    rdlk=readlink
+fi
+
+ROOT_FOLDER="$($rdlk -f $(dirname "${BASH_SOURCE[0]}")/..)"
+
+show_help() {
+    cat <<EOF
+app 0.0.1
+
+CLI for managing Tipi apps
+
+Usage: git <command> <repo> [<arguments>]
+
+Commands:
+    clone                      Clones a repo in the repo folder
+    update                     Updates the repo folder
+    get_hash                   Gets the local hash of the repo
+EOF
+}
+
+# Get a static hash based on the repo url
+function get_hash() {
+    url="${1}"
+    echo $(echo -n "${url}" | sha256sum | awk '{print $1}')
+}
+
+if [ -z ${1+x} ]; then
+    command=""
+else
+    command="$1"
+fi
+
+# Clone a repo
+if [[ "$command" = "clone" ]]; then
+    repo="$2"
+    repo_hash="$(get_hash "${repo}")"
+    echo "Cloning ${repo} to ${ROOT_FOLDER}/repos/${repo_hash}"
+    repo_dir="${ROOT_FOLDER}/repos/${repo_hash}"
+    if [ -d "${repo_dir}" ]; then
+        echo "Repo already exists"
+        exit 0
+    fi
+
+    echo "Cloning ${repo} to ${repo_dir}"
+    git clone "${repo}" "${repo_dir}"
+    echo "Done"
+    exit
+fi
+
+# Update a repo
+if [[ "$command" = "update" ]]; then
+    repo="$2"
+    repo_hash="$(get_hash "${repo}")"
+    repo_dir="${ROOT_FOLDER}/repos/${repo_hash}"
+    if [ ! -d "${repo_dir}" ]; then
+        echo "Repo does not exist"
+        exit 0
+    fi
+
+    echo "Updating ${repo} in ${repo_dir}"
+    cd "${repo_dir}"
+    git pull origin master
+    echo "Done"
+    exit
+fi
+
+if [[ "$command" = "get_hash" ]]; then
+    repo="$2"
+    echo $(get_hash "${repo}")
+    exit
+fi

scripts/start.sh

@@ -105,14 +105,9 @@ function derive_entropy() {
   printf "%s" "${identifier}" | openssl dgst -sha256 -hmac "${tipi_seed}" | sed 's/^.* //'
 }
 
-# Copy the app state if it isn't here
+# Copy the config sample if it isn't here
 if [[ ! -f "${STATE_FOLDER}/apps.json" ]]; then
-  cp "${ROOT_FOLDER}/templates/apps-sample.json" "${STATE_FOLDER}/apps.json"
-fi
-
-# Copy the user state if it isn't here
-if [[ ! -f "${STATE_FOLDER}/users.json" ]]; then
-  cp "${ROOT_FOLDER}/templates/users-sample.json" "${STATE_FOLDER}/users.json"
+  cp "${ROOT_FOLDER}/templates/config-sample.json" "${STATE_FOLDER}/config.json"
 fi
 
 # Get current dns from host
@@ -132,7 +127,6 @@ export COMPOSE_HTTP_TIMEOUT=240
 echo "Generating config files..."
 # Remove current .env file
 [[ -f "${ROOT_FOLDER}/.env" ]] && rm -f "${ROOT_FOLDER}/.env"
-[[ -f "${ROOT_FOLDER}/packages/system-api/.env" ]] && rm -f "${ROOT_FOLDER}/packages/system-api/.env"
 
 # Store paths to intermediary config files
 ENV_FILE=$(mktemp)

templates/apps-sample.json

@@ -1 +0,0 @@
-{ "installed": "" }

templates/config-sample.json

@@ -0,0 +1,3 @@
+{
+  "repo": "https://github.com/meienberger/runtipi-appstore"
+}

templates/env-sample

@@ -1,6 +1,7 @@
 # Only edit this file if you know what you are doing!
 # It will be overwritten on update.
 
+APPS_REPOSITORY=https://github.com/meienberger/runtipi-appstore
 TZ=<tz>
 INTERNAL_IP=<internal_ip>
 DNS_IP=<dns_ip>
@@ -10,5 +11,4 @@ JWT_SECRET=<jwt_secret>
 ROOT_FOLDER_HOST=<root_folder>
 NGINX_PORT=<nginx_port>
 PROXY_PORT=<proxy_port>
-
 POSTGRES_PASSWORD=<postgres_password>

templates/users-sample.json

@@ -1 +0,0 @@
-[]