chore: add loggin during start

package.json

@@ -1,6 +1,6 @@
 {
   "name": "runtipi",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "A homeserver for everyone",
   "scripts": {
     "knip": "knip",

packages/cli/assets/docker-compose.yml

@@ -39,7 +39,7 @@ services:
 
   tipi-redis:
     container_name: tipi-redis
-    image: redis:7.2.0-alpine
+    image: redis:7.2.0
     restart: on-failure
     command: redis-server --requirepass ${REDIS_PASSWORD}
     ports:

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runtipi/cli",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "",
   "main": "index.js",
   "bin": "dist/index.js",

packages/cli/src/executors/repo/repo.executors.ts

@@ -75,39 +75,35 @@ export class RepoExecutors {
 
       this.logger.info(`Pulling repo ${repoUrl} to ${repoPath}`);
 
-      await execAsync(`git config --global --add safe.directory ${repoPath}`).then(({ stdout, stderr }) => {
-        this.logger.info('------------------ git config --global --add safe.directory ------------------');
-        this.logger.error(`stderr: ${stderr}`);
-        this.logger.info(`stdout: ${stdout}`);
+      this.logger.info(`Executing: git config --global --add safe.directory ${repoPath}`);
+      await execAsync(`git config --global --add safe.directory ${repoPath}`).then(({ stderr }) => {
+        if (stderr) {
+          this.logger.error(`stderr: ${stderr}`);
+        }
       });
 
       // git config pull.rebase false
-      await execAsync(`git -C ${repoPath} config pull.rebase false`).then(({ stdout, stderr }) => {
-        this.logger.info(`------------------ git -C ${repoPath} config pull.rebase false ------------------`);
-        this.logger.error(`stderr: ${stderr}`);
-        this.logger.info(`stdout: ${stdout}`);
+      this.logger.info(`Executing: git -C ${repoPath} config pull.rebase false`);
+      await execAsync(`git -C ${repoPath} config pull.rebase false`).then(({ stderr }) => {
+        if (stderr) {
+          this.logger.error(`stderr: ${stderr}`);
+        }
       });
 
+      this.logger.info(`Executing: git -C ${repoPath} rev-parse --abbrev-ref HEAD`);
       const currentBranch = await execAsync(`git -C ${repoPath} rev-parse --abbrev-ref HEAD`).then(({ stdout }) => {
         return stdout.trim();
       });
 
-      // reset hard
-      await execAsync(`git -C ${repoPath} fetch origin && git -C ${repoPath} reset --hard origin/${currentBranch}`).then(({ stdout, stderr }) => {
-        this.logger.info(`------------------ git -C ${repoPath} reset --hard ------------------`);
-        this.logger.error(`stderr: ${stderr}`);
-        this.logger.info(`stdout: ${stdout}`);
+      this.logger.info(`Executing: git -C ${repoPath} fetch origin && git -C ${repoPath} reset --hard origin/${currentBranch}`);
+      await execAsync(`git -C ${repoPath} fetch origin && git -C ${repoPath} reset --hard origin/${currentBranch}`).then(({ stderr }) => {
+        if (stderr) {
+          this.logger.error(`stderr: ${stderr}`);
+        }
       });
 
-      const { stderr, stdout } = await execAsync(`git -C ${repoPath} pull`);
-
-      if (stderr) {
-        this.logger.error(`Error pulling repo ${repoUrl}: ${stderr}`);
-        return { success: false, message: stderr };
-      }
-
       this.logger.info(`Pulled repo ${repoUrl} to ${repoPath}`);
-      return { success: true, message: stdout };
+      return { success: true, message: '' };
     } catch (err) {
       return this.handleRepoError(err);
     }

packages/cli/src/executors/system/system.executors.ts

@@ -12,7 +12,6 @@ import { Stream } from 'stream';
 import { promisify } from 'util';
 import dotenv from 'dotenv';
 import { SystemEvent } from '@runtipi/shared';
-import { killOtherWorkers } from 'src/services/watcher/watcher';
 import chalk from 'chalk';
 import { AppExecutors } from '../app/app.executors';
 import { copySystemFiles, generateSystemEnvFile, generateTlsCertificates } from './system.helpers';
@@ -30,18 +29,21 @@ export class SystemExecutors {
 
   private readonly envFile: string;
 
+  private readonly logger;
+
   constructor() {
     this.rootFolder = process.cwd();
+    this.logger = fileLogger;
 
     this.envFile = path.join(this.rootFolder, '.env');
   }
 
   private handleSystemError = (err: unknown) => {
     if (err instanceof Error) {
-      fileLogger.error(`An error occurred: ${err.message}`);
+      this.logger.error(`An error occurred: ${err.message}`);
       return { success: false, message: err.message };
     }
-    fileLogger.error(`An error occurred: ${err}`);
+    this.logger.error(`An error occurred: ${err}`);
 
     return { success: false, message: `An error occurred: ${err}` };
   };
@@ -63,7 +65,6 @@ export class SystemExecutors {
 
     const filesAndFolders = [
       path.join(rootFolderHost, 'apps'),
-      path.join(rootFolderHost, 'app-data'),
       path.join(rootFolderHost, 'logs'),
       path.join(rootFolderHost, 'media'),
       path.join(rootFolderHost, 'repos'),
@@ -76,10 +77,12 @@ export class SystemExecutors {
 
     const files600 = [path.join(rootFolderHost, 'traefik', 'shared', 'acme.json')];
 
+    this.logger.info('Setting file permissions a+rwx on required files');
     // Give permission to read and write to all files and folders for the current user
     await Promise.all(
       filesAndFolders.map(async (fileOrFolder) => {
         if (await pathExists(fileOrFolder)) {
+          this.logger.info(`Setting permissions on ${fileOrFolder}`);
           await execAsync(`sudo chmod -R a+rwx ${fileOrFolder}`).catch(() => {
             logger.fail(`Failed to set permissions on ${fileOrFolder}`);
           });
@@ -87,9 +90,12 @@ export class SystemExecutors {
       }),
     );
 
+    this.logger.info('Setting file permissions 600 on required files');
+
     await Promise.all(
       files600.map(async (fileOrFolder) => {
         if (await pathExists(fileOrFolder)) {
+          this.logger.info(`Setting permissions on ${fileOrFolder}`);
           await execAsync(`sudo chmod 600 ${fileOrFolder}`).catch(() => {
             logger.fail(`Failed to set permissions on ${fileOrFolder}`);
           });
@@ -98,6 +104,21 @@ export class SystemExecutors {
     );
   };
 
+  public cleanLogs = async () => {
+    try {
+      const { rootFolderHost } = getEnv();
+
+      await fs.promises.rm(path.join(rootFolderHost, 'logs'), { recursive: true, force: true });
+      await fs.promises.mkdir(path.join(rootFolderHost, 'logs'));
+
+      this.logger.info('Logs cleaned successfully');
+
+      return { success: true, message: '' };
+    } catch (e) {
+      return this.handleSystemError(e);
+    }
+  };
+
   public systemInfo = async () => {
     try {
       const { rootFolderHost } = getEnv();
@@ -135,6 +156,8 @@ export class SystemExecutors {
 
       spinner.setMessage('Stopping containers...');
       spinner.start();
+
+      this.logger.info('Stopping main containers...');
       await execAsync('docker compose down --remove-orphans --rmi local');
 
       spinner.done('Tipi successfully stopped');
@@ -184,6 +207,8 @@ export class SystemExecutors {
 
       spinner.start();
       spinner.setMessage('Copying system files...');
+
+      this.logger.info('Copying system files...');
       await copySystemFiles();
 
       spinner.done('System files copied');
@@ -194,24 +219,31 @@ export class SystemExecutors {
 
       spinner.setMessage('Generating system env file...');
       spinner.start();
+      this.logger.info('Generating system env file...');
       const envMap = await generateSystemEnvFile();
       spinner.done('System env file generated');
 
       // Reload env variables after generating the env file
+      this.logger.info('Reloading env variables...');
       dotenv.config({ path: this.envFile, override: true });
 
       // Stop and Remove container tipi if exists
       spinner.setMessage('Stopping and removing containers...');
       spinner.start();
+      this.logger.info('Stopping and removing tipi-db...');
       await execAsync('docker rm -f tipi-db');
+      this.logger.info('Stopping and removing tipi-redis...');
       await execAsync('docker rm -f tipi-redis');
+      this.logger.info('Stopping and removing tipi-dashboard...');
       await execAsync('docker rm -f tipi-dashboard');
+      this.logger.info('Stopping and removing tipi-reverse-proxy...');
       await execAsync('docker rm -f tipi-reverse-proxy');
       spinner.done('Containers stopped and removed');
 
       // Pull images
       spinner.setMessage('Pulling images...');
       spinner.start();
+      this.logger.info('Pulling images new images...');
       await execAsync(`docker compose --env-file "${this.envFile}" pull`);
 
       spinner.done('Images pulled');
@@ -219,6 +251,7 @@ export class SystemExecutors {
       // Start containers
       spinner.setMessage('Starting containers...');
       spinner.start();
+      this.logger.info('Starting containers...');
       await execAsync(`docker compose --env-file "${this.envFile}" up --detach --remove-orphans --build`);
 
       spinner.done('Containers started');
@@ -227,34 +260,41 @@ export class SystemExecutors {
       spinner.setMessage('Starting watcher...');
       spinner.start();
 
+      this.logger.info('Generating TLS certificates...');
       await generateTlsCertificates({ domain: envMap.get('LOCAL_DOMAIN') });
 
+      this.logger.info('Opening log files for watcher...');
       const out = fs.openSync('./logs/watcher.log', 'a');
       const err = fs.openSync('./logs/watcher.log', 'a');
 
-      await killOtherWorkers();
-
+      this.logger.info('Starting watcher...');
       const subprocess = spawn('./runtipi-cli', [process.argv[1] as string, 'watch'], { cwd: this.rootFolder, detached: true, stdio: ['ignore', out, err] });
       subprocess.unref();
 
       spinner.done('Watcher started');
 
+      this.logger.info('Starting queue...');
       const queue = new Queue('events', { connection: { host: '127.0.0.1', port: 6379, password: envMap.get('REDIS_PASSWORD') } });
+      this.logger.info('Obliterating queue...');
       await queue.obliterate({ force: true });
 
       // Initial jobs
+      this.logger.info('Adding initial jobs to queue...');
       await queue.add(`${Math.random().toString()}_system_info`, { type: 'system', command: 'system_info' } as SystemEvent);
       await queue.add(`${Math.random().toString()}_repo_clone`, { type: 'repo', command: 'clone', url: envMap.get('APPS_REPO_URL') } as SystemEvent);
 
       // Scheduled jobs
+      this.logger.info('Adding scheduled jobs to queue...');
       await queue.add(`${Math.random().toString()}_repo_update`, { type: 'repo', command: 'update', url: envMap.get('APPS_REPO_URL') } as SystemEvent, { repeat: { pattern: '*/30 * * * *' } });
       await queue.add(`${Math.random().toString()}_system_info`, { type: 'system', command: 'system_info' } as SystemEvent, { repeat: { pattern: '* * * * *' } });
 
+      this.logger.info('Closing queue...');
       await queue.close();
 
       spinner.setMessage('Running database migrations...');
       spinner.start();
 
+      this.logger.info('Running database migrations...');
       await runPostgresMigrations({
         postgresHost: '127.0.0.1',
         postgresDatabase: envMap.get('POSTGRES_DBNAME') as string,
@@ -267,6 +307,7 @@ export class SystemExecutors {
 
       // Start all apps
       const appExecutor = new AppExecutors();
+      this.logger.info('Starting all apps...');
       await appExecutor.startAllApps();
 
       console.log(
@@ -325,14 +366,17 @@ export class SystemExecutors {
     try {
       spinner.start();
       let targetVersion = target;
+      this.logger.info(`Updating Tipi to version ${targetVersion}`);
 
       if (!targetVersion || targetVersion === 'latest') {
         spinner.setMessage('Fetching latest version...');
         const { data } = await axios.get<{ tag_name: string }>('https://api.github.com/repos/meienberger/runtipi/releases/latest');
+        this.logger.info(`Getting latest version from GitHub: ${data.tag_name}`);
         targetVersion = data.tag_name;
       }
 
       if (!semver.valid(targetVersion)) {
+        this.logger.error(`Invalid version: ${targetVersion}`);
         spinner.fail(`Invalid version: ${targetVersion}`);
         throw new Error(`Invalid version: ${targetVersion}`);
       }
@@ -347,6 +391,7 @@ export class SystemExecutors {
       const fileName = `runtipi-cli-${targetVersion}`;
       const savePath = path.join(rootFolderHost, fileName);
       const fileUrl = `https://github.com/meienberger/runtipi/releases/download/${targetVersion}/${assetName}`;
+      this.logger.info(`Downloading Tipi ${targetVersion} from ${fileUrl}`);
 
       spinner.done(`Target version: ${targetVersion}`);
       spinner.done(`Download url: ${fileUrl}`);
@@ -364,6 +409,7 @@ export class SystemExecutors {
           url: fileUrl,
           responseType: 'stream',
           onDownloadProgress: (progress) => {
+            this.logger.info(`Download progress: ${Math.round((progress.loaded / (progress.total || 0)) * 100)}%`);
             bar.update(Math.round((progress.loaded / (progress.total || 0)) * 100));
           },
         }).then((response) => {
@@ -372,21 +418,25 @@ export class SystemExecutors {
 
           writer.on('error', (err) => {
             bar.stop();
+            this.logger.error(`Failed to download Tipi: ${err}`);
             spinner.fail(`\nFailed to download Tipi ${targetVersion}`);
             reject(err);
           });
 
           writer.on('finish', () => {
+            this.logger.info('Download complete');
             bar.stop();
             resolve('');
           });
         });
       }).catch((e) => {
+        this.logger.error(`Failed to download Tipi: ${e}`);
         spinner.fail(`\nFailed to download Tipi ${targetVersion}. Please make sure this version exists on GitHub.`);
         throw e;
       });
 
       spinner.done(`Tipi ${targetVersion} downloaded`);
+      this.logger.info(`Changing permissions on ${savePath}`);
       await fs.promises.chmod(savePath, 0o755);
 
       spinner.setMessage('Replacing old cli...');
@@ -394,15 +444,18 @@ export class SystemExecutors {
 
       // Delete old cli
       if (await pathExists(path.join(rootFolderHost, 'runtipi-cli'))) {
+        this.logger.info('Deleting old cli...');
         await fs.promises.unlink(path.join(rootFolderHost, 'runtipi-cli'));
       }
 
       // Delete VERSION file
       if (await pathExists(path.join(rootFolderHost, 'VERSION'))) {
+        this.logger.info('Deleting VERSION file...');
         await fs.promises.unlink(path.join(rootFolderHost, 'VERSION'));
       }
 
       // Rename downloaded cli to runtipi-cli
+      this.logger.info('Renaming new cli to runtipi-cli...');
       await fs.promises.rename(savePath, path.join(rootFolderHost, 'runtipi-cli'));
       spinner.done('Old cli replaced');
 
@@ -410,6 +463,7 @@ export class SystemExecutors {
       // eslint-disable-next-line no-promise-executor-return
       await new Promise((resolve) => setTimeout(resolve, 3000));
 
+      this.logger.info('Starting new cli...');
       const childProcess = spawn('./runtipi-cli', [process.argv[1] as string, 'start']);
 
       childProcess.stdout.on('data', (data) => {

packages/cli/src/executors/system/system.helpers.ts

@@ -220,26 +220,32 @@ export const copySystemFiles = async () => {
   const assetsFolder = path.join('/snapshot', 'runtipi', 'packages', 'cli', 'assets');
 
   // Copy docker-compose.yml file
+  fileLogger.info('Copying file docker-compose.yml');
   await fs.promises.copyFile(path.join(assetsFolder, 'docker-compose.yml'), path.join(process.cwd(), 'docker-compose.yml'));
 
   // Copy VERSION file
+  fileLogger.info('Copying file VERSION');
   await fs.promises.copyFile(path.join(assetsFolder, 'VERSION'), path.join(process.cwd(), 'VERSION'));
 
   // Copy traefik folder from assets
+  fileLogger.info('Creating traefik folders');
   await fs.promises.mkdir(path.join(process.cwd(), 'traefik', 'dynamic'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'traefik', 'shared'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'traefik', 'tls'), { recursive: true });
 
+  fileLogger.info('Copying traefik files');
   await fs.promises.copyFile(path.join(assetsFolder, 'traefik', 'traefik.yml'), path.join(process.cwd(), 'traefik', 'traefik.yml'));
   await fs.promises.copyFile(path.join(assetsFolder, 'traefik', 'dynamic', 'dynamic.yml'), path.join(process.cwd(), 'traefik', 'dynamic', 'dynamic.yml'));
 
   // Create base folders
+  fileLogger.info('Creating base folders');
   await fs.promises.mkdir(path.join(process.cwd(), 'apps'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'app-data'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'state'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'repos'), { recursive: true });
 
   // Create media folders
+  fileLogger.info('Creating media folders');
   await fs.promises.mkdir(path.join(process.cwd(), 'media', 'torrents', 'watch'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'media', 'torrents', 'complete'), { recursive: true });
   await fs.promises.mkdir(path.join(process.cwd(), 'media', 'torrents', 'incomplete'), { recursive: true });
@@ -274,14 +280,17 @@ export const generateTlsCertificates = async (data: { domain?: string }) => {
 
   // If the certificate already exists, don't generate it again
   if (await pathExists(path.join(process.cwd(), 'traefik', 'tls', `${data.domain}.txt`))) {
+    fileLogger.info(`TLS certificate for ${data.domain} already exists`);
     return;
   }
 
   // Remove old certificates
   if (await pathExists(path.join(process.cwd(), 'traefik', 'tls', 'cert.pem'))) {
+    fileLogger.info('Removing old TLS certificate');
     await fs.promises.unlink(path.join(process.cwd(), 'traefik', 'tls', 'cert.pem'));
   }
   if (await pathExists(path.join(process.cwd(), 'traefik', 'tls', 'key.pem'))) {
+    fileLogger.info('Removing old TLS key');
     await fs.promises.unlink(path.join(process.cwd(), 'traefik', 'tls', 'key.pem'));
   }
 
@@ -289,7 +298,9 @@ export const generateTlsCertificates = async (data: { domain?: string }) => {
   const subjectAltName = `DNS:*.${data.domain},DNS:${data.domain}`;
 
   try {
+    fileLogger.info(`Generating TLS certificate for ${data.domain}`);
     await execAsync(`openssl req -x509 -newkey rsa:4096 -keyout traefik/tls/key.pem -out traefik/tls/cert.pem -days 365 -subj "${subject}" -addext "subjectAltName = ${subjectAltName}" -nodes`);
+    fileLogger.info(`Writing txt file for ${data.domain}`);
     await fs.promises.writeFile(path.join(process.cwd(), 'traefik', 'tls', `${data.domain}.txt`), '');
   } catch (error) {
     fileLogger.error(error);

packages/cli/src/index.ts

@@ -61,6 +61,14 @@ const main = async () => {
       console.log(chalk.green('✓'), 'Password reset request created. Head back to the dashboard to set a new password.');
     });
 
+  program
+    .command('clean-logs')
+    .description('Clean logs')
+    .action(async () => {
+      const systemExecutors = new SystemExecutors();
+      await systemExecutors.cleanLogs();
+    });
+
   program.parse(process.argv);
 };
 

packages/cli/src/services/watcher/watcher.ts

@@ -1,6 +1,7 @@
 import { eventSchema } from '@runtipi/shared';
 import { Worker } from 'bullmq';
 import { exec } from 'child_process';
+import IORedis from 'ioredis';
 import { promisify } from 'util';
 import { AppExecutors, RepoExecutors, SystemExecutors } from '@/executors';
 import { getEnv } from '@/utils/environment/environment';
@@ -97,6 +98,9 @@ export const killOtherWorkers = async () => {
  * Start the worker for the events queue
  */
 export const startWorker = async () => {
+  console.info('Killing other workers...');
+  await killOtherWorkers();
+
   const worker = new Worker(
     'events',
     async (job) => {
@@ -105,7 +109,7 @@ export const startWorker = async () => {
 
       return { success, stdout: message };
     },
-    { connection: { host: '127.0.0.1', port: 6379, password: getEnv().redisPassword } },
+    { connection: { host: '127.0.0.1', port: 6379, password: getEnv().redisPassword, connectTimeout: 60000 } },
   );
 
   worker.on('ready', () => {

packages/cli/src/utils/docker-helpers/docker-helpers.ts

@@ -10,8 +10,9 @@ const execAsync = promisify(exec);
 const composeUp = async (args: string[]) => {
   const { stdout, stderr } = await execAsync(`docker compose ${args.join(' ')}`);
 
-  fileLogger.info('stdout', stdout);
-  fileLogger.info('stderr', stderr);
+  if (stderr) {
+    fileLogger.error(stderr);
+  }
 
   return { stdout, stderr };
 };