[release] v0.12.0-unstable47

changelog.md

@@ -1,27 +1,31 @@
 ## Version 0.12.0
- - New Dashboard
- - New metrics gathering system
- - New alerts system
- - New notification center
- - New events manager
- - Integrated a new docker-less mode of functioning for networking
- - Added Button to force reset HTTPS cert in settings
- - New color slider with reset buttons
+ - New real time persisting andd optimized metrics monitoring system (RAM, CPU, Network, disk, requests, errors, etc...)
+ - New Dashboard with graphs for metrics, including graphs in many screens such as home, routes and servapps
+ - New customizable alerts system based on metrics in real time, with included preset for anti-crypto mining and anti memory leak 
+ - New events manager (improved logs with requests and advanced search)
+ - New notification system
+ - Added Marketplace UI to edit sources, with new display of 3rd party sources
  - Added a notification when updating a container, renewing certs, etc...
+ - Certificates now renew sooner to avoid Let's Encrypt sending emails about expiring certificates
+ - Added option to disable routes without deleting them
  - Improved icon loading speed, and added proper placeholder
- - Added lazyloading to URL and Servapp pages images
+ - Marketplace now fetch faster (removed the domain indirection to directly fetch from github)
+ - Integrated a new docker-less mode of functioning for networking
  - Added a dangerous IP detector that stops sending HTTP response to IPs that are abusing various shields features
  - Added a button in the servapp page to easily download the docker backup
+ - Added Button to force reset HTTPS cert in settings
+ - Added lazyloading to URL and Servapp pages images
+ - Fixed annoying marketplace screenshot bug (you know what I'm talking about!)
+ - New color slider with reset buttons
  - Redirect static folder to host if possible
  - New Homescreen look
- - Added option to disable routes without deleting them
  - Fixed blinking modals issues
  - Improve display or icons [fixes #121]
  - Refactored Mongo connection code [fixes #111]
  - Forward simultaneously TCP and UDP [fixes #122]
 
 ## Version 0.11.3
- - Fix missing even subscriber on export
+ - Fix missing event subscriber on export
 
 ## Version 0.11.2
  - Improve Docker exports logs

client/src/pages/config/users/formShortcuts.jsx

@@ -47,7 +47,7 @@ export const CosmosInputText = ({ name, style, value, errors, multiline, type, p
       <OutlinedInput
         id={name}
         type={type ? type : 'text'}
-        value={value || (formik && formik.values[name])}
+        value={value || (formik && getNestedValue(formik.values, name))}
         name={name}
         multiline={multiline}
         onBlur={(...ar) => {
@@ -101,7 +101,7 @@ export const CosmosInputPassword = ({ name, noStrength, type, placeholder, autoC
       <OutlinedInput
         id={name}
         type={showPassword ? 'text' : 'password'}
-        value={formik.values[name]}
+        value={getNestedValue(formik.values, name)}
         name={name}
         autoComplete={autoComplete}
         onBlur={formik.handleBlur}

client/src/pages/market/listing.jsx

@@ -1,4 +1,4 @@
-import { Box, CircularProgress, Input, InputAdornment, Stack } from "@mui/material";
+import { Box, CircularProgress, Input, InputAdornment, Stack, Tooltip } from "@mui/material";
 import { HomeBackground, TransparentHeader } from "../home";
 import { useEffect, useState } from "react";
 import * as API from "../../api";
@@ -10,18 +10,42 @@ import { Paper, Button, Chip } from '@mui/material'
 import { Link } from "react-router-dom";
 import { Link as LinkMUI } from '@mui/material'
 import DockerComposeImport from '../servapps/containers/docker-compose';
-import { AppstoreAddOutlined, SearchOutlined } from "@ant-design/icons";
+import { AppstoreAddOutlined, SearchOutlined, WarningOutlined } from "@ant-design/icons";
 import ResponsiveButton from "../../components/responseiveButton";
 import { useClientInfos } from "../../utils/hooks";
+import EditSourcesModal from "./sources";
 
 function Screenshots({ screenshots }) {
+  const aspectRatioContainerStyle = {
+    position: 'relative',
+    overflow: 'hidden',
+    paddingTop: '56.25%', // 9 / 16 = 0.5625 or 56.25%
+    height: 0,
+  };
+
+  // This will position the image correctly within the aspect ratio container
+  const imageStyle = {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    bottom: 0,
+    right: 0,
+    width: '100%',
+    height: '100%',
+    objectFit: 'cover', // This will cover the area without losing the aspect ratio
+  };
+
   return screenshots.length > 1 ? (
     <Carousel animation="slide" navButtonsAlwaysVisible={false} fullHeightHover="true" swipe={false}>
       {
-        screenshots.map((item, i) => <img style={{ maxHeight: '300px', height: '100%', maxWidth: '100%' }} key={i} src={item} />)
+        screenshots.map((item, i) => <div style={{height: "400px"}}>
+          <img style={{ maxHeight: '100%', width: '100%' }} key={i} src={item} />
+        </div>)
       }
     </Carousel>)
-    : <img src={screenshots[0]} style={{ maxHeight: '300px', height: '100%', maxWidth: '100%' }} />
+    : <div style={{height: "400px"}}>
+        <img src={screenshots[0]} style={{ maxHeight: '100%', width: '100%' }} />
+      </div>
 }
 
 function Showcases({ showcase, isDark, isAdmin }) {
@@ -122,18 +146,44 @@ const MarketPage = () => {
     // borderTop: '1px solid rgb(220,220,220)'
   };
 
-  useEffect(() => {
+  const refresh = () => {
     API.market.list().then((res) => {
       setApps(res.data.all);
       setShowcase(res.data.showcase);
     });
+  };
+
+  useEffect(() => {
+    refresh();
   }, []);
 
   let openedApp = null;
   if (appName && Object.keys(apps).length > 0) {
     openedApp = apps[appStore].find((app) => app.name === appName);
+    openedApp.appstore = appStore;
   }
 
+  let appList = apps && Object.keys(apps).reduce((acc, appstore) => {
+    const a = apps[appstore].map((app) => {
+      app.appstore = appstore;
+      return app;
+    });
+
+    return acc.concat(a);
+  }, []);
+
+  appList.sort((a, b) => {
+    if (a.name > b.name) {
+      return 1;
+    }
+
+    if (a.name < b.name) {
+      return -1;
+    }
+
+    return 0;
+  });
+
   return <>
     <HomeBackground />
     <TransparentHeader />
@@ -186,7 +236,7 @@ const MarketPage = () => {
 
           <Stack direction="row" spacing={2}>
             <img src={openedApp.icon} style={{ width: '36px', height: '36px' }} />
-            <h2>{openedApp.name}</h2>
+            <h2>{openedApp.name} <span style={{color:'grey'}}>{openedApp.appstore != 'cosmos-cloud' ? (' @ '+openedApp.appstore) : ''}</span></h2>
           </Stack>
 
           <div>
@@ -197,6 +247,14 @@ const MarketPage = () => {
             {openedApp.supported_architectures && openedApp.supported_architectures.slice(0, 8).map((tag) => <Chip label={tag} />)}
           </div>
 
+          {openedApp.appstore != 'cosmos-cloud' && <div>
+            <div>
+            <Tooltip title="This app is not hosted on the Cosmos Cloud App Store. It is not officially verified and tested.">
+                <WarningOutlined />
+              </Tooltip> <strong>source:</strong> {openedApp.appstore} 
+            </div>
+          </div>}
+          
           <div>
             <div><strong>repository:</strong> <LinkMUI href={openedApp.repository}>{openedApp.repository}</LinkMUI></div>
             <div><strong>image:</strong> <LinkMUI href={openedApp.image}>{openedApp.image}</LinkMUI></div>
@@ -259,6 +317,7 @@ const MarketPage = () => {
             >Start ServApp</ResponsiveButton>
           </Link>
           <DockerComposeImport refresh={() => { }} />
+          <EditSourcesModal onSave={refresh} />
         </Stack>
         {(!apps || !Object.keys(apps).length) && <Box style={{
           width: '100%',
@@ -275,8 +334,7 @@ const MarketPage = () => {
         </Box>}
 
         {apps && Object.keys(apps).length > 0 && <Grid2 container spacing={{ xs: 1, sm: 1, md: 2 }}>
-          {Object.keys(apps).map(appstore => apps[appstore]
-            .filter((app) => {
+          {appList.filter((app) => {
               if (!search || search.length <= 2) {
                 return true;
               }
@@ -284,17 +342,18 @@ const MarketPage = () => {
                 app.tags.join(' ').toLowerCase().includes(search.toLowerCase());
             })
             .map((app) => {
-              return <Grid2 style={{
+              return <Grid2
+              style={{
                 ...gridAnim,
                 cursor: 'pointer',
-              }} xs={12} sm={12} md={6} lg={4} xl={3} key={app.name} item><Link to={"/cosmos-ui/market-listing/" + appstore + "/" + app.name} style={{
+              }} xs={12} sm={12} md={6} lg={4} xl={3} key={app.name + app.appstore} item><Link to={"/cosmos-ui/market-listing/" + app.appstore + "/" + app.name} style={{
                 textDecoration: 'none',
               }}>
                   <div key={app.name} style={appCardStyle(theme)}>
                     <Stack spacing={3} direction={'row'} alignItems={'center'} style={{ padding: '0px 15px' }}>
                       <img src={app.icon} style={{ width: 64, height: 64 }} />
                       <Stack spacing={1}>
-                        <div style={{ fontWeight: "bold" }}>{app.name}</div>
+                        <div style={{ fontWeight: "bold" }}>{app.name}<span style={{color:'grey'}}>{app.appstore != 'cosmos-cloud' ? (' @ '+app.appstore) : ''}</span></div>
                         <div style={{
                           height: '40px',
                           overflow: 'hidden',
@@ -317,7 +376,7 @@ const MarketPage = () => {
 
                 </Link>
               </Grid2>
-            }))}
+            })}
         </Grid2>}
       </Stack>
     </Stack>

client/src/pages/market/sources.jsx

@@ -0,0 +1,183 @@
+import * as React from 'react';
+import IsLoggedIn from '../../isLoggedIn';
+import * as API from '../../api';
+import MainCard from '../../components/MainCard';
+import { Formik, Field, useFormik, FormikProvider } from 'formik';
+import * as Yup from 'yup';
+import {
+  Alert,
+  Button,
+  Checkbox,
+  FormControlLabel,
+  Grid,
+  InputLabel,
+  OutlinedInput,
+  Stack,
+  FormHelperText,
+  TextField,
+  MenuItem,
+  Skeleton,
+  CircularProgress,
+  Dialog,
+  DialogTitle,
+  DialogContent,
+  DialogContentText,
+  DialogActions,
+  Box,
+} from '@mui/material';
+import { ContainerOutlined, ExclamationCircleOutlined, InfoCircleOutlined, PlusCircleOutlined, SyncOutlined, WarningOutlined } from '@ant-design/icons';
+import PrettyTableView from '../../components/tableView/prettyTableView';
+import { DeleteButton } from '../../components/delete';
+import { CosmosCheckbox, CosmosFormDivider, CosmosInputText, CosmosSelect } from '../config/users/formShortcuts';
+import ResponsiveButton from '../../components/responseiveButton';
+
+const AlertValidationSchema = Yup.object().shape({
+  name: Yup.string().required('Name is required'),
+  trackingMetric: Yup.string().required('Tracking metric is required'),
+  conditionOperator: Yup.string().required('Condition operator is required'),
+  conditionValue: Yup.number().required('Condition value is required'),
+  period: Yup.string().required('Period is required'),
+});
+
+const EditSourcesModal = ({ onSave }) => {
+  const [config, setConfig] = React.useState(null);
+  const [open, setOpen] = React.useState(false);
+  
+  function getConfig() {
+    API.config.get().then((res) => {
+      setConfig(res.data);
+    });
+  }
+
+  React.useEffect(() => {
+    getConfig();
+  }, []);
+
+  const formik = useFormik({
+    initialValues: {
+      sources: config ? config.MarketConfig.Sources : [],
+    },
+    enableReinitialize: true, // This will reinitialize the form when `config` changes
+    // validationSchema: AlertValidationSchema,
+    onSubmit: (values) => {
+      values.sources = values.sources.filter((a) => !a.removed);
+
+      // setIsLoading(true);
+  
+      let toSave = {
+        ...config,
+        MarketConfig: {
+          ...config.MarketConfig,
+          Sources: values.sources,
+        }
+      };
+  
+      setOpen(false);
+
+      return API.config.set(toSave).then(() => {
+        onSave();
+      });
+    },
+    validate: (values) => {
+      const errors = {};
+
+      values.sources.forEach((source, index) => {
+        if (source.Name === '') {
+          errors[`sources.${index}.Name`] = 'Name is required';
+        }
+        if (source.Url === '') {
+          errors[`sources.${index}.Url`] = 'URL is required';
+        }
+
+        if (source.Name === 'cosmos-cloud' || values.sources.filter((s) => s.Name === source.Name).length > 1) {
+          errors[`sources.${index}.Name`] = 'Name must be unique';
+        }
+      });
+  
+      return errors;
+    }
+  });
+
+  return (<>
+    <Dialog open={open} onClose={() => setOpen(false)} maxWidth="sm" fullWidth>
+      <DialogTitle>Edit Sources</DialogTitle>
+      {config && <FormikProvider value={formik}>
+      <form onSubmit={formik.handleSubmit}>
+        <DialogContent>
+          <Stack spacing={2}>
+            {formik.values.sources
+            .map((action, index) => {
+              return !action.removed && <>
+                <Stack spacing={0} key={index}>
+                  <Stack direction="row" spacing={2}>
+                    <CosmosInputText
+                      name={`sources.${index}.Name`}
+                      label="Name"
+                      formik={formik}
+                    />
+                    <div style={{ flexGrow: 1 }}>
+                      <CosmosInputText
+                        name={`sources.${index}.Url`}
+                        label="URL"
+                        formik={formik}
+                      />
+                    </div>
+                    
+                    <Box  style={{
+                      height: '95px',
+                      display: 'flex',
+                      alignItems: 'center',
+                    }}>
+                      <DeleteButton
+                        onDelete={() => {
+                          formik.setFieldValue(`sources.${index}.removed`, true);
+                        }}
+                      />
+                    </Box>
+                  </Stack>
+                  <div>
+                    <FormHelperText error>{formik.errors[`sources.${index}.Name`]}</FormHelperText>
+                  </div>
+                  <div>
+                    <FormHelperText error>{formik.errors[`sources.${index}.Url`]}</FormHelperText>
+                  </div>
+                </Stack>
+              </>
+            })}
+
+            <Button
+              variant="outlined"
+              color="primary"
+              startIcon={<PlusCircleOutlined />}
+              onClick={() => {
+                formik.setFieldValue('sources', [
+                  ...formik.values.sources,
+                  {
+                    Name: '',
+                    Url: '',
+                  },
+                ]);
+              }}>
+              Add Source
+            </Button>
+          </Stack>
+        </DialogContent>
+        <DialogActions>
+          <Button onClick={() => setOpen(false)}>Cancel</Button>
+          <Button variant='contained' type="submit" disabled={formik.isSubmitting || !formik.isValid}>Save</Button>
+        </DialogActions>
+      </form>
+      </FormikProvider>}
+    </Dialog>
+
+
+    <ResponsiveButton
+      variant="outlined"
+      startIcon={<ContainerOutlined />}
+      onClick={() => setOpen(true)}
+    >Sources</ResponsiveButton>
+    </>
+  );
+};
+
+export default EditSourcesModal;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cosmos-server",
-  "version": "0.12.0-unstable46",
+  "version": "0.12.0-unstable47",
   "description": "",
   "main": "test-server.js",
   "bugs": {

readme.md

@@ -16,8 +16,7 @@
 
 [![DiscordLink](https://img.shields.io/discord/1083875833824944188?label=Discord&logo=Discord&style=flat-square)](https://discord.gg/PwMWwsrwHA) ![CircleCI](https://img.shields.io/circleci/build/github/azukaar/Cosmos-Server?token=6efd010d0f82f97175f04a6acf2dae2bbcc4063c&style=flat-square)
 
-Cosmos is a self-hosted platform for running server applications securely and with built-in privacy features. It acts as a secure gateway to your application, as well as a server manager. It aims to solve the increasingly worrying problem of vulnerable self-hosted applications and personal servers.
-
+☁️ Cosmos is the most secure and easy way to selfhost a Home Server. It acts as a secure gateway to your application, as well as a server manager. It aims to solve the increasingly worrying problem of vulnerable self-hosted applications and personal servers.
 
 <p align="center">
   <br/>
@@ -37,7 +36,7 @@ Cosmos is a self-hosted platform for running server applications securely and wi
 
 ![screenshot1](./screenshot1.png)
 
-Whether you have a **server**, a **NAS**, or a **Raspberry Pi** with applications such as **Plex**, **HomeAssistant** or even a blog, Cosmos is the perfect solution to secure them all. Simply install Cosmos on your server and connect to your applications through it to enjoy built-in security and robustness for all your services, right out of the box.
+Whether you have a **server**, a **NAS**, or a **Raspberry Pi** with applications such as **Plex**, **HomeAssistant** or even a blog, Cosmos is the perfect solution torun and secure them all. Simply install Cosmos on your server and connect to your applications through it to enjoy built-in security and robustness for all your services, right out of the box.
 
 Cosmos is a: 
 
@@ -155,7 +154,7 @@ in this command, `-v /:/mnt/host` is optional and allow to manage folders from C
 
 `--privileged` is also optional, but it is required if you use hardening software like AppArmor or SELinux, as they restrict access to the docker socket. It is also required for Constellation to work. If you don't want to use it, you can add the following capabilities: NET_ADMIN for Constellation.
 
-Once installed, simply go to `http://your-server-ip` and follow the instructions of the setup wizard.
+Once installed, simply go to `http://your-server-ip` and follow the instructions of the setup wizard. **always start the install with the browser in incognito mode** to avoid issues with your browser cache.
 
 Port 4242 is a UDP port used for the Constellation VPN.
 

src/CRON.go

@@ -123,6 +123,10 @@ func CRON() {
 		s.Every(1).Day().At("01:00").Do(checkCerts)
 		s.Every(6).Hours().Do(checkUpdatesAvailable)
 		s.Every(1).Hours().Do(utils.CleanBannedIPs)
+		s.Every(1).Day().At("00:00").Do(func() {
+			utils.CleanupByDate("notifications")
+			utils.CleanupByDate("events")
+		})
 		s.Start()
 	}()
 }

src/configapi/patch.go

@@ -95,6 +95,14 @@ func ConfigApiPatch(w http.ResponseWriter, req *http.Request) {
 
 	config.HTTPConfig.ProxyConfig.Routes = routes
 	utils.SetBaseMainConfig(config)
+
+	utils.TriggerEvent(
+		"cosmos.settings",
+		"Settings updated",
+		"success",
+		"",
+		map[string]interface{}{
+	})
 	
 	utils.RestartHTTPServer()
 		

src/configapi/set.go

@@ -40,6 +40,14 @@ func ConfigApiSet(w http.ResponseWriter, req *http.Request) {
 		request.NewInstall = config.NewInstall
 
 		utils.SetBaseMainConfig(request)
+		
+		utils.TriggerEvent(
+			"cosmos.settings",
+			"Settings updated",
+			"success",
+			"",
+			map[string]interface{}{
+		})
 
 		utils.DisconnectDB()
 		authorizationserver.Init()

src/constellation/api_devices_create.go

@@ -147,6 +147,18 @@ func DeviceCreate(w http.ResponseWriter, req *http.Request) {
 				return
 			}
 
+			utils.TriggerEvent(
+				"cosmos.constellation.device.create",
+				"Device created",
+				"success",
+				"",
+				map[string]interface{}{
+					"deviceName": deviceName,
+					"nickname": nickname,
+					"publicKey": key,
+					"ip": request.IP,
+			})
+
 			json.NewEncoder(w).Encode(map[string]interface{}{
 				"status": "OK",
 				"data": map[string]interface{}{

src/docker/api_blueprint.go

@@ -966,6 +966,20 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 	OnLog("\n")
 	OnLog(utils.DoSuccess("[OPERATION SUCCEEDED]. SERVICE STARTED\n"))
 
+	servicesNames := []string{}
+	for _, service := range serviceRequest.Services {
+		servicesNames = append(servicesNames, service.Name)
+	}
+
+	utils.TriggerEvent(
+		"cosmos.docker.compose.create",
+		"Service created",
+		"success",
+		"",
+		map[string]interface{}{
+			"services": servicesNames,
+	})
+
 	return nil
 }
 

src/docker/api_secureContainer.go

@@ -49,6 +49,16 @@ func SecureContainerRoute(w http.ResponseWriter, req *http.Request) {
 			return
 		}
 
+		utils.TriggerEvent(
+			"cosmos.docker.isolate",
+			"Container network isolation changed",
+			"success",
+			"container@"+containerName,
+			map[string]interface{}{
+				"container": containerName,
+				"status": status,
+		})
+
 		json.NewEncoder(w).Encode(map[string]interface{}{
 			"status": "OK",
 		})

src/docker/docker.go

@@ -101,6 +101,15 @@ func RecreateContainer(containerID string, containerConfig types.ContainerJSON)
 	} else {
 		return EditContainer(containerID, containerConfig, false)
 	}
+	
+	utils.TriggerEvent(
+		"cosmos.docker.recreate",
+		"Cosmos Container Recreate",
+		"success",
+		"container@" + containerID,
+		map[string]interface{}{
+			"container": containerID,
+	})
 
 	return "", nil
 }

src/docker/events.go

@@ -60,41 +60,43 @@ func DockerListenEvents() error {
 						onNetworkConnect(msg.Actor.ID)
 					}
 
-					level := "info"
-					if msg.Type == "image" {
-						level = "debug"
+					if msg.Action != "exec_create" && msg.Action != "exec_start" && msg.Action != "exec_die" {
+						level := "info"
+						if msg.Type == "image" {
+							level = "debug"
+						}
+						if msg.Action == "destroy" || msg.Action == "delete" || msg.Action == "kill" || msg.Action == "die" {
+							level = "warning"
+						}
+						if msg.Action == "create" || msg.Action == "start" {
+							level = "success"
+						}
+						
+						object := ""
+						if msg.Type == "container" {
+							object = "container@" + msg.Actor.Attributes["name"]
+						} else if msg.Type == "network" {
+							object = "network@" + msg.Actor.Attributes["name"]
+						} else if msg.Type == "image" {
+							object = "image@" + msg.Actor.Attributes["name"]
+						} else if msg.Type == "volume" && msg.Actor.Attributes["name"] != "" {
+							object = "volume@" + msg.Actor.Attributes["name"]
+						}
+						
+						utils.TriggerEvent(
+							"cosmos.docker.event." + msg.Type + "." + msg.Action,
+							"Docker Event " + msg.Type + " " + msg.Action,
+							level,
+							object,
+							map[string]interface{}{
+							"type": msg.Type,
+							"action": msg.Action,
+							"actor": msg.Actor,
+							"status": msg.Status,
+							"from": msg.From,
+							"scope": msg.Scope,
+						})
 					}
-					if msg.Action == "destroy" || msg.Action == "delete" || msg.Action == "kill" || msg.Action == "die" {
-						level = "warning"
-					}
-					if msg.Action == "create" || msg.Action == "start" {
-						level = "success"
-					}
-					
-					object := ""
-					if msg.Type == "container" {
-						object = "container@" + msg.Actor.Attributes["name"]
-					} else if msg.Type == "network" {
-						object = "network@" + msg.Actor.Attributes["name"]
-					} else if msg.Type == "image" {
-						object = "image@" + msg.Actor.Attributes["name"]
-					} else if msg.Type == "volume" && msg.Actor.Attributes["name"] != "" {
-						object = "volume@" + msg.Actor.Attributes["name"]
-					}
-					
-					utils.TriggerEvent(
-						"cosmos.docker.event." + msg.Type + "." + msg.Action,
-						"Docker Event " + msg.Type + " " + msg.Action,
-						level,
-						object,
-						map[string]interface{}{
-						"type": msg.Type,
-						"action": msg.Action,
-						"actor": msg.Actor,
-						"status": msg.Status,
-						"from": msg.From,
-						"scope": msg.Scope,
-					})
 			}
 		}
 	}()

src/httpServer.go

@@ -147,7 +147,7 @@ func tokenMiddleware(next http.Handler) http.Handler {
 	})
 }
 
-func SecureAPI(userRouter *mux.Router, public bool) {
+func SecureAPI(userRouter *mux.Router, public bool, publicCors bool) {
 	if(!public) {
 		userRouter.Use(tokenMiddleware)
 	}
@@ -162,9 +162,13 @@ func SecureAPI(userRouter *mux.Router, public bool) {
 			},
 		},
 	))
+
+	if(publicCors || public) {
+		userRouter.Use(utils.PublicCORS)
+	}
+
 	userRouter.Use(utils.MiddlewareTimeout(45 * time.Second))
-	userRouter.Use(proxy.BotDetectionMiddleware)
-	userRouter.Use(httprate.Limit(120, 1*time.Minute, 
+	userRouter.Use(httprate.Limit(180, 1*time.Minute, 
 		httprate.WithKeyFuncs(httprate.KeyByIP),
     httprate.WithLimitHandler(func(w http.ResponseWriter, r *http.Request) {
 			utils.Error("Too many requests. Throttling", nil)
@@ -334,7 +338,7 @@ func InitServer() *mux.Router {
 	}
 	
 	logoAPI := router.PathPrefix("/logo").Subrouter()
-	SecureAPI(logoAPI, true)
+	SecureAPI(logoAPI, true, true)
 	logoAPI.HandleFunc("/", SendLogo)
 	
 	
@@ -413,7 +417,7 @@ func InitServer() *mux.Router {
 		srapi.Use(utils.EnsureHostname)
 	}
 
-	SecureAPI(srapi, false)
+	SecureAPI(srapi, false, false)
 	
 	pwd,_ := os.Getwd()
 	utils.Log("Starting in " + pwd)
@@ -437,13 +441,13 @@ func InitServer() *mux.Router {
 	}))
 
 	userRouter := router.PathPrefix("/oauth2").Subrouter()
-	SecureAPI(userRouter, false)
+	SecureAPI(userRouter, false, true)
 
 	serverRouter := router.PathPrefix("/oauth2").Subrouter()
-	SecureAPI(serverRouter, true)
+	SecureAPI(serverRouter, true, true)
 
 	wellKnownRouter := router.PathPrefix("/").Subrouter()
-	SecureAPI(wellKnownRouter, true)
+	SecureAPI(wellKnownRouter, true, true)
 
 	authorizationserver.RegisterHandlers(wellKnownRouter, userRouter, serverRouter)
 

src/market/index.go

@@ -3,6 +3,7 @@ package market
 import (
 	"net/http"
 	"encoding/json"
+	"fmt"
 	"github.com/azukaar/cosmos-server/src/utils" 
 )
 
@@ -17,8 +18,23 @@ func MarketGet(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if(req.Method == "GET") {
+		config := utils.GetMainConfig()
+		configSourcesList := config.MarketConfig.Sources
+		configSources := map[string]bool{
+			"cosmos-cloud": true,
+		}
+		for _, source := range configSourcesList {
+			configSources[source.Name] = true
+		}
+
+		utils.Debug(fmt.Sprintf("MarketGet: Config sources: %v", configSources))
+
+		Init()
+
 		err := updateCache(w, req)
 		if err != nil {
+			utils.Error("MarketGet: Error while updating cache", err)
+			utils.HTTPError(w, "Error while updating cache", http.StatusInternalServerError, "MK002")
 			return
 		}
 		
@@ -28,19 +44,23 @@ func MarketGet(w http.ResponseWriter, req *http.Request) {
 		}
 
 		for _, market := range currentMarketcache {
+			if !configSources[market.Name] {
+				continue
+			}
+			utils.Debug(fmt.Sprintf("MarketGet: Adding market %v", market.Name))
 			results := []appDefinition{}
 			for _, app := range market.Results.All {
-				// if i < 10 {
-					results = append(results, app)
-				// } else {
-				// 	break
-				// }
+				results = append(results, app)
 			}
 			marketGetResult.All[market.Name] = results
 		}
 		
 		if len(currentMarketcache) > 0 {
-			marketGetResult.Showcase = currentMarketcache[0].Results.Showcase
+			for _, market := range currentMarketcache {
+				if market.Name == "cosmos-cloud" {
+					marketGetResult.Showcase = market.Results.Showcase
+				}
+			}
 		}
 
 		json.NewEncoder(w).Encode(map[string]interface{}{

src/market/init.go

@@ -6,23 +6,59 @@ import (
 
 func Init() {
 	config := utils.GetMainConfig()
-	currentMarketcache = []marketCacheObject{}
 	sources := config.MarketConfig.Sources
 
+	inConfig := map[string]bool{
+		"cosmos-cloud": true,
+	}
+	for _, source := range sources {
+		inConfig[source.Name] = true
+	}
+	
+	if currentMarketcache == nil {
+		currentMarketcache = []marketCacheObject{}
+	}
+
+	inCache := map[string]bool{}
+	toRemove := []string{}
+	for _, cachedMarket := range currentMarketcache {
+		inCache[cachedMarket.Name] = true
+
+		if !inConfig[cachedMarket.Name] {
+			utils.Log("MarketInit: Removing market " + cachedMarket.Name)
+			toRemove = append(toRemove, cachedMarket.Name)
+		}
+	}
+
+	// remove markets that are not in config
+	for _, name := range toRemove {
+		for index, cachedMarket := range currentMarketcache {
+			if cachedMarket.Name == name {
+				currentMarketcache = append(currentMarketcache[:index], currentMarketcache[index+1:]...)
+				break
+			}
+		}
+	}
+
 	// prepend the default market
 	defaultMarket := utils.MarketSource{
-		Url: "https://cosmos-cloud.io/repository",
+		Url: "https://azukaar.github.io/cosmos-servapps-official/index.json",
 		Name: "cosmos-cloud",
 	}
+
 	sources = append([]utils.MarketSource{defaultMarket}, sources...)
 
 	for _, marketDef := range sources {
-		market := marketCacheObject{
-			Url: marketDef.Url,
-			Name: marketDef.Name,
-		}
-		currentMarketcache = append(currentMarketcache, market)
+		// add markets that are in config but not in cache
+		if !inCache[marketDef.Name] {
+			market := marketCacheObject{
+				Url: marketDef.Url,
+				Name: marketDef.Name,
+			}
+
+			currentMarketcache = append(currentMarketcache, market)
 
-		utils.Log("MarketInit: Added market " + market.Name)
+			utils.Log("MarketInit: Added market " + market.Name)
+		}
 	}
 }

src/market/update.go

@@ -61,9 +61,24 @@ func updateCache(w http.ResponseWriter, req *http.Request) error {
 				continue
 			}
 
+			result.Source = cachedMarket.Url
+			if cachedMarket.Name != "cosmos-cloud" {
+				result.Showcase = []appDefinition{}
+			}
+
 			cachedMarket.Results = result
 			cachedMarket.LastUpdate = time.Now()
 
+			utils.TriggerEvent(
+				"cosmos.market.update",
+				"Market updated",
+				"success",
+				"",
+				map[string]interface{}{
+					"market": cachedMarket.Name,
+					"numberOfApps": len(result.All),
+			})
+
 			utils.Log("MarketUpdate: Updated market " + result.Source + " with " + string(len(result.All)) + " results")
 
 			// save to cache

src/proxy/shield.go

@@ -38,11 +38,11 @@ type smartShieldState struct {
 }
 
 type userUsedBudget struct {
-	ClientID string
-	Time float64
-	Requests int
-	Bytes int64
-	Simultaneous int
+	ClientID string `json:"clientID"`
+	Time float64 `json:"time"`
+	Requests int `json:"requests"`
+	Bytes int64 `json:"bytes"`
+	Simultaneous int `json:"simultaneous"`
 }
 
 var shield smartShieldState
@@ -379,6 +379,20 @@ func SmartShieldMiddleware(shieldID string, route utils.ProxyRouteConfig) func(h
 				lastBan := shield.GetLastBan(policy, userConsumed)
 				go metrics.PushShieldMetrics("smart-shield")
 				utils.IncrementIPAbuseCounter(clientID)
+
+				utils.TriggerEvent(
+					"cosmos.proxy.shield.abuse." + route.Name,
+					"Proxy Shield " + route.Name + " Abuse by " + clientID,
+					"warning",
+					"route@" + route.Name,
+					map[string]interface{}{
+					"route": route.Name,
+					"consumed": userConsumed,
+					"lastBan": lastBan,
+					"clientID": clientID,
+					"url": r.URL,
+				})
+
 				utils.Log("SmartShield: User is blocked due to abuse: " + fmt.Sprintf("%+v", lastBan))
 				http.Error(w, "Too many requests", http.StatusTooManyRequests)
 				return

src/user/create.go

@@ -77,6 +77,15 @@ func UserCreate(w http.ResponseWriter, req *http.Request) {
 				return 
 			} 
 			
+			utils.TriggerEvent(
+				"cosmos.user.create",
+				"User created",
+				"success",
+				"",
+				map[string]interface{}{
+					"nickname": nickname,
+			})
+
 			json.NewEncoder(w).Encode(map[string]interface{}{
 				"status": "OK",
 				"data": map[string]interface{}{

src/user/password_reset.go

@@ -86,6 +86,15 @@ func ResetPassword(w http.ResponseWriter, req *http.Request) {
 				return
 			}
 
+			utils.TriggerEvent(
+				"cosmos.user.passwordreset",
+				"Password reset sent",
+				"success",
+				"",
+				map[string]interface{}{
+					"nickname": user.Nickname,
+			})
+
 			json.NewEncoder(w).Encode(map[string]interface{}{
 				"status": "OK",
 			})

src/user/register.go

@@ -102,6 +102,15 @@ func UserRegister(w http.ResponseWriter, req *http.Request) {
 			}
 		}
 		
+		utils.TriggerEvent(
+			"cosmos.user.register",
+			"User registered",
+			"success",
+			"",
+			map[string]interface{}{
+				"nickname": nickname,
+		})
+
 		json.NewEncoder(w).Encode(map[string]interface{}{
 			"status": "OK",
 		})

src/utils/cleanup.go

@@ -0,0 +1,40 @@
+package utils
+
+import (
+	"time"
+	"strconv"
+	"context"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type CleanupObject struct {
+	Date time.Time
+}
+
+func CleanupByDate(collectionName string) {
+	c, errCo := GetCollection(GetRootAppId(), collectionName)
+	if errCo != nil {
+		MajorError("Database Cleanup", errCo)
+			return
+	}
+	
+	del, err := c.DeleteMany(context.Background(), bson.M{"Date": bson.M{"$lt": time.Now().AddDate(0, -1, 0)}})
+
+	if err != nil {
+		MajorError("Database Cleanup", err)
+		return
+	}
+
+	Log("Cleanup: " + collectionName + " " + strconv.Itoa(int(del.DeletedCount)) + " objects deleted")
+	
+	TriggerEvent(
+		"cosmos.database.cleanup",
+		"Database Cleanup of " + collectionName,
+		"success",
+		"",
+		map[string]interface{}{
+			"collection": collectionName,
+			"deleted": del.DeletedCount,
+	})
+}

src/utils/emails.go

@@ -147,5 +147,15 @@ func SendEmail(recipients []string, subject string, body string) error {
 		ServerURL,
 	))
 	
+	TriggerEvent(
+		"cosmos.email.send",
+		"Email sent",
+		"success",
+		"",
+		map[string]interface{}{
+			"recipients": recipients,
+			"subject": subject,
+	})
+
 	return send(hostPort, auth, config.EmailConfig.From, recipients, msg)
 }

src/utils/log.go

@@ -57,6 +57,16 @@ func MajorError(message string, err error) {
 		log.Println(Red + "[ERROR] " + message + " : " + errStr + Reset)
 	}
 	
+	TriggerEvent(
+		"cosmos.error",
+		"Critical Error",
+		"error",
+		"",
+		map[string]interface{}{
+			"message": message,
+			"error": errStr,
+	})
+
 	WriteNotification(Notification{
 		Recipient: "admin",
 		Title: "Server Error",

src/utils/middleware.go

@@ -163,6 +163,15 @@ func CORSHeader(origin string) func(next http.Handler) http.Handler {
 	}
 }
 
+func PublicCORS(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+
+		next.ServeHTTP(w, r)
+	})
+}
+
 func AcceptHeader(accept string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -218,6 +227,18 @@ func BlockByCountryMiddleware(blockedCountries []string, CountryBlacklistIsWhite
 						if blocked {
 							PushShieldMetrics("geo")
 							IncrementIPAbuseCounter(ip)
+
+							TriggerEvent(
+								"cosmos.proxy.shield.geo",
+								"Proxy Shield  Geo blocked",
+								"warning",
+								"",
+								map[string]interface{}{
+								"clientID": ip,
+								"country": countryCode,
+								"url": r.URL.String(),
+							})
+
 							http.Error(w, "Access denied", http.StatusForbidden)
 							return
 						}
@@ -253,6 +274,16 @@ func BlockPostWithoutReferer(next http.Handler) http.Handler {
 
 				ip, _, _ := net.SplitHostPort(r.RemoteAddr)
 				if ip != "" {
+					TriggerEvent(
+						"cosmos.proxy.shield.referer",
+						"Proxy Shield  Referer blocked",
+						"warning",
+						"",
+						map[string]interface{}{
+						"clientID": ip,
+						"url": r.URL.String(),
+					})
+
 					IncrementIPAbuseCounter(ip)
 				}
 
@@ -295,6 +326,16 @@ func EnsureHostname(next http.Handler) http.Handler {
 			
 			ip, _, _ := net.SplitHostPort(r.RemoteAddr)
 			if ip != "" {
+				TriggerEvent(
+					"cosmos.proxy.shield.hostname",
+					"Proxy Shield hostname blocked",
+					"warning",
+					"",
+					map[string]interface{}{
+					"clientID": ip,
+					"hostname": r.Host,
+					"url": r.URL.String(),
+				})
 				IncrementIPAbuseCounter(ip)
 			}
 
@@ -389,6 +430,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
 			if(!isInConstellation) {
 				if(!isUsingWhiteList) {
 					PushShieldMetrics("ip-whitelists")
+
+					TriggerEvent(
+						"cosmos.proxy.shield.whitelist",
+						"Proxy Shield IP blocked by whitelist",
+						"warning",
+						"",
+						map[string]interface{}{
+						"clientID": ip,
+						"url": r.URL.String(),
+					})
+
 					IncrementIPAbuseCounter(ip)
 					Error("Request from " + ip + " is blocked because of restrictions", nil)
 					Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList")
@@ -396,6 +448,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
 					return
 				} else if (!isInWhitelist) {
 					PushShieldMetrics("ip-whitelists")
+					
+					TriggerEvent(
+						"cosmos.proxy.shield.whitelist",
+						"Proxy Shield IP blocked by whitelist",
+						"warning",
+						"",
+						map[string]interface{}{
+						"clientID": ip,
+						"url": r.URL.String(),
+					})
+
 					IncrementIPAbuseCounter(ip)
 					Error("Request from " + ip + " is blocked because of restrictions", nil)
 					Debug("Blocked by RestrictToConstellation isInConstellation isInWhitelist")
@@ -405,6 +468,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
 			}
 		} else if(isUsingWhiteList && !isInWhitelist) {
 			PushShieldMetrics("ip-whitelists")
+
+			TriggerEvent(
+				"cosmos.proxy.shield.whitelist",
+				"Proxy Shield IP blocked by whitelist",
+				"warning",
+				"",
+				map[string]interface{}{
+				"clientID": ip,
+				"url": r.URL.String(),
+			})
+
 			IncrementIPAbuseCounter(ip)
 			Error("Request from " + ip + " is blocked because of restrictions", nil)
 			Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList isInWhitelist")