[release] v0.11.0-unstable

changelog.md

@@ -1,3 +1,12 @@
+## Version 0.11.0
+ - Disable support for X-FORWARDED-FOR incoming header (needs further testing)
+ - Docker export feature for backups on every docker event
+ - Compose Import feature now supports skipping creating existing resources
+ - Compose Import now overwrite containers if they are differents
+ - Added support for cosmos-persistent-env, to persist password when overwriting containers (useful for encrypted or password protected volumes, like databases use)
+ - Fixed bug where import compose would try to revert a previously created volume when errors occurs
+ - Terminal for import now has colours 
+
 ## Version 0.10.4
  - Encode OpenID .well-known to JSON
  - Fix incompatibility with other apps using .well-known

client/src/pages/config/users/configman.jsx

@@ -95,6 +95,7 @@ const ConfigManagement = () => {
           DNSChallengeConfig: config.HTTPConfig.DNSChallengeConfig,
           ForceHTTPSCertificateRenewal: config.HTTPConfig.ForceHTTPSCertificateRenewal,
           OverrideWildcardDomains: config.HTTPConfig.OverrideWildcardDomains,
+          UseForwardedFor: config.HTTPConfig.UseForwardedFor,
 
           Email_Enabled: config.EmailConfig.Enabled,
           Email_Host: config.EmailConfig.Host,
@@ -143,6 +144,7 @@ const ConfigManagement = () => {
               DNSChallengeConfig: values.DNSChallengeConfig,
               ForceHTTPSCertificateRenewal: values.ForceHTTPSCertificateRenewal,
               OverrideWildcardDomains: values.OverrideWildcardDomains.replace(/\s/g, ''),
+              UseForwardedFor: values.UseForwardedFor,
             },
             EmailConfig: {
               ...config.EmailConfig,
@@ -556,7 +558,13 @@ const ConfigManagement = () => {
 
               <MainCard title="Security">
                   <Grid container spacing={3}>
-                  
+
+                  {/* <CosmosCheckbox
+                    label={"Read Client IP from X-Forwarded-For header (not recommended)"}
+                    name="UseForwardedFor"
+                    formik={formik}
+                  /> */}
+
                   <CosmosFormDivider title='Geo-Blocking' />
 
                   <CosmosCheckbox

client/src/pages/constellation/vpn.jsx

@@ -73,6 +73,7 @@ export const ConstellationVPN = () => {
           connects all your devices together, and allows you to access them from anywhere.
           Please refer to the <a href="https://cosmos-cloud.io/doc/61 Constellation VPN/" target="_blank">documentation</a> for more information.
           In order to connect, please use the <a href="https://cosmos-cloud.io/clients" target="_blank">Constellation App</a>.
+          Constellation is currently free to use until the end of the beta, planned January 2024.
         </Alert>
         <MainCard title={"Constellation Setup"} content={config.constellationIP}>
           <Stack spacing={2}>

client/src/pages/servapps/containers/newService.jsx

@@ -20,6 +20,7 @@ import DockerTerminal from './terminal';
 import { Link } from 'react-router-dom';
 import { smartDockerLogConcat, tryParseProgressLog } from '../../../utils/docker';
 import { LoadingButton } from '@mui/lab';
+import LogLine from '../../../components/logLine';
 
 const preStyle = {
   backgroundColor: '#000',
@@ -59,6 +60,7 @@ const NewDockerService = ({service, refresh}) => {
   const [isDone, setIsDone] = React.useState(false);
   const [openModal, setOpenModal] = React.useState(false);
   const preRef = React.useRef(null);
+  const screenMin = useMediaQuery((theme) => theme.breakpoints.up('sm'))
 
   const installer = {...service['cosmos-installer']};
   service = {...service};
@@ -120,7 +122,7 @@ const NewDockerService = ({service, refresh}) => {
 ${JSON.stringify(service, false ,2)}`
         }
         {log.map((l) => {
-          return <div>{tryParseProgressLog(l)}</div>
+          return <LogLine message={tryParseProgressLog(l)} docker isMobile={!screenMin} />
         })}
       </pre>
     </Stack>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cosmos-server",
-  "version": "0.10.4",
+  "version": "0.11.0-unstable",
   "description": "",
   "main": "test-server.js",
   "bugs": {

src/docker/api_blueprint.go

@@ -29,6 +29,15 @@ type ContainerCreateRequestServiceNetwork struct {
 	IPV6Address string `json:"ipv6_address,omitempty"`
 }
 
+type ContainerCreateRequestContainerHealthcheck struct {
+	Test        []string `json:"test"`
+	Interval int `json:"interval"`
+	Timeout int `json:"timeout"`
+	Retries int `json:"retries"`
+	StartPeriod int `json:"start_period"`
+}
+
+
 type ContainerCreateRequestContainer struct {
 	Name 			string            `json:"container_name"`
 	Image       string            `json:"image"`
@@ -60,13 +69,7 @@ type ContainerCreateRequestContainer struct {
 	NetworkMode string `json:"network_mode,omitempty"`
 	StopSignal string `json:"stop_signal,omitempty"`
 	StopGracePeriod int `json:"stop_grace_period,omitempty"`
-	HealthCheck struct {
-		Test []string `json:"test"`
-		Interval int `json:"interval"`
-		Timeout int `json:"timeout"`
-		Retries int `json:"retries"`
-		StartPeriod int `json:"start_period"`
-	} `json:"healthcheck,omitempty"`
+	HealthCheck ContainerCreateRequestContainerHealthcheck `json:"healthcheck,omitempty"`
 	DNS []string `json:"dns,omitempty"`
 	DNSSearch []string `json:"dns_search,omitempty"`
 	ExtraHosts []string `json:"extra_hosts,omitempty"`
@@ -90,6 +93,11 @@ type ContainerCreateRequestVolume struct {
 	Target string `json:"target"`
 }
 
+type ContainerCreateRequestNetworkIPAMConfig struct {
+	Subnet string `json:"subnet"`
+	Gateway string `json:"gateway"`
+}
+
 type ContainerCreateRequestNetwork struct {
 	// name must be unique
 	Name string `json:"name"`
@@ -97,11 +105,10 @@ type ContainerCreateRequestNetwork struct {
 	Attachable bool `json:"attachable"`
 	Internal bool `json:"internal"`
 	EnableIPv6 bool `json:"enable_ipv6"`
+	Labels map[string]string `json:"labels"`
 	IPAM struct {
 		Driver string `json:"driver"`
-		Config []struct {
-			Subnet string `json:"subnet"`
-		} `json:"config"`
+		Config []ContainerCreateRequestNetworkIPAMConfig `json:"config"`
 	} `json:"ipam"`
 }
 
@@ -118,6 +125,8 @@ type DockerServiceCreateRollback struct {
 	Type string `json:"type"`
 	// name: container name, volume name, network name
 	Name string `json:"name"`
+	// was: container old settings
+	Was doctype.ContainerJSON `json:"was"`
 }
 
 func Rollback(actions []DockerServiceCreateRollback , OnLog func(string)) {
@@ -125,29 +134,54 @@ func Rollback(actions []DockerServiceCreateRollback , OnLog func(string)) {
 		action := actions[i]
 		switch action.Type {
 		case "container":
-			DockerClient.ContainerKill(DockerContext, action.Name, "SIGKILL")
-			err := DockerClient.ContainerRemove(DockerContext, action.Name, doctype.ContainerRemoveOptions{})
-			if err != nil {
-				utils.Error("Rollback: Container", err)
-			} else {
-				utils.Log(fmt.Sprintf("Rolled back container %s", action.Name))
-				OnLog(fmt.Sprintf("Rolled back container %s\n", action.Name))
+			if action.Action == "remove" {
+				utils.Log(fmt.Sprintf("Removing container %s...", action.Name))
+
+				DockerClient.ContainerKill(DockerContext, action.Name, "SIGKILL")
+				err := DockerClient.ContainerRemove(DockerContext, action.Name, doctype.ContainerRemoveOptions{})
+		
+				if err != nil {
+					utils.Error("Rollback: Container", err)
+					OnLog(utils.DoErr("Rollback: Container %s", err))
+				} else {
+					utils.Log(fmt.Sprintf("Rolled back container %s", action.Name))
+					OnLog(fmt.Sprintf("Rolled back container %s\n", action.Name))
+				}	
+			} else if action.Action == "revert" {
+				utils.Log(fmt.Sprintf("Reverting container %s...", action.Name))
+
+				// Edit Container
+				_, err := EditContainer(action.Name, action.Was, false)
+	
+				if err != nil {
+					utils.Error("Rollback: Container", err)
+					OnLog(utils.DoErr("Rollback: Container %s", err))
+				} else {
+					utils.Log(fmt.Sprintf("Rolled back container %s", action.Name))
+					OnLog(fmt.Sprintf("Rolled back container %s\n", action.Name))
+				}	
 			}
 		case "volume":
+			utils.Log(fmt.Sprintf("Removing volume %s...", action.Name))
+
 			err := DockerClient.VolumeRemove(DockerContext, action.Name, true)
 			if err != nil {
 				utils.Error("Rollback: Volume", err)
+				OnLog(utils.DoErr("Rollback: Volume %s", err))
 			} else {
 				utils.Log(fmt.Sprintf("Rolled back volume %s", action.Name))
 				OnLog(fmt.Sprintf("Rolled back volume %s\n", action.Name))
 			}
 		case "network":
+			utils.Log(fmt.Sprintf("Removing network %s...", action.Name))
+
 			if os.Getenv("HOSTNAME") != "" {
 				DockerClient.NetworkDisconnect(DockerContext, action.Name, os.Getenv("HOSTNAME"), true)
 			}
 			err := DockerClient.NetworkRemove(DockerContext, action.Name)
 			if err != nil {
 				utils.Error("Rollback: Network", err)
+				OnLog(utils.DoErr("Rollback: Network %s", err))
 			} else {
 				utils.Log(fmt.Sprintf("Rolled back network %s", action.Name))
 				OnLog(fmt.Sprintf("Rolled back network %s\n", action.Name))
@@ -245,12 +279,23 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 		OnLog(fmt.Sprintf("Creating network %s...\n", networkToCreateName))
 
 		// check if network already exists
-		_, err = DockerClient.NetworkInspect(DockerContext, networkToCreateName, doctype.NetworkInspectOptions{})
+		exNetworkDef, err := DockerClient.NetworkInspect(DockerContext, networkToCreateName, doctype.NetworkInspectOptions{})
+
 		if err == nil {
-			utils.Error("CreateService: Network", err)
-			OnLog(fmt.Sprintf("[ERROR] Network %s already exists\n", networkToCreateName))
-			Rollback(rollbackActions, OnLog)
-			return err
+			if networkToCreate.Driver == "" {
+				networkToCreate.Driver = "bridge"
+			}
+
+			if (exNetworkDef.Driver != networkToCreate.Driver) {
+				utils.Error("CreateService: Network", err)
+				OnLog(utils.DoErr("Network %s already exists with incompatible settings, cannot merge new network into it.\n", networkToCreateName))
+				Rollback(rollbackActions, OnLog)
+				return err
+			} else {
+				utils.Warn(fmt.Sprintf("Network %s already exists, skipping creation", networkToCreateName))
+				OnLog(utils.DoWarn("Network %s already exists, skipping creation\n", networkToCreateName))
+				continue
+			}
 		}
 
 		ipamConfig := make([]network.IPAMConfig, len(networkToCreate.IPAM.Config))
@@ -275,7 +320,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 
 		if err != nil {
 			utils.Error("CreateService: Rolling back changes because of -- Network", err)
-			OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Network creation error: %s\n", err.Error()))
+			OnLog(utils.DoErr("Rolling back changes because of -- Network creation error: %s\n", err.Error()))
 			Rollback(rollbackActions, OnLog)
 			return err
 		}
@@ -292,7 +337,19 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 	}
 
 	// Create volumes
-	for _, volume := range serviceRequest.Volumes {
+	for volumeName, volume := range serviceRequest.Volumes {
+		if volume.Name == "" {
+			volume.Name = volumeName
+		}
+
+		// check if volume already exists
+		_, err := DockerClient.VolumeInspect(DockerContext, volume.Name)
+		if err == nil {
+			utils.Warn(fmt.Sprintf("Volume %s already exists, skipping creation", volume.Name))
+			OnLog(utils.DoWarn("Volume %s already exists, skipping creation\n", volume.Name))
+			continue
+		}
+
 		utils.Log(fmt.Sprintf("Creating volume %s...", volume.Name))
 		OnLog(fmt.Sprintf("Creating volume %s...\n", volume.Name))
 		
@@ -303,7 +360,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 
 		if err != nil {
 			utils.Error("CreateService: Rolling back changes because of -- Volume", err)
-			OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Volume creation error: %s\n", err.Error()))
+			OnLog(utils.DoErr("Rolling back changes because of -- Volume creation error: %s\n", err.Error()))
 			Rollback(rollbackActions, OnLog)
 			return err
 		}
@@ -328,7 +385,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 		out, err := DockerPullImage(container.Image)
 		if err != nil {
 			utils.Error("CreateService: Rolling back changes because of -- Image pull", err)
-			OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Image pull error: %s\n", err.Error()))
+			OnLog(utils.DoErr("Rolling back changes because of -- Image pull error: %s\n", err.Error()))
 			Rollback(rollbackActions, OnLog)
 			return err
 		}
@@ -351,14 +408,16 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 		utils.Log(fmt.Sprintf("Checking service %s...", serviceName))
 		OnLog(fmt.Sprintf("Checking service %s...\n", serviceName))
 
-		if container.Labels["cosmos-force-network-secured"] == "true" || strings.ToLower(container.Labels["cosmos-network-name"]) == "auto" {
+		// If container request a Cosmos network, create and attach it
+		if (container.Labels["cosmos-force-network-secured"] == "true" || strings.ToLower(container.Labels["cosmos-network-name"]) == "auto") &&
+					container.Labels["cosmos-network-name"] == "" {
 			utils.Log(fmt.Sprintf("Forcing secure %s...", serviceName))
 			OnLog(fmt.Sprintf("Forcing secure %s...\n", serviceName))
 	
 			newNetwork, errNC := CreateCosmosNetwork()
 			if errNC != nil {
 				utils.Error("CreateService: Network", err)
-				OnLog(fmt.Sprintf("[ERROR] Network %s cant be created\n", newNetwork))
+				OnLog(utils.DoErr("Network %s cant be created\n", newNetwork))
 				Rollback(rollbackActions, OnLog)
 				return err
 			}
@@ -381,6 +440,18 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 			
 			utils.Log(fmt.Sprintf("Created secure network %s", newNetwork))
 			OnLog(fmt.Sprintf("Created secure network %s\n", newNetwork))
+		} else if container.Labels["cosmos-network-name"] != "" {
+			// Container has a declared a Cosmos network, check if it exists and connect to it
+			utils.Log(fmt.Sprintf("Checking declared network %s...", container.Labels["cosmos-network-name"]))
+			OnLog(fmt.Sprintf("Checking declared network %s...\n", container.Labels["cosmos-network-name"]))
+
+			_, err := DockerClient.NetworkInspect(DockerContext, container.Labels["cosmos-network-name"], doctype.NetworkInspectOptions{})
+			if err == nil {
+				utils.Log(fmt.Sprintf("Connecting to declared network %s...", container.Labels["cosmos-network-name"]))
+				OnLog(fmt.Sprintf("Connecting to declared network %s...\n", container.Labels["cosmos-network-name"]))
+	
+				AttachNetworkToCosmos(container.Labels["cosmos-network-name"])
+			}
 		}
 
 		utils.Log(fmt.Sprintf("Creating container %s...", container.Name))
@@ -499,7 +570,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 				if os.Getenv("HOSTNAME") != "" {
 					if _, err := os.Stat("/mnt/host"); os.IsNotExist(err) {
 						utils.Error("CreateService: Unable to create directory for bind mount in the host directory. Please mount the host / in Cosmos with  -v /:/mnt/host to enable folder creations, or create the bind folder yourself", err)
-						OnLog(fmt.Sprintf("[ERROR] Unable to create directory for bind mount in the host directory. Please mount the host / in Cosmos with  -v /:/mnt/host to enable folder creations, or create the bind folder yourself: %s\n", err.Error()))
+						OnLog(utils.DoErr("Unable to create directory for bind mount in the host directory. Please mount the host / in Cosmos with  -v /:/mnt/host to enable folder creations, or create the bind folder yourself: %s\n", err.Error()))
 					} else {
 						newSource = "/mnt/host" + newSource
 					}
@@ -516,7 +587,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 
 					if err != nil {
 						utils.Error("CreateService: Unable to create directory for bind mount. Make sure parent directories exist, and that Cosmos has permissions to create directories in the host directory", err)
-						OnLog(fmt.Sprintf("[ERROR] Unable to create directory for bind mount. Make sure parent directories exist, and that Cosmos has permissions to create directories in the host directory: %s\n", err.Error()))
+						OnLog(utils.DoErr("Unable to create directory for bind mount. Make sure parent directories exist, and that Cosmos has permissions to create directories in the host directory: %s\n", err.Error()))
 						Rollback(rollbackActions, OnLog)
 						return err
 					}
@@ -526,7 +597,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 						err = os.Chown(newSource, container.UID, container.GID)
 						if err != nil {
 							utils.Error("CreateService: Unable to change ownership of directory", err)
-							OnLog(fmt.Sprintf("[ERROR] Unable to change ownership of directory: " + err.Error()))
+							OnLog(utils.DoErr("Unable to change ownership of directory: " + err.Error()))
 						}
 					} else if container.User != "" && strings.Contains(container.User, ":") { 
 						uidgid := strings.Split(container.User, ":")
@@ -535,21 +606,21 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 						err = os.Chown(newSource, uid, gid)
 						if err != nil {
 							utils.Error("CreateService: Unable to change ownership of directory", err)
-							OnLog(fmt.Sprintf("[ERROR] Unable to change ownership of directory: " + err.Error()))
+							OnLog(utils.DoErr("Unable to change ownership of directory: " + err.Error()))
 						}
 					} else if container.User != "" {
 						// Change the ownership of the directory to the container.User
 						userInfo, err := user.Lookup(container.User)
 						if err != nil {
 							utils.Error("CreateService: Unable to lookup user", err)
-							OnLog(fmt.Sprintf("[ERROR] Unable to lookup user " + container.User + ". " +err.Error()))
+							OnLog(utils.DoErr("Unable to lookup user " + container.User + ". " +err.Error()))
 						} else {
 							uid, _ := strconv.Atoi(userInfo.Uid)
 							gid, _ := strconv.Atoi(userInfo.Gid)
 							err = os.Chown(newSource, uid, gid)
 							if err != nil {
 								utils.Error("CreateService: Unable to change ownership of directory", err)
-								OnLog(fmt.Sprintf("[ERROR] Unable to change ownership of directory: " + err.Error()))
+								OnLog(utils.DoErr("Unable to change ownership of directory: " + err.Error()))
 							}
 						}	
 					}
@@ -604,21 +675,89 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 		networkingConfig := &network.NetworkingConfig{
 			EndpointsConfig: make(map[string]*network.EndpointSettings),
 		}
+
+		// check if container exist
+		existingContainer, err := DockerClient.ContainerInspect(DockerContext, container.Name)
+		if err == nil {
+			utils.Warn("CreateService: Container " + container.Name + " already exist, overwriting.")
+			OnLog(utils.DoWarn("Container " + container.Name + " already exist, overwriting.\n"))
+	
+			// Edit Container
+			newConfig := doctype.ContainerJSON{}
+			newConfig.ContainerJSONBase = new(doctype.ContainerJSONBase)
+			newConfig.Config = containerConfig
+			newConfig.HostConfig = hostConfig
+
+			// check if there are persistent env var
+			if containerConfig.Labels["cosmos-persistent-env"] != "" {
+				// split env vars
+				envVars := strings.Split(containerConfig.Labels["cosmos-persistent-env"], ",")
+				// get existing env vars
+				existingEnvVars := existingContainer.Config.Env
+				// loop through env vars
+				for _, envVar := range envVars {
+					envVar = strings.TrimSpace(envVar)
+					
+					// check if env var already exist
+					exists := false
+					existingEnvVarValue := ""
+					for _, existingEnvVar := range existingEnvVars {
+						if strings.HasPrefix(existingEnvVar, envVar + "=") {
+							exists = true
+							existingEnvVarValue = existingEnvVar
+							break
+						}
+					}
+					// if it exist, copy value to new container
+					if exists {
+						wasReplace := false
+						for i, newEnvVar := range newConfig.Config.Env {
+							if strings.HasPrefix(newEnvVar, envVar + "=") {
+								newConfig.Config.Env[i] = envVar + "=" + strings.TrimPrefix(existingEnvVarValue, envVar + "=")
+								wasReplace = true
+								break
+							}
+						}
+						if !wasReplace {
+							newConfig.Config.Env = append(newConfig.Config.Env, envVar + "=" + strings.TrimPrefix(existingEnvVarValue, envVar + "="))
+						}
+					}
+				}
+			}
+			
+			_, errEdit := EditContainer(container.Name, newConfig, false)
+
+			if errEdit != nil {
+				utils.Error("CreateService: Rolling back changes because of -- Container", err)
+				OnLog(utils.DoErr("Rolling back changes because of -- Container creation error: "+err.Error()))
+				Rollback(rollbackActions, OnLog)
+				return err
+			}
+
+			// rollback action
 		
-		_, err = DockerClient.ContainerCreate(DockerContext, containerConfig, hostConfig, networkingConfig, nil, container.Name)
-		
-		if err != nil {
-			utils.Error("CreateService: Rolling back changes because of -- Container", err)
-			OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Container creation error: "+err.Error()))
-			Rollback(rollbackActions, OnLog)
-			return err
-		}
+			rollbackActions = append(rollbackActions, DockerServiceCreateRollback{
+				Action: "revert",
+				Type:   "container",
+				Name:   container.Name,
+				Was: existingContainer,
+			})
+		} else {
+			_, err = DockerClient.ContainerCreate(DockerContext, containerConfig, hostConfig, networkingConfig, nil, container.Name)
+
+			if err != nil {
+				utils.Error("CreateService: Rolling back changes because of -- Container", err)
+				OnLog(utils.DoErr("Rolling back changes because of -- Container creation error: "+err.Error()))
+				Rollback(rollbackActions, OnLog)
+				return err
+			}
 		
-		rollbackActions = append(rollbackActions, DockerServiceCreateRollback{
-			Action: "remove",
-			Type:   "container",
-			Name:   container.Name,
-		})
+			rollbackActions = append(rollbackActions, DockerServiceCreateRollback{
+				Action: "remove",
+				Type:   "container",
+				Name:   container.Name,
+			})
+		}	
 
 		// connect to networks
 		for netName, netConfig := range container.Networks {
@@ -628,17 +767,19 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 				IPAddress:   netConfig.IPV4Address,
 				GlobalIPv6Address: netConfig.IPV6Address,
 			})
-			if err != nil {
+			if err != nil && !strings.Contains(err.Error(), "already exists in network") {
 				utils.Error("CreateService: Rolling back changes because of -- Network Connection -- ", err)
-				OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Network connection error: "+err.Error()))
+				OnLog(utils.DoErr("Rolling back changes because of -- Network connection error: "+err.Error()))
 				Rollback(rollbackActions, OnLog)
 				return err
+			} else if strings.Contains(err.Error(), "already exists in network") {
+				utils.Warn("CreateService: Container " + container.Name + " already connected to network " + netName + ", skipping.")
+				OnLog(utils.DoWarn("Container %s already connected to network %s, skipping.", container.Name, netName))			
 			}
 		}
 
-
 		// add routes 
-		for _, route := range container.Routes {
+		for routeIndex, route := range container.Routes {
 			// check if route already exists
 			exists := false
 			for _, configRoute := range configRoutes {
@@ -651,10 +792,15 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 				needsHTTPRestart = true
 				configRoutes = append([]utils.ProxyRouteConfig{(utils.ProxyRouteConfig)(route)}, configRoutes...)
 			} else {
-				utils.Error("CreateService: Rolling back changes because of -- Route already exist", nil)
-				OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Route already exist"))
-				Rollback(rollbackActions, OnLog)
-				return errors.New("Route already exist")
+				// utils.Error("CreateService: Rolling back changes because of -- Route already exist", nil)
+				// OnLog(utils.DoErr("Rolling back changes because of -- Route already exist"))
+				// Rollback(rollbackActions, OnLog)
+				// return errors.New("Route already exist")
+
+				//overwrite route
+				configRoutes[routeIndex] = (utils.ProxyRouteConfig)(route)
+				utils.Warn("CreateService: Route " + route.Name + " already exist, overwriting.")
+				OnLog(utils.DoWarn("Route " + route.Name + " already exist, overwriting.\n"))
 			}
 		}
 		
@@ -668,7 +814,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 			if strings.Contains(targetContainer, ":") {
 				err = errors.New("Link network cannot contain ':' please use container name only")
 				utils.Error("CreateService: Rolling back changes because of -- Link network", err)
-				OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Link network creation error: "+err.Error()))
+				OnLog(utils.DoErr("Rolling back changes because of -- Link network creation error: "+err.Error()))
 				Rollback(rollbackActions, OnLog)
 				return err
 			}
@@ -676,7 +822,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 			err = CreateLinkNetwork(container.Name, targetContainer)
 			if err != nil {
 				utils.Error("CreateService: Rolling back changes because of -- Link network", err)
-				OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Link network creation error: "+err.Error()))
+				OnLog(utils.DoErr("Rolling back changes because of -- Link network creation error: "+err.Error()))
 				Rollback(rollbackActions, OnLog)
 				return err
 			}
@@ -691,7 +837,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 	startOrder, err := ReOrderServices(serviceRequest.Services)
 	if err != nil {
 		utils.Error("CreateService: Rolling back changes because of -- Container", err)
-		OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Container creation error: "+err.Error()))
+		OnLog(utils.DoErr("Rolling back changes because of -- Container creation error: "+err.Error()))
 		Rollback(rollbackActions, OnLog)
 		return err
 	}
@@ -701,7 +847,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 		err = DockerClient.ContainerStart(DockerContext, container.Name, doctype.ContainerStartOptions{})
 		if err != nil {
 			utils.Error("CreateService: Start Container", err)
-			OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Container start error: "+err.Error()))
+			OnLog(utils.DoErr("Rolling back changes because of -- Container start error: "+err.Error()))
 			Rollback(rollbackActions, OnLog)
 			return err
 		}
@@ -739,7 +885,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 			
 				if err != nil {
 					utils.Error("CreateService: Post Install", err)
-					OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Post install error: "+err.Error()))
+					OnLog(utils.DoErr("Rolling back changes because of -- Post install error: "+err.Error()))
 					Rollback(rollbackActions, OnLog)
 					return err
 				}
@@ -748,7 +894,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 				response, err := DockerClient.ContainerExecAttach(DockerContext, execResponse.ID, doctype.ExecStartCheck{})
 				if err != nil {
 					utils.Error("CreateService: Post Install", err)
-					OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Post install error: "+err.Error()))
+					OnLog(utils.DoErr("Rolling back changes because of -- Post install error: "+err.Error()))
 					Rollback(rollbackActions, OnLog)
 					return err
 				}
@@ -758,7 +904,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 				err = DockerClient.ContainerExecStart(DockerContext, execResponse.ID, doctype.ExecStartCheck{})
 				if err != nil {
 					utils.Error("CreateService: Post Install", err)
-					OnLog(fmt.Sprintf("[ERROR] Rolling back changes because of -- Post install error: "+err.Error()))
+					OnLog(utils.DoErr("Rolling back changes because of -- Post install error: "+err.Error()))
 					Rollback(rollbackActions, OnLog)
 					return err
 				}
@@ -784,7 +930,8 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string)
 
 	// After all operations
 	utils.Log("CreateService: Operation succeeded. SERVICE STARTED")
-	OnLog("[OPERATION SUCCEEDED]. SERVICE STARTED\n")
+	OnLog("\n")
+	OnLog(utils.DoSuccess("[OPERATION SUCCEEDED]. SERVICE STARTED\n"))
 
 	return nil
 }

src/docker/export.go

@@ -0,0 +1,224 @@
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+
+	"github.com/azukaar/cosmos-server/src/utils"
+	"github.com/docker/docker/api/types"
+
+	"github.com/docker/docker/api/types/mount"
+
+)
+
+func ExportDocker() {
+	errD := Connect()
+	if errD != nil {
+		utils.Error("ExportDocker - connect - ", errD)
+		return
+	}
+
+	finalBackup := DockerServiceCreateRequest{}
+	
+	// List containers
+	containers, err := DockerClient.ContainerList(DockerContext, types.ContainerListOptions{})
+	if err != nil {
+		utils.Error("ExportDocker - Cannot list containers", err)
+		return
+	}
+
+	
+	// Convert the containers into your custom format
+	var services = make(map[string]ContainerCreateRequestContainer)
+	for _, container := range containers {
+		// Fetch detailed info of each container
+		detailedInfo, err := DockerClient.ContainerInspect(DockerContext, container.ID)
+		if err != nil {
+			utils.Error("Cannot inspect container", err)
+			return
+		}
+
+		// Map the detailedInfo to your ContainerCreateRequestContainer struct
+		// Here's a simplified example, you'd need to handle all the fields
+		service := ContainerCreateRequestContainer{
+			Name:         strings.TrimPrefix(detailedInfo.Name, "/"),
+			Image:        detailedInfo.Config.Image,
+			Environment:  detailedInfo.Config.Env,
+			Labels:       detailedInfo.Config.Labels,
+			Command:      strings.Join(detailedInfo.Config.Cmd, " "),
+			Entrypoint:   strings.Join(detailedInfo.Config.Entrypoint, " "),
+			WorkingDir:   detailedInfo.Config.WorkingDir,
+			User:         detailedInfo.Config.User,
+			Tty:          detailedInfo.Config.Tty,
+			StdinOpen:    detailedInfo.Config.OpenStdin,
+			Hostname:     detailedInfo.Config.Hostname,
+			Domainname:   detailedInfo.Config.Domainname,
+			MacAddress:   detailedInfo.NetworkSettings.MacAddress,
+			NetworkMode:  string(detailedInfo.HostConfig.NetworkMode),
+			StopSignal:   detailedInfo.Config.StopSignal,
+			HealthCheck:  ContainerCreateRequestContainerHealthcheck {
+			},
+			DNS:              detailedInfo.HostConfig.DNS,
+			DNSSearch:        detailedInfo.HostConfig.DNSSearch,
+			ExtraHosts:       detailedInfo.HostConfig.ExtraHosts,
+			SecurityOpt:      detailedInfo.HostConfig.SecurityOpt,
+			StorageOpt:       detailedInfo.HostConfig.StorageOpt,
+			Sysctls:          detailedInfo.HostConfig.Sysctls,
+			Isolation:        string(detailedInfo.HostConfig.Isolation),
+			CapAdd:           detailedInfo.HostConfig.CapAdd,
+			CapDrop:          detailedInfo.HostConfig.CapDrop,
+			SysctlsMap:       detailedInfo.HostConfig.Sysctls,
+			Privileged:       detailedInfo.HostConfig.Privileged,
+			// StopGracePeriod:  int(detailedInfo.HostConfig.StopGracePeriod.Seconds()),
+			// Ports
+			Ports: func() []string {
+					ports := []string{}
+					for port, binding := range detailedInfo.NetworkSettings.Ports {
+							for _, b := range binding {
+									ports = append(ports, fmt.Sprintf("%s:%s->%s/%s", b.HostIP, b.HostPort, port.Port(), port.Proto()))
+							}
+					}
+					return ports
+			}(),
+			// Volumes
+			Volumes: func() []mount.Mount {
+					mounts := []mount.Mount{}
+					for _, m := range detailedInfo.Mounts {
+						  mount := mount.Mount{
+								Type:        m.Type,
+								Source:      m.Source,
+								Target:      m.Destination,
+								ReadOnly:    !m.RW,
+								// Consistency: mount.Consistency(m.Consistency),
+						}
+
+						if m.Type == "volume" {
+							nodata := strings.Split(strings.TrimSuffix(m.Source, "/_data"), "/")
+							mount.Source = nodata[len(nodata)-1]
+						}
+
+						mounts = append(mounts, mount)
+					}
+					return mounts
+			}(),
+			// Networks
+			Networks: func() map[string]ContainerCreateRequestServiceNetwork {
+					networks := make(map[string]ContainerCreateRequestServiceNetwork)
+					for netName, netConfig := range detailedInfo.NetworkSettings.Networks {
+							networks[netName] = ContainerCreateRequestServiceNetwork{
+									Aliases:     netConfig.Aliases,
+									IPV4Address: netConfig.IPAddress,
+									IPV6Address: netConfig.GlobalIPv6Address,
+							}
+					}
+					return networks
+			}(),
+
+			DependsOn:      []string{},  // This is not directly available from inspect. It's part of docker-compose.
+			RestartPolicy:  string(detailedInfo.HostConfig.RestartPolicy.Name),
+			Devices:        func() []string {
+					var devices []string
+					for _, device := range detailedInfo.HostConfig.Devices {
+							devices = append(devices, fmt.Sprintf("%s:%s", device.PathOnHost, device.PathInContainer))
+					}
+					return devices
+			}(),
+			Expose:         []string{},  // This information might need to be derived from other properties
+		}
+
+		// healthcheck
+		if detailedInfo.Config.Healthcheck != nil {
+			service.HealthCheck.Test = detailedInfo.Config.Healthcheck.Test
+			service.HealthCheck.Interval = int(detailedInfo.Config.Healthcheck.Interval.Seconds())
+			service.HealthCheck.Timeout = int(detailedInfo.Config.Healthcheck.Timeout.Seconds())
+			service.HealthCheck.Retries = detailedInfo.Config.Healthcheck.Retries
+			service.HealthCheck.StartPeriod = int(detailedInfo.Config.Healthcheck.StartPeriod.Seconds())
+		}
+
+		// user UID/GID
+		if detailedInfo.Config.User != "" {
+			parts := strings.Split(detailedInfo.Config.User, ":")
+			if len(parts) == 2 {
+				uid, err := strconv.Atoi(parts[0])
+				if err != nil {
+					panic(err)
+				}
+				gid, err := strconv.Atoi(parts[1])
+				if err != nil {
+					panic(err)
+				}
+				service.UID = uid
+				service.GID = gid
+			}
+		}
+
+		//expose 
+		// for _, port := range detailedInfo.Config.ExposedPorts {
+			
+		// }
+		
+		services[strings.TrimPrefix(detailedInfo.Name, "/")] = service
+	}
+
+	// List networks
+	networks, err := DockerClient.NetworkList(DockerContext, types.NetworkListOptions{})
+	if err != nil {
+		utils.Error("Cannot list networks", err)
+		return
+	}
+
+	finalBackup.Networks = make(map[string]ContainerCreateRequestNetwork)
+
+	// Convert the networks into custom format
+	for _, network := range networks {
+		if network.Name == "bridge" || network.Name == "host" || network.Name == "none" {
+			continue
+		}
+
+		// Fetch detailed info of each network
+		detailedInfo, err := DockerClient.NetworkInspect(DockerContext, network.ID, types.NetworkInspectOptions{})
+		if err != nil {
+			utils.Error("Cannot inspect network", err)
+			return
+		}
+
+		// Map the detailedInfo to ContainerCreateRequestContainer struct
+		network := ContainerCreateRequestNetwork{
+			Name:         detailedInfo.Name,
+			Driver:       detailedInfo.Driver,
+			Internal:     detailedInfo.Internal,
+			Attachable:   detailedInfo.Attachable,
+			EnableIPv6:   detailedInfo.EnableIPv6,
+			Labels:       detailedInfo.Labels,
+		}
+
+		network.IPAM.Driver = detailedInfo.IPAM.Driver
+		for _, config := range detailedInfo.IPAM.Config {
+			network.IPAM.Config = append(network.IPAM.Config, ContainerCreateRequestNetworkIPAMConfig{
+				Subnet:  config.Subnet,
+				Gateway: config.Gateway,
+			})
+		}
+
+		finalBackup.Networks[detailedInfo.Name] = network
+	}
+
+	// Convert the services map to your finalBackup struct
+	finalBackup.Services = services
+
+
+	// Convert the finalBackup struct to JSON
+	jsonData, err := json.MarshalIndent(finalBackup, "", "  ")
+	if err != nil {
+		utils.Error("Cannot marshal docker backup", err)
+	}
+
+	// Write the JSON data to a file
+	err = ioutil.WriteFile(utils.CONFIGFOLDER + "backup.cosmos-compose.json", jsonData, 0644)
+	if err != nil {
+		utils.Error("Cannot save docker backup", err)
+	}
+}

src/docker/network.go

@@ -71,7 +71,6 @@ func findAvailableSubnet() string {
 }
 
 func CreateCosmosNetwork() (string, error) {
-	// check if network exists already
 	networks, err := DockerClient.NetworkList(DockerContext, types.NetworkListOptions{})
 	if err != nil {
 		utils.Error("Docker Network List", err)

src/index.go

@@ -23,7 +23,7 @@ func main() {
 
 	go CRON()
 
-	docker.Test()
+	docker.ExportDocker()
 
 	docker.DockerListenEvents()
 

src/proxy/shield.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"fmt"
 	"net"
-	"os"
 	"math"
 	"strconv"
 )
@@ -250,8 +249,9 @@ func calculateLowestExhaustedPercentage(policy utils.SmartShieldPolicy, userCons
 
 func GetClientID(r *http.Request) string {
 	// when using Docker we need to get the real IP
+	UseForwardedFor := utils.GetMainConfig().HTTPConfig.UseForwardedFor
 
-	if os.Getenv("HOSTNAME") != "" && r.Header.Get("x-forwarded-for") != "" {
+	if UseForwardedFor && r.Header.Get("x-forwarded-for") != "" {
 		ip, _, _ := net.SplitHostPort(r.Header.Get("x-forwarded-for"))
 		utils.Debug("SmartShield: Getting client ID " + ip)
 		return ip

src/utils/log.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"log"
+	"fmt"
 )
 
 var Reset  = "\033[0m"
@@ -56,3 +57,24 @@ func Fatal(message string, err error) {
 		log.Fatal(Red + "[FATAL] " + message + " : " + errStr + Reset)
 	}
 }
+
+func DoWarn(format string, a ...interface{}) string {
+	message := fmt.Sprintf(format, a...)
+	// \033[1;33m is the ANSI code for bold yellow
+	// \033[0m resets the color
+	return fmt.Sprintf("\033[1;33m[WARN] %s\033[0m", message)
+}
+
+func DoErr(format string, a ...interface{}) string {
+	message := fmt.Sprintf(format, a...)
+	// \033[1;31m is the ANSI code for bold red
+	// \033[0m resets the color
+	return fmt.Sprintf("\033[1;31m[ERROR] %s\033[0m", message)
+}
+
+func DoSuccess(format string, a ...interface{}) string {
+	message := fmt.Sprintf(format, a...)
+	// \033[1;32m is the ANSI code for bold green
+	// \033[0m resets the color
+	return fmt.Sprintf("\033[1;32m[SUCCESS] %s\033[0m", message)
+}

src/utils/types.go

@@ -124,6 +124,7 @@ type HTTPConfig struct {
 	OverrideWildcardDomains string `validate:"omitempty,excludesall=/ "`
 	AcceptAllInsecureHostname bool
 	DNSChallengeConfig map[string]string `json:"DNSChallengeConfig,omitempty"`
+	UseForwardedFor bool
 } 
 
 const (