reverse proxy

Reserve proxy support for upstreams and a lot more improvements...

lib/general

@@ -681,7 +681,12 @@ is_url() {
 	
 	if [[ $1 =~ ^((http|https)+\:\/\/)?unix:(.+):(\/.*)?$ ]]; then
 		out="unix"
-		[[ $scheme == "unix" ]] && local host=$(echo "${1,,}" | cut -d':' -f 2 -s) || local host=$(echo "${1,,}" | cut -d':' -f 3 -s)
+		if [[ $scheme == "unix" ]]; then
+			local host="$(echo "${1,,}" | cut -d':' -f 1-2 -s):"
+		else
+			local host="$(echo "${1,,}" | cut -d':' -f 2-3 -s):"
+			local host=${host:2}
+		fi
 		local path=$(echo "${1,,}" | rev | cut -d':' -f 1 -s | rev | cut -d '/' -f 2- -s)
 		local port=""
 	elif [[ $scheme =~ ^(http|https)$ ]]; then	

lib/install

@@ -277,9 +277,10 @@ nginx_optim() {
 	sudo cp -R /opt/webinoly/templates/nginx/conf.d/* /etc/nginx/conf.d/
 	sudo cat /opt/webinoly/templates/nginx/nginx.conf > /etc/nginx/nginx.conf
 	
-	# This files are backedup by server-reset (needs to be here to avoid nginx errors)
+	# These  files are backedup by server-reset (needs to be here to avoid nginx errors)
 	[[ -f /tmp/webinoly_blockips_backup ]] && sudo mv /tmp/webinoly_blockips_backup /etc/nginx/conf.d/blockips.conf
 	[[ -f /tmp/webinoly_backup ]] && sudo mv /tmp/webinoly_backup /etc/nginx/conf.d/webinoly.conf
+	[[ -f /tmp/webinoly_upstreamproxy_backup ]] && sudo mv /tmp/webinoly_upstreamproxy_backup /etc/nginx/conf.d/upstream_proxy.conf
 
 	sudo sed -i "/client_max_body_size /c \	client_max_body_size $(check_var max-mb-uploads)m;" /etc/nginx/nginx.conf
 	sudo sed -i "/worker_rlimit_core /c \worker_rlimit_core $fd_per_process;" /etc/nginx/nginx.conf

lib/sites

@@ -592,6 +592,13 @@ deletesite() {
 		[[ ! -s /etc/nginx/conf.d/webinoly.conf ]] && sudo rm /etc/nginx/conf.d/webinoly.conf
 	fi
 	
+	# Check for Reverse Proxy upstreams
+	if [[ -f /etc/nginx/conf.d/upstream_proxy.conf ]]; then
+		local domvar=$(echo "$domain" | sed "s/[^0-9A-Za-z]/_/g")
+		sudo sed -i "/upstream ${domvar}/,/} #End/{/.*/d}" /etc/nginx/conf.d/upstream_proxy.conf
+		[[ ! -s /etc/nginx/conf.d/upstream_proxy.conf ]] && sudo rm /etc/nginx/conf.d/upstream_proxy.conf
+	fi
+	
 	echo "${gre}Site${blu} $domain ${gre}has been successfully deleted!${end}"
 }
 
@@ -632,7 +639,7 @@ deletesite_subfolder() {
 		sudo sed -i "/# WebinolyCustom/,/# WebinolyCustomEnd/{/$domain$subname-wpfc.conf/d}" /etc/nginx/sites-available/$domain
 		sudo sed -i "/# WebinolyCustom/,/# WebinolyCustomEnd/{/$domain$subname-proxy.conf/d}" /etc/nginx/sites-available/$domain
 		sudo sed -i "/# WebinolyNginxServerStart/,/# WebinolyNginxServerEnd/{\|location $subfolder/ |d}" /etc/nginx/sites-available/$domain
-			
+
 		# In case of custom cache
 		if [[ -f /etc/nginx/conf.d/webinoly.conf ]]; then
 			sudo sed -i "/^fastcgi_cache_path \/run\/nginx-cache\/$(echo $domain | sed 's/[^0-9A-Za-z]/_/g')${subname} .*/d" /etc/nginx/conf.d/webinoly.conf
@@ -646,6 +653,13 @@ deletesite_subfolder() {
 	subfolder=$subfolder_original
 	subname=$(echo $subfolder | sed "s/\//_/g")
 	
+	# Check for Reverse Proxy upstreams
+	if [[ -f /etc/nginx/conf.d/upstream_proxy.conf ]]; then
+		local domvar=$(echo "$domain$subfolder" | sed "s/[^0-9A-Za-z]/_/g")
+		sudo sed -i "/upstream ${domvar}/,/} #End/{/.*/d}" /etc/nginx/conf.d/upstream_proxy.conf
+		[[ ! -s /etc/nginx/conf.d/upstream_proxy.conf ]] && sudo rm /etc/nginx/conf.d/upstream_proxy.conf
+	fi
+	
 	sudo rm -rf /var/www/$domain/htdocs$subfolder
 	echo "${gre}Subfolder successfully removed!${end}"
 }
@@ -683,6 +697,7 @@ delete_all_sites() {
 	sudo rm -rf /etc/nginx/sites-available/!(default|$(conf_read tools-port)|*".dpkg-"*)
 	sudo rm -rf /etc/nginx/sites-enabled/!(default|$(conf_read tools-port))
 	sudo rm -rf /var/www/!(html|$(conf_read tools-port))
+	sudo rm -rf /etc/nginx/conf.d/upstream_proxy.conf
 	echo ""
 	echo "${gre}All sites and data has been deleted successfully!${end}"
 }
@@ -1258,13 +1273,22 @@ reverse_proxy() {
 	fi
 	
 	is_url $proxydata -split
-	[[ -z $url_scheme || ${url_scheme,,} == "unix" ]] && proxydata="http://${proxydata}"
-	
-	# URI part (subfolders) are not allowed because the proxy configuration we are using includes the static files location
-	# This location block is using regex which is not allowed, dedicated reverse proxy are not affected because they don't have static file location block.
-	# nginx: [emerg] "proxy_pass" cannot have URI part in location given by regular expression, or inside named location, or inside "if" statement, or inside "limit_except" block in /etc/nginx/apps.d/testa.qrokes.com-proxy.conf:31
-	if [[ -z $dedicated_reverse_proxy && -n $url_path ]]; then
-		echo "${red}[ERROR] Subfolders in your endpoint are not allowed in Reverse Proxy sites configured as websites!${end}"
+	# URL Scheme correction... Just in case of empty scheme, http is default!
+	if [[ -z $url_scheme || ${url_scheme,,} == "unix" ]]; then
+		local proxydata="http://${proxydata}"
+		url_scheme="http"
+	fi
+
+	# Fix slash at the end when is alone, no real URI path!
+	if [[ $dedicated_reverse_proxy != "simple" && $url_path == "/" ]]; then
+		local proxydata=${proxydata:0:-1}
+		unset url_path
+	fi
+
+	# URI part (subfolders) are not allowed because the proxy configuration we are using includes the static files location, this location block is using regex which is not allowed!
+	# Dedicated reverse proxy are also affected (don't have static file location block), but subfolders are not allowed in upstream servers.
+	if [[ $dedicated_reverse_proxy != "simple" && -n $url_path ]]; then
+		echo "${red}[ERROR] Subfolder in your endpoint or upstream server is not supported!${dim} (URI: $url_path)${end}"
 		exit 1
 	fi
 	
@@ -1288,21 +1312,55 @@ reverse_proxy() {
 		sudo sed -i '/BasicLocationRootFiles/,/BasicLocEnd/{/.*/d}' /etc/nginx/apps.d/$domain$subname-proxy.conf
 		sudo sed -i "/^location \/ {/c location $subfolder/ {" /etc/nginx/apps.d/$domain$subname-proxy.conf
 	fi
-	sudo sed -i "s#http://localhost:8080#${proxydata}#g" /etc/nginx/apps.d/$domain$subname-proxy.conf
 	
+	if [[ $dedicated_reverse_proxy == "simple" ]]; then
+		# Force / at the end of the url for subfolders
+		[[ -n $url_path && $url_path != "/" && $(echo "${proxydata}" | rev | cut -c-1) != "/" ]] && proxydata="${proxydata}/"
+		sudo sed -i "s#<upstream_name>#${proxydata}#g" /etc/nginx/apps.d/$domain$subname-proxy.conf
+	else
+		[[ -n $url_scheme ]] && sudo sed -i "s#proxy_pass <upstream_name>#proxy_pass ${url_scheme,,}://<upstream_name>#g" /etc/nginx/apps.d/$domain$subname-proxy.conf
+		sudo sed -i "s#<upstream_name>#$(echo "$domain$subfolder" | sed "s/[^0-9A-Za-z]/_/g")#g" /etc/nginx/apps.d/$domain$subname-proxy.conf
+		
+		# Create the upstream
+		if [[ ! -f /etc/nginx/conf.d/upstream_proxy.conf ]]; then
+			sudo touch /etc/nginx/conf.d/upstream_proxy.conf
+			sudo chmod 644 /etc/nginx/conf.d/upstream_proxy.conf
+			sudo chown -R root:root /etc/nginx/conf.d/upstream_proxy.conf
+		fi
+
+		# Upstream server needs port, if empty, default is 80, so we need to fix it in case of https.
+		[[ -z $url_port && ${url_scheme,,} == "https" ]] && local fixport=":443"
+
+		local upsvr="$(echo "${proxydata}${fixport}" | sed "s#${url_scheme}://##g")"
+		
+		echo "upstream $(echo "$domain$subfolder" | sed "s/[^0-9A-Za-z]/_/g") {
+	zone upstreams;
+	server ${upsvr};
+	#server ${upsvr} backup;
+	# When multiple servers are defined be sure the Host header is not set to one specific destination server.
+	
+	# We recommend setting the 'keepalive' parameter to twice the number of servers listed in the upstream block.
+	# The proxy_http_version directive should be set to “1.1” and the “Connection” header field should be cleared.
+	# Note also that when you specify a load-balancing algorithm in the upstream block – with the hash, ip_hash, least_conn, least_time, or random directive – the directive must appear above the keepalive directive.
+	keepalive 2;
+} #End" >> /etc/nginx/conf.d/upstream_proxy.conf
+	fi
+	
+	echo "${blu}${dim}Configuration: /etc/nginx/apps.d/$domain$subname-proxy.conf ${end}"
 	[[ $(is_ssl $domain) != "true" ]] && sudo sed -i '/CacheStaticFiles/,/expires max;/{/headers-https.conf;/d}' /etc/nginx/apps.d/$domain$subname-proxy.conf
 	
-	# Suggest / at the end of the url to pass
-	if [[ -n $dedicated_reverse_proxy && $(echo "${proxydata}" | rev | cut -c-1) != "/" ]]; then
-		echo "${dim}[INFO] Your endpoint URL doesn't have a / at the end! You should note that it can have a different behavior."
-		echo "Read: http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass ${end}"
+	# Set Host header for external sources, except when IP or Unix socket is used!
+	# Simple dedicated is also excluded because proxy_pass seems to not need it.
+	if [[ -n $upsvr && -n $proxydata && ${url_host} != "localhost" && $(is_url $url_host) != "ip" && $(is_url $url_host) != "unix" ]]; then
+		sudo sed -i "/#proxy_set_header Host /c\	proxy_set_header Host '${upsvr}';" /etc/nginx/apps.d/$domain$subname-proxy.conf
 	fi
 	
 	# Check host and port
+	echo "${blu}${dim}Checking connection with your endpoint or upstream server...${end}"
 	local code=$(wget -t 1 --timeout=5 --server-response --spider $proxydata 2>&1 | awk '/^  HTTP/{print $2}')
 	code="${code##*$'\n'}"
 	[[ -n $code ]] && local mes="(Code: ${code})"
-	[[ $code =~ ^(2|3|4)[0-9][0-9]$ ]] || echo "${red}[WARNING] Seems like your Reverse Proxy host is not responding! ${mes}${end}"
+	[[ $code =~ ^(2|3|4)[0-9][0-9]$ ]] && echo "${gre}Successful!${end}" || echo "${red}[WARNING] Seems like your Reverse Proxy host is not responding! ${mes}${end}"
 	
 	[[ -n $subfolder ]] && echo "${gre}Reverse proxy${blu} $subfolder ${gre}subfolder successfully created!${end}"
 }

lib/webin

@@ -412,6 +412,7 @@ server_reset() {
 		# Backup files will be restored in nginx_optim function
 		[[ -f /etc/nginx/conf.d/blockips.conf ]] && sudo cp -p /etc/nginx/conf.d/blockips.conf /tmp/webinoly_blockips_backup
 		[[ -f /etc/nginx/conf.d/webinoly.conf ]] && sudo cp -p /etc/nginx/conf.d/webinoly.conf /tmp/webinoly_backup
+		[[ -f /etc/nginx/conf.d/upstream_proxy.conf ]] && sudo cp -p /etc/nginx/conf.d/upstream_proxy.conf /tmp/webinoly_upstreamproxy_backup
 		
 		linux_purge
 		sudo sed -i '/WebinolyCustom/,/WebinolyCustomEnd/{/.*/d}' /etc/nginx/fastcgi_params

templates/nginx/conf.d/fastcgi.conf

@@ -19,3 +19,4 @@ fastcgi_read_timeout 30s;
 fastcgi_buffer_size 16k;
 fastcgi_buffers 16 16k;
 fastcgi_ignore_headers X-Accel-Redirect X-Accel-Expires X-Accel-Limit-Rate X-Accel-Buffering X-Accel-Charset Expires Cache-Control Set-Cookie Vary;
+#fastcgi_next_upstream error timeout;

templates/nginx/conf.d/upstream.conf

@@ -7,6 +7,11 @@
 #####################################################################################################
 
 upstream php {
+	zone upstreams 64K;
 	server 127.0.0.1:9000;
-	#keepalive 100;
+	
+	# We recommend setting the 'keepalive' parameter to twice the number of servers listed in the upstream block.
+	# For FastCGI servers, it is required to set 'fastcgi_keep_conn' for keepalive connections to work (conf.d/fastcgi.conf)
+	# Note also that when you specify a load-balancing algorithm in the upstream block – with the hash, ip_hash, least_conn, least_time, or random directive – the directive must appear above the keepalive directive.
+	keepalive 2;
 }

templates/template-site-proxy

@@ -1,6 +1,10 @@
 # REVERSE PROXY NGINX CONFIGURATION by Webinoly
 
 location / {
+	# Set the Host header to $host in case you want to preserve the original host value. (proxy_set_header Host $host;)
+	# Set the Host header to match the destination server (may be needed) to connect with external resources. (proxy_set_header Host 'your.s3.us-east-1.amazonaws.com';)
+	# The latest can't be used with multiple destinations or load-balancing, which is one of the main features of nginx upstreams.
+	# The purpose of the Host header in to indicate to the remote server which website you are trying to access and is critical if an IP address is being shared with multiple sites.
 	#proxy_set_header Host $host;
 	#proxy_set_header X-Forwarded-Host $host;
 	#proxy_set_header X-Forwarded-Server $host;
@@ -8,8 +12,12 @@ location / {
 	#proxy_set_header X-Forwarded-Proto $scheme;
 	#proxy_set_header X-Real-IP $remote_addr;
 	
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header Connection "upgrade";
+	# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Upgrade
+	# It can be used by a client to upgrade a connection from HTTP 1.1 to HTTP 2.0, or an HTTP or HTTPS connection into a WebSocket.
+	# We prefer 1.1 and keepalive because nginx don't have support for 2.0 in proxy/upstream - https://trac.nginx.org/nginx/ticket/923
+	# For Upgrade, should be set Connection "upgrade" below and keepalive disabled in the upstream.
+	#proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "";
 	proxy_http_version 1.1;
 
 	proxy_connect_timeout 300;
@@ -17,8 +25,10 @@ location / {
 	proxy_read_timeout    300;
 	proxy_ignore_headers   Set-Cookie;
 	proxy_intercept_errors on;
+	#proxy_next_upstream error timeout http_500;
 	
-	proxy_pass http://localhost:8080;
+	# Upstream defined here: conf.d/upstream_proxy.conf
+	proxy_pass <upstream_name>;
 		
 	# CacheStaticFiles
 	location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|mp4|ttf|css|rss|atom|js|jpg|jpeg|gif|png|ico|zip|tgz|gz|rar|bz2|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|woff2|m4a|cur|heic|tiff|webm|mp3|aac|webp)$ {
@@ -29,7 +39,7 @@ location / {
 		log_not_found off;
 		expires max;
 		
-		proxy_pass http://localhost:8080;
+		proxy_pass <upstream_name>;
 	}
 	
 	# BasicLocationRootFiles
@@ -37,7 +47,7 @@ location / {
 		access_log off;
 		log_not_found off;
 		
-		proxy_pass http://localhost:8080;
+		proxy_pass <upstream_name>;
 	}
 	# BasicLocEnd
 }