postgres provider: add support for load balancing

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

.github/workflows/development.yml

@@ -364,6 +364,7 @@ jobs:
           SFTPGO_DATA_PROVIDER__PORT: 26257
           SFTPGO_DATA_PROVIDER__USERNAME: root
           SFTPGO_DATA_PROVIDER__PASSWORD:
+          SFTPGO_DATA_PROVIDER__TARGET_SESSION_ATTRS: any
           SFTPGO_DATA_PROVIDER__SQL_TABLES_PREFIX: prefix_
 
   build-linux-packages:

README.md

@@ -22,6 +22,8 @@ I'd like to make SFTPGo into a sustainable long term project and would not like
 If you use SFTPGo, it is in your best interest to ensure that the project you rely on stays healthy and well maintained.
 This can only happen with your donations and [sponsorships](https://github.com/sponsors/drakkan) :heart:
 
+With sponsorships/donations we establish a channel for reciprocal access, ensuring better outcomes for both you and the project.
+
 If you just take and don't return anything back, the project will die in the long run and you will be forced to pay for a similar proprietary solution.
 
 More [info](https://github.com/drakkan/sftpgo/issues/452).

docs/full-configuration.md

@@ -242,7 +242,7 @@ The configuration file contains the following sections:
   - `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable TLS connections, 1 require TLS, 2 set TLS mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set TLS mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql`
   - `root_cert`, string. Path to the root certificate authority used to verify that the server certificate was signed by a trusted CA
   - `disable_sni`, boolean. Allows to opt out Server Name Indication (SNI) for TLS connections. Default: `false`
-  - `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed.
+  - `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed. If you explicitly set `any` the connections will be randomly distributed among the specified hosts
   - `client_cert`, string. Path to the client certificate for two-way TLS authentication
   - `client_key`,string. Path to the client key for two-way TLS authentication
   - `connection_string`, string. Provide a custom database connection string. If not empty, this connection string will be used instead of building one using the previous parameters. Leave empty for drivers `bolt` and `memory`

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-plugin v1.4.10-0.20230306173702-d78f3fc2891d
 	github.com/hashicorp/go-retryablehttp v0.7.2
-	github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d
+	github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90
 	github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
 	github.com/klauspost/compress v1.16.3
 	github.com/lestrrat-go/jwx/v2 v2.0.9
@@ -157,7 +157,7 @@ require (
 	golang.org/x/tools v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d // indirect
+	google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 // indirect
 	google.golang.org/grpc v1.54.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -230,7 +230,7 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg=
-cloud.google.com/go/kms v1.9.0 h1:b0votJQa/9DSsxgHwN33/tTLA7ZHVzfWhDCrfiXijSo=
+cloud.google.com/go/kms v1.10.0 h1:Imrtp8792uqNP9bdfPrjtUkjjqOMBcAJ2bdFaAnLhnk=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -1389,8 +1389,8 @@ github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9
 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
 github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw=
-github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d h1:ggPAJEqfHHi80w/Fx5dTntWODtTSnmkwut3b9Z5wfXs=
-github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d/go.mod h1:sU+RaYl9qnhD3Ce+mwnFii6YEPx70mCYghBzKvqq4qo=
+github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90 h1:gBugq4KF3zkdaM4oQHSfhUFAfkVQOQpbD20wNggWPP4=
+github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90/go.mod h1:sU+RaYl9qnhD3Ce+mwnFii6YEPx70mCYghBzKvqq4qo=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
@@ -2803,8 +2803,8 @@ google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ
 google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d h1:OE8TncEeAei3Tehf/P/Jdt/K+8GnTUrRY6wzYpbCes4=
-google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
+google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 h1:VmCWItVXcKboEMCwZaWge+1JLiTCQSngZeINF+wzO+g=
+google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

internal/dataprovider/mysql.go

@@ -218,8 +218,6 @@ func init() {
 }
 
 func initializeMySQLProvider() error {
-	var err error
-
 	connString, err := getMySQLConnectionString(false)
 	if err != nil {
 		return err
@@ -229,7 +227,11 @@ func initializeMySQLProvider() error {
 		return err
 	}
 	dbHandle, err := sql.Open("mysql", connString)
-	if err == nil {
+	if err != nil {
+		providerLog(logger.LevelError, "error creating mysql database handler, connection string: %q, error: %v",
+			redactedConnString, err)
+		return err
+	}
 	providerLog(logger.LevelDebug, "mysql database handle created, connection string: %q, pool size: %v",
 		redactedConnString, config.PoolSize)
 	dbHandle.SetMaxOpenConns(config.PoolSize)
@@ -241,11 +243,7 @@ func initializeMySQLProvider() error {
 	dbHandle.SetConnMaxLifetime(240 * time.Second)
 	dbHandle.SetConnMaxIdleTime(120 * time.Second)
 	provider = &MySQLProvider{dbHandle: dbHandle}
-	} else {
-		providerLog(logger.LevelError, "error creating mysql database handler, connection string: %q, error: %v",
-			redactedConnString, err)
-	}
-	return err
+	return nil
 }
 func getMySQLConnectionString(redactedPwd bool) (string, error) {
 	var connectionString string

internal/dataprovider/pgsql.go

@@ -23,11 +23,13 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"net"
+	"strconv"
 	"strings"
 	"time"
 
-	// we import pgx here to be able to disable PostgreSQL support using a build tag
-	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/stdlib"
 
 	"github.com/drakkan/sftpgo/v2/internal/logger"
 	"github.com/drakkan/sftpgo/v2/internal/version"
@@ -233,9 +235,24 @@ func init() {
 }
 
 func initializePGSQLProvider() error {
+	var dbHandle *sql.DB
+	if config.TargetSessionAttrs == "any" {
+		pgxConfig, err := pgx.ParseConfig(getPGSQLConnectionString(false))
+		if err != nil {
+			providerLog(logger.LevelError, "error parsing postgres configuration, connection string: %q, error: %v",
+				getPGSQLConnectionString(true), err)
+			return err
+		}
+		dbHandle = stdlib.OpenDB(*pgxConfig, stdlib.OptionBeforeConnect(stdlib.RandomizeHostOrderFunc))
+	} else {
 		var err error
-	dbHandle, err := sql.Open("pgx", getPGSQLConnectionString(false))
-	if err == nil {
+		dbHandle, err = sql.Open("pgx", getPGSQLConnectionString(false))
+		if err != nil {
+			providerLog(logger.LevelError, "error creating postgres database handler, connection string: %q, error: %v",
+				getPGSQLConnectionString(true), err)
+			return err
+		}
+	}
 	providerLog(logger.LevelDebug, "postgres database handle created, connection string: %q, pool size: %d",
 		getPGSQLConnectionString(true), config.PoolSize)
 	dbHandle.SetMaxOpenConns(config.PoolSize)
@@ -247,11 +264,32 @@ func initializePGSQLProvider() error {
 	dbHandle.SetConnMaxLifetime(240 * time.Second)
 	dbHandle.SetConnMaxIdleTime(120 * time.Second)
 	provider = &PGSQLProvider{dbHandle: dbHandle}
-	} else {
-		providerLog(logger.LevelError, "error creating postgres database handler, connection string: %q, error: %v",
-			getPGSQLConnectionString(true), err)
+	return nil
+}
+
+func getPGSQLHostsAndPorts(configHost string, configPort int) (string, string) {
+	var hosts, ports []string
+	defaultPort := strconv.Itoa(configPort)
+	if defaultPort == "0" {
+		defaultPort = "5432"
 	}
-	return err
+
+	for _, hostport := range strings.Split(configHost, ",") {
+		hostport = strings.TrimSpace(hostport)
+		if hostport == "" {
+			continue
+		}
+		host, port, err := net.SplitHostPort(hostport)
+		if err == nil {
+			hosts = append(hosts, host)
+			ports = append(ports, port)
+		} else {
+			hosts = append(hosts, hostport)
+			ports = append(ports, defaultPort)
+		}
+	}
+
+	return strings.Join(hosts, ","), strings.Join(ports, ",")
 }
 
 func getPGSQLConnectionString(redactedPwd bool) string {
@@ -261,8 +299,9 @@ func getPGSQLConnectionString(redactedPwd bool) string {
 		if redactedPwd && password != "" {
 			password = "[redacted]"
 		}
-		connectionString = fmt.Sprintf("host='%s' port=%d dbname='%s' user='%s' password='%s' sslmode=%s connect_timeout=10",
-			config.Host, config.Port, config.Name, config.Username, password, getSSLMode())
+		host, port := getPGSQLHostsAndPorts(config.Host, config.Port)
+		connectionString = fmt.Sprintf("host='%s' port='%s' dbname='%s' user='%s' password='%s' sslmode=%s connect_timeout=10",
+			host, port, config.Name, config.Username, password, getSSLMode())
 		if config.RootCert != "" {
 			connectionString += fmt.Sprintf(" sslrootcert='%s'", config.RootCert)
 		}

internal/dataprovider/sqlite.go

@@ -210,7 +210,6 @@ func init() {
 }
 
 func initializeSQLiteProvider(basePath string) error {
-	var err error
 	var connectionString string
 
 	if config.ConnectionString == "" {
@@ -226,15 +225,15 @@ func initializeSQLiteProvider(basePath string) error {
 		connectionString = config.ConnectionString
 	}
 	dbHandle, err := sql.Open("sqlite3", connectionString)
-	if err == nil {
+	if err != nil {
+		providerLog(logger.LevelError, "error creating sqlite database handler, connection string: %q, error: %v",
+			connectionString, err)
+		return err
+	}
 	providerLog(logger.LevelDebug, "sqlite database handle created, connection string: %q", connectionString)
 	dbHandle.SetMaxOpenConns(1)
 	provider = &SQLiteProvider{dbHandle: dbHandle}
-	} else {
-		providerLog(logger.LevelError, "error creating sqlite database handler, connection string: %q, error: %v",
-			connectionString, err)
-	}
-	return err
+	return nil
 }
 
 func (p *SQLiteProvider) checkAvailability() error {