beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 15:10:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

postgres provider: add support for load balancing

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2023-03-25 09:29:13 +01:00
parent 354fc9b3d6
commit e17068a76f
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
8 changed files with 90 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/development.yml vendored
View file

@ -364,6 +364,7 @@ jobs:
SFTPGO_DATA_PROVIDER__PORT: 26257
SFTPGO_DATA_PROVIDER__USERNAME: root
SFTPGO_DATA_PROVIDER__PASSWORD:
SFTPGO_DATA_PROVIDER__TARGET_SESSION_ATTRS: any
SFTPGO_DATA_PROVIDER__SQL_TABLES_PREFIX: prefix_
build-linux-packages:

2
README.md
View file

@ -22,6 +22,8 @@ I'd like to make SFTPGo into a sustainable long term project and would not like
If you use SFTPGo, it is in your best interest to ensure that the project you rely on stays healthy and well maintained.
This can only happen with your donations and [sponsorships](https://github.com/sponsors/drakkan) :heart:
With sponsorships/donations we establish a channel for reciprocal access, ensuring better outcomes for both you and the project.
If you just take and don't return anything back, the project will die in the long run and you will be forced to pay for a similar proprietary solution.
More [info](https://github.com/drakkan/sftpgo/issues/452).

2
docs/full-configuration.md
View file

@ -242,7 +242,7 @@ The configuration file contains the following sections:
- `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable TLS connections, 1 require TLS, 2 set TLS mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set TLS mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql`
- `root_cert`, string. Path to the root certificate authority used to verify that the server certificate was signed by a trusted CA
- `disable_sni`, boolean. Allows to opt out Server Name Indication (SNI) for TLS connections. Default: `false`
- `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed.
- `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed. If you explicitly set `any` the connections will be randomly distributed among the specified hosts
- `client_cert`, string. Path to the client certificate for two-way TLS authentication
- `client_key`,string. Path to the client key for two-way TLS authentication
- `connection_string`, string. Provide a custom database connection string. If not empty, this connection string will be used instead of building one using the previous parameters. Leave empty for drivers `bolt` and `memory`

4
go.mod
View file

@ -35,7 +35,7 @@ require (
github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-plugin v1.4.10-0.20230306173702-d78f3fc2891d
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d
github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90
github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
github.com/klauspost/compress v1.16.3
github.com/lestrrat-go/jwx/v2 v2.0.9
@ -157,7 +157,7 @@ require (
golang.org/x/tools v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d // indirect
google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect

10
go.sum
View file

@ -230,7 +230,7 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg=
cloud.google.com/go/kms v1.9.0 h1:b0votJQa/9DSsxgHwN33/tTLA7ZHVzfWhDCrfiXijSo=
cloud.google.com/go/kms v1.10.0 h1:Imrtp8792uqNP9bdfPrjtUkjjqOMBcAJ2bdFaAnLhnk=
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@ -1389,8 +1389,8 @@ github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw=
github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d h1:ggPAJEqfHHi80w/Fx5dTntWODtTSnmkwut3b9Z5wfXs=
github.com/jackc/pgx/v5 v5.3.2-0.20230311213408-9ae852eb583d/go.mod h1:sU+RaYl9qnhD3Ce+mwnFii6YEPx70mCYghBzKvqq4qo=
github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90 h1:gBugq4KF3zkdaM4oQHSfhUFAfkVQOQpbD20wNggWPP4=
github.com/jackc/pgx/v5 v5.3.2-0.20230324225134-e9d64ec29d90/go.mod h1:sU+RaYl9qnhD3Ce+mwnFii6YEPx70mCYghBzKvqq4qo=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
@ -2803,8 +2803,8 @@ google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ
google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d h1:OE8TncEeAei3Tehf/P/Jdt/K+8GnTUrRY6wzYpbCes4=
google.golang.org/genproto v0.0.0-20230322174352-cde4c949918d/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 h1:VmCWItVXcKboEMCwZaWge+1JLiTCQSngZeINF+wzO+g=
google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

30
internal/dataprovider/mysql.go
View file

@ -218,8 +218,6 @@ func init() {
}
func initializeMySQLProvider() error {
var err error
connString, err := getMySQLConnectionString(false)
if err != nil {
return err
@ -229,23 +227,23 @@ func initializeMySQLProvider() error {
return err
}
dbHandle, err := sql.Open("mysql", connString)
if err == nil {
providerLog(logger.LevelDebug, "mysql database handle created, connection string: %q, pool size: %v",
redactedConnString, config.PoolSize)
dbHandle.SetMaxOpenConns(config.PoolSize)
if config.PoolSize > 0 {
dbHandle.SetMaxIdleConns(config.PoolSize)
} else {
dbHandle.SetMaxIdleConns(2)
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &MySQLProvider{dbHandle: dbHandle}
} else {
if err != nil {
providerLog(logger.LevelError, "error creating mysql database handler, connection string: %q, error: %v",
redactedConnString, err)
return err
}
return err
providerLog(logger.LevelDebug, "mysql database handle created, connection string: %q, pool size: %v",
redactedConnString, config.PoolSize)
dbHandle.SetMaxOpenConns(config.PoolSize)
if config.PoolSize > 0 {
dbHandle.SetMaxIdleConns(config.PoolSize)
} else {
dbHandle.SetMaxIdleConns(2)
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &MySQLProvider{dbHandle: dbHandle}
return nil
}
func getMySQLConnectionString(redactedPwd bool) (string, error) {
var connectionString string

79
internal/dataprovider/pgsql.go
View file

@ -23,11 +23,13 @@ import (
"database/sql"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
// we import pgx here to be able to disable PostgreSQL support using a build tag
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/stdlib"
"github.com/drakkan/sftpgo/v2/internal/logger"
"github.com/drakkan/sftpgo/v2/internal/version"
@ -233,25 +235,61 @@ func init() {
}
func initializePGSQLProvider() error {
var err error
dbHandle, err := sql.Open("pgx", getPGSQLConnectionString(false))
if err == nil {
providerLog(logger.LevelDebug, "postgres database handle created, connection string: %q, pool size: %d",
getPGSQLConnectionString(true), config.PoolSize)
dbHandle.SetMaxOpenConns(config.PoolSize)
if config.PoolSize > 0 {
dbHandle.SetMaxIdleConns(config.PoolSize)
} else {
dbHandle.SetMaxIdleConns(2)
var dbHandle *sql.DB
if config.TargetSessionAttrs == "any" {
pgxConfig, err := pgx.ParseConfig(getPGSQLConnectionString(false))
if err != nil {
providerLog(logger.LevelError, "error parsing postgres configuration, connection string: %q, error: %v",
getPGSQLConnectionString(true), err)
return err
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &PGSQLProvider{dbHandle: dbHandle}
dbHandle = stdlib.OpenDB(*pgxConfig, stdlib.OptionBeforeConnect(stdlib.RandomizeHostOrderFunc))
} else {
providerLog(logger.LevelError, "error creating postgres database handler, connection string: %q, error: %v",
getPGSQLConnectionString(true), err)
var err error
dbHandle, err = sql.Open("pgx", getPGSQLConnectionString(false))
if err != nil {
providerLog(logger.LevelError, "error creating postgres database handler, connection string: %q, error: %v",
getPGSQLConnectionString(true), err)
return err
}
}
return err
providerLog(logger.LevelDebug, "postgres database handle created, connection string: %q, pool size: %d",
getPGSQLConnectionString(true), config.PoolSize)
dbHandle.SetMaxOpenConns(config.PoolSize)
if config.PoolSize > 0 {
dbHandle.SetMaxIdleConns(config.PoolSize)
} else {
dbHandle.SetMaxIdleConns(2)
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &PGSQLProvider{dbHandle: dbHandle}
return nil
}
func getPGSQLHostsAndPorts(configHost string, configPort int) (string, string) {
var hosts, ports []string
defaultPort := strconv.Itoa(configPort)
if defaultPort == "0" {
defaultPort = "5432"
}
for _, hostport := range strings.Split(configHost, ",") {
hostport = strings.TrimSpace(hostport)
if hostport == "" {
continue
}
host, port, err := net.SplitHostPort(hostport)
if err == nil {
hosts = append(hosts, host)
ports = append(ports, port)
} else {
hosts = append(hosts, hostport)
ports = append(ports, defaultPort)
}
}
return strings.Join(hosts, ","), strings.Join(ports, ",")
}
func getPGSQLConnectionString(redactedPwd bool) string {
@ -261,8 +299,9 @@ func getPGSQLConnectionString(redactedPwd bool) string {
if redactedPwd && password != "" {
password = "[redacted]"
}
connectionString = fmt.Sprintf("host='%s' port=%d dbname='%s' user='%s' password='%s' sslmode=%s connect_timeout=10",
config.Host, config.Port, config.Name, config.Username, password, getSSLMode())
host, port := getPGSQLHostsAndPorts(config.Host, config.Port)
connectionString = fmt.Sprintf("host='%s' port='%s' dbname='%s' user='%s' password='%s' sslmode=%s connect_timeout=10",
host, port, config.Name, config.Username, password, getSSLMode())
if config.RootCert != "" {
connectionString += fmt.Sprintf(" sslrootcert='%s'", config.RootCert)
}

13
internal/dataprovider/sqlite.go
View file

@ -210,7 +210,6 @@ func init() {
}
func initializeSQLiteProvider(basePath string) error {
var err error
var connectionString string
if config.ConnectionString == "" {
@ -226,15 +225,15 @@ func initializeSQLiteProvider(basePath string) error {
connectionString = config.ConnectionString
}
dbHandle, err := sql.Open("sqlite3", connectionString)
if err == nil {
providerLog(logger.LevelDebug, "sqlite database handle created, connection string: %q", connectionString)
dbHandle.SetMaxOpenConns(1)
provider = &SQLiteProvider{dbHandle: dbHandle}
} else {
if err != nil {
providerLog(logger.LevelError, "error creating sqlite database handler, connection string: %q, error: %v",
connectionString, err)
return err
}
return err
providerLog(logger.LevelDebug, "sqlite database handle created, connection string: %q", connectionString)
dbHandle.SetMaxOpenConns(1)
provider = &SQLiteProvider{dbHandle: dbHandle}
return nil
}
func (p *SQLiteProvider) checkAvailability() error {