From 8a4348673080dbe28d1671c068cc4ac5d16c4a14 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 2 Oct 2022 12:43:26 +0200 Subject: [PATCH] postgres driver: add multi hosts support Signed-off-by: Nicola Murino --- .github/workflows/development.yml | 48 +++++++++++++-------------- .github/workflows/docker.yml | 1 + .github/workflows/release.yml | 14 ++++---- docs/full-configuration.md | 3 +- go.mod | 5 ++- go.sum | 5 +++ internal/common/protocol_test.go | 2 +- internal/config/config.go | 34 ++++++++++--------- internal/dataprovider/dataprovider.go | 9 +++-- internal/dataprovider/pgsql.go | 29 ++++++++++++---- internal/httpd/httpd_test.go | 2 +- internal/sftpd/sftpd_test.go | 2 +- sftpgo.json | 1 + 13 files changed, 95 insertions(+), 60 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 2d03609d..dd56314f 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -32,7 +32,7 @@ jobs: - name: Build for Linux/macOS x86_64 if: startsWith(matrix.os, 'windows-') != true run: | - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo cd tests/eventsearcher go build -trimpath -ldflags "-s -w" -o eventsearcher cd - @@ -44,7 +44,7 @@ jobs: - name: Build for macOS arm64 if: startsWith(matrix.os, 'macos-') == true - run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64 + run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64 - name: Build for Windows if: startsWith(matrix.os, 'windows-') @@ -57,7 +57,7 @@ jobs: $FILE_VERSION = $LATEST_TAG.substring(1) + "." + $COMMITS_FROM_TAG go install github.com/tc-hib/go-winres@latest go-winres simply --arch amd64 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe cd tests/eventsearcher go build -trimpath -ldflags "-s -w" -o eventsearcher.exe cd ../.. @@ -69,17 +69,17 @@ jobs: $Env:GOOS='windows' $Env:GOARCH='arm64' go-winres simply --arch arm64 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe mkdir x86 $Env:GOARCH='386' go-winres simply --arch 386 --product-version $LATEST_TAG-dev-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe Remove-Item Env:\CGO_ENABLED Remove-Item Env:\GOOS Remove-Item Env:\GOARCH - name: Run test cases using SQLite provider - run: go test -v -p 1 -timeout 15m ./... -coverprofile=coverage.txt -covermode=atomic + run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -coverprofile=coverage.txt -covermode=atomic - name: Upload coverage to Codecov if: ${{ matrix.upload-coverage }} @@ -90,21 +90,21 @@ jobs: - name: Run test cases using bolt provider run: | - go test -v -p 1 -timeout 2m ./internal/config -covermode=atomic - go test -v -p 1 -timeout 5m ./internal/common -covermode=atomic - go test -v -p 1 -timeout 5m ./internal/httpd -covermode=atomic - go test -v -p 1 -timeout 8m ./internal/sftpd -covermode=atomic - go test -v -p 1 -timeout 5m ./internal/ftpd -covermode=atomic - go test -v -p 1 -timeout 5m ./internal/webdavd -covermode=atomic - go test -v -p 1 -timeout 2m ./internal/telemetry -covermode=atomic - go test -v -p 1 -timeout 2m ./internal/mfa -covermode=atomic - go test -v -p 1 -timeout 2m ./internal/command -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/config -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/common -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/httpd -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 8m ./internal/sftpd -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/ftpd -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 5m ./internal/webdavd -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/telemetry -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/mfa -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 2m ./internal/command -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: bolt SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db' - name: Run test cases using memory provider - run: go test -v -p 1 -timeout 15m ./... -covermode=atomic + run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: memory SFTPGO_DATA_PROVIDER__NAME: '' @@ -246,7 +246,7 @@ jobs: GOARCH: 386 - name: Run test cases - run: go test -v -p 1 -timeout 15m ./... -covermode=atomic + run: go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: memory SFTPGO_DATA_PROVIDER__NAME: '' @@ -310,7 +310,7 @@ jobs: - name: Build run: | - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo cd tests/eventsearcher go build -trimpath -ldflags "-s -w" -o eventsearcher cd - @@ -322,7 +322,7 @@ jobs: run: | ./sftpgo initprovider ./sftpgo resetprovider --force - go test -v -p 1 -timeout 15m ./... -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: postgresql SFTPGO_DATA_PROVIDER__NAME: sftpgo @@ -335,7 +335,7 @@ jobs: run: | ./sftpgo initprovider ./sftpgo resetprovider --force - go test -v -p 1 -timeout 15m ./... -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: mysql SFTPGO_DATA_PROVIDER__NAME: sftpgo @@ -348,7 +348,7 @@ jobs: run: | ./sftpgo initprovider ./sftpgo resetprovider --force - go test -v -p 1 -timeout 15m ./... -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: mysql SFTPGO_DATA_PROVIDER__NAME: sftpgo @@ -365,7 +365,7 @@ jobs: docker exec crdb cockroach sql --insecure -e 'create database "sftpgo"' ./sftpgo initprovider ./sftpgo resetprovider --force - go test -v -p 1 -timeout 15m ./... -covermode=atomic + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic docker stop crdb env: SFTPGO_DATA_PROVIDER__DRIVER: cockroachdb @@ -428,7 +428,7 @@ jobs: echo 'export PATH=$PATH:/usr/local/go/bin' >> build.sh echo 'go version' >> build.sh echo 'cd /usr/local/src' >> build.sh - echo 'go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh + echo 'go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh chmod 755 build.sh docker run --rm --name ubuntu-build --mount type=bind,source=`pwd`,target=/usr/local/src ${{ matrix.distro }} /usr/local/src/build.sh @@ -479,7 +479,7 @@ jobs: then export GOARM=7 fi - go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo + go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_commit.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo mkdir -p output/{init,bash_completion,zsh_completion} cp sftpgo.json output/ cp -r templates output/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dddb0434..da50018b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -169,6 +169,7 @@ jobs: COMMIT_SHA=${{ steps.info.outputs.sha }} INSTALL_OPTIONAL_PACKAGES=${{ steps.info.outputs.full }} DOWNLOAD_PLUGINS=${{ steps.info.outputs.plugins }} + FEATURES=nopgxregisterdefaulttypes labels: | org.opencontainers.image.title=SFTPGo org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional HTTP, FTP/S and WebDAV support diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5b001c3..a41b84fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,11 +72,11 @@ jobs: - name: Build for macOS x86_64 if: startsWith(matrix.os, 'windows-') != true - run: go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo + run: go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo - name: Build for macOS arm64 if: startsWith(matrix.os, 'macos-') == true - run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64 + run: CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 SDKROOT=$(xcrun --sdk macosx --show-sdk-path) go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=`git describe --always --abbrev=8 --dirty` -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo_arm64 - name: Build for Windows if: startsWith(matrix.os, 'windows-') @@ -86,17 +86,17 @@ jobs: $FILE_VERSION = $Env:SFTPGO_VERSION.substring(1) + ".0" go install github.com/tc-hib/go-winres@latest go-winres simply --arch amd64 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o sftpgo.exe mkdir arm64 $Env:CGO_ENABLED='0' $Env:GOOS='windows' $Env:GOARCH='arm64' go-winres simply --arch arm64 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\arm64\sftpgo.exe mkdir x86 $Env:GOARCH='386' go-winres simply --arch 386 --product-version $Env:SFTPGO_VERSION-$GIT_COMMIT --file-version $FILE_VERSION --file-description "SFTPGo server" --product-name SFTPGo --copyright "AGPL-3.0" --original-filename sftpgo.exe --icon .\windows-installer\icon.ico - go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe + go build -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/internal/version.date=$DATE_TIME" -o .\x86\sftpgo.exe Remove-Item Env:\CGO_ENABLED Remove-Item Env:\GOOS Remove-Item Env:\GOARCH @@ -310,7 +310,7 @@ jobs: echo 'export PATH=$PATH:/usr/local/go/bin' >> build.sh echo 'go version' >> build.sh echo 'cd /usr/local/src' >> build.sh - echo 'go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh + echo 'go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo' >> build.sh chmod 755 build.sh docker run --rm --name ubuntu-build --mount type=bind,source=`pwd`,target=/usr/local/src ${{ matrix.distro }} /usr/local/src/build.sh @@ -362,7 +362,7 @@ jobs: run: | export PATH=$PATH:/usr/local/go/bin go version - go build -buildvcs=false -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo + go build -buildvcs=false -trimpath -tags nopgxregisterdefaulttypes -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${{ steps.get_version.outputs.COMMIT }} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -o sftpgo mkdir -p output/{init,sqlite,bash_completion,zsh_completion} echo "For documentation please take a look here:" > output/README.txt echo "" >> output/README.txt diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 8d0eeef9..26abdafd 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -205,13 +205,14 @@ The configuration file contains the following sections: - **"data_provider"**, the configuration for the data provider - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `cockroachdb`, `bolt`, `memory` - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the provider dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted. If you plan to use a SQLite database over a `cifs` network share (this is not recommended in general) you must use the `nobrl` mount option otherwise you will get the `database is locked` error. Some users reported that the `bolt` provider works fine over `cifs` shares. - - `host`, string. Database host. Leave empty for drivers `sqlite`, `bolt` and `memory` + - `host`, string. Database host. For `postgresql` and `cockroachdb` drivers you can specify multiple hosts separated by commas. Leave empty for drivers `sqlite`, `bolt` and `memory` - `port`, integer. Database port. Leave empty for drivers `sqlite`, `bolt` and `memory` - `username`, string. Database user. Leave empty for drivers `sqlite`, `bolt` and `memory` - `password`, string. Database password. Leave empty for drivers `sqlite`, `bolt` and `memory` - `sslmode`, integer. Used for drivers `mysql` and `postgresql`. 0 disable TLS connections, 1 require TLS, 2 set TLS mode to `verify-ca` for driver `postgresql` and `skip-verify` for driver `mysql`, 3 set TLS mode to `verify-full` for driver `postgresql` and `preferred` for driver `mysql` - `root_cert`, string. Path to the root certificate authority used to verify that the server certificate was signed by a trusted CA - `disable_sni`, boolean. Allows to opt out Server Name Indication (SNI) for TLS connections. Default: `false` + - `target_session_attrs`, string. This is a `postgresql` and `cockroachdb` specific option. It determines whether the session must have certain properties to be acceptable. It's typically used in combination with multiple host names to select the first acceptable alternative among several hosts. Supported values: `any`, `read-write`, `read-only`, `primary`, `standby`, `prefer-standby`. If empty, `any` is assumed. - `client_cert`, string. Path to the client certificate for two-way TLS authentication - `client_key`,string. Path to the client key for two-way TLS authentication - `connection_string`, string. Provide a custom database connection string. If not empty, this connection string will be used instead of building one using the previous parameters. Leave empty for drivers `bolt` and `memory` diff --git a/go.mod b/go.mod index 82ce14d6..2663d580 100644 --- a/go.mod +++ b/go.mod @@ -34,10 +34,10 @@ require ( github.com/hashicorp/go-hclog v1.3.1 github.com/hashicorp/go-plugin v1.4.5 github.com/hashicorp/go-retryablehttp v0.7.1 + github.com/jackc/pgx/v5 v5.0.1 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 github.com/klauspost/compress v1.15.11 github.com/lestrrat-go/jwx v1.2.25 - github.com/lib/pq v1.10.7 github.com/lithammer/shortuuid/v3 v3.0.7 github.com/mattn/go-sqlite3 v1.14.15 github.com/mhale/smtpd v0.8.0 @@ -115,6 +115,8 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/kr/fs v0.1.0 // indirect @@ -123,6 +125,7 @@ require ( github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.0 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index e019b472..1f8a522c 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v63.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE= github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -990,6 +991,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -999,6 +1001,7 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1: github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= @@ -1011,6 +1014,8 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v5 v5.0.1 h1:JZu9othr7l8so2JMDAGeDUMXqERAuZpovyfl4H50tdg= +github.com/jackc/pgx/v5 v5.0.1/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 90eb3fc7..cac1bad8 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -35,7 +35,7 @@ import ( "time" _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" + _ "github.com/jackc/pgx/v5/stdlib" _ "github.com/mattn/go-sqlite3" "github.com/mhale/smtpd" "github.com/pkg/sftp" diff --git a/internal/config/config.go b/internal/config/config.go index 62c69e12..595434f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -310,22 +310,23 @@ func Init() { }, }, ProviderConf: dataprovider.Config{ - Driver: "sqlite", - Name: "sftpgo.db", - Host: "", - Port: 0, - Username: "", - Password: "", - ConnectionString: "", - SQLTablesPrefix: "", - SSLMode: 0, - DisableSNI: false, - RootCert: "", - ClientCert: "", - ClientKey: "", - TrackQuota: 2, - PoolSize: 0, - UsersBaseDir: "", + Driver: "sqlite", + Name: "sftpgo.db", + Host: "", + Port: 0, + Username: "", + Password: "", + ConnectionString: "", + SQLTablesPrefix: "", + SSLMode: 0, + DisableSNI: false, + TargetSessionAttrs: "", + RootCert: "", + ClientCert: "", + ClientKey: "", + TrackQuota: 2, + PoolSize: 0, + UsersBaseDir: "", Actions: dataprovider.ObjectsActions{ ExecuteOn: []string{}, ExecuteFor: []string{}, @@ -1941,6 +1942,7 @@ func setViperDefaults() { viper.SetDefault("data_provider.password", globalConf.ProviderConf.Password) viper.SetDefault("data_provider.sslmode", globalConf.ProviderConf.SSLMode) viper.SetDefault("data_provider.disable_sni", globalConf.ProviderConf.DisableSNI) + viper.SetDefault("data_provider.target_session_attrs", globalConf.ProviderConf.TargetSessionAttrs) viper.SetDefault("data_provider.root_cert", globalConf.ProviderConf.RootCert) viper.SetDefault("data_provider.client_cert", globalConf.ProviderConf.ClientCert) viper.SetDefault("data_provider.client_key", globalConf.ProviderConf.ClientKey) diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go index 5bdcc2ff..b5281777 100644 --- a/internal/dataprovider/dataprovider.go +++ b/internal/dataprovider/dataprovider.go @@ -322,7 +322,7 @@ type Config struct { // Database name. For driver sqlite this can be the database name relative to the config dir // or the absolute path to the SQLite database. Name string `json:"name" mapstructure:"name"` - // Database host + // Database host. For postgresql and cockroachdb driver you can specify multiple hosts separated by commas Host string `json:"host" mapstructure:"host"` // Database port Port int `json:"port" mapstructure:"port"` @@ -336,8 +336,13 @@ type Config struct { // 2 set ssl mode to verify-ca for driver postgresql and skip-verify for driver mysql. // 3 set ssl mode to verify-full for driver postgresql and preferred for driver mysql. SSLMode int `json:"sslmode" mapstructure:"sslmode"` - // Used for drivers mysql and postgresql. Set to true to disable SNI + // Used for drivers mysql, postgresql and cockroachdb. Set to true to disable SNI DisableSNI bool `json:"disable_sni" mapstructure:"disable_sni"` + // TargetSessionAttrs is a postgresql and cockroachdb specific option. + // It determines whether the session must have certain properties to be acceptable. + // It's typically used in combination with multiple host names to select the first + // acceptable alternative among several hosts + TargetSessionAttrs string `json:"target_session_attrs" mapstructure:"target_session_attrs"` // Path to the root certificate authority used to verify that the server certificate was signed by a trusted CA RootCert string `json:"root_cert" mapstructure:"root_cert"` // Path to the client certificate for two-way TLS authentication diff --git a/internal/dataprovider/pgsql.go b/internal/dataprovider/pgsql.go index 0f122590..88db99f1 100644 --- a/internal/dataprovider/pgsql.go +++ b/internal/dataprovider/pgsql.go @@ -26,8 +26,8 @@ import ( "strings" "time" - // we import lib/pq here to be able to disable PostgreSQL support using a build tag - _ "github.com/lib/pq" + // we import pgx here to be able to disable PostgreSQL support using a build tag + _ "github.com/jackc/pgx/v5/stdlib" "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/version" @@ -215,7 +215,7 @@ func init() { func initializePGSQLProvider() error { var err error - dbHandle, err := sql.Open("postgres", getPGSQLConnectionString(false)) + dbHandle, err := sql.Open("pgx", getPGSQLConnectionString(false)) if err == nil { providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v", getPGSQLConnectionString(true), config.PoolSize) @@ -252,6 +252,9 @@ func getPGSQLConnectionString(redactedPwd bool) string { if config.DisableSNI { connectionString += " sslsni=0" } + if config.TargetSessionAttrs != "" { + connectionString += fmt.Sprintf(" target_session_attrs='%s'", config.TargetSessionAttrs) + } } else { connectionString = config.ConnectionString } @@ -788,7 +791,11 @@ func downgradePgSQLDatabaseFromV23(dbHandle *sql.DB) error { func updatePgSQLDatabaseFrom19To20(dbHandle *sql.DB) error { logger.InfoToConsole("updating database schema version: 19 -> 20") providerLog(logger.LevelInfo, "updating database schema version: 19 -> 20") - sql := strings.ReplaceAll(pgsqlV20SQL, "{{events_actions}}", sqlTableEventsActions) + sql := pgsqlV20SQL + if config.Driver == CockroachDataProviderName { + sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "deleted_at" DROP DEFAULT;`, "") + } + sql = strings.ReplaceAll(sql, "{{events_actions}}", sqlTableEventsActions) sql = strings.ReplaceAll(sql, "{{events_rules}}", sqlTableEventsRules) sql = strings.ReplaceAll(sql, "{{rules_actions_mapping}}", sqlTableRulesActionsMapping) sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks) @@ -800,7 +807,12 @@ func updatePgSQLDatabaseFrom19To20(dbHandle *sql.DB) error { func updatePgSQLDatabaseFrom20To21(dbHandle *sql.DB) error { logger.InfoToConsole("updating database schema version: 20 -> 21") providerLog(logger.LevelInfo, "updating database schema version: 20 -> 21") - sql := strings.ReplaceAll(pgsqlV21SQL, "{{users}}", sqlTableUsers) + sql := pgsqlV21SQL + if config.Driver == CockroachDataProviderName { + sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "first_download" DROP DEFAULT;`, "") + sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "first_upload" DROP DEFAULT;`, "") + } + sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, true) } @@ -842,7 +854,12 @@ func downgradePgSQLDatabaseFrom21To20(dbHandle *sql.DB) error { func downgradePgSQLDatabaseFrom22To21(dbHandle *sql.DB) error { logger.InfoToConsole("downgrading database schema version: 22 -> 21") providerLog(logger.LevelInfo, "downgrading database schema version: 22 -> 21") - sql := strings.ReplaceAll(pgsqlV22DownSQL, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping) + sql := pgsqlV22DownSQL + if config.Driver == CockroachDataProviderName { + sql = strings.ReplaceAll(sql, `ALTER TABLE "{{admins_groups_mapping}}" DROP CONSTRAINT "{{prefix}}unique_admin_group_mapping";`, + `DROP INDEX "{{prefix}}unique_admin_group_mapping" CASCADE;`) + } + sql = strings.ReplaceAll(sql, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping) sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, false) } diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 6511e0c5..b78be1c6 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -41,7 +41,7 @@ import ( "github.com/go-chi/render" _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/lithammer/shortuuid/v3" _ "github.com/mattn/go-sqlite3" "github.com/mhale/smtpd" diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go index b7c57751..ae32c411 100644 --- a/internal/sftpd/sftpd_test.go +++ b/internal/sftpd/sftpd_test.go @@ -45,7 +45,7 @@ import ( "time" _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" + _ "github.com/jackc/pgx/v5/stdlib" _ "github.com/mattn/go-sqlite3" "github.com/pquerna/otp" "github.com/pquerna/otp/totp" diff --git a/sftpgo.json b/sftpgo.json index b57dd24e..e9516fe8 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -191,6 +191,7 @@ "password": "", "sslmode": 0, "disable_sni": false, + "target_session_attrs": "", "root_cert": "", "client_cert": "", "client_key": "",