From 1b1745b7f7ec78c0645dde9d460f04a5324b0d21 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Thu, 9 Feb 2023 09:33:33 +0100 Subject: [PATCH] move IP/Network lists to the data provider this is a backward incompatible change, all previous file based IP/network lists will not work anymore Signed-off-by: Nicola Murino --- .github/workflows/development.yml | 12 +- .github/workflows/docker.yml | 2 +- .github/workflows/release.yml | 2 +- Dockerfile | 2 +- Dockerfile.alpine | 2 +- Dockerfile.distroless | 2 +- README.md | 30 +- README.zh_CN.md | 6 + docker/README.md | 18 +- docs/defender.md | 29 +- docs/full-configuration.md | 21 +- docs/howto/getting-started.md | 12 +- docs/howto/two-factor-authentication.md | 4 +- docs/metrics.md | 2 +- docs/plugins.md | 2 +- docs/post-connect-hook.md | 4 +- docs/profiling.md | 2 +- docs/rate-limiting.md | 14 +- docs/rest-api.md | 6 +- docs/virtual-folders.md | 4 +- docs/web-admin.md | 4 +- go.mod | 62 +-- go.sum | 116 ++--- internal/cmd/startsubsys.go | 20 +- internal/common/common.go | 136 +++--- internal/common/common_test.go | 313 ++++++++---- internal/common/defender.go | 202 +------- internal/common/defender_test.go | 329 ++++--------- internal/common/defenderdb.go | 46 +- internal/common/defenderdb_test.go | 118 ++--- internal/common/defendermem.go | 25 +- internal/common/eventscheduler.go | 1 + internal/common/protocol_test.go | 97 +++- internal/common/ratelimiter.go | 18 +- internal/common/ratelimiter_test.go | 33 +- internal/common/tlsutils.go | 4 +- internal/config/config.go | 19 +- internal/config/config_test.go | 9 - internal/dataprovider/actions.go | 1 + internal/dataprovider/admin.go | 6 +- internal/dataprovider/bolt.go | 255 +++++++++- internal/dataprovider/dataprovider.go | 80 ++- internal/dataprovider/eventrule.go | 2 +- internal/dataprovider/iplist.go | 493 +++++++++++++++++++ internal/dataprovider/memory.go | 336 +++++++++++-- internal/dataprovider/mysql.go | 84 +++- internal/dataprovider/pgsql.go | 89 +++- internal/dataprovider/scheduler.go | 69 ++- internal/dataprovider/sqlcommon.go | 257 +++++++++- internal/dataprovider/sqlite.go | 86 +++- internal/dataprovider/sqlqueries.go | 95 ++++ internal/dataprovider/user.go | 2 +- internal/ftpd/ftpd.go | 4 +- internal/ftpd/ftpd_test.go | 10 +- internal/ftpd/server.go | 6 +- internal/httpd/api_admin.go | 3 +- internal/httpd/api_eventrule.go | 5 +- internal/httpd/api_folder.go | 3 +- internal/httpd/api_group.go | 3 +- internal/httpd/api_iplist.go | 157 ++++++ internal/httpd/api_keys.go | 3 +- internal/httpd/api_maintenance.go | 141 +++--- internal/httpd/api_role.go | 3 +- internal/httpd/api_shares.go | 3 +- internal/httpd/api_user.go | 3 +- internal/httpd/api_utils.go | 2 +- internal/httpd/httpd.go | 43 +- internal/httpd/httpd_test.go | 616 ++++++++++++++++++++++-- internal/httpd/internal_test.go | 42 +- internal/httpd/server.go | 28 +- internal/httpd/webadmin.go | 219 ++++++++- internal/httpdtest/httpdtest.go | 151 +++++- internal/metric/metric.go | 2 +- internal/metric/metric_disabled.go | 2 +- internal/service/service.go | 40 +- internal/sftpd/handler.go | 30 +- internal/sftpd/server.go | 10 +- internal/sftpd/sftpd_test.go | 16 +- internal/smtp/smtp.go | 2 +- internal/util/util.go | 24 + internal/version/version.go | 2 +- internal/vfs/folder.go | 2 +- internal/webdavd/server.go | 6 +- internal/webdavd/webdavd_test.go | 12 +- main.go | 3 - openapi/openapi.yaml | 276 ++++++++++- pkgs/build.sh | 2 +- pkgs/choco/sftpgo.nuspec | 10 +- pkgs/choco/tools/ChocolateyInstall.ps1 | 8 +- pkgs/debian/changelog | 6 + pkgs/debian/patches/config.diff | 8 +- sftpgo.json | 9 +- templates/webadmin/admin.html | 2 +- templates/webadmin/base.html | 64 ++- templates/webadmin/defender.html | 22 +- templates/webadmin/eventaction.html | 2 +- templates/webadmin/events.html | 10 +- templates/webadmin/iplist.html | 104 ++++ templates/webadmin/iplists.html | 508 +++++++++++++++++++ templates/webadmin/role.html | 4 - templates/webadmin/roles.html | 4 +- templates/webadmin/status.html | 22 + templates/webclient/editfile.html | 2 +- 103 files changed, 4958 insertions(+), 1284 deletions(-) create mode 100644 internal/dataprovider/iplist.go create mode 100644 internal/httpd/api_iplist.go create mode 100644 templates/webadmin/iplist.html create mode 100644 templates/webadmin/iplists.html diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 5d17207f..d7168be0 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -11,11 +11,11 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: [1.19] + go: ['1.20'] os: [ubuntu-latest, macos-latest] upload-coverage: [true] include: - - go: 1.19 + - go: '1.20' os: windows-latest upload-coverage: false @@ -232,7 +232,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: '1.20' - name: Build run: | @@ -252,7 +252,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: '1.20' - name: Build run: | @@ -326,7 +326,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: '1.20' - name: Build run: | @@ -546,7 +546,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: '1.20' - uses: actions/checkout@v3 - name: Run golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 14de6073..6df081f2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -160,7 +160,7 @@ jobs: if: ${{ github.event_name != 'pull_request' }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . builder: ${{ steps.builder.outputs.name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d81ce105..47eea50a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ on: tags: 'v*' env: - GO_VERSION: 1.19.4 + GO_VERSION: 1.20.0 jobs: prepare-sources-with-deps: diff --git a/Dockerfile b/Dockerfile index 6128d65b..a72fda0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-bullseye as builder +FROM golang:1.20-bullseye as builder ENV GOFLAGS="-mod=readonly" diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 29541e07..888fe948 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine3.17 AS builder +FROM golang:1.20-alpine3.17 AS builder ENV GOFLAGS="-mod=readonly" diff --git a/Dockerfile.distroless b/Dockerfile.distroless index 20d32886..34c9957c 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -1,4 +1,4 @@ -FROM golang:1.19-bullseye as builder +FROM golang:1.20-bullseye as builder ENV CGO_ENABLED=0 GOFLAGS="-mod=readonly" diff --git a/README.md b/README.md index f80557b4..909fc553 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ More [info](https://github.com/drakkan/sftpgo/issues/452). #### Silver sponsors -[Dendi logo](https://dendisoftware.com/) +[Dendi logo](https://dendisoftware.com/) #### Bronze sponsors @@ -51,12 +51,12 @@ If you report an invalid issue or ask for step-by-step support, your issue will ## Features - Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV. -- Virtual folders are supported: a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. +- Virtual folders are supported: a virtual folder can use any of the supported storage backends. So you can have, for example, a user with the S3 backend mapping a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. - Configurable [custom commands and/or HTTP hooks](./docs/custom-actions.md) on upload, pre-upload, download, pre-download, delete, pre-delete, rename, mkdir, rmdir on SSH commands and on user add, update and delete. - Virtual accounts stored within a "data provider". - SQLite, MySQL, PostgreSQL, CockroachDB, Bolt (key/value store in pure Go) and in-memory data providers are supported. - Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path. -- Per-user and per-directory virtual permissions, for each exposed path you can allow or deny: directory listing, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group/file mode and modification time. +- Per-user and per-directory virtual permissions, for each path you can allow or deny: directory listing, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group/file mode and modification time. - [REST API](./docs/rest-api.md) for users and folders management, data retention, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection. - The [Event Manager](./docs/eventmanager.md) allows to define custom workflows based on server events or schedules. - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections. @@ -92,7 +92,7 @@ If you report an invalid issue or ask for step-by-step support, your issue will - ACME protocol is supported. SFTPGo can obtain and automatically renew TLS certificates for HTTPS, WebDAV and FTPS from `Let's Encrypt` or other ACME compliant certificate authorities, using the the `HTTP-01` or `TLS-ALPN-01` [challenge types](https://letsencrypt.org/docs/challenge-types/). - Two-Way TLS authentication, aka TLS with client certificate authentication, is supported for REST API/Web Admin, FTPS and WebDAV over HTTPS. - Per-user protocols restrictions. You can configure the allowed protocols (SSH/HTTP/FTP/WebDAV) for each user. -- [Prometheus metrics](./docs/metrics.md) are exposed. +- [Prometheus metrics](./docs/metrics.md) are supported. - Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP/FTP service without losing the information about the client's address. - Easy [migration](./examples/convertusers) from Linux system user accounts. - [Portable mode](./docs/portable-mode.md): a convenient way to share a single directory on demand. @@ -222,12 +222,12 @@ To start using SFTPGo you need to create an admin user, you can do it in several SFTPGo supports upgrading from the previous release branch to the current one. Some examples for supported upgrade paths are: -- from 1.2.x to 2.0.x -- from 2.0.x to 2.1.x and so on. +- from 2.1.x to 2.2.x +- from 2.2.x to 2.3.x and so on. -For supported upgrade paths, the data and schema are migrated automatically, alternately you can use the `initprovider` command. +For supported upgrade paths, the data and schema are migrated automatically when SFTPGo starts, alternatively you can use the `initprovider` command before starting SFTPGo. -So if, for example, you want to upgrade from a version before 1.2.x to 2.0.x, you must first install version 1.2.x, update the data provider and finally install the version 2.0.x. It is recommended to always install the latest available minor version, ie do not install 1.2.0 if 1.2.2 is available. +So if, for example, you want to upgrade from 2.0.x to 2.2.x, you must first install version 2.1.x, update the data provider (automatically, by starting SFTPGo or manually using the `initprovider` command) and finally install the version 2.2.x. It is recommended to always install the latest available minor version, ie do not install 2.1.0 if 2.1.2 is available. Loading data from a provider independent JSON dump is supported from the previous release branch to the current one too. After upgrading SFTPGo it is advisable to regenerate the JSON dump from the new version. @@ -237,13 +237,13 @@ If for some reason you want to downgrade SFTPGo, you may need to downgrade your As for upgrading, SFTPGo supports downgrading from the previous release branch to the current one. -So, if you plan to downgrade from 2.0.x to 1.2.x, before uninstalling 2.0.x version, you can prepare your data provider executing the following command from the configuration directory: +So, if you plan to downgrade from 2.3.x to 2.2.x, before uninstalling 2.3.x version, you can prepare your data provider executing the following command from the configuration directory: ```shell -sftpgo revertprovider --to-version 4 +sftpgo revertprovider ``` -Take a look at the CLI usage to see the supported parameter for the `--to-version` argument and to learn how to specify a different configuration file: +Take a look at the CLI usage to learn how to specify a configuration file: ```shell sftpgo revertprovider --help @@ -253,11 +253,11 @@ The `revertprovider` command is not supported for the memory provider. Please note that we only support the current release branch and the current main branch, if you find a bug it is better to report it rather than downgrading to an older unsupported version. -## Users, groups and folders management +## Users, groups, folders and other resource management After starting SFTPGo you can manage users, groups, folders and other resources using: -- the [web based administration interface](./docs/web-admin.md) +- the [WebAdmin UI](./docs/web-admin.md) - the [REST API](./docs/rest-api.md) To support embedded data providers like `bolt` and `SQLite`, which do not support concurrent connections, we can't have a CLI that directly write users and other resources to the data provider, we always have to use the REST API. @@ -299,7 +299,7 @@ More information about custom actions can be found [here](./docs/custom-actions. ## Virtual folders -Directories outside the user home directory or based on a different storage provider can be exposed as virtual folders, more information [here](./docs/virtual-folders.md). +Directories outside the user home directory or based on a different storage provider can be mapped as virtual folders, more information [here](./docs/virtual-folders.md). ## Other hooks @@ -310,7 +310,7 @@ You can use your own hook to [check passwords](./docs/check-password-hook.md). ### S3/GCP/Azure -Each user can be mapped with a [S3 Compatible Object Storage](./docs/s3.md) /[Google Cloud Storage](./docs/google-cloud-storage.md)/[Azure Blob Storage](./docs/azure-blob-storage.md) bucket or a bucket virtual folder that is exposed over SFTP/SCP/FTP/WebDAV. +Each user can be mapped with a [S3 Compatible Object Storage](./docs/s3.md) /[Google Cloud Storage](./docs/google-cloud-storage.md)/[Azure Blob Storage](./docs/azure-blob-storage.md) bucket or a bucket virtual folder. ### SFTP backend diff --git a/README.zh_CN.md b/README.zh_CN.md index 9c209b22..38c79dd4 100644 --- a/README.zh_CN.md +++ b/README.zh_CN.md @@ -11,6 +11,8 @@ 功能齐全、高度可配置化、支持自定义 HTTP/S，FTP/S 和 WebDAV 的 SFTP 服务。 一些存储后端支持：本地文件系统、加密本地文件系统、S3（兼容）对象存储，Google Cloud 存储，Azure Blob 存储，SFTP。 +:warning: 我無法自己維護中文翻譯，這個文檔可能已經過時了 + ## 赞助商 如果你觉得 SFTPGo 有用，请考虑支持这个开源项目。 @@ -32,6 +34,10 @@ [Aledade logo](https://www.aledade.com/) +#### 銀牌贊助商 + +[Dendi logo](https://dendisoftware.com/) + #### 铜牌赞助商 [7digital logo](https://www.7digital.com/) diff --git a/docker/README.md b/docker/README.md index 1bdc81dc..559a9754 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,12 +4,12 @@ SFTPGo provides an official Docker image, it is available on both [Docker Hub](h ## Supported tags and respective Dockerfile links -- [v2.4.3, v2.4, v2, latest](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile) -- [v2.4.3-plugins, v2.4-plugins, v2-plugins, plugins](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile) -- [v2.4.3-alpine, v2.4-alpine, v2-alpine, alpine](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile.alpine) -- [v2.4.3-slim, v2.4-slim, v2-slim, slim](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile) -- [v2.4.3-alpine-slim, v2.4-alpine-slim, v2-alpine-slim, alpine-slim](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile.alpine) -- [v2.4.3-distroless-slim, v2.4-distroless-slim, v2-distroless-slim, distroless-slim](https://github.com/drakkan/sftpgo/blob/v2.4.3/Dockerfile.distroless) +- [v2.4.4, v2.4, v2, latest](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile) +- [v2.4.4-plugins, v2.4-plugins, v2-plugins, plugins](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile) +- [v2.4.4-alpine, v2.4-alpine, v2-alpine, alpine](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile.alpine) +- [v2.4.4-slim, v2.4-slim, v2-slim, slim](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile) +- [v2.4.4-alpine-slim, v2.4-alpine-slim, v2-alpine-slim, alpine-slim](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile.alpine) +- [v2.4.4-distroless-slim, v2.4-distroless-slim, v2-distroless-slim, distroless-slim](https://github.com/drakkan/sftpgo/blob/v2.4.4/Dockerfile.distroless) - [edge](../Dockerfile) - [edge-plugins](../Dockerfile) - [edge-alpine](../Dockerfile.alpine) @@ -58,7 +58,7 @@ The FTP service is now available on port 2121 and SFTP on port 2022. You can change the passive ports range (`50000-50100` by default) by setting the environment variables `SFTPGO_FTPD__PASSIVE_PORT_RANGE__START` and `SFTPGO_FTPD__PASSIVE_PORT_RANGE__END`. -It is recommended that you provide a certificate and key file to expose FTP over TLS. You should prefer SFTP to FTP even if you configure TLS, please don't blindly enable the old FTP protocol. +It is recommended that you provide a certificate and key file to enable FTP over TLS. You should prefer SFTP to FTP even if you configure TLS, please don't blindly enable the old FTP protocol. ### Enable WebDAV service @@ -75,7 +75,7 @@ docker run --name some-sftpgo \ The WebDAV service is now available on port 10080 and SFTP on port 2022. -It is recommended that you provide a certificate and key file to expose WebDAV over https. +It is recommended that you provide a certificate and key file to enable WebDAV over https. ### Container shell access and viewing SFTPGo logs @@ -116,7 +116,7 @@ Alternatively you can increase the default docker grace time to a value larger t Important note: There are several ways to store data used by applications that run in Docker containers. We encourage users of the SFTPGo images to familiarize themselves with the options available, including: - Let Docker manage the storage for SFTPGo data by [writing them to disk on the host system using its own internal volume management](https://docs.docker.com/engine/tutorials/dockervolumes/#adding-a-data-volume). This is the default and is easy and fairly transparent to the user. The downside is that the files may be hard to locate for tools and applications that run directly on the host system, i.e. outside containers. -- Create a data directory on the host system (outside the container) and [mount this to a directory visible from inside the container]((https://docs.docker.com/engine/tutorials/dockervolumes/#mount-a-host-directory-as-a-data-volume)). This places the SFTPGo files in a known location on the host system, and makes it easy for tools and applications on the host system to access the files. The downside is that the user needs to make sure that the directory exists, and that e.g. directory permissions and other security mechanisms on the host system are set up correctly. The SFTPGo image runs using `1000` as UID/GID by default. +- Create a data directory on the host system (outside the container) and [mount this to a directory visible from inside the container](https://docs.docker.com/engine/tutorials/dockervolumes/#mount-a-host-directory-as-a-data-volume). This places the SFTPGo files in a known location on the host system, and makes it easy for tools and applications on the host system to access the files. The downside is that the user needs to make sure that the directory exists, and that e.g. directory permissions and other security mechanisms on the host system are set up correctly. The SFTPGo image runs using `1000` as UID/GID by default. The Docker documentation is a good starting point for understanding the different storage options and variations, and there are multiple blogs and forum postings that discuss and give advice in this area. We will simply show the basic procedure here for the latter option above: diff --git a/docs/defender.md b/docs/defender.md index 736989ed..e0221b87 100644 --- a/docs/defender.md +++ b/docs/defender.md @@ -42,31 +42,4 @@ Using the REST API you can: - list hosts within the defender's lists - remove hosts from the defender's lists -The `defender` can also load a permanent block list and/or a safe list of ip addresses/networks from a file: - -- `safelist_file`, defines the path to a file containing a list of ip addresses and/or networks to never ban. -- `blocklist_file`, defines the path to a file containing a list of ip addresses and/or networks to always ban. - -These list must be stored as JSON conforming to the following schema: - -- `addresses`, list of strings. Each string must be a valid IPv4/IPv6 address. -- `networks`, list of strings. Each string must be a valid IPv4/IPv6 CIDR address. - -Here is a small example: - -```json -{ - "addresses":[ - "192.0.2.1", - "2001:db8::68" - ], - "networks":[ - "192.0.3.0/24", - "2001:db8:1234::/48" - ] -} -``` - -Small lists can also be set using the `safelist`/`blocklist` configuration parameters and or using environment variables. These lists will be merged with the ones specified via files, if any, so that you can set both. - -These list will be always loaded in memory (even if you use the `provider` driver) for faster lookups. The REST API queries "live" data and not these lists. +The `defender` can also check permanent block and safe lists of IP addresses/networks. You can define these lists using the WebAdmin UI or the REST API. In multi-nodes setups, the list entries propagation between nodes may take some minutes. diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 0f3d067b..c93e2eb3 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -79,9 +79,9 @@ The configuration file contains the following sections: - `post_connect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post-connect hook](./post-connect-hook.md) for more details. Leave empty to disable - `post_disconnect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post-disconnect hook](./post-disconnect-hook.md) for more details. Leave empty to disable - `data_retention_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Data retention hook](./data-retention-hook.md) for more details. Leave empty to disable - - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited. Default: 0. - - `max_per_host_connections`, integer. Maximum number of concurrent client connections from the same host (IP). If the defender is enabled, exceeding this limit will generate `score_limit_exceeded` events and thus hosts that repeatedly exceed the max allowed connections can be automatically blocked. 0 means unlimited. Default: 20. - - `whitelist_file`, string. Path to a file containing a list of IP addresses and/or networks to allow. Only the listed IPs/networks can access the configured services, all other client connections will be dropped before they even try to authenticate. The whitelist must be a JSON file with the same structure documented for the [defenders's list](./defender.md). The whitelist can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. Default: "". + - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited. Default: `0`. + - `max_per_host_connections`, integer. Maximum number of concurrent client connections from the same host (IP). If the defender is enabled, exceeding this limit will generate `score_limit_exceeded` events and thus hosts that repeatedly exceed the max allowed connections can be automatically blocked. 0 means unlimited. Default: `20`. + - `allowlist_status`, integer. Set to `1` to enable the allow list. The allow list can be populated using the WebAdmin or the REST API. If enabled, only the listed IPs/networks can access the configured services, all other client connections will be dropped before they even try to authenticate. Ensure to populate your allow list before enabling this setting. In multi-nodes setups, the list entries propagation between nodes may take some minutes. Default: `0`. - `allow_self_connections`, integer. Allow users on this instance to use other users/virtual folders on this instance as storage backend. Enable this setting if you know what you are doing. Set to `1` to enable. Default: `0`. - `defender`, struct containing the defender configuration. See [Defender](./defender.md) for more details. - `enabled`, boolean. Default `false`. @@ -96,17 +96,12 @@ The configuration file contains the following sections: - `observation_time`, integer. Defines the time window, in minutes, for tracking client errors. A host is banned if it has exceeded the defined threshold during the last observation time minutes. Default: `30`. - `entries_soft_limit`, integer. Ignored for `provider` driver. Default: `100`. - `entries_hard_limit`, integer. The number of banned IPs and host scores kept in memory will vary between the soft and hard limit for `memory` driver. If you use the `provider` driver, this setting will limit the number of entries to return when you ask for the entire host list from the defender. Default: `150`. - - `safelist_file`, string. Path to a file containing a list of ip addresses and/or networks to never ban. - - `blocklist_file`, string. Path to a file containing a list of ip addresses and/or networks to always ban. The lists can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. An host that is already banned will not be automatically unbanned if you put it inside the safe list, you have to unban it using the REST API. - - `safelist`, list of IP addresses and/or IP ranges and/or networks to never ban. Invalid entries will be silently ignored. For large lists prefer `safelist_file`. `safelist` and `safelist_file` will be merged so that you can set both. - - `blocklist`, list of IP addresses and/or IP ranges and/or networks to always ban. Invalid entries will be silently ignored.. For large lists prefer `blocklist_file`. `blocklist` and `blocklist_file` will be merged so that you can set both. - `rate_limiters`, list of structs containing the rate limiters configuration. Take a look [here](./rate-limiting.md) for more details. Each struct has the following fields: - `average`, integer. Average defines the maximum rate allowed. 0 means disabled. Default: 0 - `period`, integer. Period defines the period as milliseconds. The rate is actually defined by dividing average by period Default: 1000 (1 second). - `burst`, integer. Burst defines the maximum number of requests allowed to go through in the same arbitrarily small period of time. Default: 1 - `type`, integer. 1 means a global rate limiter, independent from the source host. 2 means a per-ip rate limiter. Default: 2 - `protocols`, list of strings. Available protocols are `SSH`, `FTP`, `DAV`, `HTTP`. By default all supported protocols are enabled - - `allow_list`, list of IP addresses and IP ranges excluded from rate limiting. Default: empty - `generate_defender_events`, boolean. If `true`, the defender is enabled, and this is not a global rate limiter, a new defender event will be generated each time the configured limit is exceeded. Default `false` - `entries_soft_limit`, integer. - `entries_hard_limit`, integer. The number of per-ip rate limiters kept in memory will vary between the soft and hard limit @@ -166,7 +161,7 @@ The configuration file contains the following sections: - `certificate_file`, string. Binding specific TLS certificate. This can be an absolute path or a path relative to the config dir. - `certificate_key_file`, string. Binding specific private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If not set the global ones will be used, if any. - `min_tls_version`, integer. Defines the minimum version of TLS to be enabled. `12` means TLS 1.2 (and therefore TLS 1.2 and TLS 1.3 will be enabled),`13` means TLS 1.3. Default: `12`. - - `force_passive_ip`, ip address. External IP address to expose for passive connections. Leave empty to autodetect. If not empty, it must be a valid IPv4 address. Default: "". + - `force_passive_ip`, ip address. External IP address for passive connections. Leave empty to autodetect. If not empty, it must be a valid IPv4 address. Default: "". - `passive_ip_overrides`, list of struct that allows to return a different passive ip based on the client IP address. Each struct has the following fields: - `networks`, list of strings. Each string must define a network in CIDR notation, for example 192.168.1.0/24. - `ip`, string. Passive IP to return if the client IP address belongs to the defined networks. Empty means autodetect. @@ -296,7 +291,7 @@ The configuration file contains the following sections:
HTTP Server -- **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface +- **"httpd"**, the configuration for the HTTP server used to serve REST API and the built-in web interfaces - `bindings`, list of structs. Each struct has the following fields: - `port`, integer. The port used for serving HTTP requests. Default: 8080. - `address`, string. Leave blank to listen on all available network interfaces. On *NIX you can specify an absolute path to listen on a Unix-domain socket Default: blank. @@ -441,7 +436,7 @@ The configuration file contains the following sections: - **mfa**, multi-factor authentication settings - `totp`, list of struct that define settings for time-based one time passwords (RFC 6238). Each struct has the following fields: - - `name`, string. Unique configuration name. This name should not be changed if there are users or admins using the configuration. The name is not exposed to the authentication apps. Default: `Default`. + - `name`, string. Unique configuration name. This name should not be changed if there are users or admins using the configuration. The name is not visible to the authentication apps. Default: `Default`. - `issuer`, string. Name of the issuing Organization/Company. Default: `SFTPGo`. - `algo`, string. Algorithm to use for HMAC. The supported algorithms are: `sha1`, `sha256`, `sha512`. Currently Google Authenticator app on iPhone seems to only support `sha1`, please check the compatibility with your target apps/device before setting a different algorithm. You can also define multiple configurations, for example one that uses `sha256` or `sha512` and another one that uses `sha1` and instruct your users to use the appropriate configuration for their devices/apps. The algorithm should not be changed if there are users or admins using the configuration. Default: `sha1`. @@ -462,7 +457,7 @@ The configuration file contains the following sections:
Plugins -- **plugins**, list of external plugins. :warning: Please note that the plugin system is experimental, the exposed configuration parameters and interfaces may change in a backward incompatible way in future. Each plugin is configured using a struct with the following fields: +- **plugins**, list of external plugins. :warning: Please note that the plugin system is experimental, the configuration parameters and interfaces may change in a backward incompatible way in future. Each plugin is configured using a struct with the following fields: - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`, `metadata`. - `notifier_options`, struct. Defines the options for notifier plugins. - `fs_events`, list of strings. Defines the filesystem events that will be notified to this plugin. @@ -587,7 +582,7 @@ When users log in, if their passwords are stored with anything other than the pr ## Telemetry Server -The telemetry server exposes the following endpoints: +The telemetry server publishes the following endpoints: - `/healthz`, health information (for health checks) - `/metrics`, Prometheus metrics diff --git a/docs/howto/getting-started.md b/docs/howto/getting-started.md index 8a246686..d75eeebb 100644 --- a/docs/howto/getting-started.md +++ b/docs/howto/getting-started.md @@ -2,7 +2,7 @@ SFTPGo allows to securely share your files over SFTP and optionally FTP/S and WebDAV too. Several storage backends are supported and they are configurable per user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. -SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. +SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, a user with the S3 backend mapping a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. In this tutorial we explore the main features and concepts using the built-in web admin interface. Advanced users can also use the SFTPGo [REST API](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml) @@ -11,7 +11,7 @@ In this tutorial we explore the main features and concepts using the built-in we - [Initial configuration](#initial-configuration) - [Creating users](#creating-users) - [Creating users with a Cloud Storage backend](#creating-users-with-a-cloud-storage-backend) - - [Creating users with a local encrypted backend (Data At Rest Encryption)](#creating-users-with-a-local-encrypted-backend-data-at-rest-Encryption) + - [Creating users with a local encrypted backend (Data At Rest Encryption)](#creating-users-with-a-local-encrypted-backend-data-at-rest-encryption) - [Virtual permissions](#virtual-permissions) - [Virtual folders](#virtual-folders) - [Groups](#groups) @@ -202,11 +202,11 @@ Suppose we created two virtual folders name `localfolder` and `minio` as you can Now, click `Users`, on the left menu, select a user and click the `Edit` icon, to update the user and associate the virtual folders. -Virtual folders must be referenced using their unique name and you can expose them on a configurable virtual path. Take a look at the following screenshot. +Virtual folders must be referenced using their unique name and you can map them on a configurable virtual path. Take a look at the following screenshot. ![Virtual Folders](./img/virtual-folders.png) -We exposed the folder named `localfolder` on the path `/vdirlocal` (this must be an absolute UNIX path on Windows too) and the folder named `minio` on the path `/vdirminio`. For `localfolder` the quota usage is included within the user quota, while for the `minio` folder we defined separate quota limits: at most 2 files and at most 100MB, whichever is reached first. +We mapped the folder named `localfolder` on the path `/vdirlocal` (this must be an absolute UNIX path on Windows too) and the folder named `minio` on the path `/vdirminio`. For `localfolder` the quota usage is included within the user quota, while for the `minio` folder we defined separate quota limits: at most 2 files and at most 100MB, whichever is reached first. The folder `minio` can be shared with other users and we can define different quota limits on a per-user basis. The folder `localfolder` is considered private since we have included its quota limits within those of the user, if we share them with other users we will break quota calculation. @@ -621,7 +621,7 @@ Restart SFTPGo to apply the changes. The FTP service is now available on port `2 You can also configure the passive ports range (`50000-50100` by default), these ports must be reachable for passive FTP to work. If your FTP server is on the private network side of a NAT configuration you have to set `force_passive_ip` to your external IP address. You may also need to open the passive port range on your firewall. -It is recommended that you provide a certificate and key file to expose FTP over TLS. You should prefer SFTP to FTP even if you configure TLS, please don't blindly enable the old FTP protocol. +It is recommended that you provide a certificate and key file to allow FTP over TLS. You should prefer SFTP to FTP even if you configure TLS, please don't blindly enable the old FTP protocol. ### Enable WebDAV service @@ -656,4 +656,4 @@ Alternatively (recommended), you can use environment variables by creating the f SFTPGO_WEBDAVD__BINDINGS__0__PORT=10080 ``` -Restart SFTPGo to apply the changes. The WebDAV service is now available on port `10080`. It is recommended that you provide a certificate and key file to expose WebDAV over https. +Restart SFTPGo to apply the changes. The WebDAV service is now available on port `10080`. It is recommended that you provide a certificate and key file to allow WebDAV over https. diff --git a/docs/howto/two-factor-authentication.md b/docs/howto/two-factor-authentication.md index 83995859..601d41e5 100644 --- a/docs/howto/two-factor-authentication.md +++ b/docs/howto/two-factor-authentication.md @@ -29,9 +29,9 @@ Two-factor authentication is enabled by default with the following settings. }, ``` -The `issuer` and `algo` are exposed to the authenticators apps. For example, you could set your company/organization name as `issuer` and an `algo` appropriate for your target apps/devices. The supported algorithms are: `sha1`, `sha256`, `sha512`. Currently Google Authenticator app on iPhone seems to only support `sha1`, please check the compatibility with your target apps/device before setting a different algorithm. +The `issuer` and `algo` are visible/used in the authenticators apps. For example, you could set your company/organization name as `issuer` and an `algo` appropriate for your target apps/devices. The supported algorithms are: `sha1`, `sha256`, `sha512`. Currently Google Authenticator app on iPhone seems to only support `sha1`, please check the compatibility with your target apps/device before setting a different algorithm. -You can also define multiple configurations, for example one that uses `sha256` or `sha512` and another one that uses `sha1` and instruct your users to use the appropriate configuration for their devices/apps. The algorithm should not be changed if there are users or admins using the configuration. The `name` is exposed to the users/admins when they select the 2FA configuration to use and it must be unique. A configuration name should not be changed if there are users or admins using it. +You can also define multiple configurations, for example one that uses `sha256` or `sha512` and another one that uses `sha1` and instruct your users to use the appropriate configuration for their devices/apps. The algorithm should not be changed if there are users or admins using the configuration. The `name` is visible to the users/admins when they select the 2FA configuration to use and it must be unique. A configuration name should not be changed if there are users or admins using it. SFTPGo can use 2FA for `HTTP`, `SSH` (SFTP, SCP) and `FTP` protocols. If you plan to use 2FA with `SSH` you have to enable the keyboard interactive authentication which is disabled by default. diff --git a/docs/metrics.md b/docs/metrics.md index 22d9bf0a..c03680e2 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,6 +1,6 @@ # Metrics -SFTPGo exposes [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint of the telemetry server. +SFTPGo supports [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint of the telemetry server. Several counters and gauges are available, for example: - Total uploads and downloads diff --git a/docs/plugins.md b/docs/plugins.md index f91ccc67..bbea847e 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -18,7 +18,7 @@ The following plugin types are supported: Full configuration details can be found [here](./full-configuration.md). -:warning: Please note that the plugin system is experimental, the exposed configuration parameters and interfaces may change in a backward incompatible way in future. +:warning: Please note that the plugin system is experimental, the configuration parameters and interfaces may change in a backward incompatible way in future. ## Available plugins diff --git a/docs/post-connect-hook.md b/docs/post-connect-hook.md index 076a6598..4fa5b046 100644 --- a/docs/post-connect-hook.md +++ b/docs/post-connect-hook.md @@ -1,6 +1,6 @@ # Post-connect hook -This hook is executed as soon as a new connection is established. It notifies the connection's IP address and protocol. Based on the received response, the connection is accepted or rejected. Combining this hook with the [Post-login hook](./post-login-hook.md) you can implement your own (even for Protocol) blacklist/whitelist of IP addresses. +This hook is executed as soon as a new connection is established. It notifies the connection's IP address and protocol. Based on the received response, the connection is accepted or rejected. Combining this hook with the [Post-login hook](./post-login-hook.md) you can implement your own (even for Protocol) blocklist/allowlist of IP addresses. The `post_connect_hook` can be defined as the absolute path of your program or an HTTP URL. @@ -17,7 +17,7 @@ The program must finish within 20 seconds. If the hook defines an HTTP URL then this URL will be invoked as HTTP GET with the following query parameters: - `ip` -- `protocol`, possible values are `SSH`, `FTP`, `DAV`, `HTTP`, `OIDC` (OpenID Connect) +- `protocol`, possible values are `SSH`, `FTP`, `DAV`, `HTTP`, `HTTPShare`, `OIDC` (OpenID Connect) The connection is accepted if the HTTP response code is `200` otherwise rejected. diff --git a/docs/profiling.md b/docs/profiling.md index 002f151b..4a091cb3 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -3,7 +3,7 @@ The built-in profiler lets you collect CPU profiles, traces, allocations and heap profiles that allow to identify and correct specific bottlenecks. You can enable the built-in profiler using `telemetry` configuration section inside the configuration file. -Profiling data are exposed via HTTP/HTTPS in the format expected by the [pprof](https://github.com/google/pprof/blob/main/doc/README.md) visualization tool. You can find the index page at the URL `/debug/pprof/`. +Profiling data are available via HTTP/HTTPS in the format expected by the [pprof](https://github.com/google/pprof/blob/main/doc/README.md) visualization tool. You can find the index page at the URL `/debug/pprof/`. The following profiles are available, you can obtain them via HTTP GET requests: diff --git a/docs/rate-limiting.md b/docs/rate-limiting.md index 2b077fa4..f7a187f5 100644 --- a/docs/rate-limiting.md +++ b/docs/rate-limiting.md @@ -22,19 +22,7 @@ You can also define two types of rate limiters: If you configure a per-host rate limiter, SFTPGo will keep a rate limiter in memory for each host that connects to the service, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys. -For each rate limiter you can exclude a list of IP addresses and IP ranges by defining an `allow_list`. -The allow list supports IPv4/IPv6 address and CIDR networks, for example: - -```json -... -"allow_list": [ - "192.0.2.1", - "192.168.1.0/24", - "2001:db8::68", - "2001:db8:1234::/48" -], -... -``` +You can exclude a list of IP addresses and IP ranges from rate limiters by adding them to rate limites allow list using the WebAdmin UI or the REST API. In multi-nodes setups, the list entries propagation between nodes may take some minutes. You can defines how many rate limiters as you want, but keep in mind that if you defines multiple rate limiters each request will be checked against all the configured limiters and so it can potentially be delayed multiple times. Let's clarify with an example, here is a configuration that defines a global rate limiter and a per-host rate limiter for the FTP protocol: diff --git a/docs/rest-api.md b/docs/rest-api.md index d5f473e0..0793ada4 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -1,10 +1,10 @@ # REST API -SFTPGo exposes REST API to manage, backup, and restore users and folders, data retention, and to get real time reports of the active connections with the ability to forcibly close a connection. +SFTPGo supports REST API to manage, backup, and restore users and folders, data retention, and to get real time reports of the active connections with the ability to forcibly close a connection. If quota tracking is enabled in the configuration file, then the used size and number of files are updated each time a file is added/removed. If files are added/removed not using SFTP/SCP, or if you change `track_quota` from `2` to `1`, you can rescan the users home dir and update the used quota using the REST API. -REST API are protected using JSON Web Tokens (JWT) authentication and can be exposed over HTTPS. You can also configure client certificate authentication in addition to JWT. +REST API are protected using JSON Web Tokens (JWT) authentication and can be served over HTTPS. You can also configure client certificate authentication in addition to JWT. You can get a JWT token using the `/api/v2/token` endpoint, you need to authenticate using HTTP Basic authentication and the credentials of an active administrator. Here is a sample response: @@ -99,7 +99,7 @@ You can find an example script that shows how to manage data retention [here](.. :warning: Deleting files is an irreversible action, please make sure you fully understand what you are doing before using this feature, you may have users with overlapping home directories or virtual folders shared between multiple users, it is relatively easy to inadvertently delete files you need. -The OpenAPI 3 schema for the exposed API can be found inside the source tree: [openapi.yaml](../openapi/openapi.yaml "OpenAPI 3 specs"). You can render the schema and try the API using the `/openapi` endpoint. SFTPGo uses by default [Swagger UI](https://github.com/swagger-api/swagger-ui), you can use another renderer just by copying it to the defined OpenAPI path. +The OpenAPI 3 schema for the supported APIs can be found inside the source tree: [openapi.yaml](../openapi/openapi.yaml "OpenAPI 3 specs"). You can render the schema and try the API using the `/openapi` endpoint. SFTPGo uses by default [Swagger UI](https://github.com/swagger-api/swagger-ui), you can use another renderer just by copying it to the defined OpenAPI path. You can also explore the schema on [Stoplight](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml). diff --git a/docs/virtual-folders.md b/docs/virtual-folders.md index 9010597c..4a6c2024 100644 --- a/docs/virtual-folders.md +++ b/docs/virtual-folders.md @@ -8,9 +8,9 @@ SFTPGo will try to automatically create any missing parent directory for the con For each virtual folder, the following properties can be configured: -- `folder_name`, is the ID for an existing folder. The folder structure contains the absolute filesystem path to expose as virtual folder +- `folder_name`, is the ID for an existing folder. The folder structure contains the absolute filesystem path to map as virtual folder - `filesystem`, this way you can map a local path or a Cloud backend to mount as virtual folders -- `virtual_path`, the SFTPGo absolute path to use to expose the mapped path +- `virtual_path`, absolute path seen by SFTPGo users where the mapped path is accessible - `quota_size`, maximum size allowed as bytes. 0 means unlimited, -1 included in user quota - `quota_files`, maximum number of files allowed. 0 means unlimited, -1 included in user quota diff --git a/docs/web-admin.md b/docs/web-admin.md index 8ceab140..5a91e765 100644 --- a/docs/web-admin.md +++ b/docs/web-admin.md @@ -1,10 +1,10 @@ # Web Admin -You can easily build your own interface using the exposed [REST API](./rest-api.md). Anyway, SFTPGo also provides a basic built-in web interface that allows you to manage users, virtual folders, admins and connections. +You can easily build your own interface using the SFTPGo [REST API](./rest-api.md). Anyway, SFTPGo also provides a basic built-in web interface that allows you to manage users, virtual folders, admins and connections. With the default `httpd` configuration, the web admin is available at the following URL: [http://127.0.0.1:8080/web/admin](http://127.0.0.1:8080/web/admin) If no admin user is found within the data provider, typically after the initial installation, SFTPGo will ask you to create the first admin. You can also pre-create an admin user by loading initial data or by enabling the `create_default_admin` configuration key. Please take a look [here](./full-configuration.md) for more details. -The web interface can be exposed via HTTPS and may require mutual TLS authentication in addition to administrator credentials. +The web interface can be configured over HTTPS and to require mutual TLS authentication in addition to administrator credentials. diff --git a/go.mod b/go.mod index f75aaf7d..adee4dfa 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,22 @@ module github.com/drakkan/sftpgo/v2 -go 1.19 +go 1.20 require ( cloud.google.com/go/storage v1.29.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 - github.com/aws/aws-sdk-go-v2 v1.17.3 - github.com/aws/aws-sdk-go-v2/config v1.18.10 - github.com/aws/aws-sdk-go-v2/credentials v1.13.10 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.49 - github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.1 - github.com/aws/aws-sdk-go-v2/service/s3 v1.30.1 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.18.2 + github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2/config v1.18.12 + github.com/aws/aws-sdk-go-v2/credentials v1.13.12 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 + github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.3 + github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/cockroachdb/cockroach-go/v2 v2.2.20 github.com/coreos/go-oidc/v3 v3.5.0 @@ -53,28 +53,28 @@ require ( github.com/rs/xid v1.4.0 github.com/rs/zerolog v1.29.0 github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0 - github.com/shirou/gopsutil/v3 v3.22.12 + github.com/shirou/gopsutil/v3 v3.23.1 github.com/spf13/afero v1.9.3 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 - github.com/studio-b12/gowebdav v0.0.0-20221109171924-60ec5ad56012 + github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 github.com/subosito/gotenv v1.4.2 github.com/unrolled/secure v1.13.0 github.com/wagslane/go-password-validator v0.3.0 github.com/wneessen/go-mail v0.3.8 github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a - go.etcd.io/bbolt v1.3.6 + go.etcd.io/bbolt v1.3.7 go.uber.org/automaxprocs v1.5.1 gocloud.dev v0.28.0 golang.org/x/crypto v0.5.0 golang.org/x/net v0.5.0 golang.org/x/oauth2 v0.4.0 - golang.org/x/sys v0.4.0 - golang.org/x/term v0.4.0 + golang.org/x/sys v0.5.0 + golang.org/x/term v0.5.0 golang.org/x/time v0.3.0 - google.golang.org/api v0.108.0 - gopkg.in/natefinch/lumberjack.v2 v2.0.0 + google.golang.org/api v0.109.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( @@ -85,16 +85,16 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/ajg/form v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect @@ -152,13 +152,13 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa // indirect - google.golang.org/grpc v1.52.3 // indirect + google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 // indirect + google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4bfab52c..b97adec3 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+Q github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= @@ -453,7 +453,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9s github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= @@ -533,68 +532,68 @@ github.com/aws/aws-sdk-go v1.44.128/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX github.com/aws/aws-sdk-go v1.44.151/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= -github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= -github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= +github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9/go.mod h1:vCmV1q1VK8eoQJ5+aYE7PkK1K6v41qJ5pJdK3ggCDvg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= -github.com/aws/aws-sdk-go-v2/config v1.18.10 h1:Znce11DWswdh+5kOsIp+QaNfY9igp1QUN+fZHCKmeCI= -github.com/aws/aws-sdk-go-v2/config v1.18.10/go.mod h1:VATKco+pl+Qe1WW+RzvZTlPPe/09Gg9+vM0ZXsqb16k= +github.com/aws/aws-sdk-go-v2/config v1.18.12 h1:fKs/I4wccmfrNRO9rdrbMO1NgLxct6H9rNMiPdBxHWw= +github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.10 h1:T4Y39IhelTLg1f3xiKJssThnFxsndS8B6OnmcXtKK+8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.10/go.mod h1:tqAm4JmQaShel+Qi38hmd1QglSnnxaYt50k/9yGQzzc= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12 h1:Cb+HhuEnV19zHRaYYVglwvdHGMJWbdsyP4oHhw04xws= +github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 h1:3aMfcTmoXtTZnaT86QlVaYh+BRMbvrrmZwIQ5jWqCZQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42/go.mod h1:LHOsygMiW/14CkFxdXxvzKyMh3jbk/QfZVaDtCbLkl8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.49 h1:zPFhadkmXbXu3RVXTPU4HVW+g2DStMY+01cJaj//+Cw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.49/go.mod h1:N9gSChQkKpdAj7vRpfKma4ND88zoZM+v6W2lJgWrDh4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 h1:iTFYCAdKzSAjGnVIUe88Hxvix0uaBqr0Rv7qJEOX5hE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51/go.mod h1:7Grl2gV+dx9SWrUIgwwlUvU40t7+lOSbx34XwfmsTkY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 h1:r+XwaCLpIvCKjBIYy/HVZujQS9tsz5ohHG3ZIe0wKoE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKFmWoqpcOQJ4bH634SkYf3FNj/A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 h1:J4xhFd6zHhdF9jPP0FQJ6WknzBboGMBNjKOv4iTuw4A= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16/go.mod h1:XH+3h395e3WVdd6T2Z3mPxuI+x/HVtdqVOREkTiyubs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18/go.mod h1:T2Ku+STrYQ1zIkL1wMvj8P3wWQaaCMKNdz70MT2FLfE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 h1:FGvpyTg2LKEmMrLlpjOgkoNp9XF5CGeyAyo33LdqZW8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19/go.mod h1:8W88sW3PjamQpKFUQvHWWKay6ARsNvZnzU7+a4apubw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20/go.mod h1:Mp4XI/CkWGD79AQxZ5lIFlgvC0A+gl+4BmyG1F+SfNc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 h1:kv5vRAl00tozRxSnI0IszPWGXsJOyA7hmEUHFYqsyvw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22/go.mod h1:Od+GU5+Yx41gryN/ZGZzAJMZ9R1yn6lgA0fD5Lo5SkQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 h1:c5+bNdV8E4fIPteWx4HZSkqI07oY9exbfQ7JH7Yx4PI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23/go.mod h1:1jcUfF+FAOEwtIcNiHPaV4TSoZqkUIPzrohmD7fb95c= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 h1:LjFQf8hFuMO22HkV5VWGLBvmCLBCLPivUAmpdpnp4Vs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19/go.mod h1:BmQWRVkLTmyNzYPFAZgon53qKLWBNSvonugD1MrSWUs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 h1:vY5siRXvW5TrOKm2qKEf9tliBfdLxdfy0i02LOcmqUo= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21/go.mod h1:WZvNXT1XuH8dnJM0HvOlvk+RNn7NbAPvA/ACO0QarSc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 h1:ISLJ2BKXe4zzyZ7mp5ewKECiw0U7KpLgS3S6OxY9Cm0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22/go.mod h1:QFVbqK54XArazLvn2wvWMRBi/jGrWii46qbr5DyPGjc= github.com/aws/aws-sdk-go-v2/service/kms v1.19.0/go.mod h1:kZodDPTQjSH/qM6/OvyTfM5mms5JHB/EKYp5dhn/vI4= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.1 h1:IOjpqwEHMYPVfiqnH/auHvhz69/SGHYo/tFBkax5O0o= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.1/go.mod h1:DSuypbY6jb7WZSxrLuCgd7ouB5uRQ+Hg5wbt0GmgRcc= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.2 h1:7vuSkPqVqwBwSV0OJD71qqWOEFr3Hh1K0e2yOQ/JWwQ= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.2/go.mod h1:vrZVsmrC7QRNBK/W8nplI0tfJDvMl6DZAUT/pkFJiws= github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4/go.mod h1:/NHbqPRiwxSPVOB2Xr+StDEH+GWV/64WwnUjv4KYzV0= -github.com/aws/aws-sdk-go-v2/service/s3 v1.30.1 h1:kIgvVY7PHx4gIb0na/Q9gTWJWauTwhKdaqJjX8PkIY8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.30.1/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 h1:5EQWIFO+Hc8E2hFcXQJ1vm6ufl/PMt/6RVRDZRju2vM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2/go.mod h1:SXDHd6fI2RhqB7vmAzyYQCTQnpZrIprVJvYxpzW3JAM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.8/go.mod h1:k6CPuxyzO247nYEM1baEwHH1kRtosRCvgahAepaaShw= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2 h1:QDVKb2VpuwzIslzshumxksayV5GkpqT+rkVvdPVrA9E= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.3 h1:Zod/h9QcDvbrrG3jjTUp4lctRb6Qg2nj7ARC/xMsUc4= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.3/go.mod h1:hqPcyOuLU6yWIbLy3qMnQnmidgKuIEwqIlW6+chYnog= github.com/aws/aws-sdk-go-v2/service/sns v1.18.6/go.mod h1:2cPUjR63iE9MPMPJtSyzYmsTFCNrN/Xi9j0v9BL5OU0= github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15/go.mod h1:DKX/7/ZiAzHO6p6AhArnGdrV4r+d461weby8KeVtvC4= github.com/aws/aws-sdk-go-v2/service/ssm v1.33.1/go.mod h1:rEsqsZrOp9YvSGPOrcL3pR9+i/QJaWRkAYbuxMa7yCU= github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 h1:/2gzjhQowRLarkkBOGPXSRnb8sQ2RVsjdG1C/UliK/c= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.0/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 h1:lQKN/LNa3qqu2cDOQZybP7oL4nMGGiFqob0jZJaR8/4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 h1:Jfly6mRxk2ZOSlbCvZfKNS7TukSx1mIzhSsqZ/IGSZI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 h1:0bLhH6DRAqox+g0LatcjGKjjhU6Eudyys6HB6DJVPj8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1/go.mod h1:O1YSOg3aekZibh2SngvCRRG+cRHKKlYgxf/JBF/Kr/k= github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.2 h1:J/4wIaGInCEYCGhTSruxCxeoA5cy91a+JT7cHFKFSHQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.2/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 h1:s49mSnsBZEXjfGBkRfmK+nPqzT7Lt3+t2SmAKNyHblw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiOVVG8uVjGI6HaZ8WBHdgDgU= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -1802,8 +1801,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0 h1:e1OQroqX8SWV06Z270CxG2/v//Wx1026iXKTDRn5J1E= github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0/go.mod h1:3GpW3Qy8IHH6kex0ny+Y6ayeYb9OJxz8Pxh3IZgAs2E= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4= +github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA= github.com/shoenig/test v0.4.3/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1880,8 +1879,8 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/studio-b12/gowebdav v0.0.0-20221109171924-60ec5ad56012 h1:ZC+dlnsjxqrcB68nEFbIEfo4iXsog3Sg8FlXKytAjhY= -github.com/studio-b12/gowebdav v0.0.0-20221109171924-60ec5ad56012/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= +github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 h1:VsBj3UD2xyAOu7kJw6O/2jjG2UXLFoBzihqDU9Ofg9M= +github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -1954,8 +1953,9 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -2095,8 +2095,8 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2384,8 +2384,9 @@ golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2393,8 +2394,9 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2407,8 +2409,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2576,8 +2579,8 @@ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91 google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= -google.golang.org/api v0.108.0 h1:WVBc/faN0DkKtR43Q/7+tPny9ZoLZdIiAyG5Q9vFClg= -google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.109.0 h1:sW9hgHyX497PP5//NUM7nqfV8D0iDfBApqq7sOh1XR8= +google.golang.org/api v0.109.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2711,8 +2714,8 @@ google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa h1:GZXdWYIKckxQE2EcLHLvF+KLF+bIwoxGdMUxTZizueg= -google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 h1:vArvWooPH749rNHpBGgVl+U9B9dATjiEhJzcWGlovNs= +google.golang.org/genproto v0.0.0-20230202175211-008b39050e57/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2758,8 +2761,8 @@ google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= -google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2798,8 +2801,9 @@ gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/internal/cmd/startsubsys.go b/internal/cmd/startsubsys.go index 1e3018ba..1f1a21a6 100644 --- a/internal/cmd/startsubsys.go +++ b/internal/cmd/startsubsys.go @@ -85,15 +85,6 @@ Command-line flags should be specified in the Subsystem declaration. logger.Error(logSender, connectionID, "unable to load configuration: %v", err) os.Exit(1) } - dataProviderConf := config.GetProviderConf() - commonConfig := config.GetCommonConfig() - // idle connection are managed externally - commonConfig.IdleTimeout = 0 - config.SetCommonConfig(commonConfig) - if err := common.Initialize(config.GetCommonConfig(), dataProviderConf.GetShared()); err != nil { - logger.Error(logSender, connectionID, "%v", err) - os.Exit(1) - } kmsConfig := config.GetKMSConfig() if err := kmsConfig.Initialize(); err != nil { logger.Error(logSender, connectionID, "unable to initialize KMS: %v", err) @@ -115,8 +106,9 @@ Command-line flags should be specified in the Subsystem declaration. logger.Error(logSender, connectionID, "unable to initialize SMTP configuration: %v", err) os.Exit(1) } + dataProviderConf := config.GetProviderConf() if dataProviderConf.Driver == dataprovider.SQLiteDataProviderName || dataProviderConf.Driver == dataprovider.BoltDataProviderName { - logger.Debug(logSender, connectionID, "data provider %#v not supported in subsystem mode, using %#v provider", + logger.Debug(logSender, connectionID, "data provider %q not supported in subsystem mode, using %q provider", dataProviderConf.Driver, dataprovider.MemoryDataProviderName) dataProviderConf.Driver = dataprovider.MemoryDataProviderName dataProviderConf.Name = "" @@ -127,6 +119,14 @@ Command-line flags should be specified in the Subsystem declaration. logger.Error(logSender, connectionID, "unable to initialize the data provider: %v", err) os.Exit(1) } + commonConfig := config.GetCommonConfig() + // idle connection are managed externally + commonConfig.IdleTimeout = 0 + config.SetCommonConfig(commonConfig) + if err := common.Initialize(config.GetCommonConfig(), dataProviderConf.GetShared()); err != nil { + logger.Error(logSender, connectionID, "%v", err) + os.Exit(1) + } httpConfig := config.GetHTTPConfig() if err := httpConfig.Initialize(configDir); err != nil { logger.Error(logSender, connectionID, "unable to initialize http client: %v", err) diff --git a/internal/common/common.go b/internal/common/common.go index a5ece406..d846ace3 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -172,24 +172,27 @@ func Initialize(c Configuration, isShared int) error { Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute startPeriodicChecks(periodicTimeoutCheckInterval) Config.defender = nil - Config.whitelist = nil + Config.allowList = nil + Config.rateLimitersList = nil rateLimiters = make(map[string][]*rateLimiter) for _, rlCfg := range c.RateLimitersConfig { if rlCfg.isEnabled() { if err := rlCfg.validate(); err != nil { return fmt.Errorf("rate limiters initialization error: %w", err) } - allowList, err := util.ParseAllowedIPAndRanges(rlCfg.AllowList) - if err != nil { - return fmt.Errorf("unable to parse rate limiter allow list %v: %v", rlCfg.AllowList, err) - } rateLimiter := rlCfg.getLimiter() - rateLimiter.allowList = allowList for _, protocol := range rlCfg.Protocols { rateLimiters[protocol] = append(rateLimiters[protocol], rateLimiter) } } } + if len(rateLimiters) > 0 { + rateLimitersList, err := dataprovider.NewIPList(dataprovider.IPListTypeRateLimiterSafeList) + if err != nil { + return fmt.Errorf("unable to initialize ratelimiters list: %w", err) + } + Config.rateLimitersList = rateLimitersList + } if c.DefenderConfig.Enabled { if !util.Contains(supportedDefenderDrivers, c.DefenderConfig.Driver) { return fmt.Errorf("unsupported defender driver %q", c.DefenderConfig.Driver) @@ -208,15 +211,13 @@ func Initialize(c Configuration, isShared int) error { logger.Info(logSender, "", "defender initialized with config %+v", c.DefenderConfig) Config.defender = defender } - if c.WhiteListFile != "" { - whitelist := &whitelist{ - fileName: c.WhiteListFile, + if c.AllowListStatus > 0 { + allowList, err := dataprovider.NewIPList(dataprovider.IPListTypeAllowList) + if err != nil { + return fmt.Errorf("unable to initialize the allow list: %w", err) } - if err := whitelist.reload(); err != nil { - return fmt.Errorf("whitelist initialization error: %w", err) - } - logger.Info(logSender, "", "whitelist initialized from file: %#v", c.WhiteListFile) - Config.whitelist = whitelist + logger.Info(logSender, "", "allow list initialized") + Config.allowList = allowList } vfs.SetTempPath(c.TempPath) dataprovider.SetTempPath(c.TempPath) @@ -293,9 +294,15 @@ func getActiveConnections() int { // It returns an error if the time to wait exceeds the max // allowed delay func LimitRate(protocol, ip string) (time.Duration, error) { + if Config.rateLimitersList != nil { + isListed, _, err := Config.rateLimitersList.IsListed(ip, protocol) + if err == nil && isListed { + return 0, nil + } + } for _, limiter := range rateLimiters[protocol] { - if delay, err := limiter.Wait(ip); err != nil { - logger.Debug(logSender, "", "protocol %v ip %v: %v", protocol, ip, err) + if delay, err := limiter.Wait(ip, protocol); err != nil { + logger.Debug(logSender, "", "protocol %s ip %s: %v", protocol, ip, err) return delay, err } } @@ -305,21 +312,11 @@ func LimitRate(protocol, ip string) (time.Duration, error) { // Reload reloads the whitelist, the IP filter plugin and the defender's block and safe lists func Reload() error { plugin.Handler.ReloadFilter() - var errWithelist error - if Config.whitelist != nil { - errWithelist = Config.whitelist.reload() - } - if Config.defender == nil { - return errWithelist - } - if err := Config.defender.Reload(); err != nil { - return err - } - return errWithelist + return nil } // IsBanned returns true if the specified IP address is banned -func IsBanned(ip string) bool { +func IsBanned(ip, protocol string) bool { if plugin.Handler.IsIPBanned(ip) { return true } @@ -327,7 +324,7 @@ func IsBanned(ip string) bool { return false } - return Config.defender.IsBanned(ip) + return Config.defender.IsBanned(ip, protocol) } // GetDefenderBanTime returns the ban time for the given IP @@ -377,12 +374,12 @@ func GetDefenderScore(ip string) (int, error) { } // AddDefenderEvent adds the specified defender event for the given IP -func AddDefenderEvent(ip string, event HostEvent) { +func AddDefenderEvent(ip, protocol string, event HostEvent) { if Config.defender == nil { return } - Config.defender.AddEvent(ip, event) + Config.defender.AddEvent(ip, protocol, event) } func startPeriodicChecks(duration time.Duration) { @@ -449,7 +446,7 @@ type StatAttributes struct { Size int64 } -// ConnectionTransfer defines the trasfer details to expose +// ConnectionTransfer defines the trasfer details type ConnectionTransfer struct { ID int64 `json:"-"` OperationType string `json:"operation_type"` @@ -479,35 +476,6 @@ func (t *ConnectionTransfer) getConnectionTransferAsString() string { return result } -type whitelist struct { - fileName string - sync.RWMutex - list HostList -} - -func (l *whitelist) reload() error { - list, err := loadHostListFromFile(l.fileName) - if err != nil { - return err - } - if list == nil { - return errors.New("cannot accept a nil whitelist") - } - - l.Lock() - defer l.Unlock() - - l.list = *list - return nil -} - -func (l *whitelist) isAllowed(ip string) bool { - l.RLock() - defer l.RUnlock() - - return l.list.isListed(ip) -} - // Configuration defines configuration parameters common to all supported protocols type Configuration struct { // Maximum idle timeout as minutes. If a client is idle for a time that exceeds this setting it will be disconnected. @@ -578,10 +546,11 @@ type Configuration struct { MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"` // Maximum number of concurrent client connections from the same host (IP). 0 means unlimited MaxPerHostConnections int `json:"max_per_host_connections" mapstructure:"max_per_host_connections"` - // Path to a file containing a list of IP addresses and/or networks to allow. - // Only the listed IPs/networks can access the configured services, all other client connections - // will be dropped before they even try to authenticate. - WhiteListFile string `json:"whitelist_file" mapstructure:"whitelist_file"` + // Defines the status of the global allow list. 0 means disabled, 1 enabled. + // If enabled, only the listed IPs/networks can access the configured services, all other + // client connections will be dropped before they even try to authenticate. + // Ensure to enable this setting only after adding some allowed ip/networks from the WebAdmin/REST API + AllowListStatus int `json:"allowlist_status" mapstructure:"allowlist_status"` // Allow users on this instance to use other users/virtual folders on this instance as storage backend. // Enable this setting if you know what you are doing. AllowSelfConnections int `json:"allow_self_connections" mapstructure:"allow_self_connections"` @@ -592,7 +561,8 @@ type Configuration struct { idleTimeoutAsDuration time.Duration idleLoginTimeout time.Duration defender Defender - whitelist *whitelist + allowList *dataprovider.IPList + rateLimitersList *dataprovider.IPList } // IsAtomicUploadEnabled returns true if atomic upload is enabled @@ -633,6 +603,24 @@ func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Lis return nil, errors.New("proxy protocol not configured") } +// GetRateLimitersStatus returns the rate limiters status +func (c *Configuration) GetRateLimitersStatus() (bool, []string) { + enabled := false + var protocols []string + for _, rlCfg := range c.RateLimitersConfig { + if rlCfg.isEnabled() { + enabled = true + protocols = append(protocols, rlCfg.Protocols...) + } + } + return enabled, util.RemoveDuplicates(protocols, false) +} + +// IsAllowListEnabled returns true if the global allow list is enabled +func (c *Configuration) IsAllowListEnabled() bool { + return c.AllowListStatus > 0 +} + // ExecuteStartupHook runs the startup hook if defined func (c *Configuration) ExecuteStartupHook() error { if c.StartupHook == "" { @@ -941,7 +929,7 @@ func (conns *ActiveConnections) Remove(connectionID string) { logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, conn.GetProtocol(), dataprovider.ErrNoAuthTryed.Error()) metric.AddNoAuthTryed() - AddDefenderEvent(ip, HostEventNoLoginTried) + AddDefenderEvent(ip, ProtocolFTP, HostEventNoLoginTried) dataprovider.ExecutePostLoginHook(&dataprovider.User{}, dataprovider.LoginMethodNoAuthTryed, ip, conn.GetProtocol(), dataprovider.ErrNoAuthTryed) } @@ -1130,12 +1118,18 @@ func (conns *ActiveConnections) GetClientConnections() int32 { // IsNewConnectionAllowed returns an error if the maximum number of concurrent allowed // connections is exceeded or a whitelist is defined and the specified ipAddr is not listed // or the service is shutting down -func (conns *ActiveConnections) IsNewConnectionAllowed(ipAddr string) error { +func (conns *ActiveConnections) IsNewConnectionAllowed(ipAddr, protocol string) error { if isShuttingDown.Load() { return ErrShuttingDown } - if Config.whitelist != nil { - if !Config.whitelist.isAllowed(ipAddr) { + if Config.allowList != nil { + isListed, _, err := Config.allowList.IsListed(ipAddr, protocol) + if err != nil { + logger.Error(logSender, "", "unable to query allow list, connection denied, ip %q, protocol %s, err: %v", + ipAddr, protocol, err) + return ErrConnectionDenied + } + if !isListed { return ErrConnectionDenied } } @@ -1146,7 +1140,7 @@ func (conns *ActiveConnections) IsNewConnectionAllowed(ipAddr string) error { if Config.MaxPerHostConnections > 0 { if total := conns.clients.getTotalFrom(ipAddr); total > Config.MaxPerHostConnections { logger.Info(logSender, "", "active connections from %s %d/%d", ipAddr, total, Config.MaxPerHostConnections) - AddDefenderEvent(ipAddr, HostEventLimitExceeded) + AddDefenderEvent(ipAddr, protocol, HostEventLimitExceeded) return ErrConnectionDenied } } diff --git a/internal/common/common_test.go b/internal/common/common_test.go index 42c01136..3bd2f6ab 100644 --- a/internal/common/common_test.go +++ b/internal/common/common_test.go @@ -215,6 +215,66 @@ func TestConnections(t *testing.T) { Connections.RUnlock() } +func TestInitializationClosedProvider(t *testing.T) { + configCopy := Config + + providerConf := dataprovider.GetProviderConfig() + err := dataprovider.Close() + assert.NoError(t, err) + + config := Configuration{ + AllowListStatus: 1, + } + err = Initialize(config, 0) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "unable to initialize the allow list") + } + + config.AllowListStatus = 0 + config.RateLimitersConfig = []RateLimiterConfig{ + { + Average: 100, + Period: 1000, + Burst: 5, + Type: int(rateLimiterTypeGlobal), + Protocols: rateLimiterProtocolValues, + }, + } + err = Initialize(config, 0) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "unable to initialize ratelimiters list") + } + + config.RateLimitersConfig = nil + config.DefenderConfig = DefenderConfig{ + Enabled: true, + Driver: DefenderDriverProvider, + BanTime: 10, + BanTimeIncrement: 50, + Threshold: 10, + ScoreInvalid: 2, + ScoreValid: 1, + ScoreNoAuth: 2, + ObservationTime: 15, + EntriesSoftLimit: 100, + EntriesHardLimit: 150, + } + err = Initialize(config, 0) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "defender initialization error") + } + config.DefenderConfig.Driver = DefenderDriverMemory + err = Initialize(config, 0) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "defender initialization error") + } + + err = dataprovider.Initialize(providerConf, configDir, true) + assert.NoError(t, err) + + Config = configCopy +} + func TestSSHConnections(t *testing.T) { conn1, conn2 := net.Pipe() now := time.Now() @@ -298,10 +358,10 @@ func TestDefenderIntegration(t *testing.T) { assert.Nil(t, Reload()) // 192.168.1.12 is banned from the ipfilter plugin - assert.True(t, IsBanned("192.168.1.12")) + assert.True(t, IsBanned("192.168.1.12", ProtocolFTP)) - AddDefenderEvent(ip, HostEventNoLoginTried) - assert.False(t, IsBanned(ip)) + AddDefenderEvent(ip, ProtocolFTP, HostEventNoLoginTried) + assert.False(t, IsBanned(ip, ProtocolFTP)) banTime, err := GetDefenderBanTime(ip) assert.NoError(t, err) @@ -342,21 +402,13 @@ func TestDefenderIntegration(t *testing.T) { // ScoreInvalid cannot be greater than threshold assert.Error(t, err) Config.DefenderConfig.Threshold = 3 - Config.DefenderConfig.SafeListFile = filepath.Join(os.TempDir(), "sl.json") - err = os.WriteFile(Config.DefenderConfig.SafeListFile, []byte(`{}`), 0644) - assert.NoError(t, err) - defer os.Remove(Config.DefenderConfig.SafeListFile) err = Initialize(Config, 0) assert.NoError(t, err) assert.Nil(t, Reload()) - err = os.WriteFile(Config.DefenderConfig.SafeListFile, []byte(`{`), 0644) - assert.NoError(t, err) - err = Reload() - assert.Error(t, err) - AddDefenderEvent(ip, HostEventNoLoginTried) - assert.False(t, IsBanned(ip)) + AddDefenderEvent(ip, ProtocolSSH, HostEventNoLoginTried) + assert.False(t, IsBanned(ip, ProtocolSSH)) score, err = GetDefenderScore(ip) assert.NoError(t, err) assert.Equal(t, 2, score) @@ -370,9 +422,9 @@ func TestDefenderIntegration(t *testing.T) { assert.NoError(t, err) assert.Nil(t, banTime) - AddDefenderEvent(ip, HostEventLoginFailed) - AddDefenderEvent(ip, HostEventNoLoginTried) - assert.True(t, IsBanned(ip)) + AddDefenderEvent(ip, ProtocolHTTP, HostEventLoginFailed) + AddDefenderEvent(ip, ProtocolHTTP, HostEventNoLoginTried) + assert.True(t, IsBanned(ip, ProtocolHTTP)) score, err = GetDefenderScore(ip) assert.NoError(t, err) assert.Equal(t, 0, score) @@ -398,9 +450,31 @@ func TestDefenderIntegration(t *testing.T) { } func TestRateLimitersIntegration(t *testing.T) { - // by default defender is nil configCopy := Config + enabled, protocols := Config.GetRateLimitersStatus() + assert.False(t, enabled) + assert.Len(t, protocols, 0) + + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "172.16.24.7/32", + Type: dataprovider.IPListTypeRateLimiterSafeList, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "172.16.0.0/16", + Type: dataprovider.IPListTypeRateLimiterSafeList, + Mode: dataprovider.ListModeAllow, + }, + } + + for idx := range entries { + e := entries[idx] + err := dataprovider.AddIPListEntry(&e, "", "", "") + assert.NoError(t, err) + } + Config.RateLimitersConfig = []RateLimiterConfig{ { Average: 100, @@ -423,16 +497,10 @@ func TestRateLimitersIntegration(t *testing.T) { err := Initialize(Config, 0) assert.Error(t, err) Config.RateLimitersConfig[0].Period = 1000 - Config.RateLimitersConfig[0].AllowList = []string{"1.1.1", "1.1.1.2"} - err = Initialize(Config, 0) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "unable to parse rate limiter allow list") - } - Config.RateLimitersConfig[0].AllowList = []string{"172.16.24.7"} - Config.RateLimitersConfig[1].AllowList = []string{"172.16.0.0/16"} err = Initialize(Config, 0) assert.NoError(t, err) + assert.NotNil(t, Config.rateLimitersList) assert.Len(t, rateLimiters, 4) assert.Len(t, rateLimiters[ProtocolSSH], 1) @@ -440,9 +508,17 @@ func TestRateLimitersIntegration(t *testing.T) { assert.Len(t, rateLimiters[ProtocolWebDAV], 2) assert.Len(t, rateLimiters[ProtocolHTTP], 1) + enabled, protocols = Config.GetRateLimitersStatus() + assert.True(t, enabled) + assert.Len(t, protocols, 4) + assert.Contains(t, protocols, ProtocolFTP) + assert.Contains(t, protocols, ProtocolSSH) + assert.Contains(t, protocols, ProtocolHTTP) + assert.Contains(t, protocols, ProtocolWebDAV) + source1 := "127.1.1.1" source2 := "127.1.1.2" - source3 := "172.16.24.7" // whitelisted + source3 := "172.16.24.7" // in safelist _, err = LimitRate(ProtocolSSH, source1) assert.NoError(t, err) @@ -465,59 +541,12 @@ func TestRateLimitersIntegration(t *testing.T) { _, err = LimitRate(ProtocolWebDAV, source3) assert.NoError(t, err) } - - Config = configCopy -} - -func TestWhitelist(t *testing.T) { - configCopy := Config - - Config.whitelist = &whitelist{} - err := Config.whitelist.reload() - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "cannot accept a nil whitelist") + for _, e := range entries { + err := dataprovider.DeleteIPListEntry(e.IPOrNet, e.Type, "", "", "") + assert.NoError(t, err) } - wlFile := filepath.Join(os.TempDir(), "wl.json") - Config.WhiteListFile = wlFile - - err = os.WriteFile(wlFile, []byte(`invalid list file`), 0664) - assert.NoError(t, err) - err = Initialize(Config, 0) - assert.Error(t, err) - - wl := HostListFile{ - IPAddresses: []string{"172.18.1.1", "172.18.1.2"}, - CIDRNetworks: []string{"10.8.7.0/24"}, - } - data, err := json.Marshal(wl) - assert.NoError(t, err) - err = os.WriteFile(wlFile, data, 0664) - assert.NoError(t, err) - defer os.Remove(wlFile) - - err = Initialize(Config, 0) - assert.NoError(t, err) - - assert.NoError(t, Connections.IsNewConnectionAllowed("172.18.1.1")) - assert.Error(t, Connections.IsNewConnectionAllowed("172.18.1.3")) - assert.NoError(t, Connections.IsNewConnectionAllowed("10.8.7.3")) - assert.Error(t, Connections.IsNewConnectionAllowed("10.8.8.2")) - - wl.IPAddresses = append(wl.IPAddresses, "172.18.1.3") - wl.CIDRNetworks = append(wl.CIDRNetworks, "10.8.8.0/24") - data, err = json.Marshal(wl) - assert.NoError(t, err) - err = os.WriteFile(wlFile, data, 0664) - assert.NoError(t, err) - assert.Error(t, Connections.IsNewConnectionAllowed("10.8.8.3")) - - err = Reload() - assert.NoError(t, err) - assert.NoError(t, Connections.IsNewConnectionAllowed("10.8.8.3")) - assert.NoError(t, Connections.IsNewConnectionAllowed("172.18.1.3")) - assert.NoError(t, Connections.IsNewConnectionAllowed("172.18.1.2")) - assert.Error(t, Connections.IsNewConnectionAllowed("172.18.1.12")) + assert.Nil(t, configCopy.rateLimitersList) Config = configCopy } @@ -551,12 +580,12 @@ func TestMaxConnections(t *testing.T) { Config.MaxPerHostConnections = 0 ipAddr := "192.168.7.8" - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolFTP)) Config.MaxTotalConnections = 1 Config.MaxPerHostConnections = perHost - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolHTTP)) c := NewBaseConnection("id", ProtocolSFTP, "", "", dataprovider.User{}) fakeConn := &fakeConnection{ BaseConnection: c, @@ -564,18 +593,18 @@ func TestMaxConnections(t *testing.T) { err := Connections.Add(fakeConn) assert.NoError(t, err) assert.Len(t, Connections.GetStats(""), 1) - assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolSSH)) res := Connections.Close(fakeConn.GetID(), "") assert.True(t, res) assert.Eventually(t, func() bool { return len(Connections.GetStats("")) == 0 }, 300*time.Millisecond, 50*time.Millisecond) - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolSSH)) Connections.AddClientConnection(ipAddr) Connections.AddClientConnection(ipAddr) - assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolSSH)) Connections.RemoveClientConnection(ipAddr) - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolWebDAV)) Connections.RemoveClientConnection(ipAddr) Config.MaxTotalConnections = oldValue @@ -615,13 +644,13 @@ func TestMaxConnectionPerHost(t *testing.T) { ipAddr := "192.168.9.9" Connections.AddClientConnection(ipAddr) - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolSSH)) Connections.AddClientConnection(ipAddr) - assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.NoError(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolWebDAV)) Connections.AddClientConnection(ipAddr) - assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr)) + assert.Error(t, Connections.IsNewConnectionAllowed(ipAddr, ProtocolFTP)) assert.Equal(t, int32(3), Connections.GetClientConnections()) Connections.RemoveClientConnection(ipAddr) @@ -725,7 +754,7 @@ func TestCloseConnection(t *testing.T) { fakeConn := &fakeConnection{ BaseConnection: c, } - assert.NoError(t, Connections.IsNewConnectionAllowed("127.0.0.1")) + assert.NoError(t, Connections.IsNewConnectionAllowed("127.0.0.1", ProtocolHTTP)) err := Connections.Add(fakeConn) assert.NoError(t, err) assert.Len(t, Connections.GetStats(""), 1) @@ -1440,6 +1469,118 @@ func TestMetadataAPIRole(t *testing.T) { require.Len(t, ActiveMetadataChecks.Get(""), 0) } +func TestIPList(t *testing.T) { + type test struct { + ip string + protocol string + expectedMatch bool + expectedMode int + expectedErr bool + } + + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "192.168.0.0/25", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.0.128/25", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Protocols: 3, + }, + { + IPOrNet: "192.168.2.128/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + Protocols: 5, + }, + { + IPOrNet: "::/0", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Protocols: 4, + }, + { + IPOrNet: "2001:4860:4860::8888/120", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Protocols: 1, + }, + { + IPOrNet: "2001:4860:4860::8988/120", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + Protocols: 3, + }, + { + IPOrNet: "::1/128", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + } + ipList, err := dataprovider.NewIPList(dataprovider.IPListTypeDefender) + require.NoError(t, err) + for idx := range entries { + e := entries[idx] + err := dataprovider.AddIPListEntry(&e, "", "", "") + assert.NoError(t, err) + } + tests := []test{ + {ip: "1.1.1.1", protocol: ProtocolSSH, expectedMatch: false, expectedMode: 0, expectedErr: false}, + {ip: "invalid ip", protocol: ProtocolSSH, expectedMatch: false, expectedMode: 0, expectedErr: true}, + {ip: "192.168.0.1", protocol: ProtocolFTP, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "192.168.0.2", protocol: ProtocolHTTP, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "192.168.0.3", protocol: ProtocolWebDAV, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "192.168.0.4", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "192.168.0.156", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeDeny, expectedErr: false}, + {ip: "192.168.0.158", protocol: ProtocolFTP, expectedMatch: true, expectedMode: dataprovider.ListModeDeny, expectedErr: false}, + {ip: "192.168.0.158", protocol: ProtocolHTTP, expectedMatch: false, expectedMode: 0, expectedErr: false}, + {ip: "192.168.2.128", protocol: ProtocolHTTP, expectedMatch: false, expectedMode: 0, expectedErr: false}, + {ip: "192.168.2.128", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "::2", protocol: ProtocolSSH, expectedMatch: false, expectedMode: 0, expectedErr: false}, + {ip: "::2", protocol: ProtocolWebDAV, expectedMatch: true, expectedMode: dataprovider.ListModeDeny, expectedErr: false}, + {ip: "::1", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "::1", protocol: ProtocolHTTP, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "2001:4860:4860:0000:0000:0000:0000:8889", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeDeny, expectedErr: false}, + {ip: "2001:4860:4860:0000:0000:0000:0000:8889", protocol: ProtocolFTP, expectedMatch: false, expectedMode: 0, expectedErr: false}, + {ip: "2001:4860:4860:0000:0000:0000:0000:8989", protocol: ProtocolFTP, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "2001:4860:4860:0000:0000:0000:0000:89F1", protocol: ProtocolSSH, expectedMatch: true, expectedMode: dataprovider.ListModeAllow, expectedErr: false}, + {ip: "2001:4860:4860:0000:0000:0000:0000:89F1", protocol: ProtocolHTTP, expectedMatch: false, expectedMode: 0, expectedErr: false}, + } + + for _, tc := range tests { + match, mode, err := ipList.IsListed(tc.ip, tc.protocol) + if tc.expectedErr { + assert.Error(t, err, "ip %s, protocol %s", tc.ip, tc.protocol) + } else { + assert.NoError(t, err, "ip %s, protocol %s", tc.ip, tc.protocol) + } + assert.Equal(t, tc.expectedMatch, match, "ip %s, protocol %s", tc.ip, tc.protocol) + assert.Equal(t, tc.expectedMode, mode, "ip %s, protocol %s", tc.ip, tc.protocol) + } + + ipList.DisableMemoryMode() + + for _, tc := range tests { + match, mode, err := ipList.IsListed(tc.ip, tc.protocol) + if tc.expectedErr { + assert.Error(t, err, "ip %s, protocol %s", tc.ip, tc.protocol) + } else { + assert.NoError(t, err, "ip %s, protocol %s", tc.ip, tc.protocol) + } + assert.Equal(t, tc.expectedMatch, match, "ip %s, protocol %s", tc.ip, tc.protocol) + assert.Equal(t, tc.expectedMode, mode, "ip %s, protocol %s", tc.ip, tc.protocol) + } + + for _, e := range entries { + err := dataprovider.DeleteIPListEntry(e.IPOrNet, e.Type, "", "", "") + assert.NoError(t, err) + } +} + func BenchmarkBcryptHashing(b *testing.B) { bcryptPassword := "bcryptpassword" for i := 0; i < b.N; i++ { diff --git a/internal/common/defender.go b/internal/common/defender.go index b1ad2257..bac934f8 100644 --- a/internal/common/defender.go +++ b/internal/common/defender.go @@ -15,19 +15,10 @@ package common import ( - "encoding/json" "fmt" - "net" - "os" - "strings" - "sync" "time" - "github.com/yl2chen/cidranger" - "github.com/drakkan/sftpgo/v2/internal/dataprovider" - "github.com/drakkan/sftpgo/v2/internal/logger" - "github.com/drakkan/sftpgo/v2/internal/util" ) // HostEvent is the enumerable for the supported host events @@ -55,12 +46,12 @@ var ( type Defender interface { GetHosts() ([]dataprovider.DefenderEntry, error) GetHost(ip string) (dataprovider.DefenderEntry, error) - AddEvent(ip string, event HostEvent) - IsBanned(ip string) bool + AddEvent(ip, protocol string, event HostEvent) + IsBanned(ip, protocol string) bool + IsSafe(ip, protocol string) bool GetBanTime(ip string) (*time.Time, error) GetScore(ip string) (int, error) DeleteHost(ip string) bool - Reload() error } // DefenderConfig defines the "defender" configuration @@ -98,59 +89,33 @@ type DefenderConfig struct { // to return when you request for the entire host list from the defender EntriesSoftLimit int `json:"entries_soft_limit" mapstructure:"entries_soft_limit"` EntriesHardLimit int `json:"entries_hard_limit" mapstructure:"entries_hard_limit"` - // Path to a file containing a list of IP addresses and/or networks to never ban - SafeListFile string `json:"safelist_file" mapstructure:"safelist_file"` - // Path to a file containing a list of IP addresses and/or networks to always ban - BlockListFile string `json:"blocklist_file" mapstructure:"blocklist_file"` - // List of IP addresses and/or networks to never ban. - // For large lists prefer SafeListFile - SafeList []string `json:"safelist" mapstructure:"safelist"` - // List of IP addresses and/or networks to always ban. - // For large lists prefer BlockListFile - BlockList []string `json:"blocklist" mapstructure:"blocklist"` } type baseDefender struct { config *DefenderConfig - sync.RWMutex - safeList *HostList - blockList *HostList + ipList *dataprovider.IPList } -// Reload reloads block and safe lists -func (d *baseDefender) Reload() error { - blockList, err := loadHostListFromFile(d.config.BlockListFile) +func (d *baseDefender) isBanned(ip, protocol string) bool { + isListed, mode, err := d.ipList.IsListed(ip, protocol) if err != nil { - return err + return false } - blockList = addEntriesToList(d.config.BlockList, blockList, "blocklist") - - d.Lock() - d.blockList = blockList - d.Unlock() - - safeList, err := loadHostListFromFile(d.config.SafeListFile) - if err != nil { - return err - } - safeList = addEntriesToList(d.config.SafeList, safeList, "safelist") - - d.Lock() - d.safeList = safeList - d.Unlock() - - return nil -} - -func (d *baseDefender) isBanned(ip string) bool { - if d.blockList != nil && d.blockList.isListed(ip) { - // permanent ban + if isListed && mode == dataprovider.ListModeDeny { return true } return false } +func (d *baseDefender) IsSafe(ip, protocol string) bool { + isListed, mode, err := d.ipList.IsListed(ip, protocol) + if err == nil && isListed && mode == dataprovider.ListModeAllow { + return true + } + return false +} + func (d *baseDefender) getScore(event HostEvent) int { var score int @@ -167,31 +132,6 @@ func (d *baseDefender) getScore(event HostEvent) int { return score } -// HostListFile defines the structure expected for safe/block list files -type HostListFile struct { - IPAddresses []string `json:"addresses"` - CIDRNetworks []string `json:"networks"` -} - -// HostList defines the structure used to keep the HostListFile in memory -type HostList struct { - IPAddresses map[string]bool - Ranges cidranger.Ranger -} - -func (h *HostList) isListed(ip string) bool { - if _, ok := h.IPAddresses[ip]; ok { - return true - } - - ok, err := h.Ranges.Contains(net.ParseIP(ip)) - if err != nil { - return false - } - - return ok -} - type hostEvent struct { dateTime time.Time score int @@ -259,113 +199,3 @@ func (c *DefenderConfig) validate() error { return nil } - -func loadHostListFromFile(name string) (*HostList, error) { - if name == "" { - return nil, nil - } - if !util.IsFileInputValid(name) { - return nil, fmt.Errorf("invalid host list file name %#v", name) - } - - info, err := os.Stat(name) - if err != nil { - return nil, err - } - - // opinionated max size, you should avoid big host lists - if info.Size() > 1048576*5 { // 5MB - return nil, fmt.Errorf("host list file %#v is too big: %v bytes", name, info.Size()) - } - - content, err := os.ReadFile(name) - if err != nil { - return nil, fmt.Errorf("unable to read input file %#v: %v", name, err) - } - - var hostList HostListFile - - err = json.Unmarshal(content, &hostList) - if err != nil { - return nil, err - } - - if len(hostList.CIDRNetworks) > 0 || len(hostList.IPAddresses) > 0 { - result := &HostList{ - IPAddresses: make(map[string]bool), - Ranges: cidranger.NewPCTrieRanger(), - } - ipCount := 0 - cdrCount := 0 - for _, ip := range hostList.IPAddresses { - if net.ParseIP(ip) == nil { - logger.Warn(logSender, "", "unable to parse IP %#v", ip) - continue - } - result.IPAddresses[ip] = true - ipCount++ - } - for _, cidrNet := range hostList.CIDRNetworks { - _, network, err := net.ParseCIDR(cidrNet) - if err != nil { - logger.Warn(logSender, "", "unable to parse CIDR network %#v: %v", cidrNet, err) - continue - } - err = result.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)) - if err == nil { - cdrCount++ - } - } - - logger.Info(logSender, "", "list %#v loaded, ip addresses loaded: %v/%v networks loaded: %v/%v", - name, ipCount, len(hostList.IPAddresses), cdrCount, len(hostList.CIDRNetworks)) - return result, nil - } - - return nil, nil -} - -func addEntriesToList(entries []string, hostList *HostList, listName string) *HostList { - if len(entries) == 0 { - return hostList - } - - if hostList == nil { - hostList = &HostList{ - IPAddresses: make(map[string]bool), - Ranges: cidranger.NewPCTrieRanger(), - } - } - ipCount := 0 - ipLoaded := 0 - cdrCount := 0 - cdrLoaded := 0 - - for _, entry := range entries { - entry = strings.TrimSpace(entry) - if strings.LastIndex(entry, "/") > 0 { - cdrCount++ - _, network, err := net.ParseCIDR(entry) - if err != nil { - logger.Warn(logSender, "", "unable to parse CIDR network %#v: %v", entry, err) - continue - } - err = hostList.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)) - if err == nil { - cdrLoaded++ - } - } else { - ipCount++ - if net.ParseIP(entry) == nil { - logger.Warn(logSender, "", "unable to parse IP %#v", entry) - continue - } - hostList.IPAddresses[entry] = true - ipLoaded++ - } - } - logger.Info(logSender, "", "%s from config loaded, ip addresses loaded: %v/%v networks loaded: %v/%v", - listName, ipLoaded, ipCount, cdrLoaded, cdrCount) - - return hostList -} diff --git a/internal/common/defender_test.go b/internal/common/defender_test.go index 407f0063..e81ba808 100644 --- a/internal/common/defender_test.go +++ b/internal/common/defender_test.go @@ -15,45 +15,88 @@ package common import ( - "crypto/rand" "encoding/hex" - "encoding/json" "fmt" "net" - "os" - "path/filepath" - "runtime" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/yl2chen/cidranger" + + "github.com/drakkan/sftpgo/v2/internal/dataprovider" ) func TestBasicDefender(t *testing.T) { - bl := HostListFile{ - IPAddresses: []string{"172.16.1.1", "172.16.1.2"}, - CIDRNetworks: []string{"10.8.0.0/24"}, + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "172.16.1.1/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "172.16.1.2/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "10.8.0.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "192.168.1.1/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "192.168.1.2/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "10.8.9.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "172.16.1.3/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "172.16.1.4/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.8.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.1.3/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.1.4/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.9.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, } - sl := HostListFile{ - IPAddresses: []string{"172.16.1.3", "172.16.1.4"}, - CIDRNetworks: []string{"192.168.8.0/24"}, + + for idx := range entries { + e := entries[idx] + err := dataprovider.AddIPListEntry(&e, "", "", "") + assert.NoError(t, err) } - blFile := filepath.Join(os.TempDir(), "bl.json") - slFile := filepath.Join(os.TempDir(), "sl.json") - - data, err := json.Marshal(bl) - assert.NoError(t, err) - - err = os.WriteFile(blFile, data, os.ModePerm) - assert.NoError(t, err) - - data, err = json.Marshal(sl) - assert.NoError(t, err) - - err = os.WriteFile(slFile, data, os.ModePerm) - assert.NoError(t, err) config := &DefenderConfig{ Enabled: true, @@ -67,31 +110,21 @@ func TestBasicDefender(t *testing.T) { ObservationTime: 15, EntriesSoftLimit: 1, EntriesHardLimit: 2, - SafeListFile: "slFile", - BlockListFile: "blFile", - SafeList: []string{"192.168.1.3", "192.168.1.4", "192.168.9.0/24"}, - BlockList: []string{"192.168.1.1", "192.168.1.2", "10.8.9.0/24"}, } - _, err = newInMemoryDefender(config) - assert.Error(t, err) - config.BlockListFile = blFile - _, err = newInMemoryDefender(config) - assert.Error(t, err) - config.SafeListFile = slFile d, err := newInMemoryDefender(config) assert.NoError(t, err) defender := d.(*memoryDefender) - assert.True(t, defender.IsBanned("172.16.1.1")) - assert.True(t, defender.IsBanned("192.168.1.1")) - assert.False(t, defender.IsBanned("172.16.1.10")) - assert.False(t, defender.IsBanned("192.168.1.10")) - assert.False(t, defender.IsBanned("10.8.2.3")) - assert.False(t, defender.IsBanned("10.9.2.3")) - assert.True(t, defender.IsBanned("10.8.0.3")) - assert.True(t, defender.IsBanned("10.8.9.3")) - assert.False(t, defender.IsBanned("invalid ip")) + assert.True(t, defender.IsBanned("172.16.1.1", ProtocolSSH)) + assert.True(t, defender.IsBanned("192.168.1.1", ProtocolFTP)) + assert.False(t, defender.IsBanned("172.16.1.10", ProtocolSSH)) + assert.False(t, defender.IsBanned("192.168.1.10", ProtocolSSH)) + assert.False(t, defender.IsBanned("10.8.2.3", ProtocolSSH)) + assert.False(t, defender.IsBanned("10.9.2.3", ProtocolSSH)) + assert.True(t, defender.IsBanned("10.8.0.3", ProtocolSSH)) + assert.True(t, defender.IsBanned("10.8.9.3", ProtocolSSH)) + assert.False(t, defender.IsBanned("invalid ip", ProtocolSSH)) assert.Equal(t, 0, defender.countBanned()) assert.Equal(t, 0, defender.countHosts()) hosts, err := defender.GetHosts() @@ -100,15 +133,15 @@ func TestBasicDefender(t *testing.T) { _, err = defender.GetHost("10.8.0.4") assert.Error(t, err) - defender.AddEvent("172.16.1.4", HostEventLoginFailed) - defender.AddEvent("192.168.1.4", HostEventLoginFailed) - defender.AddEvent("192.168.8.4", HostEventUserNotFound) - defender.AddEvent("172.16.1.3", HostEventLimitExceeded) - defender.AddEvent("192.168.1.3", HostEventLimitExceeded) + defender.AddEvent("172.16.1.4", ProtocolSSH, HostEventLoginFailed) + defender.AddEvent("192.168.1.4", ProtocolSSH, HostEventLoginFailed) + defender.AddEvent("192.168.8.4", ProtocolSSH, HostEventUserNotFound) + defender.AddEvent("172.16.1.3", ProtocolSSH, HostEventLimitExceeded) + defender.AddEvent("192.168.1.3", ProtocolSSH, HostEventLimitExceeded) assert.Equal(t, 0, defender.countHosts()) testIP := "12.34.56.78" - defender.AddEvent(testIP, HostEventLoginFailed) + defender.AddEvent(testIP, ProtocolSSH, HostEventLoginFailed) assert.Equal(t, 1, defender.countHosts()) assert.Equal(t, 0, defender.countBanned()) score, err := defender.GetScore(testIP) @@ -128,7 +161,7 @@ func TestBasicDefender(t *testing.T) { banTime, err := defender.GetBanTime(testIP) assert.NoError(t, err) assert.Nil(t, banTime) - defender.AddEvent(testIP, HostEventLimitExceeded) + defender.AddEvent(testIP, ProtocolSSH, HostEventLimitExceeded) assert.Equal(t, 1, defender.countHosts()) assert.Equal(t, 0, defender.countBanned()) score, err = defender.GetScore(testIP) @@ -141,8 +174,8 @@ func TestBasicDefender(t *testing.T) { assert.True(t, hosts[0].BanTime.IsZero()) assert.Empty(t, hosts[0].GetBanTime()) } - defender.AddEvent(testIP, HostEventUserNotFound) - defender.AddEvent(testIP, HostEventNoLoginTried) + defender.AddEvent(testIP, ProtocolSSH, HostEventUserNotFound) + defender.AddEvent(testIP, ProtocolSSH, HostEventNoLoginTried) assert.Equal(t, 0, defender.countHosts()) assert.Equal(t, 1, defender.countBanned()) score, err = defender.GetScore(testIP) @@ -169,11 +202,11 @@ func TestBasicDefender(t *testing.T) { testIP2 := "12.34.56.80" testIP3 := "12.34.56.81" - defender.AddEvent(testIP1, HostEventNoLoginTried) - defender.AddEvent(testIP2, HostEventNoLoginTried) + defender.AddEvent(testIP1, ProtocolSSH, HostEventNoLoginTried) + defender.AddEvent(testIP2, ProtocolSSH, HostEventNoLoginTried) assert.Equal(t, 2, defender.countHosts()) time.Sleep(20 * time.Millisecond) - defender.AddEvent(testIP3, HostEventNoLoginTried) + defender.AddEvent(testIP3, ProtocolSSH, HostEventNoLoginTried) assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts()) // testIP1 and testIP2 should be removed assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts()) @@ -187,8 +220,8 @@ func TestBasicDefender(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 2, score) - defender.AddEvent(testIP3, HostEventNoLoginTried) - defender.AddEvent(testIP3, HostEventNoLoginTried) + defender.AddEvent(testIP3, ProtocolSSH, HostEventNoLoginTried) + defender.AddEvent(testIP3, ProtocolSSH, HostEventNoLoginTried) // IP3 is now banned banTime, err = defender.GetBanTime(testIP3) assert.NoError(t, err) @@ -197,7 +230,7 @@ func TestBasicDefender(t *testing.T) { time.Sleep(20 * time.Millisecond) for i := 0; i < 3; i++ { - defender.AddEvent(testIP1, HostEventNoLoginTried) + defender.AddEvent(testIP1, ProtocolSSH, HostEventNoLoginTried) } assert.Equal(t, 0, defender.countHosts()) assert.Equal(t, config.EntriesSoftLimit, defender.countBanned()) @@ -212,9 +245,9 @@ func TestBasicDefender(t *testing.T) { assert.NotNil(t, banTime) for i := 0; i < 3; i++ { - defender.AddEvent(testIP, HostEventNoLoginTried) + defender.AddEvent(testIP, ProtocolSSH, HostEventNoLoginTried) time.Sleep(10 * time.Millisecond) - defender.AddEvent(testIP3, HostEventNoLoginTried) + defender.AddEvent(testIP3, ProtocolSSH, HostEventNoLoginTried) } assert.Equal(t, 0, defender.countHosts()) assert.Equal(t, defender.config.EntriesSoftLimit, defender.countBanned()) @@ -222,7 +255,7 @@ func TestBasicDefender(t *testing.T) { banTime, err = defender.GetBanTime(testIP3) assert.NoError(t, err) if assert.NotNil(t, banTime) { - assert.True(t, defender.IsBanned(testIP3)) + assert.True(t, defender.IsBanned(testIP3, ProtocolFTP)) // ban time should increase newBanTime, err := defender.GetBanTime(testIP3) assert.NoError(t, err) @@ -232,10 +265,10 @@ func TestBasicDefender(t *testing.T) { assert.True(t, defender.DeleteHost(testIP3)) assert.False(t, defender.DeleteHost(testIP3)) - err = os.Remove(slFile) - assert.NoError(t, err) - err = os.Remove(blFile) - assert.NoError(t, err) + for _, e := range entries { + err := dataprovider.DeleteIPListEntry(e.IPOrNet, e.Type, "", "", "") + assert.NoError(t, err) + } } func TestExpiredHostBans(t *testing.T) { @@ -265,14 +298,14 @@ func TestExpiredHostBans(t *testing.T) { assert.NoError(t, err) assert.Len(t, res, 0) - assert.False(t, defender.IsBanned(testIP)) + assert.False(t, defender.IsBanned(testIP, ProtocolFTP)) _, err = defender.GetHost(testIP) assert.Error(t, err) _, ok := defender.banned[testIP] assert.True(t, ok) // now add an event for an expired banned ip, it should be removed - defender.AddEvent(testIP, HostEventLoginFailed) - assert.False(t, defender.IsBanned(testIP)) + defender.AddEvent(testIP, ProtocolFTP, HostEventLoginFailed) + assert.False(t, defender.IsBanned(testIP, ProtocolFTP)) entry, err := defender.GetHost(testIP) assert.NoError(t, err) assert.Equal(t, testIP, entry.IP) @@ -314,94 +347,6 @@ func TestExpiredHostBans(t *testing.T) { assert.True(t, ok) } -func TestLoadHostListFromFile(t *testing.T) { - _, err := loadHostListFromFile(".") - assert.Error(t, err) - - hostsFilePath := filepath.Join(os.TempDir(), "hostfile") - content := make([]byte, 1048576*6) - _, err = rand.Read(content) - assert.NoError(t, err) - - err = os.WriteFile(hostsFilePath, content, os.ModePerm) - assert.NoError(t, err) - - _, err = loadHostListFromFile(hostsFilePath) - assert.Error(t, err) - - hl := HostListFile{ - IPAddresses: []string{}, - CIDRNetworks: []string{}, - } - - asJSON, err := json.Marshal(hl) - assert.NoError(t, err) - err = os.WriteFile(hostsFilePath, asJSON, os.ModePerm) - assert.NoError(t, err) - - hostList, err := loadHostListFromFile(hostsFilePath) - assert.NoError(t, err) - assert.Nil(t, hostList) - - hl.IPAddresses = append(hl.IPAddresses, "invalidip") - asJSON, err = json.Marshal(hl) - assert.NoError(t, err) - err = os.WriteFile(hostsFilePath, asJSON, os.ModePerm) - assert.NoError(t, err) - - hostList, err = loadHostListFromFile(hostsFilePath) - assert.NoError(t, err) - assert.Len(t, hostList.IPAddresses, 0) - - hl.IPAddresses = nil - hl.CIDRNetworks = append(hl.CIDRNetworks, "invalid net") - - asJSON, err = json.Marshal(hl) - assert.NoError(t, err) - err = os.WriteFile(hostsFilePath, asJSON, os.ModePerm) - assert.NoError(t, err) - - hostList, err = loadHostListFromFile(hostsFilePath) - assert.NoError(t, err) - assert.NotNil(t, hostList) - assert.Len(t, hostList.IPAddresses, 0) - assert.Equal(t, 0, hostList.Ranges.Len()) - - if runtime.GOOS != osWindows { - err = os.Chmod(hostsFilePath, 0111) - assert.NoError(t, err) - - _, err = loadHostListFromFile(hostsFilePath) - assert.Error(t, err) - - err = os.Chmod(hostsFilePath, 0644) - assert.NoError(t, err) - } - - err = os.WriteFile(hostsFilePath, []byte("non json content"), os.ModePerm) - assert.NoError(t, err) - _, err = loadHostListFromFile(hostsFilePath) - assert.Error(t, err) - - err = os.Remove(hostsFilePath) - assert.NoError(t, err) -} - -func TestAddEntriesToHostList(t *testing.T) { - name := "testList" - hostlist := addEntriesToList([]string{"192.168.6.1", "10.7.0.0/25"}, nil, name) - require.NotNil(t, hostlist) - assert.True(t, hostlist.isListed("192.168.6.1")) - assert.False(t, hostlist.isListed("192.168.6.2")) - assert.True(t, hostlist.isListed("10.7.0.28")) - assert.False(t, hostlist.isListed("10.7.0.129")) - // load invalid values - hostlist = addEntriesToList([]string{"invalidip", "invalidnet/24"}, nil, name) - require.NotNil(t, hostlist) - assert.Len(t, hostlist.IPAddresses, 0) - assert.Equal(t, 0, hostlist.Ranges.Len()) -} - func TestDefenderCleanup(t *testing.T) { d := memoryDefender{ baseDefender: baseDefender{ @@ -577,7 +522,7 @@ func BenchmarkDefenderBannedSearch(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - d.IsBanned("192.168.1.1") + d.IsBanned("192.168.1.1", ProtocolSSH) } } @@ -593,7 +538,7 @@ func BenchmarkCleanup(b *testing.B) { for i := 0; i < b.N; i++ { for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { - d.AddEvent(ip.String(), HostEventLoginFailed) + d.AddEvent(ip.String(), ProtocolSSH, HostEventLoginFailed) if d.countHosts() > d.config.EntriesHardLimit { panic("too many hosts") } @@ -604,72 +549,10 @@ func BenchmarkCleanup(b *testing.B) { } } -func BenchmarkDefenderBannedSearchWithBlockList(b *testing.B) { - d := getDefenderForBench() - - d.blockList = &HostList{ - IPAddresses: make(map[string]bool), - Ranges: cidranger.NewPCTrieRanger(), - } - - ip, ipnet, err := net.ParseCIDR("129.8.0.0/12") // 1048574 ip addresses - if err != nil { - panic(err) - } - - for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { - d.banned[ip.String()] = time.Now().Add(10 * time.Minute) - d.blockList.IPAddresses[ip.String()] = true - } - - for i := 0; i < 255; i++ { - cidr := fmt.Sprintf("10.8.%v.1/24", i) - _, network, _ := net.ParseCIDR(cidr) - if err := d.blockList.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { - panic(err) - } - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - d.IsBanned("192.168.1.1") - } -} - -func BenchmarkHostListSearch(b *testing.B) { - hostlist := &HostList{ - IPAddresses: make(map[string]bool), - Ranges: cidranger.NewPCTrieRanger(), - } - - ip, ipnet, _ := net.ParseCIDR("172.16.0.0/16") - - for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { - hostlist.IPAddresses[ip.String()] = true - } - - for i := 0; i < 255; i++ { - cidr := fmt.Sprintf("10.8.%v.1/24", i) - _, network, _ := net.ParseCIDR(cidr) - if err := hostlist.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { - panic(err) - } - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - if hostlist.isListed("192.167.1.2") { - panic("should not be listed") - } - } -} - func BenchmarkCIDRanger(b *testing.B) { ranger := cidranger.NewPCTrieRanger() for i := 0; i < 255; i++ { - cidr := fmt.Sprintf("192.168.%v.1/24", i) + cidr := fmt.Sprintf("192.168.%d.1/24", i) _, network, _ := net.ParseCIDR(cidr) if err := ranger.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { panic(err) @@ -689,7 +572,7 @@ func BenchmarkCIDRanger(b *testing.B) { func BenchmarkNetContains(b *testing.B) { var nets []*net.IPNet for i := 0; i < 255; i++ { - cidr := fmt.Sprintf("192.168.%v.1/24", i) + cidr := fmt.Sprintf("192.168.%d.1/24", i) _, network, _ := net.ParseCIDR(cidr) nets = append(nets, network) } diff --git a/internal/common/defenderdb.go b/internal/common/defenderdb.go index 6d26611e..0c5fd770 100644 --- a/internal/common/defenderdb.go +++ b/internal/common/defenderdb.go @@ -15,6 +15,7 @@ package common import ( + "sync/atomic" "time" "github.com/drakkan/sftpgo/v2/internal/dataprovider" @@ -24,7 +25,7 @@ import ( type dbDefender struct { baseDefender - lastCleanup time.Time + lastCleanup atomic.Int64 } func newDBDefender(config *DefenderConfig) (Defender, error) { @@ -32,16 +33,17 @@ func newDBDefender(config *DefenderConfig) (Defender, error) { if err != nil { return nil, err } + ipList, err := dataprovider.NewIPList(dataprovider.IPListTypeDefender) + if err != nil { + return nil, err + } defender := &dbDefender{ baseDefender: baseDefender{ config: config, + ipList: ipList, }, - lastCleanup: time.Time{}, - } - - if err := defender.Reload(); err != nil { - return nil, err } + defender.lastCleanup.Store(0) return defender, nil } @@ -59,13 +61,10 @@ func (d *dbDefender) GetHost(ip string) (dataprovider.DefenderEntry, error) { // IsBanned returns true if the specified IP is banned // and increase ban time if the IP is found. // This method must be called as soon as the client connects -func (d *dbDefender) IsBanned(ip string) bool { - d.RLock() - if d.baseDefender.isBanned(ip) { - d.RUnlock() +func (d *dbDefender) IsBanned(ip, protocol string) bool { + if d.baseDefender.isBanned(ip, protocol) { return true } - d.RUnlock() _, err := dataprovider.IsDefenderHostBanned(ip) if err != nil { @@ -90,13 +89,10 @@ func (d *dbDefender) DeleteHost(ip string) bool { // AddEvent adds an event for the given IP. // This method must be called for clients not yet banned -func (d *dbDefender) AddEvent(ip string, event HostEvent) { - d.RLock() - if d.safeList != nil && d.safeList.isListed(ip) { - d.RUnlock() +func (d *dbDefender) AddEvent(ip, protocol string, event HostEvent) { + if d.IsSafe(ip, protocol) { return } - d.RUnlock() score := d.baseDefender.getScore(event) @@ -165,15 +161,17 @@ func (d *dbDefender) getStartObservationTime() int64 { } func (d *dbDefender) getLastCleanup() time.Time { - d.RLock() - defer d.RUnlock() - - return d.lastCleanup + val := d.lastCleanup.Load() + if val == 0 { + return time.Time{} + } + return util.GetTimeFromMsecSinceEpoch(val) } func (d *dbDefender) setLastCleanup(when time.Time) { - d.Lock() - defer d.Unlock() - - d.lastCleanup = when + if when.IsZero() { + d.lastCleanup.Store(0) + return + } + d.lastCleanup.Store(util.GetTimeAsMsSinceEpoch(when)) } diff --git a/internal/common/defenderdb_test.go b/internal/common/defenderdb_test.go index 09d335a6..0220d09c 100644 --- a/internal/common/defenderdb_test.go +++ b/internal/common/defenderdb_test.go @@ -16,9 +16,6 @@ package common import ( "encoding/hex" - "encoding/json" - "os" - "path/filepath" "testing" "time" @@ -32,6 +29,45 @@ func TestBasicDbDefender(t *testing.T) { if !isDbDefenderSupported() { t.Skip("this test is not supported with the current database provider") } + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "172.16.1.1/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "172.16.1.2/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "10.8.0.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + }, + { + IPOrNet: "172.16.1.3/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "172.16.1.4/32", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + { + IPOrNet: "192.168.8.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeAllow, + }, + } + + for idx := range entries { + e := entries[idx] + err := dataprovider.AddIPListEntry(&e, "", "", "") + assert.NoError(t, err) + } + config := &DefenderConfig{ Enabled: true, BanTime: 10, @@ -44,61 +80,31 @@ func TestBasicDbDefender(t *testing.T) { ObservationTime: 15, EntriesSoftLimit: 1, EntriesHardLimit: 10, - SafeListFile: "slFile", - BlockListFile: "blFile", } - _, err := newDBDefender(config) - assert.Error(t, err) - - bl := HostListFile{ - IPAddresses: []string{"172.16.1.1", "172.16.1.2"}, - CIDRNetworks: []string{"10.8.0.0/24"}, - } - sl := HostListFile{ - IPAddresses: []string{"172.16.1.3", "172.16.1.4"}, - CIDRNetworks: []string{"192.168.8.0/24"}, - } - blFile := filepath.Join(os.TempDir(), "bl.json") - slFile := filepath.Join(os.TempDir(), "sl.json") - - data, err := json.Marshal(bl) - assert.NoError(t, err) - err = os.WriteFile(blFile, data, os.ModePerm) - assert.NoError(t, err) - - data, err = json.Marshal(sl) - assert.NoError(t, err) - err = os.WriteFile(slFile, data, os.ModePerm) - assert.NoError(t, err) - - config.BlockListFile = blFile - _, err = newDBDefender(config) - assert.Error(t, err) - config.SafeListFile = slFile d, err := newDBDefender(config) assert.NoError(t, err) defender := d.(*dbDefender) - assert.True(t, defender.IsBanned("172.16.1.1")) - assert.False(t, defender.IsBanned("172.16.1.10")) - assert.False(t, defender.IsBanned("10.8.1.3")) - assert.True(t, defender.IsBanned("10.8.0.4")) - assert.False(t, defender.IsBanned("invalid ip")) + assert.True(t, defender.IsBanned("172.16.1.1", ProtocolFTP)) + assert.False(t, defender.IsBanned("172.16.1.10", ProtocolSSH)) + assert.False(t, defender.IsBanned("10.8.1.3", ProtocolHTTP)) + assert.True(t, defender.IsBanned("10.8.0.4", ProtocolWebDAV)) + assert.False(t, defender.IsBanned("invalid ip", ProtocolSSH)) hosts, err := defender.GetHosts() assert.NoError(t, err) assert.Len(t, hosts, 0) _, err = defender.GetHost("10.8.0.3") assert.Error(t, err) - defender.AddEvent("172.16.1.4", HostEventLoginFailed) - defender.AddEvent("192.168.8.4", HostEventUserNotFound) - defender.AddEvent("172.16.1.3", HostEventLimitExceeded) + defender.AddEvent("172.16.1.4", ProtocolSSH, HostEventLoginFailed) + defender.AddEvent("192.168.8.4", ProtocolSSH, HostEventUserNotFound) + defender.AddEvent("172.16.1.3", ProtocolSSH, HostEventLimitExceeded) hosts, err = defender.GetHosts() assert.NoError(t, err) assert.Len(t, hosts, 0) assert.True(t, defender.getLastCleanup().IsZero()) testIP := "123.45.67.89" - defender.AddEvent(testIP, HostEventLoginFailed) + defender.AddEvent(testIP, ProtocolSSH, HostEventLoginFailed) lastCleanup := defender.getLastCleanup() assert.False(t, lastCleanup.IsZero()) score, err := defender.GetScore(testIP) @@ -118,7 +124,7 @@ func TestBasicDbDefender(t *testing.T) { banTime, err := defender.GetBanTime(testIP) assert.NoError(t, err) assert.Nil(t, banTime) - defender.AddEvent(testIP, HostEventLimitExceeded) + defender.AddEvent(testIP, ProtocolSSH, HostEventLimitExceeded) score, err = defender.GetScore(testIP) assert.NoError(t, err) assert.Equal(t, 4, score) @@ -129,8 +135,8 @@ func TestBasicDbDefender(t *testing.T) { assert.True(t, hosts[0].BanTime.IsZero()) assert.Empty(t, hosts[0].GetBanTime()) } - defender.AddEvent(testIP, HostEventNoLoginTried) - defender.AddEvent(testIP, HostEventNoLoginTried) + defender.AddEvent(testIP, ProtocolSSH, HostEventNoLoginTried) + defender.AddEvent(testIP, ProtocolSSH, HostEventNoLoginTried) score, err = defender.GetScore(testIP) assert.NoError(t, err) assert.Equal(t, 0, score) @@ -150,7 +156,7 @@ func TestBasicDbDefender(t *testing.T) { assert.Equal(t, 0, host.Score) assert.NotEmpty(t, host.GetBanTime()) // ban time should increase - assert.True(t, defender.IsBanned(testIP)) + assert.True(t, defender.IsBanned(testIP, ProtocolSSH)) newBanTime, err := defender.GetBanTime(testIP) assert.NoError(t, err) assert.True(t, newBanTime.After(*banTime)) @@ -162,9 +168,9 @@ func TestBasicDbDefender(t *testing.T) { testIP2 := "123.45.67.91" testIP3 := "123.45.67.92" for i := 0; i < 3; i++ { - defender.AddEvent(testIP, HostEventUserNotFound) - defender.AddEvent(testIP1, HostEventNoLoginTried) - defender.AddEvent(testIP2, HostEventUserNotFound) + defender.AddEvent(testIP, ProtocolSSH, HostEventUserNotFound) + defender.AddEvent(testIP1, ProtocolSSH, HostEventNoLoginTried) + defender.AddEvent(testIP2, ProtocolSSH, HostEventUserNotFound) } hosts, err = defender.GetHosts() assert.NoError(t, err) @@ -174,7 +180,7 @@ func TestBasicDbDefender(t *testing.T) { assert.False(t, host.BanTime.IsZero()) assert.NotEmpty(t, host.GetBanTime()) } - defender.AddEvent(testIP3, HostEventLoginFailed) + defender.AddEvent(testIP3, ProtocolSSH, HostEventLoginFailed) hosts, err = defender.GetHosts() assert.NoError(t, err) assert.Len(t, hosts, 4) @@ -248,10 +254,10 @@ func TestBasicDbDefender(t *testing.T) { assert.NoError(t, err) assert.Len(t, hosts, 0) - err = os.Remove(slFile) - assert.NoError(t, err) - err = os.Remove(blFile) - assert.NoError(t, err) + for _, e := range entries { + err := dataprovider.DeleteIPListEntry(e.IPOrNet, e.Type, "", "", "") + assert.NoError(t, err) + } } func TestDbDefenderCleanup(t *testing.T) { @@ -280,6 +286,8 @@ func TestDbDefenderCleanup(t *testing.T) { assert.False(t, lastCleanup.IsZero()) defender.cleanup() assert.Equal(t, lastCleanup, defender.getLastCleanup()) + defender.setLastCleanup(time.Time{}) + assert.True(t, defender.getLastCleanup().IsZero()) defender.setLastCleanup(time.Now().Add(-time.Duration(config.ObservationTime) * time.Minute * 4)) time.Sleep(20 * time.Millisecond) defender.cleanup() @@ -289,7 +297,7 @@ func TestDbDefenderCleanup(t *testing.T) { err = dataprovider.Close() assert.NoError(t, err) - lastCleanup = time.Now().Add(-time.Duration(config.ObservationTime) * time.Minute * 4) + lastCleanup = util.GetTimeFromMsecSinceEpoch(time.Now().Add(-time.Duration(config.ObservationTime) * time.Minute * 4).UnixMilli()) defender.setLastCleanup(lastCleanup) defender.cleanup() // cleanup will fail and so last cleanup should be reset to the previous value diff --git a/internal/common/defendermem.go b/internal/common/defendermem.go index 80bb65d9..3375cfda 100644 --- a/internal/common/defendermem.go +++ b/internal/common/defendermem.go @@ -16,6 +16,7 @@ package common import ( "sort" + "sync" "time" "github.com/drakkan/sftpgo/v2/internal/dataprovider" @@ -24,6 +25,7 @@ import ( type memoryDefender struct { baseDefender + sync.RWMutex // IP addresses of the clients trying to connected are stored inside hosts, // they are added to banned once the thresold is reached. // A violation from a banned host will increase the ban time @@ -37,18 +39,19 @@ func newInMemoryDefender(config *DefenderConfig) (Defender, error) { if err != nil { return nil, err } + ipList, err := dataprovider.NewIPList(dataprovider.IPListTypeDefender) + if err != nil { + return nil, err + } defender := &memoryDefender{ baseDefender: baseDefender{ config: config, + ipList: ipList, }, hosts: make(map[string]hostScore), banned: make(map[string]time.Time), } - if err := defender.Reload(); err != nil { - return nil, err - } - return defender, nil } @@ -119,7 +122,7 @@ func (d *memoryDefender) GetHost(ip string) (dataprovider.DefenderEntry, error) // IsBanned returns true if the specified IP is banned // and increase ban time if the IP is found. // This method must be called as soon as the client connects -func (d *memoryDefender) IsBanned(ip string) bool { +func (d *memoryDefender) IsBanned(ip, protocol string) bool { d.RLock() if banTime, ok := d.banned[ip]; ok { @@ -145,7 +148,7 @@ func (d *memoryDefender) IsBanned(ip string) bool { defer d.RUnlock() - return d.baseDefender.isBanned(ip) + return d.baseDefender.isBanned(ip, protocol) } // DeleteHost removes the specified IP from the defender lists @@ -168,14 +171,14 @@ func (d *memoryDefender) DeleteHost(ip string) bool { // AddEvent adds an event for the given IP. // This method must be called for clients not yet banned -func (d *memoryDefender) AddEvent(ip string, event HostEvent) { - d.Lock() - defer d.Unlock() - - if d.safeList != nil && d.safeList.isListed(ip) { +func (d *memoryDefender) AddEvent(ip, protocol string, event HostEvent) { + if d.IsSafe(ip, protocol) { return } + d.Lock() + defer d.Unlock() + // ignore events for already banned hosts if v, ok := d.banned[ip]; ok { if v.After(time.Now()) { diff --git a/internal/common/eventscheduler.go b/internal/common/eventscheduler.go index 4b734ef9..9263b027 100644 --- a/internal/common/eventscheduler.go +++ b/internal/common/eventscheduler.go @@ -37,6 +37,7 @@ func startEventScheduler() { stopEventScheduler() eventScheduler = cron.New(cron.WithLocation(time.UTC)) + eventManager.loadRules() _, err := eventScheduler.AddFunc("@every 10m", eventManager.loadRules) util.PanicOnError(err) eventScheduler.Start() diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 7edb45e7..8ec9c85f 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -109,6 +109,12 @@ func TestMain(m *testing.M) { providerConf.BackupsPath = backupsPath logger.InfoToConsole("Starting COMMON tests, provider: %v", providerConf.Driver) + err = dataprovider.Initialize(providerConf, configDir, true) + if err != nil { + logger.ErrorToConsole("error initializing data provider: %v", err) + os.Exit(1) + } + err = common.Initialize(config.GetCommonConfig(), 0) if err != nil { logger.WarnToConsole("error initializing common: %v", err) @@ -116,12 +122,6 @@ func TestMain(m *testing.M) { } common.SetCertAutoReloadMode(true) - err = dataprovider.Initialize(providerConf, configDir, true) - if err != nil { - logger.ErrorToConsole("error initializing data provider: %v", err) - os.Exit(1) - } - httpConfig := config.GetHTTPConfig() httpConfig.Timeout = 5 httpConfig.RetryMax = 0 @@ -3189,6 +3189,87 @@ func TestUserPasswordHashing(t *testing.T) { assert.NoError(t, err) } +func TestAllowList(t *testing.T) { + configCopy := common.Config + + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "172.18.1.1/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "172.18.1.2/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "10.8.7.0/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 5, + }, + { + IPOrNet: "0.0.0.0/0", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + }, + { + IPOrNet: "::/0", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + }, + } + + for _, e := range entries { + _, resp, err := httpdtest.AddIPListEntry(e, http.StatusCreated) + assert.NoError(t, err, string(resp)) + } + + common.Config.AllowListStatus = 1 + err := common.Initialize(common.Config, 0) + assert.NoError(t, err) + assert.True(t, common.Config.IsAllowListEnabled()) + + testIP := "172.18.1.1" + assert.NoError(t, common.Connections.IsNewConnectionAllowed(testIP, common.ProtocolFTP)) + entry := entries[0] + entry.Protocols = 1 + _, _, err = httpdtest.UpdateIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) + assert.Error(t, common.Connections.IsNewConnectionAllowed(testIP, common.ProtocolFTP)) + assert.NoError(t, common.Connections.IsNewConnectionAllowed(testIP, common.ProtocolSSH)) + _, err = httpdtest.RemoveIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) + entries = entries[1:] + assert.Error(t, common.Connections.IsNewConnectionAllowed(testIP, common.ProtocolSSH)) + assert.Error(t, common.Connections.IsNewConnectionAllowed("172.18.1.3", common.ProtocolSSH)) + assert.NoError(t, common.Connections.IsNewConnectionAllowed("172.18.1.3", common.ProtocolHTTP)) + + assert.NoError(t, common.Connections.IsNewConnectionAllowed("10.8.7.3", common.ProtocolWebDAV)) + assert.NoError(t, common.Connections.IsNewConnectionAllowed("10.8.7.4", common.ProtocolSSH)) + assert.Error(t, common.Connections.IsNewConnectionAllowed("10.8.7.4", common.ProtocolFTP)) + assert.NoError(t, common.Connections.IsNewConnectionAllowed("10.8.7.4", common.ProtocolHTTP)) + assert.NoError(t, common.Connections.IsNewConnectionAllowed("2001:0db8::1428:57ab", common.ProtocolHTTP)) + assert.Error(t, common.Connections.IsNewConnectionAllowed("2001:0db8::1428:57ab", common.ProtocolSSH)) + assert.Error(t, common.Connections.IsNewConnectionAllowed("10.8.8.2", common.ProtocolWebDAV)) + assert.Error(t, common.Connections.IsNewConnectionAllowed("invalid IP", common.ProtocolHTTP)) + + common.Config = configCopy + err = common.Initialize(common.Config, 0) + assert.NoError(t, err) + assert.False(t, common.Config.IsAllowListEnabled()) + + for _, e := range entries { + _, err := httpdtest.RemoveIPListEntry(e, http.StatusOK) + assert.NoError(t, err) + } +} + func TestDbDefenderErrors(t *testing.T) { if !isDbDefenderSupported() { t.Skip("this test is not supported with the current database provider") @@ -3203,7 +3284,7 @@ func TestDbDefenderErrors(t *testing.T) { hosts, err := common.GetDefenderHosts() assert.NoError(t, err) assert.Len(t, hosts, 0) - common.AddDefenderEvent(testIP, common.HostEventLimitExceeded) + common.AddDefenderEvent(testIP, common.ProtocolSSH, common.HostEventLimitExceeded) hosts, err = common.GetDefenderHosts() assert.NoError(t, err) assert.Len(t, hosts, 1) @@ -3217,7 +3298,7 @@ func TestDbDefenderErrors(t *testing.T) { err = dataprovider.Close() assert.NoError(t, err) - common.AddDefenderEvent(testIP, common.HostEventLimitExceeded) + common.AddDefenderEvent(testIP, common.ProtocolFTP, common.HostEventLimitExceeded) _, err = common.GetDefenderHosts() assert.Error(t, err) _, err = common.GetDefenderHost(testIP) diff --git a/internal/common/ratelimiter.go b/internal/common/ratelimiter.go index 0f8d523b..fea554ef 100644 --- a/internal/common/ratelimiter.go +++ b/internal/common/ratelimiter.go @@ -17,7 +17,6 @@ package common import ( "errors" "fmt" - "net" "sort" "sync" "sync/atomic" @@ -62,8 +61,6 @@ type RateLimiterConfig struct { // Available protocols are: "SFTP", "FTP", "DAV". // A rate limiter with no protocols defined is disabled Protocols []string `json:"protocols" mapstructure:"protocols"` - // AllowList defines a list of IP addresses and IP ranges excluded from rate limiting - AllowList []string `json:"allow_list" mapstructure:"mapstructure"` // If the rate limit is exceeded, the defender is enabled, and this is a per-source limiter, // a new defender event will be generated GenerateDefenderEvents bool `json:"generate_defender_events" mapstructure:"generate_defender_events"` @@ -142,23 +139,12 @@ type rateLimiter struct { globalBucket *rate.Limiter buckets sourceBuckets generateDefenderEvents bool - allowList []func(net.IP) bool } // Wait blocks until the limit allows one event to happen // or returns an error if the time to wait exceeds the max // allowed delay -func (rl *rateLimiter) Wait(source string) (time.Duration, error) { - if len(rl.allowList) > 0 { - ip := net.ParseIP(source) - if ip != nil { - for idx := range rl.allowList { - if rl.allowList[idx](ip) { - return 0, nil - } - } - } - } +func (rl *rateLimiter) Wait(source, protocol string) (time.Duration, error) { var res *rate.Reservation if rl.globalBucket != nil { res = rl.globalBucket.Reserve() @@ -177,7 +163,7 @@ func (rl *rateLimiter) Wait(source string) (time.Duration, error) { if delay > rl.maxDelay { res.Cancel() if rl.generateDefenderEvents && rl.globalBucket == nil { - AddDefenderEvent(source, HostEventLimitExceeded) + AddDefenderEvent(source, protocol, HostEventLimitExceeded) } return delay, fmt.Errorf("rate limit exceed, wait time to respect rate %v, max wait time allowed %v", delay, rl.maxDelay) } diff --git a/internal/common/ratelimiter_test.go b/internal/common/ratelimiter_test.go index 32ba3f19..ea55a4ad 100644 --- a/internal/common/ratelimiter_test.go +++ b/internal/common/ratelimiter_test.go @@ -20,8 +20,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/drakkan/sftpgo/v2/internal/util" ) func TestRateLimiterConfig(t *testing.T) { @@ -79,9 +77,9 @@ func TestRateLimiter(t *testing.T) { Protocols: rateLimiterProtocolValues, } limiter := config.getLimiter() - _, err := limiter.Wait("") + _, err := limiter.Wait("", ProtocolFTP) require.NoError(t, err) - _, err = limiter.Wait("") + _, err = limiter.Wait("", ProtocolSSH) require.Error(t, err) config.Type = int(rateLimiterTypeSource) @@ -91,28 +89,17 @@ func TestRateLimiter(t *testing.T) { limiter = config.getLimiter() source := "192.168.1.2" - _, err = limiter.Wait(source) + _, err = limiter.Wait(source, ProtocolSSH) require.NoError(t, err) - _, err = limiter.Wait(source) + _, err = limiter.Wait(source, ProtocolSSH) require.Error(t, err) // a different source should work - _, err = limiter.Wait(source + "1") - require.NoError(t, err) - - allowList := []string{"192.168.1.0/24"} - allowFuncs, err := util.ParseAllowedIPAndRanges(allowList) - assert.NoError(t, err) - limiter.allowList = allowFuncs - for i := 0; i < 5; i++ { - _, err = limiter.Wait(source) - require.NoError(t, err) - } - _, err = limiter.Wait("not an ip") + _, err = limiter.Wait(source+"1", ProtocolSSH) require.NoError(t, err) config.Burst = 0 limiter = config.getLimiter() - _, err = limiter.Wait(source) + _, err = limiter.Wait(source, ProtocolSSH) require.ErrorIs(t, err, errReserve) } @@ -131,10 +118,10 @@ func TestLimiterCleanup(t *testing.T) { source2 := "10.8.0.2" source3 := "10.8.0.3" source4 := "10.8.0.4" - _, err := limiter.Wait(source1) + _, err := limiter.Wait(source1, ProtocolSSH) assert.NoError(t, err) time.Sleep(20 * time.Millisecond) - _, err = limiter.Wait(source2) + _, err = limiter.Wait(source2, ProtocolSSH) assert.NoError(t, err) time.Sleep(20 * time.Millisecond) assert.Len(t, limiter.buckets.buckets, 2) @@ -142,7 +129,7 @@ func TestLimiterCleanup(t *testing.T) { assert.True(t, ok) _, ok = limiter.buckets.buckets[source2] assert.True(t, ok) - _, err = limiter.Wait(source3) + _, err = limiter.Wait(source3, ProtocolSSH) assert.NoError(t, err) assert.Len(t, limiter.buckets.buckets, 3) _, ok = limiter.buckets.buckets[source1] @@ -152,7 +139,7 @@ func TestLimiterCleanup(t *testing.T) { _, ok = limiter.buckets.buckets[source3] assert.True(t, ok) time.Sleep(20 * time.Millisecond) - _, err = limiter.Wait(source4) + _, err = limiter.Wait(source4, ProtocolSSH) assert.NoError(t, err) assert.Len(t, limiter.buckets.buckets, 2) _, ok = limiter.buckets.buckets[source3] diff --git a/internal/common/tlsutils.go b/internal/common/tlsutils.go index e49a04aa..19f65ba4 100644 --- a/internal/common/tlsutils.go +++ b/internal/common/tlsutils.go @@ -139,7 +139,7 @@ func (m *CertManager) IsRevoked(crt *x509.Certificate, caCrt *x509.Certificate) } for _, crl := range m.crls { - if !crl.HasExpired(time.Now()) && caCrt.CheckCRLSignature(crl) == nil { + if !crl.HasExpired(time.Now()) && caCrt.CheckCRLSignature(crl) == nil { //nolint:staticcheck for _, rc := range crl.TBSCertList.RevokedCertificates { if rc.SerialNumber.Cmp(crt.SerialNumber) == 0 { return true @@ -171,7 +171,7 @@ func (m *CertManager) LoadCRLs() error { logger.Warn(m.logSender, "", "unable to read revocation list %q", revocationList) return err } - crl, err := x509.ParseCRL(crlBytes) + crl, err := x509.ParseCRL(crlBytes) //nolint:staticcheck if err != nil { logger.Warn(m.logSender, "", "unable to parse revocation list %q", revocationList) return err diff --git a/internal/config/config.go b/internal/config/config.go index 61cf3174..9bd69961 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -153,7 +153,6 @@ var ( Burst: 1, Type: 2, Protocols: []string{common.ProtocolSSH, common.ProtocolFTP, common.ProtocolWebDAV, common.ProtocolHTTP}, - AllowList: []string{}, GenerateDefenderEvents: false, EntriesSoftLimit: 100, EntriesHardLimit: 150, @@ -210,7 +209,7 @@ func Init() { DataRetentionHook: "", MaxTotalConnections: 0, MaxPerHostConnections: 20, - WhiteListFile: "", + AllowListStatus: 0, AllowSelfConnections: 0, DefenderConfig: common.DefenderConfig{ Enabled: false, @@ -225,10 +224,6 @@ func Init() { ObservationTime: 30, EntriesSoftLimit: 100, EntriesHardLimit: 150, - SafeListFile: "", - BlockListFile: "", - SafeList: []string{}, - BlockList: []string{}, }, RateLimitersConfig: []common.RateLimiterConfig{defaultRateLimiter}, }, @@ -889,12 +884,6 @@ func getRateLimitersFromEnv(idx int) { isSet = true } - allowList, ok := lookupStringListFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__ALLOW_LIST", idx)) - if ok { - rtlConfig.AllowList = allowList - isSet = true - } - generateEvents, ok := lookupBoolFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__GENERATE_DEFENDER_EVENTS", idx)) if ok { rtlConfig.GenerateDefenderEvents = generateEvents @@ -1959,7 +1948,7 @@ func setViperDefaults() { viper.SetDefault("common.data_retention_hook", globalConf.Common.DataRetentionHook) viper.SetDefault("common.max_total_connections", globalConf.Common.MaxTotalConnections) viper.SetDefault("common.max_per_host_connections", globalConf.Common.MaxPerHostConnections) - viper.SetDefault("common.whitelist_file", globalConf.Common.WhiteListFile) + viper.SetDefault("common.allowlist_status", globalConf.Common.AllowListStatus) viper.SetDefault("common.allow_self_connections", globalConf.Common.AllowSelfConnections) viper.SetDefault("common.defender.enabled", globalConf.Common.DefenderConfig.Enabled) viper.SetDefault("common.defender.driver", globalConf.Common.DefenderConfig.Driver) @@ -1973,10 +1962,6 @@ func setViperDefaults() { viper.SetDefault("common.defender.observation_time", globalConf.Common.DefenderConfig.ObservationTime) viper.SetDefault("common.defender.entries_soft_limit", globalConf.Common.DefenderConfig.EntriesSoftLimit) viper.SetDefault("common.defender.entries_hard_limit", globalConf.Common.DefenderConfig.EntriesHardLimit) - viper.SetDefault("common.defender.safelist_file", globalConf.Common.DefenderConfig.SafeListFile) - viper.SetDefault("common.defender.blocklist_file", globalConf.Common.DefenderConfig.BlockListFile) - viper.SetDefault("common.defender.safelist", globalConf.Common.DefenderConfig.SafeList) - viper.SetDefault("common.defender.blocklist", globalConf.Common.DefenderConfig.BlockList) viper.SetDefault("acme.email", globalConf.ACME.Email) viper.SetDefault("acme.key_type", globalConf.ACME.KeyType) viper.SetDefault("acme.certs_path", globalConf.ACME.CertsPath) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4e634bf4..43780f8f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -804,9 +804,7 @@ func TestRateLimitersFromEnv(t *testing.T) { os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS", "1") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT", "50") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT", "100") - os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST", ", 172.16.2.4, ") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE", "50") - os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST", "192.168.1.1, 192.168.2.0/24") t.Cleanup(func() { os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__AVERAGE") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__PERIOD") @@ -816,9 +814,7 @@ func TestRateLimitersFromEnv(t *testing.T) { os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT") - os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE") - os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST") }) err := config.LoadConfig(configDir, "") @@ -836,12 +832,7 @@ func TestRateLimitersFromEnv(t *testing.T) { require.True(t, limiters[0].GenerateDefenderEvents) require.Equal(t, 50, limiters[0].EntriesSoftLimit) require.Equal(t, 100, limiters[0].EntriesHardLimit) - require.Len(t, limiters[0].AllowList, 1) - require.Equal(t, "172.16.2.4", limiters[0].AllowList[0]) require.Equal(t, int64(50), limiters[1].Average) - require.Len(t, limiters[1].AllowList, 2) - require.Equal(t, "192.168.1.1", limiters[1].AllowList[0]) - require.Equal(t, "192.168.2.0/24", limiters[1].AllowList[1]) // we check the default values here require.Equal(t, int64(1000), limiters[1].Period) require.Equal(t, 1, limiters[1].Burst) diff --git a/internal/dataprovider/actions.go b/internal/dataprovider/actions.go index f6963f04..f9eb9e93 100644 --- a/internal/dataprovider/actions.go +++ b/internal/dataprovider/actions.go @@ -51,6 +51,7 @@ const ( actionObjectEventAction = "event_action" actionObjectEventRule = "event_rule" actionObjectRole = "role" + actionObjectIPListEntry = "ip_list_entry" ) var ( diff --git a/internal/dataprovider/admin.go b/internal/dataprovider/admin.go index bf8064e6..edd02b24 100644 --- a/internal/dataprovider/admin.go +++ b/internal/dataprovider/admin.go @@ -57,6 +57,7 @@ const ( PermAdminViewEvents = "view_events" PermAdminManageEventRules = "manage_event_rules" PermAdminManageRoles = "manage_roles" + PermAdminManageIPLists = "manage_ip_lists" ) const ( @@ -73,9 +74,10 @@ var ( PermAdminViewUsers, PermAdminManageGroups, PermAdminViewConnections, PermAdminCloseConnections, PermAdminViewServerStatus, PermAdminManageAdmins, PermAdminManageRoles, PermAdminManageEventRules, PermAdminManageAPIKeys, PermAdminQuotaScans, PermAdminManageSystem, PermAdminManageDefender, - PermAdminViewDefender, PermAdminRetentionChecks, PermAdminMetadataChecks, PermAdminViewEvents} + PermAdminViewDefender, PermAdminManageIPLists, PermAdminRetentionChecks, PermAdminMetadataChecks, + PermAdminViewEvents} forbiddenPermsForRoleAdmins = []string{PermAdminAny, PermAdminManageAdmins, PermAdminManageSystem, - PermAdminManageEventRules, PermAdminManageRoles} + PermAdminManageEventRules, PermAdminManageIPLists, PermAdminManageRoles} ) // AdminTOTPConfig defines the time-based one time password configuration diff --git a/internal/dataprovider/bolt.go b/internal/dataprovider/bolt.go index 5fc72e41..d1c61b35 100644 --- a/internal/dataprovider/bolt.go +++ b/internal/dataprovider/bolt.go @@ -18,10 +18,12 @@ package dataprovider import ( + "bytes" "crypto/x509" "encoding/json" "errors" "fmt" + "net/netip" "path/filepath" "sort" "time" @@ -35,7 +37,7 @@ import ( ) const ( - boltDatabaseVersion = 26 + boltDatabaseVersion = 27 ) var ( @@ -48,10 +50,11 @@ var ( actionsBucket = []byte("events_actions") rulesBucket = []byte("events_rules") rolesBucket = []byte("roles") + ipListsBucket = []byte("ip_lists") dbVersionBucket = []byte("db_version") dbVersionKey = []byte("version") boltBuckets = [][]byte{usersBucket, groupsBucket, foldersBucket, adminsBucket, apiKeysBucket, - sharesBucket, actionsBucket, rulesBucket, rolesBucket, dbVersionBucket} + sharesBucket, actionsBucket, rulesBucket, rolesBucket, ipListsBucket, dbVersionBucket} ) // BoltProvider defines the auth provider for bolt key/value store @@ -85,7 +88,7 @@ func initializeBoltProvider(basePath string) error { _, e := tx.CreateBucketIfNotExists(bucket) return e }); err != nil { - providerLog(logger.LevelError, "error creating bucket %#v: %v", string(bucket), err) + providerLog(logger.LevelError, "error creating bucket %q: %v", string(bucket), err) } } @@ -2749,6 +2752,231 @@ func (p *BoltProvider) dumpRoles() ([]Role, error) { return roles, err } +func (p *BoltProvider) ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + entry := IPListEntry{ + IPOrNet: ipOrNet, + Type: listType, + } + err := p.dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + e := bucket.Get([]byte(entry.getKey())) + if e == nil { + return util.NewRecordNotFoundError(fmt.Sprintf("entry %q does not exist", entry.IPOrNet)) + } + err = json.Unmarshal(e, &entry) + if err == nil { + entry.PrepareForRendering() + } + return err + }) + return entry, err +} + +func (p *BoltProvider) addIPListEntry(entry *IPListEntry) error { + if err := entry.validate(); err != nil { + return err + } + return p.dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + if e := bucket.Get([]byte(entry.getKey())); e != nil { + return fmt.Errorf("entry %q already exists", entry.IPOrNet) + } + entry.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + entry.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + buf, err := json.Marshal(entry) + if err != nil { + return err + } + return bucket.Put([]byte(entry.getKey()), buf) + }) +} + +func (p *BoltProvider) updateIPListEntry(entry *IPListEntry) error { + if err := entry.validate(); err != nil { + return err + } + return p.dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + var e []byte + if e = bucket.Get([]byte(entry.getKey())); e == nil { + return fmt.Errorf("entry %q does not exist", entry.IPOrNet) + } + var oldEntry IPListEntry + err = json.Unmarshal(e, &oldEntry) + if err != nil { + return err + } + entry.CreatedAt = oldEntry.CreatedAt + entry.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + buf, err := json.Marshal(entry) + if err != nil { + return err + } + return bucket.Put([]byte(entry.getKey()), buf) + }) +} + +func (p *BoltProvider) deleteIPListEntry(entry IPListEntry, softDelete bool) error { + return p.dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + if e := bucket.Get([]byte(entry.getKey())); e == nil { + return fmt.Errorf("entry %q does not exist", entry.IPOrNet) + } + return bucket.Delete([]byte(entry.getKey())) + }) +} + +func (p *BoltProvider) getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + entries := make([]IPListEntry, 0, 15) + err := p.dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + prefix := []byte(fmt.Sprintf("%d_", listType)) + acceptKey := func(k []byte) bool { + return k != nil && bytes.HasPrefix(k, prefix) + } + cursor := bucket.Cursor() + if order == OrderASC { + for k, v := cursor.Seek(prefix); acceptKey(k); k, v = cursor.Next() { + var entry IPListEntry + err = json.Unmarshal(v, &entry) + if err != nil { + return err + } + if entry.satisfySearchConstraints(filter, from, order) { + entry.PrepareForRendering() + entries = append(entries, entry) + if limit > 0 && len(entries) >= limit { + break + } + } + } + } else { + for k, v := cursor.Last(); acceptKey(k); k, v = cursor.Prev() { + var entry IPListEntry + err = json.Unmarshal(v, &entry) + if err != nil { + return err + } + if entry.satisfySearchConstraints(filter, from, order) { + entry.PrepareForRendering() + entries = append(entries, entry) + if limit > 0 && len(entries) >= limit { + break + } + } + } + } + return nil + }) + return entries, err +} + +func (p *BoltProvider) getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) { + return nil, ErrNotImplemented +} + +func (p *BoltProvider) dumpIPListEntries() ([]IPListEntry, error) { + entries := make([]IPListEntry, 0, 10) + err := p.dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + if count := bucket.Stats().KeyN; count > ipListMemoryLimit { + providerLog(logger.LevelInfo, "IP lists excluded from dump, too many entries: %d", count) + return nil + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var entry IPListEntry + err = json.Unmarshal(v, &entry) + if err != nil { + return err + } + entry.PrepareForRendering() + entries = append(entries, entry) + } + return nil + }) + return entries, err +} + +func (p *BoltProvider) countIPListEntries(listType IPListType) (int64, error) { + var count int64 + err := p.dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + if listType == 0 { + count = int64(bucket.Stats().KeyN) + return nil + } + prefix := []byte(fmt.Sprintf("%d_", listType)) + cursor := bucket.Cursor() + for k, _ := cursor.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = cursor.Next() { + count++ + } + return nil + }) + return count, err +} + +func (p *BoltProvider) getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) { + entries := make([]IPListEntry, 0, 3) + ipAddr, err := netip.ParseAddr(ip) + if err != nil { + return entries, fmt.Errorf("invalid ip address %s", ip) + } + var netType int + var ipBytes []byte + if ipAddr.Is4() || ipAddr.Is4In6() { + netType = ipTypeV4 + as4 := ipAddr.As4() + ipBytes = as4[:] + } else { + netType = ipTypeV6 + as16 := ipAddr.As16() + ipBytes = as16[:] + } + err = p.dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := p.getIPListsBucket(tx) + if err != nil { + return err + } + prefix := []byte(fmt.Sprintf("%d_", listType)) + cursor := bucket.Cursor() + for k, v := cursor.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = cursor.Next() { + var entry IPListEntry + err = json.Unmarshal(v, &entry) + if err != nil { + return err + } + if entry.IPType == netType && bytes.Compare(ipBytes, entry.First) >= 0 && bytes.Compare(ipBytes, entry.Last) <= 0 { + entry.PrepareForRendering() + entries = append(entries, entry) + } + } + return nil + }) + return entries, err +} + func (p *BoltProvider) setFirstDownloadTimestamp(username string) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { bucket, err := p.getUsersBucket(tx) @@ -2833,9 +3061,9 @@ func (p *BoltProvider) migrateDatabase() error { providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err - case version == 23, version == 24, version == 25: - logger.InfoToConsole("updating database schema version: %d -> 26", version) - providerLog(logger.LevelInfo, "updating database schema version: %d -> 26", version) + case version == 23, version == 24, version == 25, version == 26: + logger.InfoToConsole("updating database schema version: %d -> 27", version) + providerLog(logger.LevelInfo, "updating database schema version: %d -> 27", version) err := p.dbHandle.Update(func(tx *bolt.Tx) error { rules, err := p.dumpEventRules() if err != nil { @@ -2845,8 +3073,8 @@ func (p *BoltProvider) migrateDatabase() error { if err != nil { return err } - for _, rule := range rules { - rule := rule // pin + for idx := range rules { + rule := rules[idx] if rule.Status == 1 { continue } @@ -2867,7 +3095,7 @@ func (p *BoltProvider) migrateDatabase() error { if err != nil { return err } - return updateBoltDatabaseVersion(p.dbHandle, 26) + return updateBoltDatabaseVersion(p.dbHandle, 27) default: if version > boltDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -3639,6 +3867,15 @@ func (p *BoltProvider) getRolesBucket(tx *bolt.Tx) (*bolt.Bucket, error) { return bucket, err } +func (p *BoltProvider) getIPListsBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + var err error + bucket := tx.Bucket(rolesBucket) + if bucket == nil { + err = fmt.Errorf("unable to find IP lists bucket, bolt database structure not correcly defined") + } + return bucket, err +} + func (p *BoltProvider) getFoldersBucket(tx *bolt.Tx) (*bolt.Bucket, error) { var err error bucket := tx.Bucket(foldersBucket) diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go index dc3bd940..67465fe2 100644 --- a/internal/dataprovider/dataprovider.go +++ b/internal/dataprovider/dataprovider.go @@ -13,7 +13,7 @@ // along with this program. If not, see . // Package dataprovider provides data access. -// It abstracts different data providers and exposes a common API. +// It abstracts different data providers using a common API. package dataprovider import ( @@ -87,7 +87,7 @@ const ( CockroachDataProviderName = "cockroachdb" // DumpVersion defines the version for the dump. // For restore/load we support the current version and the previous one - DumpVersion = 15 + DumpVersion = 16 argonPwdPrefix = "$argon2id$" bcryptPwdPrefix = "$2a$" @@ -191,6 +191,7 @@ var ( sqlTableTasks string sqlTableNodes string sqlTableRoles string + sqlTableIPLists string sqlTableSchemaVersion string argon2Params *argon2id.Params lastLoginMinDelay = 10 * time.Minute @@ -223,6 +224,7 @@ func initSQLTables() { sqlTableTasks = "tasks" sqlTableNodes = "nodes" sqlTableRoles = "roles" + sqlTableIPLists = "ip_lists" sqlTableSchemaVersion = "schema_version" } @@ -663,6 +665,7 @@ type BackupData struct { EventActions []BaseEventAction `json:"event_actions"` EventRules []EventRule `json:"event_rules"` Roles []Role `json:"roles"` + IPLists []IPListEntry `json:"ip_lists"` Version int `json:"version"` } @@ -805,6 +808,15 @@ type Provider interface { deleteRole(role Role) error getRoles(limit int, offset int, order string, minimal bool) ([]Role, error) dumpRoles() ([]Role, error) + ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) + addIPListEntry(entry *IPListEntry) error + updateIPListEntry(entry *IPListEntry) error + deleteIPListEntry(entry IPListEntry, softDelete bool) error + getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) + getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) + dumpIPListEntries() ([]IPListEntry, error) + countIPListEntries(listType IPListType) (int64, error) + getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) checkAvailability() error close() error reloadConfig() error @@ -984,16 +996,18 @@ func validateSQLTablesPrefix() error { sqlTableTasks = config.SQLTablesPrefix + sqlTableTasks sqlTableNodes = config.SQLTablesPrefix + sqlTableNodes sqlTableRoles = config.SQLTablesPrefix + sqlTableRoles + sqlTableIPLists = config.SQLTablesPrefix + sqlTableIPLists sqlTableSchemaVersion = config.SQLTablesPrefix + sqlTableSchemaVersion providerLog(logger.LevelDebug, "sql table for users %q, folders %q users folders mapping %q admins %q "+ "api keys %q shares %q defender hosts %q defender events %q transfers %q groups %q "+ "users groups mapping %q admins groups mapping %q groups folders mapping %q shared sessions %q "+ - "schema version %q events actions %q events rules %q rules actions mapping %q tasks %q nodes %q roles %q", + "schema version %q events actions %q events rules %q rules actions mapping %q tasks %q nodes %q roles %q"+ + "ip lists %q", sqlTableUsers, sqlTableFolders, sqlTableUsersFoldersMapping, sqlTableAdmins, sqlTableAPIKeys, sqlTableShares, sqlTableDefenderHosts, sqlTableDefenderEvents, sqlTableActiveTransfers, sqlTableGroups, sqlTableUsersGroupsMapping, sqlTableAdminsGroupsMapping, sqlTableGroupsFoldersMapping, sqlTableSharedSessions, sqlTableSchemaVersion, sqlTableEventsActions, sqlTableEventsRules, sqlTableRulesActionsMapping, - sqlTableTasks, sqlTableNodes, sqlTableRoles) + sqlTableTasks, sqlTableNodes, sqlTableRoles, sqlTableIPLists) } return nil } @@ -1543,6 +1557,59 @@ func ShareExists(shareID, username string) (Share, error) { return provider.shareExists(shareID, username) } +// AddIPListEntry adds a new IP list entry +func AddIPListEntry(entry *IPListEntry, executor, ipAddress, executorRole string) error { + err := provider.addIPListEntry(entry) + if err == nil { + executeAction(operationAdd, executor, ipAddress, actionObjectIPListEntry, entry.getName(), executorRole, entry) + for _, l := range inMemoryLists { + l.addEntry(entry) + } + } + return err +} + +// UpdateIPListEntry updates an existing IP list entry +func UpdateIPListEntry(entry *IPListEntry, executor, ipAddress, executorRole string) error { + err := provider.updateIPListEntry(entry) + if err == nil { + executeAction(operationUpdate, executor, ipAddress, actionObjectIPListEntry, entry.getName(), executorRole, entry) + for _, l := range inMemoryLists { + l.updateEntry(entry) + } + } + return err +} + +// DeleteIPListEntry deletes an existing IP list entry +func DeleteIPListEntry(ipOrNet string, listType IPListType, executor, ipAddress, executorRole string) error { + entry, err := provider.ipListEntryExists(ipOrNet, listType) + if err != nil { + return err + } + err = provider.deleteIPListEntry(entry, config.IsShared == 1) + if err == nil { + executeAction(operationDelete, executor, ipAddress, actionObjectIPListEntry, entry.getName(), executorRole, &entry) + for _, l := range inMemoryLists { + l.removeEntry(&entry) + } + } + return err +} + +// IPListEntryExists returns the IP list entry with the given IP/net and type if it exists +func IPListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + return provider.ipListEntryExists(ipOrNet, listType) +} + +// GetIPListEntries returns the IP list entries applying the specified criteria and search limit +func GetIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + if !util.Contains(supportedIPListType, listType) { + return nil, util.NewValidationError(fmt.Sprintf("invalid list type %d", listType)) + } + return provider.getIPListEntries(listType, filter, from, order, limit) +} + // AddRole adds a new role func AddRole(role *Role, executor, ipAddress, executorRole string) error { role.Name = config.convertName(role.Name) @@ -2235,6 +2302,10 @@ func DumpData() (BackupData, error) { if err != nil { return data, err } + ipLists, err := provider.dumpIPListEntries() + if err != nil { + return data, err + } data.Users = users data.Groups = groups data.Folders = folders @@ -2244,6 +2315,7 @@ func DumpData() (BackupData, error) { data.EventActions = actions data.EventRules = rules data.Roles = roles + data.IPLists = ipLists data.Version = DumpVersion return data, err } diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go index 33cb781b..2b237d4c 100644 --- a/internal/dataprovider/eventrule.go +++ b/internal/dataprovider/eventrule.go @@ -500,7 +500,7 @@ func (c *EventActionEmailConfig) validate() error { // FolderRetention defines a folder retention configuration type FolderRetention struct { - // Path is the exposed virtual directory path, if no other specific retention is defined, + // Path is the virtual directory path, if no other specific retention is defined, // the retention applies for sub directories too. For example if retention is defined // for the paths "/" and "/sub" then the retention for "/" is applied for any file outside // the "/sub" directory diff --git a/internal/dataprovider/iplist.go b/internal/dataprovider/iplist.go new file mode 100644 index 00000000..b9dcab97 --- /dev/null +++ b/internal/dataprovider/iplist.go @@ -0,0 +1,493 @@ +// Copyright (C) 2019-2023 Nicola Murino +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dataprovider + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "strings" + "sync" + "sync/atomic" + + "github.com/yl2chen/cidranger" + + "github.com/drakkan/sftpgo/v2/internal/logger" + "github.com/drakkan/sftpgo/v2/internal/util" +) + +const ( + // maximum number of entries to match in memory + // if the list contains more elements than this limit a + // database query will be executed + ipListMemoryLimit = 15000 +) + +var ( + inMemoryLists map[IPListType]*IPList +) + +func init() { + inMemoryLists = map[IPListType]*IPList{} +} + +// IPListType is the enumerable for the supported IP list types +type IPListType int + +// AsString returns the string representation for the list type +func (t IPListType) AsString() string { + switch t { + case IPListTypeAllowList: + return "Allow list" + case IPListTypeDefender: + return "Defender" + case IPListTypeRateLimiterSafeList: + return "Rate limiters safe list" + default: + return "" + } +} + +// Supported IP list types +const ( + IPListTypeAllowList IPListType = iota + 1 + IPListTypeDefender + IPListTypeRateLimiterSafeList +) + +// Supported IP list modes +const ( + ListModeAllow = iota + 1 + ListModeDeny +) + +const ( + ipTypeV4 = iota + 1 + ipTypeV6 +) + +var ( + supportedIPListType = []IPListType{IPListTypeAllowList, IPListTypeDefender, IPListTypeRateLimiterSafeList} +) + +// CheckIPListType returns an error if the provided IP list type is not valid +func CheckIPListType(t IPListType) error { + if !util.Contains(supportedIPListType, t) { + return util.NewValidationError(fmt.Sprintf("invalid list type %d", t)) + } + return nil +} + +// IPListEntry defines an entry for the IP addresses list +type IPListEntry struct { + IPOrNet string `json:"ipornet"` + Description string `json:"description,omitempty"` + Type IPListType `json:"type"` + Mode int `json:"mode"` + // Defines the protocols the entry applies to + // - 0 all the supported protocols + // - 1 SSH + // - 2 FTP + // - 4 WebDAV + // - 8 HTTP + // Protocols can be combined + Protocols int `json:"protocols"` + First []byte `json:"first,omitempty"` + Last []byte `json:"last,omitempty"` + IPType int `json:"ip_type,omitempty"` + // Creation time as unix timestamp in milliseconds + CreatedAt int64 `json:"created_at"` + // last update time as unix timestamp in milliseconds + UpdatedAt int64 `json:"updated_at"` + // in multi node setups we mark the rule as deleted to be able to update the cache + DeletedAt int64 `json:"-"` +} + +// PrepareForRendering prepares an IP list entry for rendering. +// It hides internal fields +func (e *IPListEntry) PrepareForRendering() { + e.First = nil + e.Last = nil + e.IPType = 0 +} + +// HasProtocol returns true if the specified protocol is defined +func (e *IPListEntry) HasProtocol(proto string) bool { + switch proto { + case protocolSSH: + return e.Protocols&1 != 0 + case protocolFTP: + return e.Protocols&2 != 0 + case protocolWebDAV: + return e.Protocols&4 != 0 + case protocolHTTP: + return e.Protocols&8 != 0 + default: + return false + } +} + +// RenderAsJSON implements the renderer interface used within plugins +func (e *IPListEntry) RenderAsJSON(reload bool) ([]byte, error) { + if reload { + entry, err := provider.ipListEntryExists(e.IPOrNet, e.Type) + if err != nil { + providerLog(logger.LevelError, "unable to reload IP list entry before rendering as json: %v", err) + return nil, err + } + entry.PrepareForRendering() + return json.Marshal(entry) + } + e.PrepareForRendering() + return json.Marshal(e) +} + +func (e *IPListEntry) getKey() string { + return fmt.Sprintf("%d_%s", e.Type, e.IPOrNet) +} + +func (e *IPListEntry) getName() string { + return e.Type.AsString() + "-" + e.IPOrNet +} + +func (e *IPListEntry) getFirst() netip.Addr { + if e.IPType == ipTypeV4 { + var a4 [4]byte + copy(a4[:], e.First) + return netip.AddrFrom4(a4) + } + var a16 [16]byte + copy(a16[:], e.First) + return netip.AddrFrom16(a16) +} + +func (e *IPListEntry) getLast() netip.Addr { + if e.IPType == ipTypeV4 { + var a4 [4]byte + copy(a4[:], e.Last) + return netip.AddrFrom4(a4) + } + var a16 [16]byte + copy(a16[:], e.Last) + return netip.AddrFrom16(a16) +} + +func (e *IPListEntry) checkProtocols() { + for _, proto := range ValidProtocols { + if !e.HasProtocol(proto) { + return + } + } + e.Protocols = 0 +} + +func (e *IPListEntry) validate() error { + if err := CheckIPListType(e.Type); err != nil { + return err + } + e.checkProtocols() + switch e.Type { + case IPListTypeDefender: + if e.Mode < ListModeAllow || e.Mode > ListModeDeny { + return util.NewValidationError(fmt.Sprintf("invalid list mode: %d", e.Mode)) + } + default: + if e.Mode != ListModeAllow { + return util.NewValidationError("invalid list mode") + } + } + e.PrepareForRendering() + if !strings.Contains(e.IPOrNet, "/") { + // parse as IP + parsed, err := netip.ParseAddr(e.IPOrNet) + if err != nil { + return util.NewValidationError(fmt.Sprintf("invalid IP %q", e.IPOrNet)) + } + if parsed.Is4() { + e.IPOrNet += "/32" + } else if parsed.Is4In6() { + e.IPOrNet = netip.AddrFrom4(parsed.As4()).String() + "/32" + } else { + e.IPOrNet += "/128" + } + } + prefix, err := netip.ParsePrefix(e.IPOrNet) + if err != nil { + return util.NewValidationError(fmt.Sprintf("invalid network %q: %v", e.IPOrNet, err)) + } + prefix = prefix.Masked() + if prefix.Addr().Is4In6() { + e.IPOrNet = fmt.Sprintf("%s/%d", netip.AddrFrom4(prefix.Addr().As4()).String(), prefix.Bits()-96) + } + // TODO: to remove when the in memory ranger switch to netip + _, _, err = net.ParseCIDR(e.IPOrNet) + if err != nil { + return util.NewValidationError(fmt.Sprintf("invalid network: %v", err)) + } + if prefix.Addr().Is4() || prefix.Addr().Is4In6() { + e.IPType = ipTypeV4 + first := prefix.Addr().As4() + last := util.GetLastIPForPrefix(prefix).As4() + e.First = first[:] + e.Last = last[:] + } else { + e.IPType = ipTypeV6 + first := prefix.Addr().As16() + last := util.GetLastIPForPrefix(prefix).As16() + e.First = first[:] + e.Last = last[:] + } + return nil +} + +func (e *IPListEntry) getACopy() IPListEntry { + first := make([]byte, len(e.First)) + copy(first, e.First) + last := make([]byte, len(e.Last)) + copy(last, e.Last) + + return IPListEntry{ + IPOrNet: e.IPOrNet, + Description: e.Description, + Type: e.Type, + Mode: e.Mode, + First: first, + Last: last, + IPType: e.IPType, + Protocols: e.Protocols, + CreatedAt: e.CreatedAt, + UpdatedAt: e.UpdatedAt, + DeletedAt: e.DeletedAt, + } +} + +// getAsRangerEntry returns the entry as cidranger.RangerEntry +func (e *IPListEntry) getAsRangerEntry() (cidranger.RangerEntry, error) { + _, network, err := net.ParseCIDR(e.IPOrNet) + if err != nil { + return nil, err + } + entry := e.getACopy() + return &rangerEntry{ + entry: &entry, + network: *network, + }, nil +} + +func (e IPListEntry) satisfySearchConstraints(filter, from, order string) bool { + if filter != "" && !strings.HasPrefix(e.IPOrNet, filter) { + return false + } + if from != "" { + if order == OrderASC { + return e.IPOrNet > from + } + return e.IPOrNet < from + } + return true +} + +type rangerEntry struct { + entry *IPListEntry + network net.IPNet +} + +func (e *rangerEntry) Network() net.IPNet { + return e.network +} + +// IPList defines an IP list +type IPList struct { + isInMemory atomic.Bool + listType IPListType + mu sync.RWMutex + Ranges cidranger.Ranger +} + +func (l *IPList) addEntry(e *IPListEntry) { + if l.listType != e.Type { + return + } + if !l.isInMemory.Load() { + return + } + entry, err := e.getAsRangerEntry() + if err != nil { + providerLog(logger.LevelError, "unable to get entry to add %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + return + } + l.mu.Lock() + defer l.mu.Unlock() + + if err := l.Ranges.Insert(entry); err != nil { + providerLog(logger.LevelError, "unable to add entry %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + return + } + if l.Ranges.Len() >= ipListMemoryLimit { + providerLog(logger.LevelError, "memory limit exceeded for list type %d, disabling memory mode", l.listType) + l.isInMemory.Store(false) + } +} + +func (l *IPList) removeEntry(e *IPListEntry) { + if l.listType != e.Type { + return + } + if !l.isInMemory.Load() { + return + } + entry, err := e.getAsRangerEntry() + if err != nil { + providerLog(logger.LevelError, "unable to get entry to remove %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + return + } + l.mu.Lock() + defer l.mu.Unlock() + + if _, err := l.Ranges.Remove(entry.Network()); err != nil { + providerLog(logger.LevelError, "unable to remove entry %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + } +} + +func (l *IPList) updateEntry(e *IPListEntry) { + if l.listType != e.Type { + return + } + if !l.isInMemory.Load() { + return + } + entry, err := e.getAsRangerEntry() + if err != nil { + providerLog(logger.LevelError, "unable to get entry to update %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + return + } + l.mu.Lock() + defer l.mu.Unlock() + + if _, err := l.Ranges.Remove(entry.Network()); err != nil { + providerLog(logger.LevelError, "unable to remove entry to update %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + return + } + if err := l.Ranges.Insert(entry); err != nil { + providerLog(logger.LevelError, "unable to add entry to update %q for list type %d, disabling memory mode, err: %v", + e.IPOrNet, l.listType, err) + l.isInMemory.Store(false) + } + if l.Ranges.Len() >= ipListMemoryLimit { + providerLog(logger.LevelError, "memory limit exceeded for list type %d, disabling memory mode", l.listType) + l.isInMemory.Store(false) + } +} + +// DisableMemoryMode disables memory mode forcing database queries +func (l *IPList) DisableMemoryMode() { + l.isInMemory.Store(false) +} + +// IsListed checks if there is a match for the specified IP and protocol. +// If there are multiple matches, the first one is returned, in no particular order, +// so the behavior is undefined +func (l *IPList) IsListed(ip, protocol string) (bool, int, error) { + if l.isInMemory.Load() { + l.mu.RLock() + defer l.mu.RUnlock() + + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return false, 0, fmt.Errorf("invalid IP %s", ip) + } + + entries, err := l.Ranges.ContainingNetworks(parsedIP) + if err != nil { + return false, 0, fmt.Errorf("unable to find containing networks for ip %q: %w", ip, err) + } + for _, e := range entries { + entry, ok := e.(*rangerEntry) + if ok { + if entry.entry.Protocols == 0 || entry.entry.HasProtocol(protocol) { + return true, entry.entry.Mode, nil + } + } + } + + return false, 0, nil + } + + entries, err := provider.getListEntriesForIP(ip, l.listType) + if err != nil { + return false, 0, err + } + for _, e := range entries { + if e.Protocols == 0 || e.HasProtocol(protocol) { + return true, e.Mode, nil + } + } + + return false, 0, nil +} + +// NewIPList returns a new IP list for the specified type +func NewIPList(listType IPListType) (*IPList, error) { + delete(inMemoryLists, listType) + count, err := provider.countIPListEntries(listType) + if err != nil { + return nil, err + } + if count < ipListMemoryLimit { + providerLog(logger.LevelInfo, "using in-memory matching for list type %d, num entries: %d", listType, count) + entries, err := provider.getIPListEntries(listType, "", "", OrderASC, 0) + if err != nil { + return nil, err + } + ipList := &IPList{ + listType: listType, + Ranges: cidranger.NewPCTrieRanger(), + } + for idx := range entries { + e := entries[idx] + entry, err := e.getAsRangerEntry() + if err != nil { + return nil, fmt.Errorf("unable to get ranger for entry %q: %w", e.IPOrNet, err) + } + if err := ipList.Ranges.Insert(entry); err != nil { + return nil, fmt.Errorf("unable to add ranger for entry %q: %w", e.IPOrNet, err) + } + } + ipList.isInMemory.Store(true) + inMemoryLists[listType] = ipList + + return ipList, nil + } + providerLog(logger.LevelInfo, "list type %d has %d entries, in-memory matching disabled", listType, count) + ipList := &IPList{ + listType: listType, + Ranges: nil, + } + ipList.isInMemory.Store(false) + return ipList, nil +} diff --git a/internal/dataprovider/memory.go b/internal/dataprovider/memory.go index 136b7a8d..3fc50d39 100644 --- a/internal/dataprovider/memory.go +++ b/internal/dataprovider/memory.go @@ -15,9 +15,11 @@ package dataprovider import ( + "bytes" "crypto/x509" "errors" "fmt" + "net/netip" "os" "path/filepath" "sort" @@ -74,6 +76,10 @@ type memoryProviderHandle struct { roles map[string]Role // slice with ordered roles roleNames []string + // map for IP List entry + ipListEntries map[string]IPListEntry + // slice with ordered IP list entries + ipListEntriesKeys []string } // MemoryProvider defines the auth provider for a memory store @@ -91,26 +97,28 @@ func initializeMemoryProvider(basePath string) { } provider = &MemoryProvider{ dbHandle: &memoryProviderHandle{ - isClosed: false, - usernames: []string{}, - users: make(map[string]User), - groupnames: []string{}, - groups: make(map[string]Group), - vfolders: make(map[string]vfs.BaseVirtualFolder), - vfoldersNames: []string{}, - admins: make(map[string]Admin), - adminsUsernames: []string{}, - apiKeys: make(map[string]APIKey), - apiKeysIDs: []string{}, - shares: make(map[string]Share), - sharesIDs: []string{}, - actions: make(map[string]BaseEventAction), - actionsNames: []string{}, - rules: make(map[string]EventRule), - rulesNames: []string{}, - roles: map[string]Role{}, - roleNames: []string{}, - configFile: configFile, + isClosed: false, + usernames: []string{}, + users: make(map[string]User), + groupnames: []string{}, + groups: make(map[string]Group), + vfolders: make(map[string]vfs.BaseVirtualFolder), + vfoldersNames: []string{}, + admins: make(map[string]Admin), + adminsUsernames: []string{}, + apiKeys: make(map[string]APIKey), + apiKeysIDs: []string{}, + shares: make(map[string]Share), + sharesIDs: []string{}, + actions: make(map[string]BaseEventAction), + actionsNames: []string{}, + rules: make(map[string]EventRule), + rulesNames: []string{}, + roles: map[string]Role{}, + roleNames: []string{}, + ipListEntries: map[string]IPListEntry{}, + ipListEntriesKeys: []string{}, + configFile: configFile, }, } if err := provider.reloadConfig(); err != nil { @@ -670,6 +678,13 @@ func (p *MemoryProvider) roleExistsInternal(name string) (Role, error) { return Role{}, util.NewRecordNotFoundError(fmt.Sprintf("role %q does not exist", name)) } +func (p *MemoryProvider) ipListEntryExistsInternal(entry *IPListEntry) (IPListEntry, error) { + if val, ok := p.dbHandle.ipListEntries[entry.getKey()]; ok { + return val.getACopy(), nil + } + return IPListEntry{}, util.NewRecordNotFoundError(fmt.Sprintf("IP list entry %q does not exist", entry.getName())) +} + func (p *MemoryProvider) addAdmin(admin *Admin) error { p.dbHandle.Lock() defer p.dbHandle.Unlock() @@ -2590,6 +2605,198 @@ func (p *MemoryProvider) dumpRoles() ([]Role, error) { return roles, nil } +func (p *MemoryProvider) ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + if p.dbHandle.isClosed { + return IPListEntry{}, errMemoryProviderClosed + } + entry, err := p.ipListEntryExistsInternal(&IPListEntry{IPOrNet: ipOrNet, Type: listType}) + if err != nil { + return entry, err + } + entry.PrepareForRendering() + return entry, nil +} + +func (p *MemoryProvider) addIPListEntry(entry *IPListEntry) error { + if err := entry.validate(); err != nil { + return err + } + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + if p.dbHandle.isClosed { + return errMemoryProviderClosed + } + _, err := p.ipListEntryExistsInternal(entry) + if err == nil { + return fmt.Errorf("entry %q already exists", entry.IPOrNet) + } + entry.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + entry.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + p.dbHandle.ipListEntries[entry.getKey()] = entry.getACopy() + p.dbHandle.ipListEntriesKeys = append(p.dbHandle.ipListEntriesKeys, entry.getKey()) + sort.Strings(p.dbHandle.ipListEntriesKeys) + return nil +} + +func (p *MemoryProvider) updateIPListEntry(entry *IPListEntry) error { + if err := entry.validate(); err != nil { + return err + } + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + if p.dbHandle.isClosed { + return errMemoryProviderClosed + } + oldEntry, err := p.ipListEntryExistsInternal(entry) + if err != nil { + return err + } + entry.CreatedAt = oldEntry.CreatedAt + entry.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + p.dbHandle.ipListEntries[entry.getKey()] = entry.getACopy() + return nil +} + +func (p *MemoryProvider) deleteIPListEntry(entry IPListEntry, softDelete bool) error { + if err := entry.validate(); err != nil { + return err + } + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + if p.dbHandle.isClosed { + return errMemoryProviderClosed + } + _, err := p.ipListEntryExistsInternal(&entry) + if err != nil { + return err + } + delete(p.dbHandle.ipListEntries, entry.getKey()) + p.dbHandle.ipListEntriesKeys = make([]string, 0, len(p.dbHandle.ipListEntries)) + for k := range p.dbHandle.ipListEntries { + p.dbHandle.ipListEntriesKeys = append(p.dbHandle.ipListEntriesKeys, k) + } + sort.Strings(p.dbHandle.ipListEntriesKeys) + return nil +} + +func (p *MemoryProvider) getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + + if p.dbHandle.isClosed { + return nil, errMemoryProviderClosed + } + entries := make([]IPListEntry, 0, 15) + if order == OrderASC { + for _, k := range p.dbHandle.ipListEntriesKeys { + e := p.dbHandle.ipListEntries[k] + if e.Type == listType && e.satisfySearchConstraints(filter, from, order) { + entry := e.getACopy() + entry.PrepareForRendering() + entries = append(entries, entry) + if limit > 0 && len(entries) >= limit { + break + } + } + } + } else { + for i := len(p.dbHandle.ipListEntriesKeys) - 1; i >= 0; i-- { + e := p.dbHandle.ipListEntries[p.dbHandle.ipListEntriesKeys[i]] + if e.Type == listType && e.satisfySearchConstraints(filter, from, order) { + entry := e.getACopy() + entry.PrepareForRendering() + entries = append(entries, entry) + if limit > 0 && len(entries) >= limit { + break + } + } + } + } + + return entries, nil +} + +func (p *MemoryProvider) getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) { + return nil, ErrNotImplemented +} + +func (p *MemoryProvider) dumpIPListEntries() ([]IPListEntry, error) { + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + + if p.dbHandle.isClosed { + return nil, errMemoryProviderClosed + } + if count := len(p.dbHandle.ipListEntriesKeys); count > ipListMemoryLimit { + providerLog(logger.LevelInfo, "IP lists excluded from dump, too many entries: %d", count) + return nil, nil + } + entries := make([]IPListEntry, 0, len(p.dbHandle.ipListEntries)) + for _, k := range p.dbHandle.ipListEntriesKeys { + e := p.dbHandle.ipListEntries[k] + entry := e.getACopy() + entry.PrepareForRendering() + entries = append(entries, entry) + } + return entries, nil +} + +func (p *MemoryProvider) countIPListEntries(listType IPListType) (int64, error) { + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + + if p.dbHandle.isClosed { + return 0, errMemoryProviderClosed + } + if listType == 0 { + return int64(len(p.dbHandle.ipListEntriesKeys)), nil + } + var count int64 + for _, k := range p.dbHandle.ipListEntriesKeys { + e := p.dbHandle.ipListEntries[k] + if e.Type == listType { + count++ + } + } + return count, nil +} + +func (p *MemoryProvider) getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) { + p.dbHandle.Lock() + defer p.dbHandle.Unlock() + + if p.dbHandle.isClosed { + return nil, errMemoryProviderClosed + } + entries := make([]IPListEntry, 0, 3) + ipAddr, err := netip.ParseAddr(ip) + if err != nil { + return entries, fmt.Errorf("invalid ip address %s", ip) + } + var netType int + var ipBytes []byte + if ipAddr.Is4() || ipAddr.Is4In6() { + netType = ipTypeV4 + as4 := ipAddr.As4() + ipBytes = as4[:] + } else { + netType = ipTypeV6 + as16 := ipAddr.As16() + ipBytes = as16[:] + } + for _, k := range p.dbHandle.ipListEntriesKeys { + e := p.dbHandle.ipListEntries[k] + if e.Type == listType && e.IPType == netType && bytes.Compare(ipBytes, e.First) >= 0 && bytes.Compare(ipBytes, e.Last) <= 0 { + entry := e.getACopy() + entry.PrepareForRendering() + entries = append(entries, entry) + } + } + return entries, nil +} + func (p *MemoryProvider) setFirstDownloadTimestamp(username string) error { p.dbHandle.Lock() defer p.dbHandle.Unlock() @@ -2720,6 +2927,8 @@ func (p *MemoryProvider) clear() { p.dbHandle.rulesNames = []string{} p.dbHandle.roles = map[string]Role{} p.dbHandle.roleNames = []string{} + p.dbHandle.ipListEntries = map[string]IPListEntry{} + p.dbHandle.ipListEntriesKeys = []string{} } func (p *MemoryProvider) reloadConfig() error { @@ -2738,8 +2947,8 @@ func (p *MemoryProvider) reloadConfig() error { providerLog(logger.LevelError, "error loading dump: %v", err) return err } - if fi.Size() > 10485760 { - err = errors.New("dump configuration file is invalid, its size must be <= 10485760 bytes") + if fi.Size() > 20971520 { + err = errors.New("dump configuration file is invalid, its size must be <= 20971520 bytes") providerLog(logger.LevelError, "error loading dump: %v", err) return err } @@ -2753,12 +2962,16 @@ func (p *MemoryProvider) reloadConfig() error { providerLog(logger.LevelError, "error loading dump: %v", err) return err } - return p.restoreDump(dump) + return p.restoreDump(&dump) } -func (p *MemoryProvider) restoreDump(dump BackupData) error { +func (p *MemoryProvider) restoreDump(dump *BackupData) error { p.clear() + if err := p.restoreIPListEntries(*dump); err != nil { + return err + } + if err := p.restoreRoles(dump); err != nil { return err } @@ -2799,10 +3012,10 @@ func (p *MemoryProvider) restoreDump(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreEventActions(dump BackupData) error { - for _, action := range dump.EventActions { +func (p *MemoryProvider) restoreEventActions(dump *BackupData) error { + for idx := range dump.EventActions { + action := dump.EventActions[idx] a, err := p.eventActionExists(action.Name) - action := action // pin if err == nil { action.ID = a.ID err = UpdateEventAction(&action, ActionExecutorSystem, "", "") @@ -2821,10 +3034,10 @@ func (p *MemoryProvider) restoreEventActions(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreEventRules(dump BackupData) error { - for _, rule := range dump.EventRules { +func (p *MemoryProvider) restoreEventRules(dump *BackupData) error { + for idx := range dump.EventRules { + rule := dump.EventRules[idx] r, err := p.eventRuleExists(rule.Name) - rule := rule // pin if dump.Version < 15 { rule.Status = 1 } @@ -2846,10 +3059,10 @@ func (p *MemoryProvider) restoreEventRules(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreShares(dump BackupData) error { - for _, share := range dump.Shares { +func (p *MemoryProvider) restoreShares(dump *BackupData) error { + for idx := range dump.Shares { + share := dump.Shares[idx] s, err := p.shareExists(share.ShareID, "") - share := share // pin share.IsRestore = true if err == nil { share.ID = s.ID @@ -2869,13 +3082,13 @@ func (p *MemoryProvider) restoreShares(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreAPIKeys(dump BackupData) error { - for _, apiKey := range dump.APIKeys { +func (p *MemoryProvider) restoreAPIKeys(dump *BackupData) error { + for idx := range dump.APIKeys { + apiKey := dump.APIKeys[idx] if apiKey.Key == "" { return fmt.Errorf("cannot restore an empty API key: %+v", apiKey) } k, err := p.apiKeyExists(apiKey.KeyID) - apiKey := apiKey // pin if err == nil { apiKey.ID = k.ID err = UpdateAPIKey(&apiKey, ActionExecutorSystem, "", "") @@ -2894,9 +3107,9 @@ func (p *MemoryProvider) restoreAPIKeys(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreAdmins(dump BackupData) error { - for _, admin := range dump.Admins { - admin := admin // pin +func (p *MemoryProvider) restoreAdmins(dump *BackupData) error { + for idx := range dump.Admins { + admin := dump.Admins[idx] admin.Username = config.convertName(admin.Username) a, err := p.adminExists(admin.Username) if err == nil { @@ -2917,9 +3130,30 @@ func (p *MemoryProvider) restoreAdmins(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreRoles(dump BackupData) error { - for _, role := range dump.Roles { - role := role // pin +func (p *MemoryProvider) restoreIPListEntries(dump BackupData) error { + for idx := range dump.IPLists { + entry := dump.IPLists[idx] + _, err := p.ipListEntryExists(entry.IPOrNet, entry.Type) + if err == nil { + err = UpdateIPListEntry(&entry, ActionExecutorSystem, "", "") + if err != nil { + providerLog(logger.LevelError, "error updating IP list entry %q: %v", entry.getName(), err) + return err + } + } else { + err = AddIPListEntry(&entry, ActionExecutorSystem, "", "") + if err != nil { + providerLog(logger.LevelError, "error adding IP list entry %q: %v", entry.getName(), err) + return err + } + } + } + return nil +} + +func (p *MemoryProvider) restoreRoles(dump *BackupData) error { + for idx := range dump.Roles { + role := dump.Roles[idx] role.Name = config.convertName(role.Name) r, err := p.roleExists(role.Name) if err == nil { @@ -2942,9 +3176,9 @@ func (p *MemoryProvider) restoreRoles(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreGroups(dump BackupData) error { - for _, group := range dump.Groups { - group := group // pin +func (p *MemoryProvider) restoreGroups(dump *BackupData) error { + for idx := range dump.Groups { + group := dump.Groups[idx] group.Name = config.convertName(group.Name) g, err := p.groupExists(group.Name) if err == nil { @@ -2966,9 +3200,9 @@ func (p *MemoryProvider) restoreGroups(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreFolders(dump BackupData) error { - for _, folder := range dump.Folders { - folder := folder // pin +func (p *MemoryProvider) restoreFolders(dump *BackupData) error { + for idx := range dump.Folders { + folder := dump.Folders[idx] folder.Name = config.convertName(folder.Name) f, err := p.getFolderByName(folder.Name) if err == nil { @@ -2990,9 +3224,9 @@ func (p *MemoryProvider) restoreFolders(dump BackupData) error { return nil } -func (p *MemoryProvider) restoreUsers(dump BackupData) error { - for _, user := range dump.Users { - user := user // pin +func (p *MemoryProvider) restoreUsers(dump *BackupData) error { + for idx := range dump.Users { + user := dump.Users[idx] user.Username = config.convertName(user.Username) u, err := p.userExists(user.Username, "") if err == nil { diff --git a/internal/dataprovider/mysql.go b/internal/dataprovider/mysql.go index dbca8dd0..e304cd85 100644 --- a/internal/dataprovider/mysql.go +++ b/internal/dataprovider/mysql.go @@ -58,6 +58,7 @@ const ( "DROP TABLE IF EXISTS `{{tasks}}` CASCADE;" + "DROP TABLE IF EXISTS `{{nodes}}` CASCADE;" + "DROP TABLE IF EXISTS `{{roles}}` CASCADE;" + + "DROP TABLE IF EXISTS `{{ip_lists}}` CASCADE;" + "DROP TABLE IF EXISTS `{{schema_version}}` CASCADE;" mysqlInitialSQL = "CREATE TABLE `{{schema_version}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `version` integer NOT NULL);" + "CREATE TABLE `{{admins}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `username` varchar(255) NOT NULL UNIQUE, " + @@ -189,6 +190,18 @@ const ( mysqlV26SQL = "ALTER TABLE `{{events_rules}}` ADD COLUMN `status` integer DEFAULT 1 NOT NULL; " + "ALTER TABLE `{{events_rules}}` ALTER COLUMN `status` DROP DEFAULT; " mysqlV26DownSQL = "ALTER TABLE `{{events_rules}}` DROP COLUMN `status`; " + mysqlV27SQL = "CREATE TABLE `{{ip_lists}}` (`id` bigint AUTO_INCREMENT NOT NULL PRIMARY KEY, `type` integer NOT NULL, " + + "`ipornet` varchar(50) NOT NULL, `mode` integer NOT NULL, `description` varchar(512) NULL, " + + "`first` VARBINARY(16) NOT NULL, `last` VARBINARY(16) NOT NULL, `ip_type` integer NOT NULL, `protocols` integer NOT NULL, " + + "`created_at` bigint NOT NULL, `updated_at` bigint NOT NULL, `deleted_at` bigint NOT NULL);" + + "ALTER TABLE `{{ip_lists}}` ADD CONSTRAINT `{{prefix}}unique_ipornet_type_mapping` UNIQUE (`type`, `ipornet`);" + + "CREATE INDEX `{{prefix}}ip_lists_type_idx` ON `{{ip_lists}}` (`type`);" + + "CREATE INDEX `{{prefix}}ip_lists_ipornet_idx` ON `{{ip_lists}}` (`ipornet`);" + + "CREATE INDEX `{{prefix}}ip_lists_ip_type_idx` ON `{{ip_lists}}` (`ip_type`);" + + "CREATE INDEX `{{prefix}}ip_lists_updated_at_idx` ON `{{ip_lists}}` (`updated_at`);" + + "CREATE INDEX `{{prefix}}ip_lists_deleted_at_idx` ON `{{ip_lists}}` (`deleted_at`);" + + "CREATE INDEX `{{prefix}}ip_lists_first_last_idx` ON `{{ip_lists}}` (`first`, `last`);" + mysqlV27DownSQL = "DROP TABLE `{{ip_lists}}` CASCADE;" ) // MySQLProvider defines the auth provider for MySQL/MariaDB database @@ -696,6 +709,42 @@ func (p *MySQLProvider) dumpRoles() ([]Role, error) { return sqlCommonDumpRoles(p.dbHandle) } +func (p *MySQLProvider) ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + return sqlCommonGetIPListEntry(ipOrNet, listType, p.dbHandle) +} + +func (p *MySQLProvider) addIPListEntry(entry *IPListEntry) error { + return sqlCommonAddIPListEntry(entry, p.dbHandle) +} + +func (p *MySQLProvider) updateIPListEntry(entry *IPListEntry) error { + return sqlCommonUpdateIPListEntry(entry, p.dbHandle) +} + +func (p *MySQLProvider) deleteIPListEntry(entry IPListEntry, softDelete bool) error { + return sqlCommonDeleteIPListEntry(entry, softDelete, p.dbHandle) +} + +func (p *MySQLProvider) getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + return sqlCommonGetIPListEntries(listType, filter, from, order, limit, p.dbHandle) +} + +func (p *MySQLProvider) getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) { + return sqlCommonGetRecentlyUpdatedIPListEntries(after, p.dbHandle) +} + +func (p *MySQLProvider) dumpIPListEntries() ([]IPListEntry, error) { + return sqlCommonDumpIPListEntries(p.dbHandle) +} + +func (p *MySQLProvider) countIPListEntries(listType IPListType) (int64, error) { + return sqlCommonCountIPListEntries(listType, p.dbHandle) +} + +func (p *MySQLProvider) getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) { + return sqlCommonGetListEntriesForIP(ip, listType, p.dbHandle) +} + func (p *MySQLProvider) setFirstDownloadTimestamp(username string) error { return sqlCommonSetFirstDownloadTimestamp(username, p.dbHandle) } @@ -749,6 +798,8 @@ func (p *MySQLProvider) migrateDatabase() error { //nolint:dupl return updateMySQLDatabaseFromV24(p.dbHandle) case version == 25: return updateMySQLDatabaseFromV25(p.dbHandle) + case version == 26: + return updateMySQLDatabaseFromV26(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -777,6 +828,8 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error { return downgradeMySQLDatabaseFromV25(p.dbHandle) case 26: return downgradeMySQLDatabaseFromV26(p.dbHandle) + case 27: + return downgradeMySQLDatabaseFromV27(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -802,7 +855,14 @@ func updateMySQLDatabaseFromV24(dbHandle *sql.DB) error { } func updateMySQLDatabaseFromV25(dbHandle *sql.DB) error { - return updateMySQLDatabaseFrom25To26(dbHandle) + if err := updateMySQLDatabaseFrom25To26(dbHandle); err != nil { + return err + } + return updateMySQLDatabaseFromV26(dbHandle) +} + +func updateMySQLDatabaseFromV26(dbHandle *sql.DB) error { + return updateMySQLDatabaseFrom26To27(dbHandle) } func downgradeMySQLDatabaseFromV24(dbHandle *sql.DB) error { @@ -823,6 +883,13 @@ func downgradeMySQLDatabaseFromV26(dbHandle *sql.DB) error { return downgradeMySQLDatabaseFromV25(dbHandle) } +func downgradeMySQLDatabaseFromV27(dbHandle *sql.DB) error { + if err := downgradeMySQLDatabaseFrom27To26(dbHandle); err != nil { + return err + } + return downgradeMySQLDatabaseFromV26(dbHandle) +} + func updateMySQLDatabaseFrom23To24(dbHandle *sql.DB) error { logger.InfoToConsole("updating database schema version: 23 -> 24") providerLog(logger.LevelInfo, "updating database schema version: 23 -> 24") @@ -847,6 +914,14 @@ func updateMySQLDatabaseFrom25To26(dbHandle *sql.DB) error { return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 26, true) } +func updateMySQLDatabaseFrom26To27(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database schema version: 26 -> 27") + providerLog(logger.LevelInfo, "updating database schema version: 26 -> 27") + sql := strings.ReplaceAll(mysqlV27SQL, "{{ip_lists}}", sqlTableIPLists) + sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 27, true) +} + func downgradeMySQLDatabaseFrom24To23(dbHandle *sql.DB) error { logger.InfoToConsole("downgrading database schema version: 24 -> 23") providerLog(logger.LevelInfo, "downgrading database schema version: 24 -> 23") @@ -870,3 +945,10 @@ func downgradeMySQLDatabaseFrom26To25(dbHandle *sql.DB) error { sql := strings.ReplaceAll(mysqlV26DownSQL, "{{events_rules}}", sqlTableEventsRules) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 25, false) } + +func downgradeMySQLDatabaseFrom27To26(dbHandle *sql.DB) error { + logger.InfoToConsole("downgrading database schema version: 27 -> 26") + providerLog(logger.LevelInfo, "downgrading database schema version: 27 -> 26") + sql := strings.ReplaceAll(mysqlV27DownSQL, "{{ip_lists}}", sqlTableIPLists) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 26, false) +} diff --git a/internal/dataprovider/pgsql.go b/internal/dataprovider/pgsql.go index 0f9bb387..9d666a07 100644 --- a/internal/dataprovider/pgsql.go +++ b/internal/dataprovider/pgsql.go @@ -56,6 +56,7 @@ DROP TABLE IF EXISTS "{{events_rules}}" CASCADE; DROP TABLE IF EXISTS "{{tasks}}" CASCADE; DROP TABLE IF EXISTS "{{nodes}}" CASCADE; DROP TABLE IF EXISTS "{{roles}}" CASCADE; +DROP TABLE IF EXISTS "{{ip_lists}}" CASCADE; DROP TABLE IF EXISTS "{{schema_version}}" CASCADE; ` pgsqlInitial = `CREATE TABLE "{{schema_version}}" ("id" serial NOT NULL PRIMARY KEY, "version" integer NOT NULL); @@ -202,6 +203,19 @@ ALTER TABLE "{{users}}" ALTER COLUMN "last_password_change" DROP DEFAULT; ALTER TABLE "{{events_rules}}" ALTER COLUMN "status" DROP DEFAULT; ` pgsqlV26DownSQL = `ALTER TABLE "{{events_rules}}" DROP COLUMN "status" CASCADE;` + pgsqlV27SQL = `CREATE TABLE "{{ip_lists}}" ("id" bigserial NOT NULL PRIMARY KEY, "type" integer NOT NULL, +"ipornet" varchar(50) NOT NULL, "mode" integer NOT NULL, "description" varchar(512) NULL, "first" inet NOT NULL, +"last" inet NOT NULL, "ip_type" integer NOT NULL, "protocols" integer NOT NULL, "created_at" bigint NOT NULL, +"updated_at" bigint NOT NULL, "deleted_at" bigint NOT NULL); +ALTER TABLE "{{ip_lists}}" ADD CONSTRAINT "{{prefix}}unique_ipornet_type_mapping" UNIQUE ("type", "ipornet"); +CREATE INDEX "{{prefix}}ip_lists_type_idx" ON "{{ip_lists}}" ("type"); +CREATE INDEX "{{prefix}}ip_lists_ipornet_idx" ON "{{ip_lists}}" ("ipornet"); +CREATE INDEX "{{prefix}}ip_lists_ipornet_like_idx" ON "{{ip_lists}}" ("ipornet" varchar_pattern_ops); +CREATE INDEX "{{prefix}}ip_lists_updated_at_idx" ON "{{ip_lists}}" ("updated_at"); +CREATE INDEX "{{prefix}}ip_lists_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at"); +CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last"); +` + pgsqlV27DownSQL = `DROP TABLE "{{ip_lists}}" CASCADE;` ) // PGSQLProvider defines the auth provider for PostgreSQL database @@ -668,6 +682,42 @@ func (p *PGSQLProvider) dumpRoles() ([]Role, error) { return sqlCommonDumpRoles(p.dbHandle) } +func (p *PGSQLProvider) ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + return sqlCommonGetIPListEntry(ipOrNet, listType, p.dbHandle) +} + +func (p *PGSQLProvider) addIPListEntry(entry *IPListEntry) error { + return sqlCommonAddIPListEntry(entry, p.dbHandle) +} + +func (p *PGSQLProvider) updateIPListEntry(entry *IPListEntry) error { + return sqlCommonUpdateIPListEntry(entry, p.dbHandle) +} + +func (p *PGSQLProvider) deleteIPListEntry(entry IPListEntry, softDelete bool) error { + return sqlCommonDeleteIPListEntry(entry, softDelete, p.dbHandle) +} + +func (p *PGSQLProvider) getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + return sqlCommonGetIPListEntries(listType, filter, from, order, limit, p.dbHandle) +} + +func (p *PGSQLProvider) getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) { + return sqlCommonGetRecentlyUpdatedIPListEntries(after, p.dbHandle) +} + +func (p *PGSQLProvider) dumpIPListEntries() ([]IPListEntry, error) { + return sqlCommonDumpIPListEntries(p.dbHandle) +} + +func (p *PGSQLProvider) countIPListEntries(listType IPListType) (int64, error) { + return sqlCommonCountIPListEntries(listType, p.dbHandle) +} + +func (p *PGSQLProvider) getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) { + return sqlCommonGetListEntriesForIP(ip, listType, p.dbHandle) +} + func (p *PGSQLProvider) setFirstDownloadTimestamp(username string) error { return sqlCommonSetFirstDownloadTimestamp(username, p.dbHandle) } @@ -721,6 +771,8 @@ func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl return updatePgSQLDatabaseFromV24(p.dbHandle) case version == 25: return updatePgSQLDatabaseFromV25(p.dbHandle) + case version == 26: + return updatePgSQLDatabaseFromV26(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -749,6 +801,8 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error { return downgradePgSQLDatabaseFromV25(p.dbHandle) case 26: return downgradePgSQLDatabaseFromV26(p.dbHandle) + case 27: + return downgradePgSQLDatabaseFromV27(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -774,7 +828,14 @@ func updatePgSQLDatabaseFromV24(dbHandle *sql.DB) error { } func updatePgSQLDatabaseFromV25(dbHandle *sql.DB) error { - return updatePgSQLDatabaseFrom25To26(dbHandle) + if err := updatePgSQLDatabaseFrom25To26(dbHandle); err != nil { + return err + } + return updatePgSQLDatabaseFromV26(dbHandle) +} + +func updatePgSQLDatabaseFromV26(dbHandle *sql.DB) error { + return updatePgSQLDatabaseFrom26To27(dbHandle) } func downgradePgSQLDatabaseFromV24(dbHandle *sql.DB) error { @@ -795,6 +856,13 @@ func downgradePgSQLDatabaseFromV26(dbHandle *sql.DB) error { return downgradePgSQLDatabaseFromV25(dbHandle) } +func downgradePgSQLDatabaseFromV27(dbHandle *sql.DB) error { + if err := downgradePgSQLDatabaseFrom27To26(dbHandle); err != nil { + return err + } + return downgradePgSQLDatabaseFromV26(dbHandle) +} + func updatePgSQLDatabaseFrom23To24(dbHandle *sql.DB) error { logger.InfoToConsole("updating database schema version: 23 -> 24") providerLog(logger.LevelInfo, "updating database schema version: 23 -> 24") @@ -827,6 +895,18 @@ func updatePgSQLDatabaseFrom25To26(dbHandle *sql.DB) error { return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 26, true) } +func updatePgSQLDatabaseFrom26To27(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database schema version: 26 -> 27") + providerLog(logger.LevelInfo, "updating database schema version: 26 -> 27") + sql := pgsqlV27SQL + if config.Driver == CockroachDataProviderName { + sql = strings.ReplaceAll(sql, `CREATE INDEX "{{prefix}}ip_lists_ipornet_like_idx" ON "{{ip_lists}}" ("ipornet" varchar_pattern_ops);`, "") + } + sql = strings.ReplaceAll(sql, "{{ip_lists}}", sqlTableIPLists) + sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 27, true) +} + func downgradePgSQLDatabaseFrom24To23(dbHandle *sql.DB) error { logger.InfoToConsole("downgrading database schema version: 24 -> 23") providerLog(logger.LevelInfo, "downgrading database schema version: 24 -> 23") @@ -850,3 +930,10 @@ func downgradePgSQLDatabaseFrom26To25(dbHandle *sql.DB) error { sql := strings.ReplaceAll(pgsqlV26DownSQL, "{{events_rules}}", sqlTableEventsRules) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 25, false) } + +func downgradePgSQLDatabaseFrom27To26(dbHandle *sql.DB) error { + logger.InfoToConsole("downgrading database schema version: 27 -> 26") + providerLog(logger.LevelInfo, "downgrading database schema version: 27 -> 26") + sql := strings.ReplaceAll(pgsqlV27DownSQL, "{{ip_lists}}", sqlTableIPLists) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 26, false) +} diff --git a/internal/dataprovider/scheduler.go b/internal/dataprovider/scheduler.go index 14966448..b543cb5c 100644 --- a/internal/dataprovider/scheduler.go +++ b/internal/dataprovider/scheduler.go @@ -27,8 +27,9 @@ import ( ) var ( - scheduler *cron.Cron - lastUserCacheUpdate atomic.Int64 + scheduler *cron.Cron + lastUserCacheUpdate atomic.Int64 + lastIPListsCacheUpdate atomic.Int64 // used for bolt and memory providers, so we avoid iterating all users/rules // to find recently modified ones lastUserUpdate atomic.Int64 @@ -54,9 +55,6 @@ func startScheduler() error { if err != nil { return err } - if fnReloadRules != nil { - fnReloadRules() - } if currentNode != nil { _, err = scheduler.AddFunc("@every 30m", func() { err := provider.cleanupNodes() @@ -76,6 +74,7 @@ func startScheduler() error { func addScheduledCacheUpdates() error { lastUserCacheUpdate.Store(util.GetTimeAsMsSinceEpoch(time.Now())) + lastIPListsCacheUpdate.Store(util.GetTimeAsMsSinceEpoch(time.Now())) _, err := scheduler.AddFunc("@every 10m", checkCacheUpdates) if err != nil { return fmt.Errorf("unable to schedule cache updates: %w", err) @@ -99,14 +98,24 @@ func checkDataprovider() { } func checkCacheUpdates() { - providerLog(logger.LevelDebug, "start user cache check, update time %v", util.GetTimeFromMsecSinceEpoch(lastUserCacheUpdate.Load())) + checkUserCache() + checkIPListEntryCache() +} + +func checkUserCache() { + lastCheck := lastUserCacheUpdate.Load() + providerLog(logger.LevelDebug, "start user cache check, update time %v", util.GetTimeFromMsecSinceEpoch(lastCheck)) checkTime := util.GetTimeAsMsSinceEpoch(time.Now()) - users, err := provider.getRecentlyUpdatedUsers(lastUserCacheUpdate.Load()) + if config.IsShared == 1 { + lastCheck -= 5000 + } + users, err := provider.getRecentlyUpdatedUsers(lastCheck) if err != nil { providerLog(logger.LevelError, "unable to get recently updated users: %v", err) return } - for _, user := range users { + for idx := range users { + user := users[idx] providerLog(logger.LevelDebug, "invalidate caches for user %q", user.Username) if user.DeletedAt > 0 { deletedAt := util.GetTimeFromMsecSinceEpoch(user.DeletedAt) @@ -121,11 +130,53 @@ func checkCacheUpdates() { } cachedPasswords.Remove(user.Username) } - lastUserCacheUpdate.Store(checkTime) providerLog(logger.LevelDebug, "end user cache check, new update time %v", util.GetTimeFromMsecSinceEpoch(lastUserCacheUpdate.Load())) } +func checkIPListEntryCache() { + if config.IsShared != 1 { + return + } + hasMemoryLists := false + for _, l := range inMemoryLists { + if l.isInMemory.Load() { + hasMemoryLists = true + break + } + } + if !hasMemoryLists { + return + } + providerLog(logger.LevelDebug, "start IP list cache check, update time %v", util.GetTimeFromMsecSinceEpoch(lastIPListsCacheUpdate.Load())) + checkTime := util.GetTimeAsMsSinceEpoch(time.Now()) + entries, err := provider.getRecentlyUpdatedIPListEntries(lastIPListsCacheUpdate.Load() - 5000) + if err != nil { + providerLog(logger.LevelError, "unable to get recently updated IP list entries: %v", err) + return + } + for idx := range entries { + e := entries[idx] + providerLog(logger.LevelDebug, "update cache for IP list entry %q", e.getName()) + if e.DeletedAt > 0 { + deletedAt := util.GetTimeFromMsecSinceEpoch(e.DeletedAt) + if deletedAt.Add(30 * time.Minute).Before(time.Now()) { + providerLog(logger.LevelDebug, "removing IP list entry %q deleted at %s", e.getName(), deletedAt) + go provider.deleteIPListEntry(e, false) //nolint:errcheck + } + for _, l := range inMemoryLists { + l.removeEntry(&e) + } + } else { + for _, l := range inMemoryLists { + l.updateEntry(&e) + } + } + } + lastIPListsCacheUpdate.Store(checkTime) + providerLog(logger.LevelDebug, "end IP list entries cache check, new update time %v", util.GetTimeFromMsecSinceEpoch(lastIPListsCacheUpdate.Load())) +} + func setLastUserUpdate() { lastUserUpdate.Store(util.GetTimeAsMsSinceEpoch(time.Now())) } diff --git a/internal/dataprovider/sqlcommon.go b/internal/dataprovider/sqlcommon.go index 07163911..732bfce1 100644 --- a/internal/dataprovider/sqlcommon.go +++ b/internal/dataprovider/sqlcommon.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "net/netip" "runtime/debug" "strings" "time" @@ -34,7 +35,7 @@ import ( ) const ( - sqlDatabaseVersion = 26 + sqlDatabaseVersion = 27 defaultSQLQueryTimeout = 10 * time.Second longSQLQueryTimeout = 60 * time.Second ) @@ -79,6 +80,7 @@ func sqlReplaceAll(sql string) string { sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks) sql = strings.ReplaceAll(sql, "{{nodes}}", sqlTableNodes) sql = strings.ReplaceAll(sql, "{{roles}}", sqlTableRoles) + sql = strings.ReplaceAll(sql, "{{ip_lists}}", sqlTableIPLists) sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix) return sql } @@ -538,6 +540,241 @@ func sqlCommonDumpAdmins(dbHandle sqlQuerier) ([]Admin, error) { return getAdminsWithGroups(ctx, admins, dbHandle) } +func sqlCommonGetIPListEntry(ipOrNet string, listType IPListType, dbHandle sqlQuerier) (IPListEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := getIPListEntryQuery() + row := dbHandle.QueryRowContext(ctx, q, listType, ipOrNet) + return getIPListEntryFromDbRow(row) +} + +func sqlCommonDumpIPListEntries(dbHandle *sql.DB) ([]IPListEntry, error) { + count, err := sqlCommonCountIPListEntries(0, dbHandle) + if err != nil { + return nil, err + } + if count > ipListMemoryLimit { + providerLog(logger.LevelInfo, "IP lists excluded from dump, too many entries: %d", count) + return nil, nil + } + entries := make([]IPListEntry, 0, 100) + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getDumpListEntriesQuery() + + rows, err := dbHandle.QueryContext(ctx, q) + if err != nil { + return entries, err + } + defer rows.Close() + + for rows.Next() { + entry, err := getIPListEntryFromDbRow(rows) + if err != nil { + return entries, err + } + entries = append(entries, entry) + } + return entries, rows.Err() +} + +func sqlCommonCountIPListEntries(listType IPListType, dbHandle *sql.DB) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + var q string + var args []any + if listType == 0 { + q = getCountAllIPListEntriesQuery() + } else { + q = getCountIPListEntriesQuery() + args = append(args, listType) + } + var count int64 + err := dbHandle.QueryRowContext(ctx, q, args...).Scan(&count) + return count, err +} + +func sqlCommonGetIPListEntries(listType IPListType, filter, from, order string, limit int, dbHandle sqlQuerier) ([]IPListEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := getIPListEntriesQuery(filter, from, order, limit) + args := []any{listType} + if from != "" { + args = append(args, from) + } + if filter != "" { + args = append(args, filter+"%") + } + if limit > 0 { + args = append(args, limit) + } + entries := make([]IPListEntry, 0, limit) + rows, err := dbHandle.QueryContext(ctx, q, args...) + if err != nil { + return entries, err + } + defer rows.Close() + + for rows.Next() { + entry, err := getIPListEntryFromDbRow(rows) + if err != nil { + return entries, err + } + entries = append(entries, entry) + } + return entries, rows.Err() +} + +func sqlCommonGetRecentlyUpdatedIPListEntries(after int64, dbHandle sqlQuerier) ([]IPListEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getRecentlyUpdatedIPListQuery() + entries := make([]IPListEntry, 0, 5) + rows, err := dbHandle.QueryContext(ctx, q, after) + if err != nil { + return entries, err + } + defer rows.Close() + + for rows.Next() { + entry, err := getIPListEntryFromDbRow(rows) + if err != nil { + return entries, err + } + entries = append(entries, entry) + } + return entries, rows.Err() +} + +func sqlCommonGetListEntriesForIP(ip string, listType IPListType, dbHandle sqlQuerier) ([]IPListEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + var rows *sql.Rows + var err error + + entries := make([]IPListEntry, 0, 2) + if config.Driver == PGSQLDataProviderName || config.Driver == CockroachDataProviderName { + rows, err = dbHandle.QueryContext(ctx, getIPListEntriesForIPQueryPg(), listType, ip) + if err != nil { + return entries, err + } + } else { + ipAddr, err := netip.ParseAddr(ip) + if err != nil { + return entries, fmt.Errorf("invalid ip address %s", ip) + } + var netType int + var ipBytes []byte + if ipAddr.Is4() || ipAddr.Is4In6() { + netType = ipTypeV4 + as4 := ipAddr.As4() + ipBytes = as4[:] + } else { + netType = ipTypeV6 + as16 := ipAddr.As16() + ipBytes = as16[:] + } + rows, err = dbHandle.QueryContext(ctx, getIPListEntriesForIPQueryNoPg(), listType, netType, ipBytes) + if err != nil { + return entries, err + } + } + defer rows.Close() + + for rows.Next() { + entry, err := getIPListEntryFromDbRow(rows) + if err != nil { + return entries, err + } + entries = append(entries, entry) + } + return entries, rows.Err() +} + +func sqlCommonAddIPListEntry(entry *IPListEntry, dbHandle *sql.DB) error { + if err := entry.validate(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + var err error + q := getAddIPListEntryQuery() + first := entry.getFirst() + last := entry.getLast() + var netType int + if first.Is4() { + netType = ipTypeV4 + } else { + netType = ipTypeV6 + } + if config.IsShared == 1 { + return sqlCommonExecuteTx(ctx, dbHandle, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, getRemoveSoftDeletedIPListEntryQuery(), entry.Type, entry.IPOrNet) + if err != nil { + return err + } + if config.Driver == PGSQLDataProviderName || config.Driver == CockroachDataProviderName { + _, err = tx.ExecContext(ctx, q, entry.Type, entry.IPOrNet, first.String(), last.String(), + netType, entry.Protocols, entry.Description, entry.Mode, util.GetTimeAsMsSinceEpoch(time.Now()), + util.GetTimeAsMsSinceEpoch(time.Now())) + } else { + _, err = tx.ExecContext(ctx, q, entry.Type, entry.IPOrNet, entry.First, entry.Last, + netType, entry.Protocols, entry.Description, entry.Mode, util.GetTimeAsMsSinceEpoch(time.Now()), + util.GetTimeAsMsSinceEpoch(time.Now())) + } + return err + }) + } + if config.Driver == PGSQLDataProviderName || config.Driver == CockroachDataProviderName { + _, err = dbHandle.ExecContext(ctx, q, entry.Type, entry.IPOrNet, first.String(), last.String(), + netType, entry.Protocols, entry.Description, entry.Mode, util.GetTimeAsMsSinceEpoch(time.Now()), + util.GetTimeAsMsSinceEpoch(time.Now())) + } else { + _, err = dbHandle.ExecContext(ctx, q, entry.Type, entry.IPOrNet, entry.First, entry.Last, + netType, entry.Protocols, entry.Description, entry.Mode, util.GetTimeAsMsSinceEpoch(time.Now()), + util.GetTimeAsMsSinceEpoch(time.Now())) + } + return err +} + +func sqlCommonUpdateIPListEntry(entry *IPListEntry, dbHandle *sql.DB) error { + if err := entry.validate(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := getUpdateIPListEntryQuery() + _, err := dbHandle.ExecContext(ctx, q, entry.Mode, entry.Protocols, entry.Description, + util.GetTimeAsMsSinceEpoch(time.Now()), entry.Type, entry.IPOrNet) + return err +} + +func sqlCommonDeleteIPListEntry(entry IPListEntry, softDelete bool, dbHandle *sql.DB) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := getDeleteIPListEntryQuery(softDelete) + var args []any + if softDelete { + ts := util.GetTimeAsMsSinceEpoch(time.Now()) + args = append(args, ts, ts) + } + args = append(args, entry.Type, entry.IPOrNet) + res, err := dbHandle.ExecContext(ctx, q, args...) + if err != nil { + return err + } + return sqlCommonRequireRowAffected(res) +} + func sqlCommonGetRoleByName(name string, dbHandle sqlQuerier) (Role, error) { ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) defer cancel() @@ -1872,6 +2109,24 @@ func getEventRuleFromDbRow(row sqlScanner) (EventRule, error) { return rule, nil } +func getIPListEntryFromDbRow(row sqlScanner) (IPListEntry, error) { + var entry IPListEntry + var description sql.NullString + + err := row.Scan(&entry.Type, &entry.IPOrNet, &entry.Mode, &entry.Protocols, &description, + &entry.CreatedAt, &entry.UpdatedAt, &entry.DeletedAt) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return entry, util.NewRecordNotFoundError(err.Error()) + } + return entry, err + } + if description.Valid { + entry.Description = description.String + } + return entry, err +} + func getRoleFromDbRow(row sqlScanner) (Role, error) { var role Role var description sql.NullString diff --git a/internal/dataprovider/sqlite.go b/internal/dataprovider/sqlite.go index bee4df3e..06d2827c 100644 --- a/internal/dataprovider/sqlite.go +++ b/internal/dataprovider/sqlite.go @@ -57,6 +57,7 @@ DROP TABLE IF EXISTS "{{events_rules}}"; DROP TABLE IF EXISTS "{{events_actions}}"; DROP TABLE IF EXISTS "{{tasks}}"; DROP TABLE IF EXISTS "{{roles}}"; +DROP TABLE IF EXISTS "{{ip_lists}}"; DROP TABLE IF EXISTS "{{schema_version}}"; ` sqliteInitialSQL = `CREATE TABLE "{{schema_version}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "version" integer NOT NULL); @@ -179,6 +180,19 @@ DROP TABLE "{{roles}}"; sqliteV25DownSQL = `ALTER TABLE "{{users}}" DROP COLUMN "last_password_change";` sqliteV26SQL = `ALTER TABLE "{{events_rules}}" ADD COLUMN "status" integer DEFAULT 1 NOT NULL;` sqliteV26DownSQL = `ALTER TABLE "{{events_rules}}" DROP COLUMN "status";` + sqliteV27SQL = `CREATE TABLE "{{ip_lists}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, +"type" integer NOT NULL, "ipornet" varchar(50) NOT NULL, "mode" integer NOT NULL, "description" varchar(512) NULL, +"first" BLOB NOT NULL, "last" BLOB NOT NULL, "ip_type" integer NOT NULL, "protocols" integer NOT NULL, +"created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "deleted_at" bigint NOT NULL, +CONSTRAINT "{{prefix}}unique_ipornet_type_mapping" UNIQUE ("type", "ipornet")); +CREATE INDEX "{{prefix}}ip_lists_type_idx" ON "{{ip_lists}}" ("type"); +CREATE INDEX "{{prefix}}ip_lists_ipornet_idx" ON "{{ip_lists}}" ("ipornet"); +CREATE INDEX "{{prefix}}ip_lists_ip_type_idx" ON "{{ip_lists}}" ("ip_type"); +CREATE INDEX "{{prefix}}ip_lists_ip_updated_at_idx" ON "{{ip_lists}}" ("updated_at"); +CREATE INDEX "{{prefix}}ip_lists_ip_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at"); +CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last"); +` + sqliteV27DownSQL = `DROP TABLE "{{ip_lists}}";` ) // SQLiteProvider defines the auth provider for SQLite database @@ -624,6 +638,42 @@ func (p *SQLiteProvider) dumpRoles() ([]Role, error) { return sqlCommonDumpRoles(p.dbHandle) } +func (p *SQLiteProvider) ipListEntryExists(ipOrNet string, listType IPListType) (IPListEntry, error) { + return sqlCommonGetIPListEntry(ipOrNet, listType, p.dbHandle) +} + +func (p *SQLiteProvider) addIPListEntry(entry *IPListEntry) error { + return sqlCommonAddIPListEntry(entry, p.dbHandle) +} + +func (p *SQLiteProvider) updateIPListEntry(entry *IPListEntry) error { + return sqlCommonUpdateIPListEntry(entry, p.dbHandle) +} + +func (p *SQLiteProvider) deleteIPListEntry(entry IPListEntry, softDelete bool) error { + return sqlCommonDeleteIPListEntry(entry, softDelete, p.dbHandle) +} + +func (p *SQLiteProvider) getIPListEntries(listType IPListType, filter, from, order string, limit int) ([]IPListEntry, error) { + return sqlCommonGetIPListEntries(listType, filter, from, order, limit, p.dbHandle) +} + +func (p *SQLiteProvider) getRecentlyUpdatedIPListEntries(after int64) ([]IPListEntry, error) { + return sqlCommonGetRecentlyUpdatedIPListEntries(after, p.dbHandle) +} + +func (p *SQLiteProvider) dumpIPListEntries() ([]IPListEntry, error) { + return sqlCommonDumpIPListEntries(p.dbHandle) +} + +func (p *SQLiteProvider) countIPListEntries(listType IPListType) (int64, error) { + return sqlCommonCountIPListEntries(listType, p.dbHandle) +} + +func (p *SQLiteProvider) getListEntriesForIP(ip string, listType IPListType) ([]IPListEntry, error) { + return sqlCommonGetListEntriesForIP(ip, listType, p.dbHandle) +} + func (p *SQLiteProvider) setFirstDownloadTimestamp(username string) error { return sqlCommonSetFirstDownloadTimestamp(username, p.dbHandle) } @@ -652,7 +702,6 @@ func (p *SQLiteProvider) initializeDatabase() error { logger.InfoToConsole("creating initial database schema, version 23") providerLog(logger.LevelInfo, "creating initial database schema, version 23") sql := sqlReplaceAll(sqliteInitialSQL) - return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 23, true) } @@ -677,6 +726,8 @@ func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl return updateSQLiteDatabaseFromV24(p.dbHandle) case version == 25: return updateSQLiteDatabaseFromV25(p.dbHandle) + case version == 26: + return updateSQLiteDatabaseFromV26(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -705,6 +756,8 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error { return downgradeSQLiteDatabaseFromV25(p.dbHandle) case 26: return downgradeSQLiteDatabaseFromV26(p.dbHandle) + case 27: + return downgradeSQLiteDatabaseFromV27(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -730,7 +783,14 @@ func updateSQLiteDatabaseFromV24(dbHandle *sql.DB) error { } func updateSQLiteDatabaseFromV25(dbHandle *sql.DB) error { - return updateSQLiteDatabaseFrom25To26(dbHandle) + if err := updateSQLiteDatabaseFrom25To26(dbHandle); err != nil { + return err + } + return updateSQLiteDatabaseFromV26(dbHandle) +} + +func updateSQLiteDatabaseFromV26(dbHandle *sql.DB) error { + return updateSQLiteDatabaseFrom26To27(dbHandle) } func downgradeSQLiteDatabaseFromV24(dbHandle *sql.DB) error { @@ -751,6 +811,13 @@ func downgradeSQLiteDatabaseFromV26(dbHandle *sql.DB) error { return downgradeSQLiteDatabaseFromV25(dbHandle) } +func downgradeSQLiteDatabaseFromV27(dbHandle *sql.DB) error { + if err := downgradeSQLiteDatabaseFrom27To26(dbHandle); err != nil { + return err + } + return downgradeSQLiteDatabaseFromV26(dbHandle) +} + func updateSQLiteDatabaseFrom23To24(dbHandle *sql.DB) error { logger.InfoToConsole("updating database schema version: 23 -> 24") providerLog(logger.LevelInfo, "updating database schema version: 23 -> 24") @@ -775,6 +842,14 @@ func updateSQLiteDatabaseFrom25To26(dbHandle *sql.DB) error { return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 26, true) } +func updateSQLiteDatabaseFrom26To27(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database schema version: 26 -> 27") + providerLog(logger.LevelInfo, "updating database schema version: 26 -> 27") + sql := strings.ReplaceAll(sqliteV27SQL, "{{ip_lists}}", sqlTableIPLists) + sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 27, true) +} + func downgradeSQLiteDatabaseFrom24To23(dbHandle *sql.DB) error { logger.InfoToConsole("downgrading database schema version: 24 -> 23") providerLog(logger.LevelInfo, "downgrading database schema version: 24 -> 23") @@ -799,6 +874,13 @@ func downgradeSQLiteDatabaseFrom26To25(dbHandle *sql.DB) error { return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 25, false) } +func downgradeSQLiteDatabaseFrom27To26(dbHandle *sql.DB) error { + logger.InfoToConsole("downgrading database schema version: 27 -> 26") + providerLog(logger.LevelInfo, "downgrading database schema version: 27 -> 26") + sql := strings.ReplaceAll(sqliteV27DownSQL, "{{ip_lists}}", sqlTableIPLists) + return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 26, false) +} + /*func setPragmaFK(dbHandle *sql.DB, value string) error { ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) defer cancel() diff --git a/internal/dataprovider/sqlqueries.go b/internal/dataprovider/sqlqueries.go index 747f9dc0..3aea828c 100644 --- a/internal/dataprovider/sqlqueries.go +++ b/internal/dataprovider/sqlqueries.go @@ -36,6 +36,7 @@ const ( selectGroupFields = "id,name,description,created_at,updated_at,user_settings" selectEventActionFields = "id,name,description,type,options" selectRoleFields = "id,name,description,created_at,updated_at" + selectIPListEntryFields = "type,ipornet,mode,protocols,description,created_at,updated_at,deleted_at" selectMinimalFields = "id,name" ) @@ -179,6 +180,100 @@ func getDefenderEventsCleanupQuery() string { return fmt.Sprintf(`DELETE FROM %s WHERE date_time < %s`, sqlTableDefenderEvents, sqlPlaceholders[0]) } +func getIPListEntryQuery() string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE type = %s AND ipornet = %s AND deleted_at = 0`, + selectIPListEntryFields, sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1]) +} + +func getIPListEntriesQuery(filter, from, order string, limit int) string { + var sb strings.Builder + var idx int + + sb.WriteString("SELECT ") + sb.WriteString(selectIPListEntryFields) + sb.WriteString(" FROM ") + sb.WriteString(sqlTableIPLists) + sb.WriteString(" WHERE type = ") + sb.WriteString(sqlPlaceholders[idx]) + idx++ + if from != "" { + if order == OrderASC { + sb.WriteString(" AND ipornet > ") + } else { + sb.WriteString(" AND ipornet < ") + } + sb.WriteString(sqlPlaceholders[idx]) + idx++ + } + if filter != "" { + sb.WriteString(" AND ipornet LIKE ") + sb.WriteString(sqlPlaceholders[idx]) + idx++ + } + sb.WriteString(" AND deleted_at = 0 ") + sb.WriteString(" ORDER BY ipornet ") + sb.WriteString(order) + if limit > 0 { + sb.WriteString(" LIMIT ") + sb.WriteString(sqlPlaceholders[idx]) + } + return sb.String() +} + +func getCountIPListEntriesQuery() string { + return fmt.Sprintf(`SELECT count(ipornet) FROM %s WHERE type = %s AND deleted_at = 0`, sqlTableIPLists, sqlPlaceholders[0]) +} + +func getCountAllIPListEntriesQuery() string { + return fmt.Sprintf(`SELECT count(ipornet) FROM %s WHERE deleted_at = 0`, sqlTableIPLists) +} + +func getIPListEntriesForIPQueryPg() string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE type = %s AND deleted_at = 0 AND %s::inet BETWEEN first AND last`, + selectIPListEntryFields, sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1]) +} + +func getIPListEntriesForIPQueryNoPg() string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE type = %s AND deleted_at = 0 AND ip_type = %s AND %s BETWEEN first AND last`, + selectIPListEntryFields, sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2]) +} + +func getRecentlyUpdatedIPListQuery() string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE updated_at >= %s OR deleted_at > 0`, + selectIPListEntryFields, sqlTableIPLists, sqlPlaceholders[0]) +} + +func getDumpListEntriesQuery() string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE deleted_at = 0`, selectIPListEntryFields, sqlTableIPLists) +} + +func getAddIPListEntryQuery() string { + return fmt.Sprintf(`INSERT INTO %s (type,ipornet,first,last,ip_type,protocols,description,mode,created_at,updated_at,deleted_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,0)`, sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1], + sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], + sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9]) +} + +func getUpdateIPListEntryQuery() string { + return fmt.Sprintf(`UPDATE %s SET mode=%s,protocols=%s,description=%s,updated_at=%s WHERE type = %s AND ipornet = %s`, + sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], + sqlPlaceholders[4], sqlPlaceholders[5]) +} + +func getDeleteIPListEntryQuery(softDelete bool) string { + if softDelete { + return fmt.Sprintf(`UPDATE %s SET updated_at=%s,deleted_at=%s WHERE type = %s AND ipornet = %s`, + sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3]) + } + return fmt.Sprintf(`DELETE FROM %s WHERE type = %s AND ipornet = %s`, + sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1]) +} + +func getRemoveSoftDeletedIPListEntryQuery() string { + return fmt.Sprintf(`DELETE FROM %s WHERE type = %s AND ipornet = %s AND deleted_at > 0`, + sqlTableIPLists, sqlPlaceholders[0], sqlPlaceholders[1]) +} + func getRoleByNameQuery() string { return fmt.Sprintf(`SELECT %s FROM %s WHERE name = %s`, selectRoleFields, sqlTableRoles, sqlPlaceholders[0]) diff --git a/internal/dataprovider/user.go b/internal/dataprovider/user.go index bbef850b..5566069a 100644 --- a/internal/dataprovider/user.go +++ b/internal/dataprovider/user.go @@ -473,7 +473,7 @@ func (u *User) SetEmptySecrets() { } // GetPermissionsForPath returns the permissions for the given path. -// The path must be a SFTPGo exposed path +// The path must be a SFTPGo virtual path func (u *User) GetPermissionsForPath(p string) []string { permissions := []string{} if perms, ok := u.Permissions["/"]; ok { diff --git a/internal/ftpd/ftpd.go b/internal/ftpd/ftpd.go index 618cd53c..c215473a 100644 --- a/internal/ftpd/ftpd.go +++ b/internal/ftpd/ftpd.go @@ -68,9 +68,9 @@ type Binding struct { CertificateKeyFile string `json:"certificate_key_file" mapstructure:"certificate_key_file"` // Defines the minimum TLS version. 13 means TLS 1.3, default is TLS 1.2 MinTLSVersion int `json:"min_tls_version" mapstructure:"min_tls_version"` - // External IP address to expose for passive connections. + // External IP address for passive connections. ForcePassiveIP string `json:"force_passive_ip" mapstructure:"force_passive_ip"` - // PassiveIPOverrides allows to define different IP addresses to expose for passive connections + // PassiveIPOverrides allows to define different IP addresses for passive connections // based on the client IP address PassiveIPOverrides []PassiveIPOverride `json:"passive_ip_overrides" mapstructure:"passive_ip_overrides"` // Set to 1 to require client certificate authentication. diff --git a/internal/ftpd/ftpd_test.go b/internal/ftpd/ftpd_test.go index 5fb29ba5..81ebf3f8 100644 --- a/internal/ftpd/ftpd_test.go +++ b/internal/ftpd/ftpd_test.go @@ -309,16 +309,16 @@ func TestMain(m *testing.M) { os.Exit(1) } - err = common.Initialize(commonConf, 0) - if err != nil { - logger.WarnToConsole("error initializing common: %v", err) - os.Exit(1) - } err = dataprovider.Initialize(providerConf, configDir, true) if err != nil { logger.ErrorToConsole("error initializing data provider: %v", err) os.Exit(1) } + err = common.Initialize(commonConf, 0) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } httpConfig := config.GetHTTPConfig() httpConfig.Initialize(configDir) //nolint:errcheck diff --git a/internal/ftpd/server.go b/internal/ftpd/server.go index f0e9764d..ad2d384a 100644 --- a/internal/ftpd/server.go +++ b/internal/ftpd/server.go @@ -160,11 +160,11 @@ func (s *Server) ClientConnected(cc ftpserver.ClientContext) (string, error) { cc.SetDebug(s.binding.Debug) ipAddr := util.GetIPFromRemoteAddress(cc.RemoteAddr().String()) common.Connections.AddClientConnection(ipAddr) - if common.IsBanned(ipAddr) { + if common.IsBanned(ipAddr, common.ProtocolFTP) { logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection refused, ip %#v is banned", ipAddr) return "Access denied: banned client IP", common.ErrConnectionDenied } - if err := common.Connections.IsNewConnectionAllowed(ipAddr); err != nil { + if err := common.Connections.IsNewConnectionAllowed(ipAddr, common.ProtocolFTP); err != nil { logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection not allowed from ip %q: %v", ipAddr, err) return "Access denied", err } @@ -429,7 +429,7 @@ func updateLoginMetrics(user *dataprovider.User, ip, loginMethod string, err err if errors.Is(err, util.ErrNotFound) { event = common.HostEventUserNotFound } - common.AddDefenderEvent(ip, event) + common.AddDefenderEvent(ip, common.ProtocolFTP, event) } metric.AddLoginResult(loginMethod, err) dataprovider.ExecutePostLoginHook(user, loginMethod, ip, common.ProtocolFTP, err) diff --git a/internal/httpd/api_admin.go b/internal/httpd/api_admin.go index 05ba7fa6..3062655d 100644 --- a/internal/httpd/api_admin.go +++ b/internal/httpd/api_admin.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "github.com/go-chi/jwtauth/v5" "github.com/go-chi/render" @@ -83,7 +84,7 @@ func addAdmin(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", adminPath, admin.Username)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", adminPath, url.PathEscape(admin.Username))) renderAdmin(w, r, admin.Username, http.StatusCreated) } diff --git a/internal/httpd/api_eventrule.go b/internal/httpd/api_eventrule.go index b362acc8..cb8dc40f 100644 --- a/internal/httpd/api_eventrule.go +++ b/internal/httpd/api_eventrule.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "github.com/go-chi/render" @@ -82,7 +83,7 @@ func addEventAction(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", eventActionsPath, action.Name)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", eventActionsPath, url.PathEscape(action.Name))) renderEventAction(w, r, action.Name, http.StatusCreated) } @@ -197,7 +198,7 @@ func addEventRule(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", eventRulesPath, rule.Name)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", eventRulesPath, url.PathEscape(rule.Name))) renderEventRule(w, r, rule.Name, http.StatusCreated) } diff --git a/internal/httpd/api_folder.go b/internal/httpd/api_folder.go index 461e2922..ac9035b3 100644 --- a/internal/httpd/api_folder.go +++ b/internal/httpd/api_folder.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "github.com/go-chi/render" @@ -60,7 +61,7 @@ func addFolder(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", folderPath, folder.Name)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", folderPath, url.PathEscape(folder.Name))) renderFolder(w, r, folder.Name, http.StatusCreated) } diff --git a/internal/httpd/api_group.go b/internal/httpd/api_group.go index 87f5bc6d..ced86d4e 100644 --- a/internal/httpd/api_group.go +++ b/internal/httpd/api_group.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "github.com/go-chi/render" @@ -59,7 +60,7 @@ func addGroup(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", groupPath, group.Name)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", groupPath, url.PathEscape(group.Name))) renderGroup(w, r, group.Name, http.StatusCreated) } diff --git a/internal/httpd/api_iplist.go b/internal/httpd/api_iplist.go new file mode 100644 index 00000000..3e4d3553 --- /dev/null +++ b/internal/httpd/api_iplist.go @@ -0,0 +1,157 @@ +// Copyright (C) 2019-2023 Nicola Murino +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package httpd + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + + "github.com/drakkan/sftpgo/v2/internal/dataprovider" + "github.com/drakkan/sftpgo/v2/internal/util" +) + +func getIPListEntries(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + limit, _, order, err := getSearchFilters(w, r) + if err != nil { + return + } + listType, _, err := getIPListPathParams(r) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + entries, err := dataprovider.GetIPListEntries(listType, r.URL.Query().Get("filter"), r.URL.Query().Get("from"), + order, limit) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + render.JSON(w, r, entries) +} + +func getIPListEntry(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + listType, ipOrNet, err := getIPListPathParams(r) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + + entry, err := dataprovider.IPListEntryExists(ipOrNet, listType) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + render.JSON(w, r, entry) +} + +func addIPListEntry(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } + var entry dataprovider.IPListEntry + err = render.DecodeJSON(r.Body, &entry) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + err = dataprovider.AddIPListEntry(&entry, claims.Username, util.GetIPFromRemoteAddress(r.RemoteAddr), claims.Role) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + w.Header().Add("Location", fmt.Sprintf("%s/%d/%s", ipListsPath, entry.Type, url.PathEscape(entry.IPOrNet))) + sendAPIResponse(w, r, nil, "Entry added", http.StatusCreated) +} + +func updateIPListEntry(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } + listType, ipOrNet, err := getIPListPathParams(r) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + entry, err := dataprovider.IPListEntryExists(ipOrNet, listType) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + var updatedEntry dataprovider.IPListEntry + err = render.DecodeJSON(r.Body, &updatedEntry) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + updatedEntry.Type = entry.Type + updatedEntry.IPOrNet = entry.IPOrNet + err = dataprovider.UpdateIPListEntry(&updatedEntry, claims.Username, util.GetIPFromRemoteAddress(r.RemoteAddr), claims.Role) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + sendAPIResponse(w, r, nil, "Entry updated", http.StatusOK) +} + +func deleteIPListEntry(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } + listType, ipOrNet, err := getIPListPathParams(r) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + err = dataprovider.DeleteIPListEntry(ipOrNet, listType, claims.Username, util.GetIPFromRemoteAddress(r.RemoteAddr), + claims.Role) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + sendAPIResponse(w, r, err, "Entry deleted", http.StatusOK) +} + +func getIPListPathParams(r *http.Request) (dataprovider.IPListType, string, error) { + listTypeString := chi.URLParam(r, "type") + listType, err := strconv.Atoi(listTypeString) + if err != nil { + return dataprovider.IPListType(listType), "", errors.New("invalid list type") + } + if err := dataprovider.CheckIPListType(dataprovider.IPListType(listType)); err != nil { + return dataprovider.IPListType(listType), "", err + } + return dataprovider.IPListType(listType), getURLParam(r, "ipornet"), nil +} diff --git a/internal/httpd/api_keys.go b/internal/httpd/api_keys.go index 71495fe0..98bbc20b 100644 --- a/internal/httpd/api_keys.go +++ b/internal/httpd/api_keys.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "github.com/go-chi/render" @@ -78,7 +79,7 @@ func addAPIKey(w http.ResponseWriter, r *http.Request) { response := make(map[string]string) response["message"] = "API key created. This is the only time the API key is visible, please save it." response["key"] = apiKey.DisplayKey() - w.Header().Add("Location", fmt.Sprintf("%v/%v", apiKeysPath, apiKey.KeyID)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", apiKeysPath, url.PathEscape(apiKey.KeyID))) w.Header().Add("X-Object-ID", apiKey.KeyID) ctx := context.WithValue(r.Context(), render.StatusCtxKey, http.StatusCreated) render.JSON(w, r.WithContext(ctx), response) diff --git a/internal/httpd/api_maintenance.go b/internal/httpd/api_maintenance.go index 8017575e..c847515f 100644 --- a/internal/httpd/api_maintenance.go +++ b/internal/httpd/api_maintenance.go @@ -39,10 +39,10 @@ func validateBackupFile(outputFile string) (string, error) { return "", errors.New("invalid or missing output-file") } if filepath.IsAbs(outputFile) { - return "", fmt.Errorf("invalid output-file %#v: it must be a relative path", outputFile) + return "", fmt.Errorf("invalid output-file %q: it must be a relative path", outputFile) } if strings.Contains(outputFile, "..") { - return "", fmt.Errorf("invalid output-file %#v", outputFile) + return "", fmt.Errorf("invalid output-file %q", outputFile) } outputFile = filepath.Join(dataprovider.GetBackupsPath(), outputFile) return outputFile, nil @@ -71,16 +71,16 @@ func dumpData(w http.ResponseWriter, r *http.Request) { err = os.MkdirAll(filepath.Dir(outputFile), 0700) if err != nil { - logger.Error(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile) + logger.Error(logSender, "", "dumping data error: %v, output file: %q", err, outputFile) sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - logger.Debug(logSender, "", "dumping data to: %#v", outputFile) + logger.Debug(logSender, "", "dumping data to: %q", outputFile) } backup, err := dataprovider.DumpData() if err != nil { - logger.Error(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile) + logger.Error(logSender, "", "dumping data error: %v, output file: %q", err, outputFile) sendAPIResponse(w, r, err, "", getRespStatus(err)) return } @@ -101,11 +101,11 @@ func dumpData(w http.ResponseWriter, r *http.Request) { err = os.WriteFile(outputFile, dump, 0600) } if err != nil { - logger.Warn(logSender, "", "dumping data error: %v, output file: %#v", err, outputFile) + logger.Warn(logSender, "", "dumping data error: %v, output file: %q", err, outputFile) sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - logger.Debug(logSender, "", "dumping data completed, output file: %#v, error: %v", outputFile, err) + logger.Debug(logSender, "", "dumping data completed, output file: %q, error: %v", outputFile, err) sendAPIResponse(w, r, err, "Data saved", http.StatusOK) } @@ -150,7 +150,8 @@ func loadData(w http.ResponseWriter, r *http.Request) { return } if !filepath.IsAbs(inputFile) { - sendAPIResponse(w, r, fmt.Errorf("invalid input_file %#v: it must be an absolute path", inputFile), "", http.StatusBadRequest) + sendAPIResponse(w, r, fmt.Errorf("invalid input_file %q: it must be an absolute path", inputFile), "", + http.StatusBadRequest) return } fi, err := os.Stat(inputFile) @@ -159,7 +160,7 @@ func loadData(w http.ResponseWriter, r *http.Request) { return } if fi.Size() > MaxRestoreSize { - sendAPIResponse(w, r, err, fmt.Sprintf("Unable to restore input file: %#v size too big: %v/%v bytes", + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to restore input file: %q size too big: %d/%d bytes", inputFile, fi.Size(), MaxRestoreSize), http.StatusBadRequest) return } @@ -182,6 +183,10 @@ func restoreBackup(content []byte, inputFile string, scanQuota, mode int, execut return util.NewValidationError(fmt.Sprintf("unable to parse backup content: %v", err)) } + if err = RestoreIPListEntries(dump.IPLists, inputFile, mode, executor, ipAddress, role); err != nil { + return err + } + if err = RestoreRoles(dump.Roles, inputFile, mode, executor, ipAddress, role); err != nil { return err } @@ -251,29 +256,29 @@ func getLoaddataOptions(r *http.Request) (string, int, int, error) { // RestoreFolders restores the specified folders func RestoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, mode, scanQuota int, executor, ipAddress, role string) error { - for _, folder := range folders { - folder := folder // pin + for idx := range folders { + folder := folders[idx] f, err := dataprovider.GetFolderByName(folder.Name) if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing folder %#v not updated", folder.Name) + logger.Debug(logSender, "", "loaddata mode 1, existing folder %q not updated", folder.Name) continue } folder.ID = f.ID folder.Name = f.Name err = dataprovider.UpdateFolder(&folder, f.Users, f.Groups, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing folder %#v, dump file: %#v, error: %v", folder.Name, inputFile, err) + logger.Debug(logSender, "", "restoring existing folder %q, dump file: %q, error: %v", folder.Name, inputFile, err) } else { folder.Users = nil err = dataprovider.AddFolder(&folder, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new folder %#v, dump file: %#v, error: %v", folder.Name, inputFile, err) + logger.Debug(logSender, "", "adding new folder %q, dump file: %q, error: %v", folder.Name, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore folder %#v: %w", folder.Name, err) + return fmt.Errorf("unable to restore folder %q: %w", folder.Name, err) } if scanQuota >= 1 { if common.QuotaScans.AddVFolderQuotaScan(folder.Name) { - logger.Debug(logSender, "", "starting quota scan for restored folder: %#v", folder.Name) + logger.Debug(logSender, "", "starting quota scan for restored folder: %q", folder.Name) go doFolderQuotaScan(folder) //nolint:errcheck } } @@ -285,21 +290,21 @@ func RestoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, mode, sca func RestoreShares(shares []dataprovider.Share, inputFile string, mode int, executor, ipAddress, role string, ) error { - for _, share := range shares { - share := share // pin + for idx := range shares { + share := shares[idx] share.IsRestore = true s, err := dataprovider.ShareExists(share.ShareID, "") if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing share %#v not updated", share.ShareID) + logger.Debug(logSender, "", "loaddata mode 1, existing share %q not updated", share.ShareID) continue } share.ID = s.ID err = dataprovider.UpdateShare(&share, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing share %#v, dump file: %#v, error: %v", share.ShareID, inputFile, err) + logger.Debug(logSender, "", "restoring existing share %q, dump file: %q, error: %v", share.ShareID, inputFile, err) } else { err = dataprovider.AddShare(&share, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new share %#v, dump file: %#v, error: %v", share.ShareID, inputFile, err) + logger.Debug(logSender, "", "adding new share %q, dump file: %q, error: %v", share.ShareID, inputFile, err) } if err != nil { return fmt.Errorf("unable to restore share %q: %w", share.ShareID, err) @@ -310,8 +315,8 @@ func RestoreShares(shares []dataprovider.Share, inputFile string, mode int, exec // RestoreEventActions restores the specified event actions func RestoreEventActions(actions []dataprovider.BaseEventAction, inputFile string, mode int, executor, ipAddress, role string) error { - for _, action := range actions { - action := action // pin + for idx := range actions { + action := actions[idx] a, err := dataprovider.EventActionExists(action.Name) if err == nil { if mode == 1 { @@ -336,8 +341,8 @@ func RestoreEventActions(actions []dataprovider.BaseEventAction, inputFile strin func RestoreEventRules(rules []dataprovider.EventRule, inputFile string, mode int, executor, ipAddress, role string, dumpVersion int, ) error { - for _, rule := range rules { - rule := rule // pin + for idx := range rules { + rule := rules[idx] if dumpVersion < 15 { rule.Status = 1 } @@ -363,8 +368,8 @@ func RestoreEventRules(rules []dataprovider.EventRule, inputFile string, mode in // RestoreAPIKeys restores the specified API keys func RestoreAPIKeys(apiKeys []dataprovider.APIKey, inputFile string, mode int, executor, ipAddress, role string) error { - for _, apiKey := range apiKeys { - apiKey := apiKey // pin + for idx := range apiKeys { + apiKey := apiKeys[idx] if apiKey.Key == "" { logger.Warn(logSender, "", "cannot restore empty API key") return fmt.Errorf("cannot restore an empty API key: %+v", apiKey) @@ -372,18 +377,18 @@ func RestoreAPIKeys(apiKeys []dataprovider.APIKey, inputFile string, mode int, e k, err := dataprovider.APIKeyExists(apiKey.KeyID) if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing API key %#v not updated", apiKey.KeyID) + logger.Debug(logSender, "", "loaddata mode 1, existing API key %q not updated", apiKey.KeyID) continue } apiKey.ID = k.ID err = dataprovider.UpdateAPIKey(&apiKey, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing API key %#v, dump file: %#v, error: %v", apiKey.KeyID, inputFile, err) + logger.Debug(logSender, "", "restoring existing API key %q, dump file: %q, error: %v", apiKey.KeyID, inputFile, err) } else { err = dataprovider.AddAPIKey(&apiKey, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new API key %#v, dump file: %#v, error: %v", apiKey.KeyID, inputFile, err) + logger.Debug(logSender, "", "adding new API key %q, dump file: %q, error: %v", apiKey.KeyID, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore API key %#v: %w", apiKey.KeyID, err) + return fmt.Errorf("unable to restore API key %q: %w", apiKey.KeyID, err) } } return nil @@ -391,34 +396,62 @@ func RestoreAPIKeys(apiKeys []dataprovider.APIKey, inputFile string, mode int, e // RestoreAdmins restores the specified admins func RestoreAdmins(admins []dataprovider.Admin, inputFile string, mode int, executor, ipAddress, role string) error { - for _, admin := range admins { - admin := admin // pin + for idx := range admins { + admin := admins[idx] a, err := dataprovider.AdminExists(admin.Username) if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing admin %#v not updated", a.Username) + logger.Debug(logSender, "", "loaddata mode 1, existing admin %q not updated", a.Username) continue } admin.ID = a.ID admin.Username = a.Username err = dataprovider.UpdateAdmin(&admin, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing admin %#v, dump file: %#v, error: %v", admin.Username, inputFile, err) + logger.Debug(logSender, "", "restoring existing admin %q, dump file: %q, error: %v", admin.Username, inputFile, err) } else { err = dataprovider.AddAdmin(&admin, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new admin %#v, dump file: %#v, error: %v", admin.Username, inputFile, err) + logger.Debug(logSender, "", "adding new admin %q, dump file: %q, error: %v", admin.Username, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore admin %#v: %w", admin.Username, err) + return fmt.Errorf("unable to restore admin %q: %w", admin.Username, err) } } return nil } +// RestoreIPListEntries restores the specified IP list entries +func RestoreIPListEntries(entries []dataprovider.IPListEntry, inputFile string, mode int, executor, ipAddress, + executorRole string, +) error { + for idx := range entries { + entry := entries[idx] + e, err := dataprovider.IPListEntryExists(entry.IPOrNet, entry.Type) + if err == nil { + if mode == 1 { + logger.Debug(logSender, "", "loaddata mode 1, existing IP list entry %s-%s not updated", + e.Type.AsString(), e.IPOrNet) + continue + } + err = dataprovider.UpdateIPListEntry(&entry, executor, ipAddress, executorRole) + logger.Debug(logSender, "", "restoring existing IP list entry: %s-%s, dump file: %q, error: %v", + entry.Type.AsString(), entry.IPOrNet, inputFile, err) + } else { + err = dataprovider.AddIPListEntry(&entry, executor, ipAddress, executorRole) + logger.Debug(logSender, "", "adding new IP list entry %s-%s, dump file: %q, error: %v", + entry.Type.AsString(), entry.IPOrNet, inputFile, err) + } + if err != nil { + return fmt.Errorf("unable to restore IP list entry %s-%s: %w", entry.Type.AsString(), entry.IPOrNet, err) + } + } + return nil +} + // RestoreRoles restores the specified roles func RestoreRoles(roles []dataprovider.Role, inputFile string, mode int, executor, ipAddress, executorRole string) error { - for _, role := range roles { - role := role // pin + for idx := range roles { + role := roles[idx] r, err := dataprovider.RoleExists(role.Name) if err == nil { if mode == 1 { @@ -427,13 +460,13 @@ func RestoreRoles(roles []dataprovider.Role, inputFile string, mode int, executo } role.ID = r.ID err = dataprovider.UpdateRole(&role, executor, ipAddress, executorRole) - logger.Debug(logSender, "", "restoring existing role: %q, dump file: %#v, error: %v", role.Name, inputFile, err) + logger.Debug(logSender, "", "restoring existing role: %q, dump file: %q, error: %v", role.Name, inputFile, err) } else { err = dataprovider.AddRole(&role, executor, ipAddress, executorRole) logger.Debug(logSender, "", "adding new role: %q, dump file: %q, error: %v", role.Name, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore role %#v: %w", role.Name, err) + return fmt.Errorf("unable to restore role %q: %w", role.Name, err) } } return nil @@ -441,24 +474,24 @@ func RestoreRoles(roles []dataprovider.Role, inputFile string, mode int, executo // RestoreGroups restores the specified groups func RestoreGroups(groups []dataprovider.Group, inputFile string, mode int, executor, ipAddress, role string) error { - for _, group := range groups { - group := group // pin + for idx := range groups { + group := groups[idx] g, err := dataprovider.GroupExists(group.Name) if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing group %#v not updated", g.Name) + logger.Debug(logSender, "", "loaddata mode 1, existing group %q not updated", g.Name) continue } group.ID = g.ID group.Name = g.Name err = dataprovider.UpdateGroup(&group, g.Users, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing group: %#v, dump file: %#v, error: %v", group.Name, inputFile, err) + logger.Debug(logSender, "", "restoring existing group: %q, dump file: %q, error: %v", group.Name, inputFile, err) } else { err = dataprovider.AddGroup(&group, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new group: %#v, dump file: %#v, error: %v", group.Name, inputFile, err) + logger.Debug(logSender, "", "adding new group: %q, dump file: %q, error: %v", group.Name, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore group %#v: %w", group.Name, err) + return fmt.Errorf("unable to restore group %q: %w", group.Name, err) } } return nil @@ -466,31 +499,31 @@ func RestoreGroups(groups []dataprovider.Group, inputFile string, mode int, exec // RestoreUsers restores the specified users func RestoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota int, executor, ipAddress, role string) error { - for _, user := range users { - user := user // pin + for idx := range users { + user := users[idx] u, err := dataprovider.UserExists(user.Username, "") if err == nil { if mode == 1 { - logger.Debug(logSender, "", "loaddata mode 1, existing user %#v not updated", u.Username) + logger.Debug(logSender, "", "loaddata mode 1, existing user %q not updated", u.Username) continue } user.ID = u.ID user.Username = u.Username err = dataprovider.UpdateUser(&user, executor, ipAddress, role) - logger.Debug(logSender, "", "restoring existing user: %#v, dump file: %#v, error: %v", user.Username, inputFile, err) + logger.Debug(logSender, "", "restoring existing user: %q, dump file: %q, error: %v", user.Username, inputFile, err) if mode == 2 && err == nil { disconnectUser(user.Username, executor, role) } } else { err = dataprovider.AddUser(&user, executor, ipAddress, role) - logger.Debug(logSender, "", "adding new user: %#v, dump file: %#v, error: %v", user.Username, inputFile, err) + logger.Debug(logSender, "", "adding new user: %q, dump file: %q, error: %v", user.Username, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restore user %#v: %w", user.Username, err) + return fmt.Errorf("unable to restore user %q: %w", user.Username, err) } if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) { if common.QuotaScans.AddUserQuotaScan(user.Username, user.Role) { - logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username) + logger.Debug(logSender, "", "starting quota scan for restored user: %q", user.Username) go doUserQuotaScan(user) //nolint:errcheck } } diff --git a/internal/httpd/api_role.go b/internal/httpd/api_role.go index b3a483d6..5958642e 100644 --- a/internal/httpd/api_role.go +++ b/internal/httpd/api_role.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "github.com/go-chi/render" @@ -59,7 +60,7 @@ func addRole(w http.ResponseWriter, r *http.Request) { if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) } else { - w.Header().Add("Location", fmt.Sprintf("%s/%s", rolesPath, role.Name)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", rolesPath, url.PathEscape(role.Name))) renderRole(w, r, role.Name, http.StatusCreated) } } diff --git a/internal/httpd/api_shares.go b/internal/httpd/api_shares.go index 613af261..ea7549b5 100644 --- a/internal/httpd/api_shares.go +++ b/internal/httpd/api_shares.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "os" "path" "strings" @@ -112,7 +113,7 @@ func addShare(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%v/%v", userSharesPath, share.ShareID)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", userSharesPath, url.PathEscape(share.ShareID))) w.Header().Add("X-Object-ID", share.ShareID) sendAPIResponse(w, r, nil, "Share created", http.StatusCreated) } diff --git a/internal/httpd/api_user.go b/internal/httpd/api_user.go index eb8fd3c7..b780e9fc 100644 --- a/internal/httpd/api_user.go +++ b/internal/httpd/api_user.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "time" @@ -114,7 +115,7 @@ func addUser(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - w.Header().Add("Location", fmt.Sprintf("%s/%s", userPath, user.Username)) + w.Header().Add("Location", fmt.Sprintf("%s/%s", userPath, url.PathEscape(user.Username))) renderUser(w, r, user.Username, claims.Role, http.StatusCreated) } diff --git a/internal/httpd/api_utils.go b/internal/httpd/api_utils.go index 1846e254..c55707d2 100644 --- a/internal/httpd/api_utils.go +++ b/internal/httpd/api_utils.go @@ -603,7 +603,7 @@ func updateLoginMetrics(user *dataprovider.User, loginMethod, ip string, err err if errors.Is(err, util.ErrNotFound) { event = common.HostEventUserNotFound } - common.AddDefenderEvent(ip, event) + common.AddDefenderEvent(ip, common.ProtocolHTTP, event) } metric.AddLoginResult(loginMethod, err) dataprovider.ExecutePostLoginHook(user, loginMethod, ip, protocol, err) diff --git a/internal/httpd/httpd.go b/internal/httpd/httpd.go index d88c0640..50ed8a38 100644 --- a/internal/httpd/httpd.go +++ b/internal/httpd/httpd.go @@ -13,7 +13,7 @@ // along with this program. If not, see . // Package httpd implements REST API and Web interface for SFTPGo. -// The OpenAPI 3 schema for the exposed API can be found inside the source tree: +// The OpenAPI 3 schema for the supported API can be found inside the source tree: // https://github.com/drakkan/sftpgo/blob/main/openapi/openapi.yaml package httpd @@ -93,6 +93,7 @@ const ( eventActionsPath = "/api/v2/eventactions" eventRulesPath = "/api/v2/eventrules" rolesPath = "/api/v2/roles" + ipListsPath = "/api/v2/iplists" healthzPath = "/healthz" robotsTxtPath = "/robots.txt" webRootPathDefault = "/" @@ -139,6 +140,8 @@ const ( webTemplateUserDefault = "/web/admin/template/user" webTemplateFolderDefault = "/web/admin/template/folder" webDefenderPathDefault = "/web/admin/defender" + webIPListsPathDefault = "/web/admin/ip-lists" + webIPListPathDefault = "/web/admin/ip-list" webDefenderHostsPathDefault = "/web/admin/defender/hosts" webEventsPathDefault = "/web/admin/events" webEventsFsSearchPathDefault = "/web/admin/events/fs" @@ -171,11 +174,11 @@ const ( webStaticFilesPathDefault = "/static" webOpenAPIPathDefault = "/openapi" // MaxRestoreSize defines the max size for the loaddata input file - MaxRestoreSize = 10485760 // 10 MB - maxRequestSize = 1048576 // 1MB - maxLoginBodySize = 262144 // 256 KB - httpdMaxEditFileSize = 1048576 // 1 MB - maxMultipartMem = 10485760 // 10 MB + MaxRestoreSize = 20 * 1048576 // 20 MB + maxRequestSize = 1048576 // 1MB + maxLoginBodySize = 262144 // 256 KB + httpdMaxEditFileSize = 1048576 // 1 MB + maxMultipartMem = 10 * 1048576 // 10 MB osWindows = "windows" otpHeaderCode = "X-SFTPGO-OTP" mTimeHeader = "X-SFTPGO-MTIME" @@ -231,6 +234,8 @@ var ( webTemplateUser string webTemplateFolder string webDefenderPath string + webIPListPath string + webIPListsPath string webEventsPath string webEventsFsSearchPath string webEventsProviderSearchPath string @@ -636,6 +641,20 @@ type defenderStatus struct { IsActive bool `json:"is_active"` } +type allowListStatus struct { + IsActive bool `json:"is_active"` +} + +type rateLimiters struct { + IsActive bool `json:"is_active"` + Protocols []string `json:"protocols"` +} + +// GetProtocolsAsString returns the enabled protocols as comma separated string +func (r *rateLimiters) GetProtocolsAsString() string { + return strings.Join(r.Protocols, ", ") +} + // ServicesStatus keep the state of the running services type ServicesStatus struct { SSH sftpd.ServiceStatus `json:"ssh"` @@ -644,6 +663,8 @@ type ServicesStatus struct { DataProvider dataprovider.ProviderStatus `json:"data_provider"` Defender defenderStatus `json:"defender"` MFA mfa.ServiceStatus `json:"mfa"` + AllowList allowListStatus `json:"allow_list"` + RateLimiters rateLimiters `json:"rate_limiters"` } // SetupConfig defines the configuration parameters for the initial web admin setup @@ -924,6 +945,7 @@ func getConfigPath(name, configDir string) string { } func getServicesStatus() *ServicesStatus { + rtlEnabled, rtlProtocols := common.Config.GetRateLimitersStatus() status := &ServicesStatus{ SSH: sftpd.GetStatus(), FTP: ftpd.GetStatus(), @@ -933,6 +955,13 @@ func getServicesStatus() *ServicesStatus { IsActive: common.Config.DefenderConfig.Enabled, }, MFA: mfa.GetStatus(), + AllowList: allowListStatus{ + IsActive: common.Config.IsAllowListEnabled(), + }, + RateLimiters: rateLimiters{ + IsActive: rtlEnabled, + Protocols: rtlProtocols, + }, } return status } @@ -1035,6 +1064,8 @@ func updateWebAdminURLs(baseURL string) { webTemplateFolder = path.Join(baseURL, webTemplateFolderDefault) webDefenderHostsPath = path.Join(baseURL, webDefenderHostsPathDefault) webDefenderPath = path.Join(baseURL, webDefenderPathDefault) + webIPListPath = path.Join(baseURL, webIPListPathDefault) + webIPListsPath = path.Join(baseURL, webIPListsPathDefault) webEventsPath = path.Join(baseURL, webEventsPathDefault) webEventsFsSearchPath = path.Join(baseURL, webEventsFsSearchPathDefault) webEventsProviderSearchPath = path.Join(baseURL, webEventsProviderSearchPathDefault) diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index db5cb8e7..9e22d4f5 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -127,6 +127,7 @@ const ( eventActionsPath = "/api/v2/eventactions" eventRulesPath = "/api/v2/eventrules" rolesPath = "/api/v2/roles" + ipListsPath = "/api/v2/iplists" healthzPath = "/healthz" robotsTxtPath = "/robots.txt" webBasePath = "/web" @@ -151,6 +152,8 @@ const ( webTemplateUser = "/web/admin/template/user" webTemplateFolder = "/web/admin/template/folder" webDefenderPath = "/web/admin/defender" + webIPListsPath = "/web/admin/ip-lists" + webIPListPath = "/web/admin/ip-list" webAdminTwoFactorPath = "/web/admin/twofactor" webAdminTwoFactorRecoveryPath = "/web/admin/twofactor-recovery" webAdminMFAPath = "/web/admin/mfa" @@ -330,12 +333,6 @@ func TestMain(m *testing.M) { providerConf := config.GetProviderConf() logger.InfoToConsole("Starting HTTPD tests, provider: %v", providerConf.Driver) - err = common.Initialize(config.GetCommonConfig(), 0) - if err != nil { - logger.WarnToConsole("error initializing common: %v", err) - os.Exit(1) - } - backupsPath = filepath.Join(os.TempDir(), "test_backups") providerConf.BackupsPath = backupsPath err = os.MkdirAll(backupsPath, os.ModePerm) @@ -350,6 +347,12 @@ func TestMain(m *testing.M) { os.Exit(1) } + err = common.Initialize(config.GetCommonConfig(), 0) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } + postConnectPath = filepath.Join(homeBasePath, "postconnect.sh") preActionPath = filepath.Join(homeBasePath, "preaction.sh") @@ -1281,6 +1284,206 @@ func TestGroupSettingsOverride(t *testing.T) { assert.NoError(t, err) } +func TestBasicIPListEntriesHandling(t *testing.T) { + entry := dataprovider.IPListEntry{ + IPOrNet: "::ffff:12.34.56.78", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Description: "test desc", + } + _, _, err := httpdtest.GetIPListEntry(entry.IPOrNet, -1, http.StatusBadRequest) + assert.NoError(t, err) + _, _, err = httpdtest.UpdateIPListEntry(entry, http.StatusNotFound) + assert.NoError(t, err) + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusCreated) + assert.Error(t, err) + // IPv4 address in IPv6 will be converted to standard IPv4 + entry1, _, err := httpdtest.GetIPListEntry("12.34.56.78/32", dataprovider.IPListTypeAllowList, http.StatusOK) + assert.NoError(t, err) + + entry = dataprovider.IPListEntry{ + IPOrNet: "192.168.0.0/24", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + } + entry2, _, err := httpdtest.AddIPListEntry(entry, http.StatusCreated) + assert.NoError(t, err) + // adding the same entry again should fail + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusInternalServerError) + assert.NoError(t, err) + // adding an entry with an invalid IP should fail + entry.IPOrNet = "invalid" + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + // adding an entry with an incompatible mode should fail + entry.IPOrNet = entry2.IPOrNet + entry.Mode = -1 + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + entry.Type = -1 + _, _, err = httpdtest.UpdateIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + entry = dataprovider.IPListEntry{ + IPOrNet: "2001:4860:4860::8888/120", + Type: dataprovider.IPListTypeRateLimiterSafeList, + Mode: dataprovider.ListModeDeny, + } + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + entry.Mode = dataprovider.ListModeAllow + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusCreated) + assert.NoError(t, err) + entry.Protocols = 3 + entry3, _, err := httpdtest.UpdateIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) + entry.Mode = dataprovider.ListModeDeny + _, _, err = httpdtest.UpdateIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + + for _, tt := range []dataprovider.IPListType{dataprovider.IPListTypeAllowList, dataprovider.IPListTypeDefender, dataprovider.IPListTypeRateLimiterSafeList} { + entries, _, err := httpdtest.GetIPListEntries(tt, "", "", dataprovider.OrderASC, 0, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, entries, 1) { + switch tt { + case dataprovider.IPListTypeAllowList: + assert.Equal(t, entry1, entries[0]) + case dataprovider.IPListTypeDefender: + assert.Equal(t, entry2, entries[0]) + case dataprovider.IPListTypeRateLimiterSafeList: + assert.Equal(t, entry3, entries[0]) + } + } + } + + _, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "", "", "invalid order", 0, http.StatusBadRequest) + assert.NoError(t, err) + _, _, err = httpdtest.GetIPListEntries(-1, "", "", dataprovider.OrderASC, 0, http.StatusBadRequest) + assert.NoError(t, err) + + _, err = httpdtest.RemoveIPListEntry(entry1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry1, http.StatusNotFound) + assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry2, http.StatusOK) + assert.NoError(t, err) + entry2.Type = -1 + _, err = httpdtest.RemoveIPListEntry(entry2, http.StatusBadRequest) + assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry3, http.StatusOK) + assert.NoError(t, err) +} + +func TestSearchIPListEntries(t *testing.T) { + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "192.168.0.0/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "192.168.0.1/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "192.168.0.2/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 5, + }, + { + IPOrNet: "192.168.0.3/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + }, + { + IPOrNet: "10.8.0.0/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 3, + }, + { + IPOrNet: "10.8.1.0/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + }, + { + IPOrNet: "10.8.2.0/24", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 1, + }, + } + + for _, e := range entries { + _, _, err := httpdtest.AddIPListEntry(e, http.StatusCreated) + assert.NoError(t, err) + } + + results, _, err := httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "", "", dataprovider.OrderASC, 20, http.StatusOK) + assert.NoError(t, err) + if assert.Equal(t, len(entries), len(results)) { + assert.Equal(t, "10.8.0.0/24", results[0].IPOrNet) + } + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "", "", dataprovider.OrderDESC, 20, http.StatusOK) + assert.NoError(t, err) + if assert.Equal(t, len(entries), len(results)) { + assert.Equal(t, "192.168.0.3/24", results[0].IPOrNet) + } + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "", "192.168.0.1/24", dataprovider.OrderASC, 1, http.StatusOK) + assert.NoError(t, err) + if assert.Equal(t, 1, len(results), results) { + assert.Equal(t, "192.168.0.2/24", results[0].IPOrNet) + } + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "", "10.8.2.0/24", dataprovider.OrderDESC, 1, http.StatusOK) + assert.NoError(t, err) + if assert.Equal(t, 1, len(results), results) { + assert.Equal(t, "10.8.1.0/24", results[0].IPOrNet) + } + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "10.", "", dataprovider.OrderASC, 20, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 3, len(results)) + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "192", "", dataprovider.OrderASC, 20, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 4, len(results)) + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "1", "", dataprovider.OrderASC, 20, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 7, len(results)) + results, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeAllowList, "108", "", dataprovider.OrderASC, 20, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 0, len(results)) + + for _, e := range entries { + _, err := httpdtest.RemoveIPListEntry(e, http.StatusOK) + assert.NoError(t, err) + } +} + +func TestIPListEntriesValidation(t *testing.T) { + entry := dataprovider.IPListEntry{ + IPOrNet: "::ffff:34.56.78.90/120", + Type: -1, + Mode: dataprovider.ListModeDeny, + } + _, resp, err := httpdtest.AddIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + assert.Contains(t, string(resp), "invalid list type") + entry.Type = dataprovider.IPListTypeRateLimiterSafeList + _, resp, err = httpdtest.AddIPListEntry(entry, http.StatusBadRequest) + assert.NoError(t, err) + assert.Contains(t, string(resp), "invalid list mode") + entry.Type = dataprovider.IPListTypeDefender + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusCreated) + assert.Error(t, err) + entry.IPOrNet = "34.56.78.0/24" + _, err = httpdtest.RemoveIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) +} + func TestBasicActionRulesHandling(t *testing.T) { actionName := "test action" a := dataprovider.BaseEventAction{ @@ -6503,6 +6706,8 @@ func TestProviderErrors(t *testing.T) { assert.NoError(t, err) _, _, err = httpdtest.GetEventRules(1, 0, http.StatusInternalServerError) assert.NoError(t, err) + _, _, err = httpdtest.GetIPListEntries(dataprovider.IPListTypeDefender, "", "", dataprovider.OrderASC, 10, http.StatusInternalServerError) + assert.NoError(t, err) _, _, err = httpdtest.GetRoles(1, 0, http.StatusInternalServerError) assert.NoError(t, err) _, _, err = httpdtest.UpdateRole(getTestRole(), http.StatusInternalServerError) @@ -6696,6 +6901,22 @@ func TestProviderErrors(t *testing.T) { assert.NoError(t, err) _, _, err = httpdtest.Loaddata(backupFilePath, "", "", http.StatusInternalServerError) assert.NoError(t, err) + backupData = dataprovider.BackupData{ + IPLists: []dataprovider.IPListEntry{ + { + IPOrNet: "192.168.1.1/24", + Type: dataprovider.IPListTypeRateLimiterSafeList, + Mode: dataprovider.ListModeAllow, + }, + }, + Version: dataprovider.DumpVersion, + } + backupContent, err = json.Marshal(backupData) + assert.NoError(t, err) + err = os.WriteFile(backupFilePath, backupContent, os.ModePerm) + assert.NoError(t, err) + _, _, err = httpdtest.Loaddata(backupFilePath, "", "", http.StatusInternalServerError) + assert.NoError(t, err) err = os.Remove(backupFilePath) assert.NoError(t, err) @@ -6973,11 +7194,11 @@ func TestDefenderAPI(t *testing.T) { _, err = httpdtest.RemoveDefenderHostByIP(ip, http.StatusNotFound) assert.NoError(t, err) - common.AddDefenderEvent(ip, common.HostEventNoLoginTried) + common.AddDefenderEvent(ip, common.ProtocolHTTP, common.HostEventNoLoginTried) hosts, _, err = httpdtest.GetDefenderHosts(http.StatusOK) assert.NoError(t, err) assert.Len(t, hosts, 0) - common.AddDefenderEvent(ip, common.HostEventUserNotFound) + common.AddDefenderEvent(ip, common.ProtocolHTTP, common.HostEventUserNotFound) hosts, _, err = httpdtest.GetDefenderHosts(http.StatusOK) assert.NoError(t, err) if assert.Len(t, hosts, 1) { @@ -6991,7 +7212,7 @@ func TestDefenderAPI(t *testing.T) { assert.Empty(t, host.GetBanTime()) assert.Equal(t, 2, host.Score) - common.AddDefenderEvent(ip, common.HostEventUserNotFound) + common.AddDefenderEvent(ip, common.ProtocolHTTP, common.HostEventUserNotFound) hosts, _, err = httpdtest.GetDefenderHosts(http.StatusOK) assert.NoError(t, err) if assert.Len(t, hosts, 1) { @@ -7011,8 +7232,8 @@ func TestDefenderAPI(t *testing.T) { _, _, err = httpdtest.GetDefenderHostByIP(ip, http.StatusNotFound) assert.NoError(t, err) - common.AddDefenderEvent(ip, common.HostEventUserNotFound) - common.AddDefenderEvent(ip, common.HostEventUserNotFound) + common.AddDefenderEvent(ip, common.ProtocolHTTP, common.HostEventUserNotFound) + common.AddDefenderEvent(ip, common.ProtocolHTTP, common.HostEventUserNotFound) hosts, _, err = httpdtest.GetDefenderHosts(http.StatusOK) assert.NoError(t, err) assert.Len(t, hosts, 1) @@ -7289,6 +7510,13 @@ func TestLoaddata(t *testing.T) { Name: group.Name, }, } + ipListEntry := dataprovider.IPListEntry{ + IPOrNet: "172.16.2.4/32", + Description: "entry desc", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Protocols: 3, + } apiKey := dataprovider.APIKey{ Name: util.GenerateUniqueID(), Scope: dataprovider.APIKeyScopeAdmin, @@ -7361,6 +7589,7 @@ func TestLoaddata(t *testing.T) { backupData.Shares = append(backupData.Shares, share) backupData.EventActions = append(backupData.EventActions, action) backupData.EventRules = append(backupData.EventRules, rule) + backupData.IPLists = append(backupData.IPLists, ipListEntry) backupContent, err := json.Marshal(backupData) assert.NoError(t, err) backupFilePath := filepath.Join(backupsPath, "backup.json") @@ -7412,6 +7641,14 @@ func TestLoaddata(t *testing.T) { action, _, err = httpdtest.GetEventActionByName(action.Name, http.StatusOK) assert.NoError(t, err) + entry, _, err := httpdtest.GetIPListEntry(ipListEntry.IPOrNet, ipListEntry.Type, http.StatusOK) + assert.NoError(t, err) + assert.Greater(t, entry.CreatedAt, int64(0)) + assert.Greater(t, entry.UpdatedAt, int64(0)) + assert.Equal(t, ipListEntry.Description, entry.Description) + assert.Equal(t, ipListEntry.Protocols, entry.Protocols) + assert.Equal(t, ipListEntry.Mode, entry.Mode) + rule, _, err = httpdtest.GetEventRuleByName(rule.Name, http.StatusOK) assert.NoError(t, err) assert.Equal(t, 1, rule.Status) @@ -7493,10 +7730,12 @@ func TestLoaddata(t *testing.T) { assert.NoError(t, err) _, err = httpdtest.RemoveRole(role, http.StatusOK) assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) err = os.Remove(backupFilePath) assert.NoError(t, err) - err = createTestFile(backupFilePath, 10485761) + err = createTestFile(backupFilePath, 20*1048576+1) assert.NoError(t, err) _, _, err = httpdtest.Loaddata(backupFilePath, "1", "0", http.StatusBadRequest) assert.NoError(t, err) @@ -7581,6 +7820,13 @@ func TestLoaddataMode(t *testing.T) { }, }, } + ipListEntry := dataprovider.IPListEntry{ + IPOrNet: "10.8.3.9/32", + Description: "note", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Protocols: 7, + } backupData := dataprovider.BackupData{ Version: dataprovider.DumpVersion, } @@ -7606,6 +7852,7 @@ func TestLoaddataMode(t *testing.T) { } backupData.APIKeys = append(backupData.APIKeys, apiKey) backupData.Shares = append(backupData.Shares, share) + backupData.IPLists = append(backupData.IPLists, ipListEntry) backupContent, _ := json.Marshal(backupData) backupFilePath := filepath.Join(backupsPath, "backup.json") err := os.WriteFile(backupFilePath, backupContent, os.ModePerm) @@ -7676,6 +7923,13 @@ func TestLoaddataMode(t *testing.T) { rule, _, err = httpdtest.UpdateEventRule(rule, http.StatusOK) assert.NoError(t, err) + entry, _, err := httpdtest.GetIPListEntry(ipListEntry.IPOrNet, ipListEntry.Type, http.StatusOK) + assert.NoError(t, err) + oldEntryDesc := entry.Description + entry.Description = "new note" + entry, _, err = httpdtest.UpdateIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) + backupData.Folders = []vfs.BaseVirtualFolder{ { MappedPath: mappedPath, @@ -7700,6 +7954,9 @@ func TestLoaddataMode(t *testing.T) { rule, _, err = httpdtest.GetEventRuleByName(rule.Name, http.StatusOK) assert.NoError(t, err) assert.NotEqual(t, oldRuleDesc, rule.Description) + entry, _, err = httpdtest.GetIPListEntry(ipListEntry.IPOrNet, ipListEntry.Type, http.StatusOK) + assert.NoError(t, err) + assert.NotEqual(t, oldEntryDesc, entry.Description) c := common.NewBaseConnection("connID", common.ProtocolFTP, "", "", user) fakeConn := &fakeConnection{ @@ -7757,6 +8014,8 @@ func TestLoaddataMode(t *testing.T) { assert.NoError(t, err) _, err = httpdtest.RemoveRole(role, http.StatusOK) assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) err = os.Remove(backupFilePath) assert.NoError(t, err) } @@ -7946,6 +8205,48 @@ func TestAddRoleInvalidJsonMock(t *testing.T) { checkResponseCode(t, http.StatusBadRequest, rr) } +func TestIPListEntriesErrorsMock(t *testing.T) { + token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) + assert.NoError(t, err) + req, err := http.NewRequest(http.MethodGet, ipListsPath+"/a/b", nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr := executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "invalid list type") + req, err = http.NewRequest(http.MethodGet, ipListsPath+"/invalid", nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "invalid list type") + + reqBody := bytes.NewBuffer([]byte("{")) + req, err = http.NewRequest(http.MethodPost, ipListsPath+"/2", reqBody) + assert.NoError(t, err) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + + entry := dataprovider.IPListEntry{ + IPOrNet: "172.120.1.1/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + } + _, _, err = httpdtest.AddIPListEntry(entry, http.StatusCreated) + assert.NoError(t, err) + + req, err = http.NewRequest(http.MethodPut, path.Join(ipListsPath, "1", url.PathEscape(entry.IPOrNet)), reqBody) + assert.NoError(t, err) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + + _, err = httpdtest.RemoveIPListEntry(entry, http.StatusOK) + assert.NoError(t, err) +} + func TestRoleErrorsMock(t *testing.T) { token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) assert.NoError(t, err) @@ -15903,23 +16204,38 @@ func TestWebAdminSetupMock(t *testing.T) { os.Setenv("SFTPGO_DATA_PROVIDER__CREATE_DEFAULT_ADMIN", "1") } -func TestWhitelist(t *testing.T) { +func TestAllowList(t *testing.T) { configCopy := common.Config - common.Config.MaxTotalConnections = 1 - wlFile := filepath.Join(os.TempDir(), "wl.json") - common.Config.WhiteListFile = wlFile - wl := common.HostListFile{ - IPAddresses: []string{"172.120.1.1", "172.120.1.2"}, - CIDRNetworks: []string{"192.8.7.0/22"}, + entries := []dataprovider.IPListEntry{ + { + IPOrNet: "172.120.1.1/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "172.120.1.2/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 0, + }, + { + IPOrNet: "192.8.7.0/22", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + }, } - data, err := json.Marshal(wl) - assert.NoError(t, err) - err = os.WriteFile(wlFile, data, 0664) - assert.NoError(t, err) - defer os.Remove(wlFile) - err = common.Initialize(common.Config, 0) + for _, e := range entries { + _, _, err := httpdtest.AddIPListEntry(e, http.StatusCreated) + assert.NoError(t, err) + } + + common.Config.MaxTotalConnections = 1 + common.Config.AllowListStatus = 1 + err := common.Initialize(common.Config, 0) assert.NoError(t, err) req, _ := http.NewRequest(http.MethodGet, webLoginPath, nil) @@ -15931,7 +16247,8 @@ func TestWhitelist(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) - req.RemoteAddr = "172.120.1.3" + testIP := "172.120.1.3" + req.RemoteAddr = testIP rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) assert.Contains(t, rr.Body.String(), common.ErrConnectionDenied.Error()) @@ -15940,21 +16257,35 @@ func TestWhitelist(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) - wl.IPAddresses = append(wl.IPAddresses, "172.120.1.3") - data, err = json.Marshal(wl) - assert.NoError(t, err) - err = os.WriteFile(wlFile, data, 0664) - assert.NoError(t, err) - err = common.Reload() + entry := dataprovider.IPListEntry{ + IPOrNet: "172.120.1.3/32", + Type: dataprovider.IPListTypeAllowList, + Mode: dataprovider.ListModeAllow, + Protocols: 8, + } + err = dataprovider.AddIPListEntry(&entry, "", "", "") assert.NoError(t, err) - req.RemoteAddr = "172.120.1.3" + req.RemoteAddr = testIP rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) + err = dataprovider.DeleteIPListEntry(entry.IPOrNet, entry.Type, "", "", "") + assert.NoError(t, err) + + req.RemoteAddr = testIP + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + assert.Contains(t, rr.Body.String(), common.ErrConnectionDenied.Error()) + common.Config = configCopy err = common.Initialize(common.Config, 0) assert.NoError(t, err) + + for _, e := range entries { + _, err := httpdtest.RemoveIPListEntry(e, http.StatusOK) + assert.NoError(t, err) + } } func TestWebAdminLoginMock(t *testing.T) { @@ -17178,7 +17509,7 @@ func TestRenderDefenderPageMock(t *testing.T) { setJWTCookieForReq(req, token) rr := executeRequest(req) checkResponseCode(t, http.StatusOK, rr) - assert.Contains(t, rr.Body.String(), "View and manage blocklist") + assert.Contains(t, rr.Body.String(), "View and manage auto blocklist") } func TestWebAdminBasicMock(t *testing.T) { @@ -20876,6 +21207,192 @@ func TestWebEventRule(t *testing.T) { checkResponseCode(t, http.StatusOK, rr) } +func TestWebIPListEntries(t *testing.T) { + webToken, err := getJWTWebTokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) + assert.NoError(t, err) + csrfToken, err := getCSRFToken(httpBaseURL + webLoginPath) + assert.NoError(t, err) + + req, err := http.NewRequest(http.MethodGet, webIPListPath+"/mode", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + + req, err = http.NewRequest(http.MethodGet, webIPListPath+"/mode/a", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + + req, err = http.NewRequest(http.MethodGet, webIPListPath+"/1/a", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodGet, webIPListPath+"/1", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + + req, err = http.NewRequest(http.MethodGet, webIPListsPath, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + + entry := dataprovider.IPListEntry{ + IPOrNet: "12.34.56.78/20", + Type: dataprovider.IPListTypeDefender, + Mode: dataprovider.ListModeDeny, + Description: "note", + Protocols: 5, + } + form := make(url.Values) + form.Set("ipornet", entry.IPOrNet) + form.Set("description", entry.Description) + form.Set("mode", "a") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/mode", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "invalid list type") + + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + assert.Contains(t, rr.Body.String(), "unable to verify form token") + + form.Set(csrfFormToken, csrfToken) + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/2", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Contains(t, rr.Body.String(), "invalid mode") + + form.Set("mode", "2") + form.Set("protocols", "a") + form.Add("protocols", "1") + form.Add("protocols", "4") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/2", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr) + + entry1, _, err := httpdtest.GetIPListEntry(entry.IPOrNet, dataprovider.IPListTypeDefender, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, entry.Description, entry1.Description) + assert.Equal(t, entry.Mode, entry1.Mode) + assert.Equal(t, entry.Protocols, entry1.Protocols) + + form.Set("ipornet", "1111.11.11.11") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Contains(t, rr.Body.String(), "invalid IP") + + form.Set("ipornet", entry.IPOrNet) + form.Set("mode", "invalid") // ignored for list type 1 + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1", bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr) + + entry2, _, err := httpdtest.GetIPListEntry(entry.IPOrNet, dataprovider.IPListTypeAllowList, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, entry.Description, entry2.Description) + assert.Equal(t, dataprovider.ListModeAllow, entry2.Mode) + assert.Equal(t, entry.Protocols, entry2.Protocols) + + req, err = http.NewRequest(http.MethodGet, webIPListPath+"/1/"+url.PathEscape(entry2.IPOrNet), nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + + form.Set("protocols", "1") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1/"+url.PathEscape(entry.IPOrNet), + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr) + entry2, _, err = httpdtest.GetIPListEntry(entry.IPOrNet, dataprovider.IPListTypeAllowList, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, entry.Description, entry2.Description) + assert.Equal(t, dataprovider.ListModeAllow, entry2.Mode) + assert.Equal(t, 1, entry2.Protocols) + + form.Del(csrfFormToken) + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1/"+url.PathEscape(entry.IPOrNet), + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + assert.Contains(t, rr.Body.String(), "unable to verify form token") + + form.Set(csrfFormToken, csrfToken) + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/a/"+url.PathEscape(entry.IPOrNet), + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1/"+url.PathEscape(entry.IPOrNet)+"a", + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + form.Set("mode", "a") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/2/"+url.PathEscape(entry.IPOrNet), + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Contains(t, rr.Body.String(), "invalid mode") + + form.Set("mode", "100") + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/2/"+url.PathEscape(entry.IPOrNet), + bytes.NewBuffer([]byte(form.Encode()))) + assert.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Contains(t, rr.Body.String(), "invalid list mode") + + _, err = httpdtest.RemoveIPListEntry(entry1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveIPListEntry(entry2, http.StatusOK) + assert.NoError(t, err) +} + func TestWebRole(t *testing.T) { webToken, err := getJWTWebTokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) assert.NoError(t, err) @@ -22324,6 +22841,18 @@ func TestProviderClosedMock(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) + req, err = http.NewRequest(http.MethodGet, webIPListPath+"/1/a", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + + req, err = http.NewRequest(http.MethodPost, webIPListPath+"/1/a", nil) + assert.NoError(t, err) + setJWTCookieForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + req, err = http.NewRequest(http.MethodGet, webAdminRolesPath, nil) assert.NoError(t, err) setJWTCookieForReq(req, token) @@ -22387,12 +22916,31 @@ func TestWebConnectionsMock(t *testing.T) { } func TestGetWebStatusMock(t *testing.T) { + oldConfig := config.GetCommonConfig() + + cfg := config.GetCommonConfig() + cfg.RateLimitersConfig = []common.RateLimiterConfig{ + { + Average: 1, + Period: 1000, + Burst: 1, + Type: 1, + Protocols: []string{common.ProtocolFTP}, + }, + } + + err := common.Initialize(cfg, 0) + assert.NoError(t, err) + token, err := getJWTWebTokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) assert.NoError(t, err) req, _ := http.NewRequest(http.MethodGet, webStatusPath, nil) setJWTCookieForReq(req, token) rr := executeRequest(req) checkResponseCode(t, http.StatusOK, rr) + + err = common.Initialize(oldConfig, 0) + assert.NoError(t, err) } func TestStaticFilesMock(t *testing.T) { diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go index 8666c8dd..d9bb5fdc 100644 --- a/internal/httpd/internal_test.go +++ b/internal/httpd/internal_test.go @@ -756,6 +756,21 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() + addIPListEntry(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + + rr = httptest.NewRecorder() + updateIPListEntry(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + + rr = httptest.NewRecorder() + deleteIPListEntry(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() server.handleGetWebUsers(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -811,6 +826,11 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "invalid token claims") + rr = httptest.NewRecorder() + server.handleWebUpdateIPListEntryPost(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "invalid token claims") + rr = httptest.NewRecorder() server.handleWebClientTwoFactorRecoveryPost(rr, req) assert.Equal(t, http.StatusNotFound, rr.Code) @@ -826,6 +846,22 @@ func TestInvalidToken(t *testing.T) { rr = httptest.NewRecorder() server.handleWebAdminTwoFactorPost(rr, req) assert.Equal(t, http.StatusNotFound, rr.Code) + + rr = httptest.NewRecorder() + server.handleWebUpdateIPListEntryPost(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "invalid token claims") + + form := make(url.Values) + req, _ = http.NewRequest(http.MethodPost, webIPListPath+"/1", bytes.NewBuffer([]byte(form.Encode()))) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rctx = chi.NewRouteContext() + rctx.URLParams.Add("type", "1") + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) + rr = httptest.NewRecorder() + server.handleWebAddIPListEntryPost(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code, rr.Body.String()) + assert.Contains(t, rr.Body.String(), "invalid token claims") } func TestUpdateWebAdminInvalidClaims(t *testing.T) { @@ -1046,6 +1082,11 @@ func TestCreateTokenError(t *testing.T) { _, err = getEventRuleFromPostFields(req) assert.Error(t, err) + req, _ = http.NewRequest(http.MethodPost, webIPListPath+"/1?a=a%C3%AO%GG", nil) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + _, err = getIPListEntryFromPostFields(req, dataprovider.IPListTypeAllowList) + assert.Error(t, err) + req, _ = http.NewRequest(http.MethodPost, webClientLoginPath+"?a=a%C3%AO%GG", bytes.NewBuffer([]byte(form.Encode()))) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") rr = httptest.NewRecorder() @@ -1135,7 +1176,6 @@ func TestCreateTokenError(t *testing.T) { assert.Contains(t, rr.Body.String(), "invalid URL escape") req, _ = http.NewRequest(http.MethodPost, webChangeClientPwdPath+"?a=a%K3%AO%GA", bytes.NewBuffer([]byte(form.Encode()))) - _, err = getShareFromPostFields(req) if assert.Error(t, err) { assert.Contains(t, err.Error(), "invalid URL escape") diff --git a/internal/httpd/server.go b/internal/httpd/server.go index 49153ef8..6a78d0a9 100644 --- a/internal/httpd/server.go +++ b/internal/httpd/server.go @@ -1038,12 +1038,12 @@ func (s *httpdServer) checkConnection(next http.Handler) http.Handler { common.Connections.AddClientConnection(ipAddr) defer common.Connections.RemoveClientConnection(ipAddr) - if err := common.Connections.IsNewConnectionAllowed(ipAddr); err != nil { + if err := common.Connections.IsNewConnectionAllowed(ipAddr, common.ProtocolHTTP); err != nil { logger.Log(logger.LevelDebug, common.ProtocolHTTP, "", "connection not allowed from ip %q: %v", ipAddr, err) s.sendForbiddenResponse(w, r, err.Error()) return } - if common.IsBanned(ipAddr) { + if common.IsBanned(ipAddr, common.ProtocolHTTP) { s.sendForbiddenResponse(w, r, "your IP address is banned") return } @@ -1189,7 +1189,7 @@ func (s *httpdServer) initializeRouter() { }) if s.enableRESTAPI { - // share API exposed to external users + // share API available to external users s.router.Get(sharesPath+"/{id}", s.downloadFromShare) s.router.Post(sharesPath+"/{id}", s.uploadFilesToShare) s.router.Post(sharesPath+"/{id}/{name}", s.uploadFileToShare) @@ -1304,10 +1304,15 @@ func (s *httpdServer) initializeRouter() { router.With(s.checkPerm(dataprovider.PermAdminManageEventRules)).Delete(eventRulesPath+"/{name}", deleteEventRule) router.With(s.checkPerm(dataprovider.PermAdminManageEventRules)).Post(eventRulesPath+"/run/{name}", runOnDemandRule) router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Get(rolesPath, getRoles) - router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Get(rolesPath+"/{name}", getRoleByName) router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Post(rolesPath, addRole) + router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Get(rolesPath+"/{name}", getRoleByName) router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Put(rolesPath+"/{name}", updateRole) router.With(s.checkPerm(dataprovider.PermAdminManageRoles)).Delete(rolesPath+"/{name}", deleteRole) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists), compressor.Handler).Get(ipListsPath+"/{type}", getIPListEntries) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Post(ipListsPath+"/{type}", addIPListEntry) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Get(ipListsPath+"/{type}/{ipornet}", getIPListEntry) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Put(ipListsPath+"/{type}/{ipornet}", updateIPListEntry) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Delete(ipListsPath+"/{type}/{ipornet}", deleteIPListEntry) }) s.router.Get(userTokenPath, s.getUserToken) @@ -1441,7 +1446,7 @@ func (s *httpdServer) setupWebClientRoutes() { s.jwtAuthenticatorPartial(tokenAudienceWebClientPartial)). Post(webClientTwoFactorRecoveryPath, s.handleWebClientTwoFactorRecoveryPost) } - // share routes exposed to external users + // share routes available to external users s.router.Get(webClientPubSharesPath+"/{id}", s.downloadFromShare) s.router.Get(webClientPubSharesPath+"/{id}/partial", s.handleClientSharePartialDownload) s.router.Get(webClientPubSharesPath+"/{id}/browse", s.handleShareGetFiles) @@ -1677,6 +1682,19 @@ func (s *httpdServer) setupWebAdminRoutes() { Get(webEventsFsSearchPath, searchFsEvents) router.With(s.checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler, s.refreshCookie). Get(webEventsProviderSearchPath, searchProviderEvents) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Get(webIPListsPath, s.handleWebIPListsPage) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists), compressor.Handler, s.refreshCookie). + Get(webIPListsPath+"/{type}", getIPListEntries) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Get(webIPListPath+"/{type}", + s.handleWebAddIPListEntryGet) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Post(webIPListPath+"/{type}", + s.handleWebAddIPListEntryPost) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Get(webIPListPath+"/{type}/{ipornet}", + s.handleWebUpdateIPListEntryGet) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists)).Post(webIPListPath+"/{type}/{ipornet}", + s.handleWebUpdateIPListEntryPost) + router.With(s.checkPerm(dataprovider.PermAdminManageIPLists), verifyCSRFHeader). + Delete(webIPListPath+"/{type}/{ipornet}", deleteIPListEntry) }) } } diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index 1a993373..d9a1833c 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -92,6 +92,8 @@ const ( templateStatus = "status.html" templateLogin = "login.html" templateDefender = "defender.html" + templateIPLists = "iplists.html" + templateIPList = "iplist.html" templateProfile = "profile.html" templateChangePwd = "changepassword.html" templateMaintenance = "maintenance.html" @@ -109,7 +111,8 @@ const ( pageProfileTitle = "My profile" pageChangePwdTitle = "Change password" pageMaintenanceTitle = "Maintenance" - pageDefenderTitle = "Defender" + pageDefenderTitle = "Auto Blocklist" + pageIPListsTitle = "IP Lists" pageEventsTitle = "Logs" pageForgotPwdTitle = "SFTPGo Admin - Forgot password" pageResetPwdTitle = "SFTPGo Admin - Reset password" @@ -138,6 +141,8 @@ type basePage struct { FolderURL string FolderTemplateURL string DefenderURL string + IPListsURL string + IPListURL string EventsURL string LogoutURL string ProfileURL string @@ -164,10 +169,12 @@ type basePage struct { StatusTitle string MaintenanceTitle string DefenderTitle string + IPListsTitle string EventsTitle string Version string CSRFToken string IsEventManagerPage bool + IsIPManagerPage bool HasDefender bool HasSearcher bool HasExternalLogin bool @@ -292,6 +299,21 @@ type defenderHostsPage struct { DefenderHostsURL string } +type ipListsPage struct { + basePage + IPListsSearchURL string + RateLimitersStatus bool + RateLimitersProtocols string + IsAllowListEnabled bool +} + +type ipListPage struct { + basePage + Entry *dataprovider.IPListEntry + Error string + Mode genericPageMode +} + type setupPage struct { basePage Username string @@ -479,6 +501,16 @@ func loadAdminTemplates(templatesPath string) { filepath.Join(templatesPath, templateAdminDir, templateBase), filepath.Join(templatesPath, templateAdminDir, templateDefender), } + ipListsPaths := []string{ + filepath.Join(templatesPath, templateCommonDir, templateCommonCSS), + filepath.Join(templatesPath, templateAdminDir, templateBase), + filepath.Join(templatesPath, templateAdminDir, templateIPLists), + } + ipListPaths := []string{ + filepath.Join(templatesPath, templateCommonDir, templateCommonCSS), + filepath.Join(templatesPath, templateAdminDir, templateBase), + filepath.Join(templatesPath, templateAdminDir, templateIPList), + } mfaPaths := []string{ filepath.Join(templatesPath, templateCommonDir, templateCommonCSS), filepath.Join(templatesPath, templateAdminDir, templateBase), @@ -552,6 +584,8 @@ func loadAdminTemplates(templatesPath string) { changePwdTmpl := util.LoadTemplate(nil, changePwdPaths...) maintenanceTmpl := util.LoadTemplate(nil, maintenancePaths...) defenderTmpl := util.LoadTemplate(nil, defenderPaths...) + ipListsTmpl := util.LoadTemplate(nil, ipListsPaths...) + ipListTmpl := util.LoadTemplate(nil, ipListPaths...) mfaTmpl := util.LoadTemplate(nil, mfaPaths...) twoFactorTmpl := util.LoadTemplate(nil, twoFactorPaths...) twoFactorRecoveryTmpl := util.LoadTemplate(nil, twoFactorRecoveryPaths...) @@ -582,6 +616,8 @@ func loadAdminTemplates(templatesPath string) { adminTemplates[templateChangePwd] = changePwdTmpl adminTemplates[templateMaintenance] = maintenanceTmpl adminTemplates[templateDefender] = defenderTmpl + adminTemplates[templateIPLists] = ipListsTmpl + adminTemplates[templateIPList] = ipListTmpl adminTemplates[templateMFA] = mfaTmpl adminTemplates[templateTwoFactor] = twoFactorTmpl adminTemplates[templateTwoFactorRecovery] = twoFactorRecoveryTmpl @@ -609,6 +645,19 @@ func isEventManagerResource(currentURL string) bool { return false } +func isIPListsResource(currentURL string) bool { + if currentURL == webDefenderPath { + return true + } + if currentURL == webIPListsPath { + return true + } + if strings.HasPrefix(currentURL, webIPListPath+"/") { + return true + } + return false +} + func (s *httpdServer) getBasePageData(title, currentURL string, r *http.Request) basePage { var csrfToken string if currentURL != "" { @@ -628,6 +677,8 @@ func (s *httpdServer) getBasePageData(title, currentURL string, r *http.Request) FolderURL: webFolderPath, FolderTemplateURL: webTemplateFolder, DefenderURL: webDefenderPath, + IPListsURL: webIPListsPath, + IPListURL: webIPListPath, EventsURL: webEventsPath, LogoutURL: webLogoutPath, ProfileURL: webAdminProfilePath, @@ -656,10 +707,12 @@ func (s *httpdServer) getBasePageData(title, currentURL string, r *http.Request) StatusTitle: pageStatusTitle, MaintenanceTitle: pageMaintenanceTitle, DefenderTitle: pageDefenderTitle, + IPListsTitle: pageIPListsTitle, EventsTitle: pageEventsTitle, Version: version.GetAsString(), LoggedAdmin: getAdminFromToken(r), IsEventManagerPage: isEventManagerResource(currentURL), + IsIPManagerPage: isIPListsResource(currentURL), HasDefender: common.Config.DefenderConfig.Enabled, HasSearcher: plugin.Handler.HasSearcher(), HasExternalLogin: isLoggedInWithOIDC(r), @@ -937,6 +990,27 @@ func (s *httpdServer) renderUserPage(w http.ResponseWriter, r *http.Request, use renderAdminTemplate(w, templateUser, data) } +func (s *httpdServer) renderIPListPage(w http.ResponseWriter, r *http.Request, entry dataprovider.IPListEntry, + mode genericPageMode, error string, +) { + var title, currentURL string + switch mode { + case genericPageModeAdd: + title = "Add a new IP List entry" + currentURL = fmt.Sprintf("%s/%d", webIPListPath, entry.Type) + case genericPageModeUpdate: + title = "Update IP List entry" + currentURL = fmt.Sprintf("%s/%d/%s", webIPListPath, entry.Type, url.PathEscape(entry.IPOrNet)) + } + data := ipListPage{ + basePage: s.getBasePageData(title, currentURL, r), + Error: error, + Entry: &entry, + Mode: mode, + } + renderAdminTemplate(w, templateIPList, data) +} + func (s *httpdServer) renderRolePage(w http.ResponseWriter, r *http.Request, role dataprovider.Role, mode genericPageMode, error string, ) { @@ -2378,6 +2452,36 @@ func getRoleFromPostFields(r *http.Request) (dataprovider.Role, error) { }, nil } +func getIPListEntryFromPostFields(r *http.Request, listType dataprovider.IPListType) (dataprovider.IPListEntry, error) { + err := r.ParseForm() + if err != nil { + return dataprovider.IPListEntry{}, err + } + var mode int + if listType == dataprovider.IPListTypeDefender { + mode, err = strconv.Atoi(r.Form.Get("mode")) + if err != nil { + return dataprovider.IPListEntry{}, fmt.Errorf("invalid mode: %w", err) + } + } else { + mode = 1 + } + protocols := 0 + for _, proto := range r.Form["protocols"] { + p, err := strconv.Atoi(proto) + if err == nil { + protocols += p + } + } + + return dataprovider.IPListEntry{ + IPOrNet: r.Form.Get("ipornet"), + Mode: mode, + Protocols: protocols, + Description: r.Form.Get("description"), + }, nil +} + func (s *httpdServer) handleWebAdminForgotPwd(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) if !smtp.IsEnabled() { @@ -3673,7 +3777,7 @@ func (s *httpdServer) handleWebUpdateRolePost(w http.ResponseWriter, r *http.Req updatedRole, err := getRoleFromPostFields(r) if err != nil { - s.renderRolePage(w, r, role, genericPageModeAdd, err.Error()) + s.renderRolePage(w, r, role, genericPageModeUpdate, err.Error()) return } ipAddr := util.GetIPFromRemoteAddress(r.RemoteAddr) @@ -3701,3 +3805,114 @@ func (s *httpdServer) handleWebGetEvents(w http.ResponseWriter, r *http.Request) } renderAdminTemplate(w, templateEvents, data) } + +func (s *httpdServer) handleWebIPListsPage(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + rtlStatus, rtlProtocols := common.Config.GetRateLimitersStatus() + data := ipListsPage{ + basePage: s.getBasePageData(pageIPListsTitle, webIPListsPath, r), + RateLimitersStatus: rtlStatus, + RateLimitersProtocols: strings.Join(rtlProtocols, ", "), + IsAllowListEnabled: common.Config.IsAllowListEnabled(), + } + + renderAdminTemplate(w, templateIPLists, data) +} + +func (s *httpdServer) handleWebAddIPListEntryGet(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + listType, _, err := getIPListPathParams(r) + if err != nil { + s.renderBadRequestPage(w, r, err) + return + } + s.renderIPListPage(w, r, dataprovider.IPListEntry{Type: listType}, genericPageModeAdd, "") +} + +func (s *httpdServer) handleWebAddIPListEntryPost(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + listType, _, err := getIPListPathParams(r) + if err != nil { + s.renderBadRequestPage(w, r, err) + return + } + entry, err := getIPListEntryFromPostFields(r, listType) + if err != nil { + s.renderIPListPage(w, r, entry, genericPageModeAdd, err.Error()) + return + } + entry.Type = listType + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + s.renderBadRequestPage(w, r, errors.New("invalid token claims")) + return + } + ipAddr := util.GetIPFromRemoteAddress(r.RemoteAddr) + if err := verifyCSRFToken(r.Form.Get(csrfFormToken), ipAddr); err != nil { + s.renderForbiddenPage(w, r, err.Error()) + return + } + err = dataprovider.AddIPListEntry(&entry, claims.Username, ipAddr, claims.Role) + if err != nil { + s.renderIPListPage(w, r, entry, genericPageModeAdd, err.Error()) + return + } + http.Redirect(w, r, webIPListsPath, http.StatusSeeOther) +} + +func (s *httpdServer) handleWebUpdateIPListEntryGet(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + listType, ipOrNet, err := getIPListPathParams(r) + if err != nil { + s.renderBadRequestPage(w, r, err) + return + } + entry, err := dataprovider.IPListEntryExists(ipOrNet, listType) + if err == nil { + s.renderIPListPage(w, r, entry, genericPageModeUpdate, "") + } else if errors.Is(err, util.ErrNotFound) { + s.renderNotFoundPage(w, r, err) + } else { + s.renderInternalServerErrorPage(w, r, err) + } +} + +func (s *httpdServer) handleWebUpdateIPListEntryPost(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + s.renderBadRequestPage(w, r, errors.New("invalid token claims")) + return + } + listType, ipOrNet, err := getIPListPathParams(r) + if err != nil { + s.renderBadRequestPage(w, r, err) + return + } + entry, err := dataprovider.IPListEntryExists(ipOrNet, listType) + if errors.Is(err, util.ErrNotFound) { + s.renderNotFoundPage(w, r, err) + return + } else if err != nil { + s.renderInternalServerErrorPage(w, r, err) + return + } + updatedEntry, err := getIPListEntryFromPostFields(r, listType) + if err != nil { + s.renderIPListPage(w, r, entry, genericPageModeUpdate, err.Error()) + return + } + ipAddr := util.GetIPFromRemoteAddress(r.RemoteAddr) + if err := verifyCSRFToken(r.Form.Get(csrfFormToken), ipAddr); err != nil { + s.renderForbiddenPage(w, r, err.Error()) + return + } + updatedEntry.Type = listType + updatedEntry.IPOrNet = ipOrNet + err = dataprovider.UpdateIPListEntry(&updatedEntry, claims.Username, ipAddr, claims.Role) + if err != nil { + s.renderIPListPage(w, r, entry, genericPageModeUpdate, err.Error()) + return + } + http.Redirect(w, r, webIPListsPath, http.StatusSeeOther) +} diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index f9b7ab70..d79ad09b 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -12,7 +12,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -// Package httpdtest provides utilities for testing the exposed REST API. +// Package httpdtest provides utilities for testing the supported REST API. package httpdtest import ( @@ -63,6 +63,7 @@ const ( eventActionsPath = "/api/v2/eventactions" eventRulesPath = "/api/v2/eventrules" rolesPath = "/api/v2/roles" + ipListsPath = "/api/v2/iplists" ) const ( @@ -478,6 +479,129 @@ func GetRoles(limit, offset int64, expectedStatusCode int) ([]dataprovider.Role, return roles, body, err } +// AddIPListEntry adds a new IP list entry and checks the received HTTP Status code against expectedStatusCode. +func AddIPListEntry(entry dataprovider.IPListEntry, expectedStatusCode int) (dataprovider.IPListEntry, []byte, error) { + var newEntry dataprovider.IPListEntry + var body []byte + + asJSON, _ := json.Marshal(entry) + resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(ipListsPath, strconv.Itoa(int(entry.Type))), + bytes.NewBuffer(asJSON), "application/json", getDefaultToken()) + if err != nil { + return newEntry, body, err + } + defer resp.Body.Close() + + err = checkResponse(resp.StatusCode, expectedStatusCode) + if expectedStatusCode != http.StatusCreated { + body, _ = getResponseBody(resp) + return newEntry, body, err + } + if err == nil { + newEntry, body, err = GetIPListEntry(entry.IPOrNet, entry.Type, http.StatusOK) + } + if err == nil { + err = checkIPListEntry(entry, newEntry) + } + return newEntry, body, err +} + +// UpdateIPListEntry updates an existing IP list entry and checks the received HTTP Status code against expectedStatusCode +func UpdateIPListEntry(entry dataprovider.IPListEntry, expectedStatusCode int) (dataprovider.IPListEntry, []byte, error) { + var newEntry dataprovider.IPListEntry + var body []byte + + asJSON, _ := json.Marshal(entry) + resp, err := sendHTTPRequest(http.MethodPut, buildURLRelativeToBase(ipListsPath, fmt.Sprintf("%d", entry.Type), + url.PathEscape(entry.IPOrNet)), bytes.NewBuffer(asJSON), + "application/json", getDefaultToken()) + if err != nil { + return newEntry, body, err + } + defer resp.Body.Close() + + body, _ = getResponseBody(resp) + err = checkResponse(resp.StatusCode, expectedStatusCode) + if expectedStatusCode != http.StatusOK { + return newEntry, body, err + } + if err == nil { + newEntry, body, err = GetIPListEntry(entry.IPOrNet, entry.Type, http.StatusOK) + } + if err == nil { + err = checkIPListEntry(entry, newEntry) + } + return newEntry, body, err +} + +// RemoveIPListEntry removes an existing IP list entry and checks the received HTTP Status code against expectedStatusCode. +func RemoveIPListEntry(entry dataprovider.IPListEntry, expectedStatusCode int) ([]byte, error) { + var body []byte + resp, err := sendHTTPRequest(http.MethodDelete, buildURLRelativeToBase(ipListsPath, fmt.Sprintf("%d", entry.Type), + url.PathEscape(entry.IPOrNet)), nil, "", getDefaultToken()) + if err != nil { + return body, err + } + defer resp.Body.Close() + body, _ = getResponseBody(resp) + return body, checkResponse(resp.StatusCode, expectedStatusCode) +} + +// GetIPListEntry returns an IP list entry matching the specified parameters, if exists, +// and checks the received HTTP Status code against expectedStatusCode. +func GetIPListEntry(ipOrNet string, listType dataprovider.IPListType, expectedStatusCode int, +) (dataprovider.IPListEntry, []byte, error) { + var entry dataprovider.IPListEntry + var body []byte + resp, err := sendHTTPRequest(http.MethodGet, buildURLRelativeToBase(ipListsPath, fmt.Sprintf("%d", listType), url.PathEscape(ipOrNet)), + nil, "", getDefaultToken()) + if err != nil { + return entry, body, err + } + defer resp.Body.Close() + err = checkResponse(resp.StatusCode, expectedStatusCode) + if err == nil && expectedStatusCode == http.StatusOK { + err = render.DecodeJSON(resp.Body, &entry) + } else { + body, _ = getResponseBody(resp) + } + return entry, body, err +} + +// GetIPListEntries returns a list of IP list entries and checks the received HTTP Status code against expectedStatusCode. +func GetIPListEntries(listType dataprovider.IPListType, filter, from, order string, limit int64, + expectedStatusCode int, +) ([]dataprovider.IPListEntry, []byte, error) { + var entries []dataprovider.IPListEntry + var body []byte + + url, err := url.Parse(buildURLRelativeToBase(ipListsPath, strconv.Itoa(int(listType)))) + if err != nil { + return entries, body, err + } + q := url.Query() + q.Add("filter", filter) + q.Add("from", from) + q.Add("order", order) + if limit > 0 { + q.Add("limit", strconv.FormatInt(limit, 10)) + } + url.RawQuery = q.Encode() + resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "", getDefaultToken()) + if err != nil { + return entries, body, err + } + defer resp.Body.Close() + + err = checkResponse(resp.StatusCode, expectedStatusCode) + if err == nil && expectedStatusCode == http.StatusOK { + err = render.DecodeJSON(resp.Body, &entries) + } else { + body, _ = getResponseBody(resp) + } + return entries, body, err +} + // AddAdmin adds a new admin and checks the received HTTP Status code against expectedStatusCode. func AddAdmin(admin dataprovider.Admin, expectedStatusCode int) (dataprovider.Admin, []byte, error) { var newAdmin dataprovider.Admin @@ -1641,6 +1765,31 @@ func checkEventRule(expected, actual dataprovider.EventRule) error { return checkEventRuleActions(expected.Actions, actual.Actions) } +func checkIPListEntry(expected, actual dataprovider.IPListEntry) error { + if expected.IPOrNet != actual.IPOrNet { + return errors.New("ipornet mismatch") + } + if expected.Description != actual.Description { + return errors.New("description mismatch") + } + if expected.Type != actual.Type { + return errors.New("type mismatch") + } + if expected.Mode != actual.Mode { + return errors.New("mode mismatch") + } + if expected.Protocols != actual.Protocols { + return errors.New("protocols mismatch") + } + if actual.CreatedAt == 0 { + return errors.New("created_at unset") + } + if actual.UpdatedAt == 0 { + return errors.New("updated_at unset") + } + return nil +} + func checkRole(expected, actual dataprovider.Role) error { if expected.ID <= 0 { if actual.ID <= 0 { diff --git a/internal/metric/metric.go b/internal/metric/metric.go index de1c990c..534e9840 100644 --- a/internal/metric/metric.go +++ b/internal/metric/metric.go @@ -640,7 +640,7 @@ var ( }) ) -// AddMetricsEndpoint exposes metrics to the specified endpoint +// AddMetricsEndpoint publishes metrics to the specified endpoint func AddMetricsEndpoint(metricsPath string, handler chi.Router) { handler.Handle(metricsPath, promhttp.Handler()) } diff --git a/internal/metric/metric_disabled.go b/internal/metric/metric_disabled.go index 854cd0eb..507897be 100644 --- a/internal/metric/metric_disabled.go +++ b/internal/metric/metric_disabled.go @@ -13,7 +13,7 @@ func init() { version.AddFeature("-metrics") } -// AddMetricsEndpoint exposes metrics to the specified endpoint +// AddMetricsEndpoint publishes metrics to the specified endpoint func AddMetricsEndpoint(_ string, _ chi.Router) {} // TransferCompleted updates metrics after an upload or a download diff --git a/internal/service/service.go b/internal/service/service.go index b1786853..16588821 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -121,14 +121,8 @@ func (s *Service) Start(disableAWSInstallationCode bool) error { func (s *Service) initializeServices(disableAWSInstallationCode bool) error { providerConf := config.GetProviderConf() - err := common.Initialize(config.GetCommonConfig(), providerConf.GetShared()) - if err != nil { - logger.Error(logSender, "", "%v", err) - logger.ErrorToConsole("%v", err) - return err - } kmsConfig := config.GetKMSConfig() - err = kmsConfig.Initialize() + err := kmsConfig.Initialize() if err != nil { logger.Error(logSender, "", "unable to initialize KMS: %v", err) logger.ErrorToConsole("unable to initialize KMS: %v", err) @@ -159,6 +153,12 @@ func (s *Service) initializeServices(disableAWSInstallationCode bool) error { logger.ErrorToConsole("error initializing data provider: %v", err) return err } + err = common.Initialize(config.GetCommonConfig(), providerConf.GetShared()) + if err != nil { + logger.Error(logSender, "", "%v", err) + logger.ErrorToConsole("%v", err) + return err + } if s.PortableMode == 1 { // create the user for portable mode @@ -319,7 +319,7 @@ func (s *Service) LoadInitialData() error { return fmt.Errorf("unable to stat file %#v: %w", s.LoadDataFrom, err) } if info.Size() > httpd.MaxRestoreSize { - return fmt.Errorf("unable to restore input file %#v size too big: %v/%v bytes", + return fmt.Errorf("unable to restore input file %q size too big: %d/%d bytes", s.LoadDataFrom, info.Size(), httpd.MaxRestoreSize) } content, err := os.ReadFile(s.LoadDataFrom) @@ -350,42 +350,46 @@ func (s *Service) LoadInitialData() error { } func (s *Service) restoreDump(dump *dataprovider.BackupData) error { - err := httpd.RestoreRoles(dump.Roles, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") + err := httpd.RestoreIPListEntries(dump.IPLists, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore roles from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore IP list entries from file %q: %v", s.LoadDataFrom, err) + } + err = httpd.RestoreRoles(dump.Roles, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") + if err != nil { + return fmt.Errorf("unable to restore roles from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreFolders(dump.Folders, s.LoadDataFrom, s.LoadDataMode, s.LoadDataQuotaScan, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore folders from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore folders from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreGroups(dump.Groups, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore groups from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore groups from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreUsers(dump.Users, s.LoadDataFrom, s.LoadDataMode, s.LoadDataQuotaScan, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore users from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore users from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreAdmins(dump.Admins, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore admins from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore admins from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreAPIKeys(dump.APIKeys, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore API keys from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore API keys from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreShares(dump.Shares, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore API keys from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore API keys from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreEventActions(dump.EventActions, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "") if err != nil { - return fmt.Errorf("unable to restore event actions from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore event actions from file %q: %v", s.LoadDataFrom, err) } err = httpd.RestoreEventRules(dump.EventRules, s.LoadDataFrom, s.LoadDataMode, dataprovider.ActionExecutorSystem, "", "", dump.Version) if err != nil { - return fmt.Errorf("unable to restore event rules from file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to restore event rules from file %q: %v", s.LoadDataFrom, err) } return nil } diff --git a/internal/sftpd/handler.go b/internal/sftpd/handler.go index 4d915cbc..af8659be 100644 --- a/internal/sftpd/handler.go +++ b/internal/sftpd/handler.go @@ -84,7 +84,7 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { } if ok, policy := c.User.IsFileAllowed(request.Filepath); !ok { - c.Log(logger.LevelWarn, "reading file %#v is not allowed", request.Filepath) + c.Log(logger.LevelWarn, "reading file %q is not allowed", request.Filepath) return nil, c.GetErrorForDeniedFile(policy) } @@ -94,13 +94,13 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { } if _, err := common.ExecutePreAction(c.BaseConnection, common.OperationPreDownload, p, request.Filepath, 0, 0); err != nil { - c.Log(logger.LevelDebug, "download for file %#v denied by pre action: %v", request.Filepath, err) + c.Log(logger.LevelDebug, "download for file %q denied by pre action: %v", request.Filepath, err) return nil, c.GetPermissionDeniedError() } file, r, cancelFn, err := fs.Open(p, 0) if err != nil { - c.Log(logger.LevelError, "could not open file %#v for reading: %+v", p, err) + c.Log(logger.LevelError, "could not open file %q for reading: %+v", p, err) return nil, c.GetFsError(fs, err) } @@ -125,7 +125,7 @@ func (c *Connection) handleFilewrite(request *sftp.Request) (sftp.WriterAtReader c.UpdateLastActivity() if ok, _ := c.User.IsFileAllowed(request.Filepath); !ok { - c.Log(logger.LevelWarn, "writing file %#v is not allowed", request.Filepath) + c.Log(logger.LevelWarn, "writing file %q is not allowed", request.Filepath) return nil, c.GetPermissionDeniedError() } @@ -160,13 +160,13 @@ func (c *Connection) handleFilewrite(request *sftp.Request) (sftp.WriterAtReader } if statErr != nil { - c.Log(logger.LevelError, "error performing file stat %#v: %+v", p, statErr) + c.Log(logger.LevelError, "error performing file stat %q: %+v", p, statErr) return nil, c.GetFsError(fs, statErr) } // This happen if we upload a file that has the same name of an existing directory if stat.IsDir() { - c.Log(logger.LevelError, "attempted to open a directory for writing to: %#v", p) + c.Log(logger.LevelError, "attempted to open a directory for writing to: %q", p) return nil, sftp.ErrSSHFxOpUnsupported } @@ -255,7 +255,7 @@ func (c *Connection) Readlink(filePath string) (string, error) { s, err := fs.Readlink(p) if err != nil { - c.Log(logger.LevelDebug, "error running readlink on path %#v: %+v", p, err) + c.Log(logger.LevelDebug, "error running readlink on path %q: %+v", p, err) return "", c.GetFsError(fs, err) } @@ -383,11 +383,11 @@ func (c *Connection) handleSFTPRemove(request *sftp.Request) error { var fi os.FileInfo if fi, err = fs.Lstat(fsPath); err != nil { - c.Log(logger.LevelDebug, "failed to remove file %#v: stat error: %+v", fsPath, err) + c.Log(logger.LevelDebug, "failed to remove file %q: stat error: %+v", fsPath, err) return c.GetFsError(fs, err) } if fi.IsDir() && fi.Mode()&os.ModeSymlink == 0 { - c.Log(logger.LevelDebug, "cannot remove %#v is not a file/symlink", fsPath) + c.Log(logger.LevelDebug, "cannot remove %q is not a file/symlink", fsPath) return sftp.ErrSSHFxFailure } @@ -402,14 +402,14 @@ func (c *Connection) handleSFTPUploadToNewFile(fs vfs.Fs, pflags sftp.FileOpenFl } if _, err := common.ExecutePreAction(c.BaseConnection, common.OperationPreUpload, resolvedPath, requestPath, 0, 0); err != nil { - c.Log(logger.LevelDebug, "upload for file %#v denied by pre action: %v", requestPath, err) + c.Log(logger.LevelDebug, "upload for file %q denied by pre action: %v", requestPath, err) return nil, c.GetPermissionDeniedError() } osFlags := getOSOpenFlags(pflags) file, w, cancelFn, err := fs.Create(filePath, osFlags) if err != nil { - c.Log(logger.LevelError, "error creating file %#vm os flags %v, pflags %+v: %+v", resolvedPath, osFlags, pflags, err) + c.Log(logger.LevelError, "error creating file %q, os flags %d, pflags %+v: %+v", resolvedPath, osFlags, pflags, err) return nil, c.GetFsError(fs, err) } @@ -450,14 +450,14 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO } if _, err := common.ExecutePreAction(c.BaseConnection, common.OperationPreUpload, resolvedPath, requestPath, fileSize, osFlags); err != nil { - c.Log(logger.LevelDebug, "upload for file %#v denied by pre action: %v", requestPath, err) + c.Log(logger.LevelDebug, "upload for file %q denied by pre action: %v", requestPath, err) return nil, c.GetPermissionDeniedError() } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { _, _, err = fs.Rename(resolvedPath, filePath) if err != nil { - c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", + c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %+v", resolvedPath, filePath, err) return nil, c.GetFsError(fs, err) } @@ -465,7 +465,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO file, w, cancelFn, err := fs.Create(filePath, osFlags) if err != nil { - c.Log(logger.LevelError, "error opening existing file, os flags %v, pflags: %+v, source: %#v, err: %+v", + c.Log(logger.LevelError, "error opening existing file, os flags %v, pflags: %+v, source: %q, err: %+v", osFlags, pflags, filePath, err) return nil, c.GetFsError(fs, err) } @@ -473,7 +473,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO initialSize := int64(0) truncatedSize := int64(0) // bytes truncated and not included in quota if isResume { - c.Log(logger.LevelDebug, "resuming upload requested, file path %#v initial size: %v has append flag %v", + c.Log(logger.LevelDebug, "resuming upload requested, file path %q initial size: %d, has append flag %t", filePath, fileSize, pflags.Append) // enforce min write offset only if the client passed the APPEND flag if pflags.Append { diff --git a/internal/sftpd/server.go b/internal/sftpd/server.go index ca150bc8..49e35b1c 100644 --- a/internal/sftpd/server.go +++ b/internal/sftpd/server.go @@ -505,11 +505,11 @@ func (c *Configuration) configureKeyboardInteractiveAuth(serverConfig *ssh.Serve } func canAcceptConnection(ip string) bool { - if common.IsBanned(ip) { + if common.IsBanned(ip, common.ProtocolSSH) { logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, ip %#v is banned", ip) return false } - if err := common.Connections.IsNewConnectionAllowed(ip); err != nil { + if err := common.Connections.IsNewConnectionAllowed(ip, common.ProtocolSSH); err != nil { logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection not allowed from ip %q: %v", ip, err) return false } @@ -700,7 +700,7 @@ func checkAuthError(ip string, err error) { if errors.Is(err, util.ErrNotFound) { event = common.HostEventUserNotFound } - common.AddDefenderEvent(ip, event) + common.AddDefenderEvent(ip, common.ProtocolSSH, event) return } } @@ -708,7 +708,7 @@ func checkAuthError(ip string, err error) { } else { logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, common.ProtocolSSH, err.Error()) metric.AddNoAuthTryed() - common.AddDefenderEvent(ip, common.HostEventNoLoginTried) + common.AddDefenderEvent(ip, common.ProtocolSSH, common.HostEventNoLoginTried) dataprovider.ExecutePostLoginHook(&dataprovider.User{}, dataprovider.LoginMethodNoAuthTryed, ip, common.ProtocolSSH, err) } } @@ -1159,7 +1159,7 @@ func updateLoginMetrics(user *dataprovider.User, ip, method string, err error) { if errors.Is(err, util.ErrNotFound) { event = common.HostEventUserNotFound } - common.AddDefenderEvent(ip, event) + common.AddDefenderEvent(ip, common.ProtocolSSH, event) } } metric.AddLoginResult(method, err) diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go index 625d1d37..63737943 100644 --- a/internal/sftpd/sftpd_test.go +++ b/internal/sftpd/sftpd_test.go @@ -213,18 +213,18 @@ func TestMain(m *testing.M) { scriptArgs = "$@" } - err = common.Initialize(commonConf, 0) - if err != nil { - logger.WarnToConsole("error initializing common: %v", err) - os.Exit(1) - } - err = dataprovider.Initialize(providerConf, configDir, true) if err != nil { logger.ErrorToConsole("error initializing data provider: %v", err) os.Exit(1) } + err = common.Initialize(commonConf, 0) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } + httpConfig := config.GetHTTPConfig() httpConfig.Initialize(configDir) //nolint:errcheck kmsConfig := config.GetKMSConfig() @@ -3066,7 +3066,7 @@ func TestPreLoginUserCreation(t *testing.T) { err = dataprovider.Initialize(providerConf, configDir, true) assert.NoError(t, err) - user, _, err := httpdtest.GetUserByUsername(defaultUsername, http.StatusNotFound) + _, _, err = httpdtest.GetUserByUsername(defaultUsername, http.StatusNotFound) assert.NoError(t, err) conn, client, err := getSftpClient(u, usePubKey) if assert.NoError(t, err) { @@ -3074,7 +3074,7 @@ func TestPreLoginUserCreation(t *testing.T) { defer client.Close() assert.NoError(t, checkBasicSFTP(client)) } - user, _, err = httpdtest.GetUserByUsername(defaultUsername, http.StatusOK) + user, _, err := httpdtest.GetUserByUsername(defaultUsername, http.StatusOK) assert.NoError(t, err) _, err = httpdtest.RemoveUser(user, http.StatusOK) assert.NoError(t, err) diff --git a/internal/smtp/smtp.go b/internal/smtp/smtp.go index d19b5e72..25d4db6a 100644 --- a/internal/smtp/smtp.go +++ b/internal/smtp/smtp.go @@ -120,7 +120,7 @@ func (c *Config) Initialize(configDir string) error { } func (c *Config) getMailClientOptions() []mail.Option { - options := []mail.Option{mail.WithPort(c.Port)} + options := []mail.Option{mail.WithPort(c.Port), mail.WithoutNoop()} switch c.Encryption { case 1: diff --git a/internal/util/util.go b/internal/util/util.go index a00ba7df..e20969a9 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -32,6 +32,7 @@ import ( "math" "net" "net/http" + "net/netip" "net/url" "os" "path" @@ -775,3 +776,26 @@ func GetAbsolutePath(name string) (string, error) { } return filepath.Join(curDir, name), nil } + +// GetLastIPForPrefix returns the last IP for the given prefix +// https://github.com/go4org/netipx/blob/8449b0a6169f5140fb0340cb4fc0de4c9b281ef6/netipx.go#L173 +func GetLastIPForPrefix(p netip.Prefix) netip.Addr { + if !p.IsValid() { + return netip.Addr{} + } + a16 := p.Addr().As16() + var off uint8 + var bits uint8 = 128 + if p.Addr().Is4() { + off = 12 + bits = 32 + } + for b := uint8(p.Bits()); b < bits; b++ { + byteNum, bitInByte := b/8, 7-(b%8) + a16[off+byteNum] |= 1 << uint(bitInByte) + } + if p.Addr().Is4() { + return netip.AddrFrom16(a16).Unmap() + } + return netip.AddrFrom16(a16) // doesn't unmap +} diff --git a/internal/version/version.go b/internal/version/version.go index 2b5972d7..4073f319 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -17,7 +17,7 @@ package version import "strings" -const version = "2.4.3-dev" +const version = "2.4.4-dev" var ( commit = "" diff --git a/internal/vfs/folder.go b/internal/vfs/folder.go index 6fbc37d3..4d131a6f 100644 --- a/internal/vfs/folder.go +++ b/internal/vfs/folder.go @@ -176,7 +176,7 @@ func (v *BaseVirtualFolder) hasPathPlaceholder() bool { return false } -// VirtualFolder defines a mapping between an SFTPGo exposed virtual path and a +// VirtualFolder defines a mapping between an SFTPGo virtual path and a // filesystem path outside the user home directory. // The specified paths must be absolute and the virtual path cannot be "/", // it must be a sub directory. The parent directory for the specified virtual diff --git a/internal/webdavd/server.go b/internal/webdavd/server.go index e43664b8..a782ffc6 100644 --- a/internal/webdavd/server.go +++ b/internal/webdavd/server.go @@ -165,12 +165,12 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { common.Connections.AddClientConnection(ipAddr) defer common.Connections.RemoveClientConnection(ipAddr) - if err := common.Connections.IsNewConnectionAllowed(ipAddr); err != nil { + if err := common.Connections.IsNewConnectionAllowed(ipAddr, common.ProtocolWebDAV); err != nil { logger.Log(logger.LevelDebug, common.ProtocolWebDAV, "", "connection not allowed from ip %q: %v", ipAddr, err) http.Error(w, err.Error(), http.StatusServiceUnavailable) return } - if common.IsBanned(ipAddr) { + if common.IsBanned(ipAddr, common.ProtocolWebDAV) { http.Error(w, common.ErrConnectionDenied.Error(), http.StatusForbidden) return } @@ -413,7 +413,7 @@ func updateLoginMetrics(user *dataprovider.User, ip, loginMethod string, err err if errors.Is(err, util.ErrNotFound) { event = common.HostEventUserNotFound } - common.AddDefenderEvent(ip, event) + common.AddDefenderEvent(ip, common.ProtocolWebDAV, event) } metric.AddLoginResult(loginMethod, err) dataprovider.ExecutePostLoginHook(user, loginMethod, ip, common.ProtocolWebDAV, err) diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go index c1d3be3f..e5bb5d48 100644 --- a/internal/webdavd/webdavd_test.go +++ b/internal/webdavd/webdavd_test.go @@ -319,18 +319,18 @@ func TestMain(m *testing.M) { os.Exit(1) } - err = common.Initialize(commonConf, 0) - if err != nil { - logger.WarnToConsole("error initializing common: %v", err) - os.Exit(1) - } - err = dataprovider.Initialize(providerConf, configDir, true) if err != nil { logger.ErrorToConsole("error initializing data provider: %v", err) os.Exit(1) } + err = common.Initialize(commonConf, 0) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } + httpConfig := config.GetHTTPConfig() httpConfig.Initialize(configDir) //nolint:errcheck kmsConfig := config.GetKMSConfig() diff --git a/main.go b/main.go index 67821d3c..a372b2ab 100644 --- a/main.go +++ b/main.go @@ -21,8 +21,6 @@ package main // import "github.com/drakkan/sftpgo" import ( "fmt" - "math/rand" - "time" "go.uber.org/automaxprocs/maxprocs" @@ -34,6 +32,5 @@ func main() { fmt.Printf("error setting max procs: %v\n", err) undo() } - rand.Seed(time.Now().UnixNano()) cmd.Execute() } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index bb728be2..608c636a 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -6,6 +6,7 @@ tags: - name: admins - name: API keys - name: connections + - name: IP Lists - name: defender - name: quota - name: folders @@ -23,12 +24,12 @@ info: description: | SFTPGo allows you to securely share your files over SFTP and optionally over HTTP/S, FTP/S and WebDAV as well. Several storage backends are supported and they are configurable per-user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. - SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a Google Cloud Storage bucket (or part of it) on a specified path and an encrypted local filesystem on another one. + SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, a user with the S3 backend mapping a Google Cloud Storage bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. SFTPGo supports groups to simplify the administration of multiple accounts by letting you assign settings once to a group, instead of multiple times to each individual user. The SFTPGo WebClient allows end users to change their credentials, browse and manage their files in the browser and setup two-factor authentication which works with Authy, Google Authenticator and other compatible apps. From the WebClient each authorized user can also create HTTP/S links to externally share files and folders securely, by setting limits to the number of downloads/uploads, protecting the share with a password, limiting access by source IP address, setting an automatic expiration date. - version: 2.4.3-dev + version: 2.4.4-dev contact: name: API support url: 'https://github.com/drakkan/sftpgo' @@ -783,6 +784,204 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' + /iplists/{type}: + parameters: + - name: type + in: path + description: IP list type + required: true + schema: + $ref: '#/components/schemas/IPListType' + get: + tags: + - IP Lists + summary: Get IP list entries + description: Returns an array with one or more IP list entry + operationId: get_ip_list_entries + parameters: + - in: query + name: filter + schema: + type: string + description: restrict results to ipornet matching or starting with this filter + - in: query + name: from + schema: + type: string + description: ipornet to start from + required: false + - in: query + name: limit + schema: + type: integer + minimum: 1 + maximum: 500 + default: 100 + required: false + description: 'The maximum number of items to return. Max value is 500, default is 100' + - in: query + name: order + required: false + description: Ordering entries by ipornet field. Default ASC + schema: + type: string + enum: + - ASC + - DESC + example: ASC + responses: + '200': + description: successful operation + content: + application/json; charset=utf-8: + schema: + type: array + items: + $ref: '#/components/schemas/IPListEntry' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + post: + tags: + - IP Lists + summary: Add a new IP list entry + description: Add an IP address or a CIDR network to a supported list + operationId: add_ip_list_entry + requestBody: + required: true + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/IPListEntry' + responses: + '201': + description: successful operation + headers: + Location: + schema: + type: string + description: 'URI of the newly created object' + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/ApiResponse' + example: + message: Entry added + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + /iplists/{type}/{ipornet}: + parameters: + - name: type + in: path + description: IP list type + required: true + schema: + $ref: '#/components/schemas/IPListType' + - name: ipornet + in: path + required: true + schema: + type: string + get: + tags: + - IP Lists + summary: Find entry by ipornet + description: Returns the entry with the given ipornet if it exists. + operationId: get_ip_list_by_ipornet + responses: + '200': + description: successful operation + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/IPListEntry' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + put: + tags: + - IP Lists + summary: Update IP list entry + description: Updates an existing IP list entry + operationId: update_ip_list_entry + requestBody: + required: true + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/IPListEntry' + responses: + '200': + description: successful operation + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/ApiResponse' + example: + message: Entry updated + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + delete: + tags: + - IP Lists + summary: Delete IP list entry + description: Deletes an existing IP list entry + operationId: delete_ip_list_entry + responses: + '200': + description: successful operation + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/ApiResponse' + example: + message: Entry deleted + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' /defender/hosts: get: tags: @@ -4616,7 +4815,8 @@ components: - metadata_checks - view_events - manage_event_rules - - manager_roles + - manage_roles + - manage_ip_lists description: | Admin permissions: * `*` - all permissions are granted @@ -4638,7 +4838,8 @@ components: * `metadata_checks` - view and start metadata checks is allowed * `view_events` - view and search filesystem and provider events is allowed * `manage_event_rules` - manage event actions and rules is allowed - * `manager_roles` - manage roles is allowed + * `manage_roles` - manage roles is allowed + * `manage_ip_lists` - manage global and ratelimter allow lists and defender block and safe lists is allowed FsProviders: type: integer enum: @@ -4903,6 +5104,26 @@ components: TLS version: * `12` - TLS 1.2 * `13` - TLS 1.3 + IPListType: + type: integer + enum: + - 1 + - 2 + - 3 + description: > + IP List types: + * `1` - allow list + * `2` - defender + * `3` - rate limiter safe list + IPListMode: + type: integer + enum: + - 1 + - 2 + description: > + IP list modes + * `1` - allow + * `2` - deny, supported for defender list type only TOTPConfig: type: object properties: @@ -4948,7 +5169,7 @@ components: properties: path: type: string - description: 'exposed virtual path, if no other specific filter is defined, the filter applies for sub directories too. For example if filters are defined for the paths "/" and "/sub" then the filters for "/" are applied for any file outside the "/sub" directory' + description: 'virtual path as seen by users, if no other specific filter is defined, the filter applies for sub directories too. For example if filters are defined for the paths "/" and "/sub" then the filters for "/" are applied for any file outside the "/sub" directory' allowed_patterns: type: array items: @@ -5665,7 +5886,7 @@ components: description: Last user login as unix timestamp in milliseconds. It is saved at most once every 10 minutes role: type: string - description: 'If set the admin can only administer users with the same role. Role admins cannot have the following permissions: "manage_admins", "manage_apikeys", "manage_system", "manage_event_rules", "manage_roles"' + description: 'If set the admin can only administer users with the same role. Role admins cannot have the following permissions: "manage_admins", "manage_apikeys", "manage_system", "manage_event_rules", "manage_roles", "manage_ip_lists"' AdminProfile: type: object properties: @@ -5823,7 +6044,7 @@ components: properties: path: type: string - description: 'exposed virtual directory path, if no other specific retention is defined, the retention applies for sub directories too. For example if retention is defined for the paths "/" and "/sub" then the retention for "/" is applied for any file outside the "/sub" directory' + description: 'virtual directory path as seen by users, if no other specific retention is defined, the retention applies for sub directories too. For example if retention is defined for the paths "/" and "/sub" then the retention for "/" is applied for any file outside the "/sub" directory' example: '/' retention: type: integer @@ -5985,7 +6206,7 @@ components: $ref: '#/components/schemas/TLSVersions' force_passive_ip: type: string - description: External IP address to expose for passive connections + description: External IP address for passive connections passive_ip_overrides: type: array items: @@ -6107,6 +6328,21 @@ components: type: boolean mfa: $ref: '#/components/schemas/MFAStatus' + allow_list: + type: object + properties: + is_active: + type: boolean + rate_limiters: + type: object + properties: + is_active: + type: boolean + protocols: + type: array + items: + type: string + example: SSH Share: type: object properties: @@ -6827,6 +7063,30 @@ components: type: array items: $ref: '#/components/schemas/EventActionMinimal' + IPListEntry: + type: object + properties: + ipornet: + type: string + description: IP address or network in CIDR format, for example `192.168.1.2/32`, `192.168.0.0/24`, `2001:db8::/32` + description: + type: string + description: optional description + type: + $ref: '#/components/schemas/IPListType' + mode: + $ref: '#/components/schemas/IPListMode' + protocols: + type: integer + description: Defines the protocol the entry applies to. `0` means all the supported protocols, 1 SSH, 2 FTP, 4 WebDAV, 8 HTTP. Protocols can be combined, for example 3 means SSH and FTP + created_at: + type: integer + format: int64 + description: creation time as unix timestamp in milliseconds + updated_at: + type: integer + format: int64 + description: last update time as unix timestamp in millisecond ApiResponse: type: object properties: diff --git a/pkgs/build.sh b/pkgs/build.sh index 56280e2c..50e70001 100755 --- a/pkgs/build.sh +++ b/pkgs/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -NFPM_VERSION=2.24.0 +NFPM_VERSION=2.25.0 NFPM_ARCH=${NFPM_ARCH:-amd64} if [ -z ${SFTPGO_VERSION} ] then diff --git a/pkgs/choco/sftpgo.nuspec b/pkgs/choco/sftpgo.nuspec index b03a02d3..1e78f7c8 100644 --- a/pkgs/choco/sftpgo.nuspec +++ b/pkgs/choco/sftpgo.nuspec @@ -3,24 +3,24 @@ sftpgo - 2.4.3 + 2.4.4 https://github.com/drakkan/sftpgo/tree/main/pkgs/choco asheroto SFTPGo Nicola Murino https://github.com/drakkan/sftpgo - https://cdn.statically.io/gh/drakkan/sftpgo/v2.4.3/static/img/logo.png + https://cdn.statically.io/gh/drakkan/sftpgo/v2.4.4/static/img/logo.png https://github.com/drakkan/sftpgo/blob/main/LICENSE false https://github.com/drakkan/sftpgo - https://github.com/drakkan/sftpgo/tree/v2.4.3/docs + https://github.com/drakkan/sftpgo/tree/v2.4.4/docs https://github.com/drakkan/sftpgo/issues sftp sftp-server ftp webdav s3 azure-blob google-cloud-storage cloud-storage scp data-at-rest-encryption multi-factor-authentication multi-step-authentication Fully featured and highly configurable SFTP server with optional HTTP/S,FTP/S and WebDAV support. SFTPGo allows you to securely share your files over SFTP and optionally over HTTP/S, FTP/S and WebDAV as well. Several storage backends are supported and they are configurable per-user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. -SFTPGo also supports virtual folders. A virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. +SFTPGo also supports virtual folders. A virtual folder can use any of the supported storage backends. So you can have, for example, a user with the S3 backend mapping a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. SFTPGo allows to create HTTP/S links to externally share files and folders securely, by setting limits to the number of downloads/uploads, protecting the share with a password, limiting access by source IP address, setting an automatic expiration date. @@ -32,7 +32,7 @@ You can find more info [here](https://github.com/drakkan/sftpgo). * This package installs SFTPGo as Windows Service. * After the first installation please take a look at the [Getting Started Guide](https://github.com/drakkan/sftpgo/blob/main/docs/howto/getting-started.md). - https://github.com/drakkan/sftpgo/releases/tag/v2.4.3 + https://github.com/drakkan/sftpgo/releases/tag/v2.4.4 diff --git a/pkgs/choco/tools/ChocolateyInstall.ps1 b/pkgs/choco/tools/ChocolateyInstall.ps1 index 06499a6a..bd901ddc 100644 --- a/pkgs/choco/tools/ChocolateyInstall.ps1 +++ b/pkgs/choco/tools/ChocolateyInstall.ps1 @@ -1,8 +1,8 @@ $ErrorActionPreference = 'Stop' $packageName = 'sftpgo' $softwareName = 'SFTPGo' -$url = 'https://github.com/drakkan/sftpgo/releases/download/v2.4.3/sftpgo_v2.4.3_windows_x86_64.exe' -$checksum = '7DF2DBC5EEBC859E4DE1832D1124F9872C1394A1C9D00EBA44F89E5EDABFFE4F' +$url = 'https://github.com/drakkan/sftpgo/releases/download/v2.4.4/sftpgo_v2.4.4_windows_x86_64.exe' +$checksum = 'E43B7097B2099ACE95D336694DAEEF65646D4078FC045011DD3C6A2F07A30B46' $silentArgs = '/VERYSILENT' $validExitCodes = @(0) @@ -47,8 +47,8 @@ Write-Output "" Write-Output "General information (README) location:" Write-Output "`thttps://github.com/drakkan/sftpgo" Write-Output "Getting started guide location:" -Write-Output "`thttps://github.com/drakkan/sftpgo/blob/v2.4.3/docs/howto/getting-started.md" +Write-Output "`thttps://github.com/drakkan/sftpgo/blob/v2.4.4/docs/howto/getting-started.md" Write-Output "Detailed information (docs folder) location:" -Write-Output "`thttps://github.com/drakkan/sftpgo/tree/v2.4.3/docs" +Write-Output "`thttps://github.com/drakkan/sftpgo/tree/v2.4.4/docs" Write-Output "" Write-Output "---------------------------" \ No newline at end of file diff --git a/pkgs/debian/changelog b/pkgs/debian/changelog index 543a2926..04b5871c 100644 --- a/pkgs/debian/changelog +++ b/pkgs/debian/changelog @@ -1,3 +1,9 @@ +sftpgo (2.4.4-1ppa1) bionic; urgency=medium + + * New upstream release + + -- Nicola Murino Sat, 04 Feb 2023 17:29:22 +0100 + sftpgo (2.4.3-1ppa1) bionic; urgency=medium * New upstream release diff --git a/pkgs/debian/patches/config.diff b/pkgs/debian/patches/config.diff index 85f99b49..acfcbfc4 100644 --- a/pkgs/debian/patches/config.diff +++ b/pkgs/debian/patches/config.diff @@ -2,7 +2,7 @@ Index: sftpgo/sftpgo.json =================================================================== --- sftpgo.orig/sftpgo.json +++ sftpgo/sftpgo.json -@@ -59,7 +59,7 @@ +@@ -60,7 +60,7 @@ "domains": [], "email": "", "key_type": "4096", @@ -11,7 +11,7 @@ Index: sftpgo/sftpgo.json "ca_endpoint": "https://acme-v02.api.letsencrypt.org/directory", "renew_days": 30, "http01_challenge": { -@@ -186,7 +186,7 @@ +@@ -187,7 +187,7 @@ }, "data_provider": { "driver": "sqlite", @@ -20,7 +20,7 @@ Index: sftpgo/sftpgo.json "host": "", "port": 0, "username": "", -@@ -202,7 +202,7 @@ +@@ -203,7 +203,7 @@ "track_quota": 2, "delayed_quota_update": 0, "pool_size": 0, @@ -29,7 +29,7 @@ Index: sftpgo/sftpgo.json "actions": { "execute_on": [], "execute_for": [], -@@ -244,7 +244,7 @@ +@@ -245,7 +245,7 @@ "port": 0, "proto": "http" }, diff --git a/sftpgo.json b/sftpgo.json index 254f3404..bd7fd34e 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -18,7 +18,7 @@ "data_retention_hook": "", "max_total_connections": 0, "max_per_host_connections": 20, - "whitelist_file": "", + "allowlist_status": 0, "allow_self_connections": 0, "defender": { "enabled": false, @@ -32,11 +32,7 @@ "score_no_auth": 0, "observation_time": 30, "entries_soft_limit": 100, - "entries_hard_limit": 150, - "safelist_file": "", - "blocklist_file": "", - "safelist": [], - "blocklist": [] + "entries_hard_limit": 150 }, "rate_limiters": [ { @@ -50,7 +46,6 @@ "DAV", "HTTP" ], - "allow_list": [], "generate_defender_events": false, "entries_soft_limit": 100, "entries_hard_limit": 150 diff --git a/templates/webadmin/admin.html b/templates/webadmin/admin.html index cff24d1d..336f7096 100644 --- a/templates/webadmin/admin.html +++ b/templates/webadmin/admin.html @@ -101,7 +101,7 @@ along with this program. If not, see . Role
-
Setting a role limit the administrator to only manage users with the same role. Role administrators cannot have the following permissions: "manage_admins", "manage_roles", "manage_event_rules", "manage_apikeys", "manage_system"
+
Setting a role limit the administrator to only manage users with the same role. Role administrators cannot have the following permissions: "manage_admins", "manage_roles", "manage_event_rules", "manage_apikeys", "manage_system", "manage_ip_lists"
diff --git a/templates/webadmin/base.html b/templates/webadmin/base.html index 41b6b67d..be75904d 100644 --- a/templates/webadmin/base.html +++ b/templates/webadmin/base.html @@ -73,7 +73,17 @@ along with this program. If not, see . {{.UsersTitle}} + {{ end }} + {{ if .LoggedAdmin.HasPermission "manage_groups"}} +
  • + + + {{.GroupsTitle}} +
    • + {{end}} + + {{ if .LoggedAdmin.HasPermission "view_users"}}
  • @@ -81,11 +91,11 @@ along with this program. If not, see .
    • {{end}} - {{ if .LoggedAdmin.HasPermission "manage_groups"}} -
  • - - - {{.GroupsTitle}} + {{ if .LoggedAdmin.HasPermission "view_conns"}} +
  • + + + {{.ConnectionsTitle}}
    • {{end}} @@ -105,27 +115,23 @@ along with this program. If not, see . {{end}} - {{ if .LoggedAdmin.HasPermission "view_conns"}} -
  • - - - {{.ConnectionsTitle}} -
    • - {{end}} - - {{ if and .HasDefender (.LoggedAdmin.HasPermission "view_defender")}} -
  • - + {{ if or (.LoggedAdmin.HasPermission "manage_ip_lists") (and .HasDefender (.LoggedAdmin.HasPermission "view_defender"))}} +
  • + - {{.DefenderTitle}} -
    • - {{end}} - - {{ if .LoggedAdmin.HasPermission "manage_roles"}} -
  • - - - {{.RolesTitle}} + IP Manager + +
    +
    + {{ if .LoggedAdmin.HasPermission "manage_ip_lists"}} + {{.IPListsTitle}} + {{end}} + {{ if and .HasDefender (.LoggedAdmin.HasPermission "view_defender")}} + {{.DefenderTitle}} + {{end}} +
    +
    • {{end}} @@ -137,6 +143,14 @@ along with this program. If not, see . {{end}} + {{ if .LoggedAdmin.HasPermission "manage_roles"}} +
  • + + + {{.RolesTitle}} +
    • + {{end}} + {{ if and .HasSearcher (.LoggedAdmin.HasPermission "view_events")}}
  • diff --git a/templates/webadmin/defender.html b/templates/webadmin/defender.html index 65f46d2d..5b8cee37 100644 --- a/templates/webadmin/defender.html +++ b/templates/webadmin/defender.html @@ -31,7 +31,7 @@ along with this program. If not, see .
    • -
    View and manage blocklist
    +
    View and manage auto blocklist
    @@ -63,7 +63,7 @@ along with this program. If not, see .
    -
    Do you want to remoce the selected blocklist entry?
    +
    Do you want to remoce the selected entry?
    +
    +
    +
    +
    + +
    + + + + + + + + + +
    IP/NetworkProtocolsModeNote
    +
    + +     + +
    + +{{end}} + +{{define "dialog"}} + +{{end}} + +{{define "extra_js"}} + + + + + + + + + + + +{{end}} \ No newline at end of file diff --git a/templates/webadmin/role.html b/templates/webadmin/role.html index 0859ef17..947e21d9 100644 --- a/templates/webadmin/role.html +++ b/templates/webadmin/role.html @@ -17,10 +17,6 @@ along with this program. If not, see . {{define "title"}}{{.Title}}{{end}} -{{define "extra_css"}} - -{{end}} - {{define "page_body"}}
    diff --git a/templates/webadmin/roles.html b/templates/webadmin/roles.html index c6a42339..2ee90c29 100644 --- a/templates/webadmin/roles.html +++ b/templates/webadmin/roles.html @@ -150,8 +150,8 @@ along with this program. If not, see . name: 'edit', titleAttr: "Edit", action: function (e, dt, node, config) { - var roleName = table.row({ selected: true }).data()[0]; - var path = '{{.RoleURL}}' + "/" + fixedEncodeURIComponent(roleName); + let roleName = table.row({ selected: true }).data()[0]; + let path = '{{.RoleURL}}' + "/" + fixedEncodeURIComponent(roleName); window.location.href = path; }, enabled: false diff --git a/templates/webadmin/status.html b/templates/webadmin/status.html index 5a2d4d5d..0a28057d 100644 --- a/templates/webadmin/status.html +++ b/templates/webadmin/status.html @@ -100,6 +100,15 @@ along with this program. If not, see .
    +
    +
    +
    Allow list
    +

    + Status: {{ if .Status.AllowList.IsActive}}"Enabled"{{else}}"Disabled"{{end}} +

    +
    +
    +
    Defender
    @@ -109,6 +118,19 @@ along with this program. If not, see .
    +
    +
    +
    Rate limiters
    +

    + Status: {{ if .Status.RateLimiters.IsActive}}"Enabled"{{else}}"Disabled"{{end}} + {{if .Status.RateLimiters.IsActive}} +
    + Protocols: {{.Status.RateLimiters.GetProtocolsAsString}} + {{end}} +

    +
    +
    +
    Multi-factor authentication
    diff --git a/templates/webclient/editfile.html b/templates/webclient/editfile.html index 1b14225b..0f33ed89 100644 --- a/templates/webclient/editfile.html +++ b/templates/webclient/editfile.html @@ -32,7 +32,7 @@ along with this program. If not, see . {{end}} {{define "additionalnavitems"}}
  • -