diff --git a/Dockerfile b/Dockerfile
index c44c44cd..a72fda0f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,7 @@ ARG DOWNLOAD_PLUGINS=false
 
 RUN if [ "${DOWNLOAD_PLUGINS}" = "true" ]; then apt-get update && apt-get install --no-install-recommends -y curl && ./docker/scripts/download-plugins.sh; fi
 
-RUN apt-get update && apt-get install --no-install-recommends -y openssh-server libcap2-bin && setcap cap_net_bind_service=+ep /workspace/sftpgo && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install --no-install-recommends -y openssh-server && rm -rf /var/lib/apt/lists/*
 
 FROM debian:bullseye-slim
 
diff --git a/Dockerfile.alpine b/Dockerfile.alpine
index f80d0d16..9ed6c919 100644
--- a/Dockerfile.alpine
+++ b/Dockerfile.alpine
@@ -25,7 +25,7 @@ RUN set -xe && \
     export COMMIT_SHA=${COMMIT_SHA:-$(git describe --always --abbrev=8 --dirty)} && \
     go build $(if [ -n "${FEATURES}" ]; then echo "-tags ${FEATURES}"; fi) -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/internal/version.commit=${COMMIT_SHA} -X github.com/drakkan/sftpgo/v2/internal/version.date=`date -u +%FT%TZ`" -v -o sftpgo
 
-RUN apk add --update --no-cache openssh-client-common libcap && setcap cap_net_bind_service=+ep /workspace/sftpgo
+RUN apk add --update --no-cache openssh-client-common
 
 FROM alpine:3.18
 
diff --git a/Dockerfile.distroless b/Dockerfile.distroless
index ec5cd5d9..34c9957c 100644
--- a/Dockerfile.distroless
+++ b/Dockerfile.distroless
@@ -28,7 +28,7 @@ RUN sed -i 's|"users_base_dir": "",|"users_base_dir": "/srv/sftpgo/data",|' sftp
     sed -i 's|"backups"|"/srv/sftpgo/backups"|' sftpgo.json && \
     sed -i 's|"sqlite"|"bolt"|' sftpgo.json
 
-RUN apt-get update && apt-get install --no-install-recommends -y media-types openssh-server libcap2-bin && setcap cap_net_bind_service=+ep /workspace/sftpgo && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install --no-install-recommends -y media-types openssh-server && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /etc/sftpgo /var/lib/sftpgo /srv/sftpgo