From 676286182abe1c1449f2b17ddd1839806fc68a43 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Thu, 3 Nov 2022 08:31:40 +0100 Subject: [PATCH] webdav: always open files for reading in lazy mode Signed-off-by: Nicola Murino --- docs/webdav.md | 3 +- go.mod | 10 +-- go.sum | 104 +++++++++++++++++++--- internal/dataprovider/cacheduser.go | 2 +- internal/webdavd/file.go | 110 +++++++++++++++-------- internal/webdavd/handler.go | 21 +---- internal/webdavd/internal_test.go | 132 +++++++++++++++++++++------- internal/webdavd/server.go | 2 +- internal/webdavd/webdavd_test.go | 52 +++++++---- pkgs/build.sh | 2 +- 10 files changed, 314 insertions(+), 124 deletions(-) diff --git a/docs/webdav.md b/docs/webdav.md index 25922579..37a82d3a 100644 --- a/docs/webdav.md +++ b/docs/webdav.md @@ -24,8 +24,7 @@ If you use WebDAV behind a reverse proxy ensure to preserve the `Host` header or Know issues: - removing a directory tree on Cloud Storage backends could generate a `not found` error when removing the last (virtual) directory. This happens if the client cycles the directories tree itself and removes files and directories one by one instead of issuing a single remove command -- the used [WebDAV library](https://pkg.go.dev/golang.org/x/net/webdav?tab=doc) asks to open a file to execute a `stat` and sometimes reads some bytes to find the content type. Stat calls are executed before and after a download too, so to be able to properly list a directory you need to grant both `list` and `download` permissions and to be able to upload files you need to gran both `list` and `upload` permissions -- the used [WebDAV library](https://pkg.go.dev/golang.org/x/net/webdav?tab=doc) not always returns a proper error code/message, most of the times it simply returns `Method not Allowed`. I'll try to improve the library error codes in the future +- to be able to properly list a directory you need to grant both `list` and `download` permissions and to be able to upload files you need to gran both `list` and `upload` permissions - if a file or a directory cannot be accessed, for example due to OS permissions issues or because a mapped path for a virtual folder is a missing, it will be omitted from the directory listing. If there is a different error then the whole directory listing will fail. This behavior is different from SFTP/FTP where you will be able to see the problematic file/directory in the directory listing, you will only get an error if you try to access it - if you use the native Windows client please check its usage and pay particular attention to the [registry settings](https://docs.microsoft.com/en-us/iis/publish/using-webdav/using-the-webdav-redirector#webdav-redirector-registry-settings). The default file size limit is 50MB and if you don't configure SFTPGo to use HTTPS you have to set `BasicAuthLevel` to `2` diff --git a/go.mod b/go.mod index 1f45d5e7..e1bd9444 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 github.com/cockroachdb/cockroach-go/v2 v2.2.16 github.com/coreos/go-oidc/v3 v3.4.0 + github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 github.com/fclairamb/ftpserverlib v0.20.1-0.20221012093027-95be4ae0c9a6 github.com/fclairamb/go-log v0.4.1 @@ -46,7 +47,7 @@ require ( github.com/pires/go-proxyproto v0.6.2 github.com/pkg/sftp v1.13.6-0.20221020054726-e4133ab7e9bd github.com/pquerna/otp v1.3.0 - github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_golang v1.13.1 github.com/robfig/cron/v3 v3.0.1 github.com/rs/cors v1.8.3-0.20220619195839-da52b0701de5 github.com/rs/xid v1.4.0 @@ -57,7 +58,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.8.1 - github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 + github.com/studio-b12/gowebdav v0.0.0-20221102155456-200a600c0272 github.com/subosito/gotenv v1.4.1 github.com/unrolled/secure v1.13.0 github.com/wagslane/go-password-validator v0.3.0 @@ -71,7 +72,7 @@ require ( golang.org/x/oauth2 v0.1.0 golang.org/x/sys v0.1.0 golang.org/x/time v0.1.0 - google.golang.org/api v0.101.0 + google.golang.org/api v0.102.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -112,7 +113,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.6.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect @@ -172,5 +173,4 @@ require ( replace ( github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221020054403-a265c1cba3cb - golang.org/x/net => github.com/drakkan/net v0.0.0-20221026175805-eaebd725b308 ) diff --git a/go.sum b/go.sum index 71ada88e..68a36362 100644 --- a/go.sum +++ b/go.sum @@ -542,8 +542,8 @@ github.com/drakkan/crypto v0.0.0-20221020054403-a265c1cba3cb h1:ex3x8ir969oV6bQ8 github.com/drakkan/crypto v0.0.0-20221020054403-a265c1cba3cb/go.mod h1:IBSs4ri4rdTqz2QKcpTKpwKMdM+WJ7atZeL9lCu2swQ= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= -github.com/drakkan/net v0.0.0-20221026175805-eaebd725b308 h1:F7OUb3MgSa2SuY5mdmseihGXhVhxv1OCuKHrona2eh0= -github.com/drakkan/net v0.0.0-20221026175805-eaebd725b308/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ= +github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -864,8 +864,8 @@ github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0 github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= -github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -1371,8 +1371,8 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= +github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1536,8 +1536,8 @@ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 h1:Wd8WDEEusB5+En4PiRWJp1cP59QLNsQun+mOTW8+s6s= -github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= +github.com/studio-b12/gowebdav v0.0.0-20221102155456-200a600c0272 h1:dXbdJSdxf0EnR4SkcsfRNuGCvoEk9lavXbSCFXN2gJc= +github.com/studio-b12/gowebdav v0.0.0-20221102155456-200a600c0272/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -1746,6 +1746,84 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1863,6 +1941,7 @@ golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1877,9 +1956,13 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1925,6 +2008,7 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2109,8 +2193,8 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.101.0 h1:lJPPeEBIRxGpGLwnBTam1NPEM8Z2BmmXEd3z812pjwM= -google.golang.org/api v0.101.0/go.mod h1:CjxAAWWt3A3VrUE2IGDY2bgK5qhoG/OkyWVlYcP05MY= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/dataprovider/cacheduser.go b/internal/dataprovider/cacheduser.go index 3363fed7..67cbcc0c 100644 --- a/internal/dataprovider/cacheduser.go +++ b/internal/dataprovider/cacheduser.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "golang.org/x/net/webdav" + "github.com/drakkan/webdav" "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/util" diff --git a/internal/webdavd/file.go b/internal/webdavd/file.go index adb92c00..f6ffeaf8 100644 --- a/internal/webdavd/file.go +++ b/internal/webdavd/file.go @@ -26,8 +26,8 @@ import ( "sync/atomic" "time" + "github.com/drakkan/webdav" "github.com/eikenb/pipeat" - "golang.org/x/net/webdav" "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" @@ -84,6 +84,9 @@ type webDavFileInfo struct { // ContentType implements webdav.ContentTyper interface func (fi *webDavFileInfo) ContentType(ctx context.Context) (string, error) { extension := path.Ext(fi.virtualPath) + if extension == "" || extension == ".dat" { + return "application/octet-stream", nil + } contentType := mime.TypeByExtension(extension) if contentType != "" { return contentType, nil @@ -105,20 +108,19 @@ func (f *webDavFile) Readdir(count int) ([]os.FileInfo, error) { if !f.Connection.User.HasPerm(dataprovider.PermListItems, f.GetVirtualPath()) { return nil, f.Connection.GetPermissionDeniedError() } - fileInfos, err := f.Connection.ListDir(f.GetVirtualPath()) + entries, err := f.Connection.ListDir(f.GetVirtualPath()) if err != nil { return nil, err } - result := make([]os.FileInfo, 0, len(fileInfos)) - for _, fileInfo := range fileInfos { - result = append(result, &webDavFileInfo{ - FileInfo: fileInfo, + for idx, info := range entries { + entries[idx] = &webDavFileInfo{ + FileInfo: info, Fs: f.Fs, - virtualPath: path.Join(f.GetVirtualPath(), fileInfo.Name()), - fsPath: f.Fs.Join(f.GetFsPath(), fileInfo.Name()), - }) + virtualPath: path.Join(f.GetVirtualPath(), info.Name()), + fsPath: f.Fs.Join(f.GetFsPath(), info.Name()), + } } - return result, nil + return entries, nil } // Stat the handle @@ -131,7 +133,7 @@ func (f *webDavFile) Stat() (os.FileInfo, error) { f.Unlock() if f.GetType() == common.TransferUpload && errUpload == nil { info := &webDavFileInfo{ - FileInfo: vfs.NewFileInfo(f.GetFsPath(), false, f.BytesReceived.Load(), time.Unix(0, 0), false), + FileInfo: vfs.NewFileInfo(f.GetFsPath(), false, f.BytesReceived.Load(), time.Now(), false), Fs: f.Fs, virtualPath: f.GetVirtualPath(), fsPath: f.GetFsPath(), @@ -154,33 +156,38 @@ func (f *webDavFile) Stat() (os.FileInfo, error) { return fi, nil } +func (f *webDavFile) checkFirstRead() error { + if !f.Connection.User.HasPerm(dataprovider.PermDownload, path.Dir(f.GetVirtualPath())) { + return f.Connection.GetPermissionDeniedError() + } + transferQuota := f.BaseTransfer.GetTransferQuota() + if !transferQuota.HasDownloadSpace() { + f.Connection.Log(logger.LevelInfo, "denying file read due to quota limits") + return f.Connection.GetReadQuotaExceededError() + } + if ok, policy := f.Connection.User.IsFileAllowed(f.GetVirtualPath()); !ok { + f.Connection.Log(logger.LevelWarn, "reading file %#v is not allowed", f.GetVirtualPath()) + return f.Connection.GetErrorForDeniedFile(policy) + } + err := common.ExecutePreAction(f.Connection, common.OperationPreDownload, f.GetFsPath(), f.GetVirtualPath(), 0, 0) + if err != nil { + f.Connection.Log(logger.LevelDebug, "download for file %#v denied by pre action: %v", f.GetVirtualPath(), err) + return f.Connection.GetPermissionDeniedError() + } + f.readTryed.Store(true) + return nil +} + // Read reads the contents to downloads. func (f *webDavFile) Read(p []byte) (n int, err error) { if f.AbortTransfer.Load() { return 0, errTransferAborted } if !f.readTryed.Load() { - if !f.Connection.User.HasPerm(dataprovider.PermDownload, path.Dir(f.GetVirtualPath())) { - return 0, f.Connection.GetPermissionDeniedError() + if err := f.checkFirstRead(); err != nil { + return 0, err } - transferQuota := f.BaseTransfer.GetTransferQuota() - if !transferQuota.HasDownloadSpace() { - f.Connection.Log(logger.LevelInfo, "denying file read due to quota limits") - return 0, f.Connection.GetReadQuotaExceededError() - } - - if ok, policy := f.Connection.User.IsFileAllowed(f.GetVirtualPath()); !ok { - f.Connection.Log(logger.LevelWarn, "reading file %#v is not allowed", f.GetVirtualPath()) - return 0, f.Connection.GetErrorForDeniedFile(policy) - } - err := common.ExecutePreAction(f.Connection, common.OperationPreDownload, f.GetFsPath(), f.GetVirtualPath(), 0, 0) - if err != nil { - f.Connection.Log(logger.LevelDebug, "download for file %#v denied by pre action: %v", f.GetVirtualPath(), err) - return 0, f.Connection.GetPermissionDeniedError() - } - f.readTryed.Store(true) } - f.Connection.UpdateLastActivity() // the file is read sequentially we don't need to check for concurrent reads and so @@ -190,10 +197,16 @@ func (f *webDavFile) Read(p []byte) (n int, err error) { f.TransferError(common.ErrOpUnsupported) return 0, common.ErrOpUnsupported } - _, r, cancelFn, e := f.Fs.Open(f.GetFsPath(), 0) + file, r, cancelFn, e := f.Fs.Open(f.GetFsPath(), 0) f.Lock() if e == nil { - f.reader = r + if file != nil { + f.File = file + f.writer = f.File + f.reader = f.File + } else if r != nil { + f.reader = r + } f.BaseTransfer.SetCancelFn(cancelFn) } f.ErrTransfer = e @@ -263,18 +276,41 @@ func (f *webDavFile) updateTransferQuotaOnSeek() { } } +func (f *webDavFile) checkFile() error { + if f.File == nil && vfs.IsLocalOrUnbufferedSFTPFs(f.Fs) { + file, _, _, err := f.Fs.Open(f.GetFsPath(), 0) + if err != nil { + f.Connection.Log(logger.LevelWarn, "could not open file %q for seeking: %v", + f.GetFsPath(), err) + f.TransferError(err) + return err + } + f.File = file + f.reader = file + f.writer = file + } + return nil +} + +func (f *webDavFile) seekFile(offset int64, whence int) (int64, error) { + ret, err := f.File.Seek(offset, whence) + if err != nil { + f.TransferError(err) + } + return ret, err +} + // Seek sets the offset for the next Read or Write on the writer to offset, // interpreted according to whence: 0 means relative to the origin of the file, // 1 means relative to the current offset, and 2 means relative to the end. // It returns the new offset and an error, if any. func (f *webDavFile) Seek(offset int64, whence int) (int64, error) { f.Connection.UpdateLastActivity() + if err := f.checkFile(); err != nil { + return 0, err + } if f.File != nil { - ret, err := f.File.Seek(offset, whence) - if err != nil { - f.TransferError(err) - } - return ret, err + return f.seekFile(offset, whence) } if f.GetType() == common.TransferDownload { readOffset := f.startOffset + f.BytesSent.Load() diff --git a/internal/webdavd/handler.go b/internal/webdavd/handler.go index e90852f0..40b92db6 100644 --- a/internal/webdavd/handler.go +++ b/internal/webdavd/handler.go @@ -21,8 +21,7 @@ import ( "path" "strings" - "github.com/eikenb/pipeat" - "golang.org/x/net/webdav" + "github.com/drakkan/webdav" "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" @@ -134,25 +133,13 @@ func (c *Connection) OpenFile(ctx context.Context, name string, flag int, perm o } func (c *Connection) getFile(fs vfs.Fs, fsPath, virtualPath string) (webdav.File, error) { - var err error - var file vfs.File - var r *pipeat.PipeReaderAt var cancelFn func() - // for cloud fs we open the file when we receive the first read to avoid to download the first part of - // the file if it was opened only to do a stat or a readdir and so it is not a real download - if vfs.IsLocalOrUnbufferedSFTPFs(fs) { - file, r, cancelFn, err = fs.Open(fsPath, 0) - if err != nil { - c.Log(logger.LevelError, "could not open file %#v for reading: %+v", fsPath, err) - return nil, c.GetFsError(fs, err) - } - } - - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, fsPath, virtualPath, + // we open the file when we receive the first read so we only open the file if necessary + baseTransfer := common.NewBaseTransfer(nil, c.BaseConnection, cancelFn, fsPath, fsPath, virtualPath, common.TransferDownload, 0, 0, 0, 0, false, fs, c.GetTransferQuota()) - return newWebDavFile(baseTransfer, nil, r), nil + return newWebDavFile(baseTransfer, nil, nil), nil } func (c *Connection) putFile(fs vfs.Fs, fsPath, virtualPath string) (webdav.File, error) { diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index f0e568d9..9d2de3b4 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -30,10 +30,10 @@ import ( "testing" "time" + "github.com/drakkan/webdav" "github.com/eikenb/pipeat" "github.com/sftpgo/sdk" "github.com/stretchr/testify/assert" - "golang.org/x/net/webdav" "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" @@ -286,7 +286,10 @@ func (fs *MockOsFs) Name() string { // Open returns nil func (fs *MockOsFs) Open(name string, offset int64) (vfs.File, *pipeat.PipeReaderAt, func(), error) { - return nil, fs.reader, nil, nil + if fs.reader != nil { + return nil, fs.reader, nil, nil + } + return fs.Fs.Open(name, offset) } // IsUploadResumeSupported returns true if resuming uploads is supported @@ -314,14 +317,18 @@ func (fs *MockOsFs) Rename(source, target string) error { // GetMimeType returns the content type func (fs *MockOsFs) GetMimeType(name string) (string, error) { + if fs.err != nil { + return "", fs.err + } return "application/custom-mime", nil } -func newMockOsFs(atomicUpload bool, connectionID, rootDir string, reader *pipeat.PipeReaderAt) vfs.Fs { +func newMockOsFs(atomicUpload bool, connectionID, rootDir string, reader *pipeat.PipeReaderAt, err error) vfs.Fs { return &MockOsFs{ Fs: vfs.NewOsFs(connectionID, rootDir, ""), isAtomicUploadSupported: atomicUpload, reader: reader, + err: err, } } @@ -552,38 +559,31 @@ func TestFileAccessErrors(t *testing.T) { missingPath := "missing path" fsMissingPath := filepath.Join(user.HomeDir, missingPath) err := connection.RemoveAll(ctx, missingPath) - if assert.Error(t, err) { - assert.EqualError(t, err, os.ErrNotExist.Error()) - } - _, err = connection.getFile(fs, fsMissingPath, missingPath) - if assert.Error(t, err) { - assert.EqualError(t, err, os.ErrNotExist.Error()) - } - _, err = connection.getFile(fs, fsMissingPath, missingPath) - if assert.Error(t, err) { - assert.EqualError(t, err, os.ErrNotExist.Error()) - } + assert.ErrorIs(t, err, os.ErrNotExist) + davFile, err := connection.getFile(fs, fsMissingPath, missingPath) + assert.NoError(t, err) + buf := make([]byte, 64) + _, err = davFile.Read(buf) + assert.ErrorIs(t, err, os.ErrNotExist) + err = davFile.Close() + assert.ErrorIs(t, err, os.ErrNotExist) p := filepath.Join(user.HomeDir, "adir", missingPath) _, err = connection.handleUploadToNewFile(fs, p, p, path.Join("adir", missingPath)) - if assert.Error(t, err) { - assert.EqualError(t, err, os.ErrNotExist.Error()) - } + assert.ErrorIs(t, err, os.ErrNotExist) _, err = connection.handleUploadToExistingFile(fs, p, "_"+p, 0, path.Join("adir", missingPath)) if assert.Error(t, err) { assert.ErrorIs(t, err, os.ErrNotExist) } - fs = newMockOsFs(false, fs.ConnectionID(), user.HomeDir, nil) + fs = newMockOsFs(false, fs.ConnectionID(), user.HomeDir, nil, nil) _, err = connection.handleUploadToExistingFile(fs, p, p, 0, path.Join("adir", missingPath)) - if assert.Error(t, err) { - assert.ErrorIs(t, err, os.ErrNotExist) - } + assert.ErrorIs(t, err, os.ErrNotExist) f, err := os.CreateTemp("", "temp") assert.NoError(t, err) err = f.Close() assert.NoError(t, err) - davFile, err := connection.handleUploadToExistingFile(fs, f.Name(), f.Name(), 123, f.Name()) + davFile, err = connection.handleUploadToExistingFile(fs, f.Name(), f.Name(), 123, f.Name()) if assert.NoError(t, err) { transfer := davFile.(*webDavFile) transfers := connection.GetTransfers() @@ -650,9 +650,9 @@ func TestContentType(t *testing.T) { } testFilePath := filepath.Join(user.HomeDir, testFile) ctx := context.Background() - baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, + baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown", common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) - fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil) + fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, nil) err := os.WriteFile(testFilePath, []byte(""), os.ModePerm) assert.NoError(t, err) davFile := newWebDavFile(baseTransfer, nil, nil) @@ -668,6 +668,8 @@ func TestContentType(t *testing.T) { err = davFile.Close() assert.NoError(t, err) + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown1", + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) davFile = newWebDavFile(baseTransfer, nil, nil) davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") fi, err = davFile.Stat() @@ -679,9 +681,53 @@ func TestContentType(t *testing.T) { err = davFile.Close() assert.NoError(t, err) - fi.(*webDavFileInfo).fsPath = "missing" - _, err = fi.(*webDavFileInfo).ContentType(ctx) - assert.EqualError(t, err, webdav.ErrNotImplemented.Error()) + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) + davFile = newWebDavFile(baseTransfer, nil, nil) + davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") + fi, err = davFile.Stat() + if assert.NoError(t, err) { + ctype, err := fi.(*webDavFileInfo).ContentType(ctx) + assert.NoError(t, err) + assert.Equal(t, "application/octet-stream", ctype) + } + err = davFile.Close() + assert.NoError(t, err) + + for i := 0; i < 2; i++ { + // the second time the cache will be used + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".custom", + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) + davFile = newWebDavFile(baseTransfer, nil, nil) + davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") + fi, err = davFile.Stat() + if assert.NoError(t, err) { + ctype, err := fi.(*webDavFileInfo).ContentType(ctx) + assert.NoError(t, err) + assert.Equal(t, "text/plain; charset=utf-8", ctype) + } + err = davFile.Close() + assert.NoError(t, err) + } + + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown2", + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) + fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid) + davFile = newWebDavFile(baseTransfer, nil, nil) + davFile.Fs = fs + fi, err = davFile.Stat() + if assert.NoError(t, err) { + ctype, err := fi.(*webDavFileInfo).ContentType(ctx) + assert.EqualError(t, err, webdav.ErrNotImplemented.Error(), "unexpected content type %q", ctype) + } + cache := mimeCache{ + maxSize: 10, + mimeTypes: map[string]string{}, + } + cache.addMimeToCache("", "") + cache.RLock() + assert.Len(t, cache.mimeTypes, 0) + cache.RUnlock() err = os.Remove(testFilePath) assert.NoError(t, err) @@ -750,7 +796,7 @@ func TestTransferReadWriteErrors(t *testing.T) { r, w, err = pipeat.Pipe() assert.NoError(t, err) - mockFs := newMockOsFs(false, fs.ConnectionID(), user.HomeDir, r) + mockFs := newMockOsFs(false, fs.ConnectionID(), user.HomeDir, r, nil) baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, 0, false, mockFs, dataprovider.TransferQuota{}) davFile = newWebDavFile(baseTransfer, nil, nil) @@ -790,7 +836,7 @@ func TestTransferSeek(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := newMockOsFs(true, "connID", user.HomeDir, nil, nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -831,6 +877,8 @@ func TestTransferSeek(t *testing.T) { res, err := davFile.Seek(0, io.SeekStart) assert.NoError(t, err) assert.Equal(t, int64(0), res) + err = davFile.Close() + assert.NoError(t, err) davFile.Connection.RemoveTransfer(davFile.BaseTransfer) davFile = newWebDavFile(baseTransfer, nil, nil) @@ -838,7 +886,9 @@ func TestTransferSeek(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(len(testFileContents)), res) err = davFile.updateStatInfo() - assert.Nil(t, err) + assert.NoError(t, err) + err = davFile.Close() + assert.NoError(t, err) baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100}) @@ -847,26 +897,42 @@ func TestTransferSeek(t *testing.T) { assert.True(t, fs.IsNotExist(err)) davFile.Connection.RemoveTransfer(davFile.BaseTransfer) + fs = vfs.NewOsFs(fs.ConnectionID(), user.GetHomeDir(), "") + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100}) + davFile = newWebDavFile(baseTransfer, nil, nil) + _, err = davFile.Seek(0, io.SeekEnd) + assert.True(t, fs.IsNotExist(err)) + davFile.Connection.RemoveTransfer(davFile.BaseTransfer) + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100}) davFile = newWebDavFile(baseTransfer, nil, nil) davFile.reader = f - davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), nil) + r, _, err := pipeat.Pipe() + assert.NoError(t, err) + davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), r, nil) res, err = davFile.Seek(2, io.SeekStart) assert.NoError(t, err) assert.Equal(t, int64(2), res) + err = davFile.Close() + assert.NoError(t, err) + r, _, err = pipeat.Pipe() + assert.NoError(t, err) davFile = newWebDavFile(baseTransfer, nil, nil) - davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), nil) + davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), r, nil) res, err = davFile.Seek(2, io.SeekEnd) assert.NoError(t, err) assert.Equal(t, int64(5), res) + err = davFile.Close() + assert.NoError(t, err) baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100}) davFile = newWebDavFile(baseTransfer, nil, nil) - davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), nil) + davFile.Fs = newMockOsFs(true, fs.ConnectionID(), user.GetHomeDir(), nil, nil) res, err = davFile.Seek(2, io.SeekEnd) assert.True(t, fs.IsNotExist(err)) assert.Equal(t, int64(0), res) diff --git a/internal/webdavd/server.go b/internal/webdavd/server.go index 130c6057..8290b24e 100644 --- a/internal/webdavd/server.go +++ b/internal/webdavd/server.go @@ -28,10 +28,10 @@ import ( "runtime/debug" "time" + "github.com/drakkan/webdav" "github.com/go-chi/chi/v5/middleware" "github.com/rs/cors" "github.com/rs/xid" - "golang.org/x/net/webdav" "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go index 02002562..bd3d1c08 100644 --- a/internal/webdavd/webdavd_test.go +++ b/internal/webdavd/webdavd_test.go @@ -2117,6 +2117,19 @@ func TestBytesRangeRequests(t *testing.T) { assert.Equal(t, "file", string(bodyBytes)) } } + // seek on a missing file + remotePath = fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName+"_missing") + req, err = http.NewRequest(http.MethodGet, remotePath, nil) + if assert.NoError(t, err) { + httpClient := httpclient.GetHTTPClient() + req.SetBasicAuth(user.Username, defaultPassword) + req.Header.Set("Range", "bytes=5-") + resp, err := httpClient.Do(req) + if assert.NoError(t, err) { + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + } + } err = os.Remove(testFilePath) assert.NoError(t, err) @@ -2339,28 +2352,24 @@ func TestOsErrors(t *testing.T) { info, err := client.Stat(vdir) assert.NoError(t, err) assert.True(t, info.IsDir()) - // now remove the folder mapped to vdir. It should not appear in directory listing + // now remove the folder mapped to vdir. It still appear in directory listing + // virtual folders are automatically added err = os.RemoveAll(mappedPath) assert.NoError(t, err) files, err = client.ReadDir(".") assert.NoError(t, err) - assert.Len(t, files, 0) + assert.Len(t, files, 1) err = createTestFile(filepath.Join(user.GetHomeDir(), testFileName), 32768) assert.NoError(t, err) files, err = client.ReadDir(".") assert.NoError(t, err) - if assert.Len(t, files, 1) { - assert.Equal(t, testFileName, files[0].Name()) - } - if runtime.GOOS != osWindows { - // if the file cannot be accessed it should not appear in directory listing - err = os.Chmod(filepath.Join(user.GetHomeDir(), testFileName), 0001) - assert.NoError(t, err) - files, err = client.ReadDir(".") - assert.NoError(t, err) - assert.Len(t, files, 0) - err = os.Chmod(filepath.Join(user.GetHomeDir(), testFileName), os.ModePerm) - assert.NoError(t, err) + if assert.Len(t, files, 2) { + var names []string + for _, info := range files { + names = append(names, info.Name()) + } + assert.Contains(t, names, testFileName) + assert.Contains(t, names, "vdir") } _, err = httpdtest.RemoveUser(user, http.StatusOK) @@ -2816,9 +2825,18 @@ func TestSFTPLoopVirtualFolders(t *testing.T) { contents, err := client.ReadDir("/") assert.NoError(t, err) - if assert.Len(t, contents, 1) { - assert.Equal(t, testDir, contents[0].Name()) - assert.True(t, contents[0].IsDir()) + if assert.Len(t, contents, 2) { + expected := 0 + for _, info := range contents { + switch info.Name() { + case testDir, "vdir": + assert.True(t, info.IsDir()) + expected++ + default: + t.Errorf("unexpected file/dir %q", info.Name()) + } + } + assert.Equal(t, expected, 2) } _, err = httpdtest.RemoveUser(user1, http.StatusOK) diff --git a/pkgs/build.sh b/pkgs/build.sh index 20f5aafa..71c40370 100755 --- a/pkgs/build.sh +++ b/pkgs/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -NFPM_VERSION=2.20.0 +NFPM_VERSION=2.21.0 NFPM_ARCH=${NFPM_ARCH:-amd64} if [ -z ${SFTPGO_VERSION} ] then