httpfs.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788
  1. // Copyright (C) 2019-2023 Nicola Murino
  2. //
  3. // This program is free software: you can redistribute it and/or modify
  4. // it under the terms of the GNU Affero General Public License as published
  5. // by the Free Software Foundation, version 3.
  6. //
  7. // This program is distributed in the hope that it will be useful,
  8. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. // GNU Affero General Public License for more details.
  11. //
  12. // You should have received a copy of the GNU Affero General Public License
  13. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  14. package vfs
  15. import (
  16. "context"
  17. "crypto/tls"
  18. "encoding/json"
  19. "errors"
  20. "fmt"
  21. "io"
  22. "io/fs"
  23. "mime"
  24. "net"
  25. "net/http"
  26. "net/url"
  27. "os"
  28. "path"
  29. "path/filepath"
  30. "strings"
  31. "time"
  32. "github.com/eikenb/pipeat"
  33. "github.com/pkg/sftp"
  34. "github.com/sftpgo/sdk"
  35. "github.com/drakkan/sftpgo/v2/internal/kms"
  36. "github.com/drakkan/sftpgo/v2/internal/logger"
  37. "github.com/drakkan/sftpgo/v2/internal/metric"
  38. "github.com/drakkan/sftpgo/v2/internal/util"
  39. )
  40. const (
  41. // httpFsName is the name for the HTTP Fs implementation
  42. httpFsName = "httpfs"
  43. )
  44. var (
  45. supportedEndpointSchema = []string{"http://", "https://"}
  46. )
  47. // HTTPFsConfig defines the configuration for HTTP based filesystem
  48. type HTTPFsConfig struct {
  49. sdk.BaseHTTPFsConfig
  50. Password *kms.Secret `json:"password,omitempty"`
  51. APIKey *kms.Secret `json:"api_key,omitempty"`
  52. }
  53. func (c *HTTPFsConfig) isUnixDomainSocket() bool {
  54. return strings.HasPrefix(c.Endpoint, "http://unix") || strings.HasPrefix(c.Endpoint, "https://unix")
  55. }
  56. // HideConfidentialData hides confidential data
  57. func (c *HTTPFsConfig) HideConfidentialData() {
  58. if c.Password != nil {
  59. c.Password.Hide()
  60. }
  61. if c.APIKey != nil {
  62. c.APIKey.Hide()
  63. }
  64. }
  65. func (c *HTTPFsConfig) setNilSecretsIfEmpty() {
  66. if c.Password != nil && c.Password.IsEmpty() {
  67. c.Password = nil
  68. }
  69. if c.APIKey != nil && c.APIKey.IsEmpty() {
  70. c.APIKey = nil
  71. }
  72. }
  73. func (c *HTTPFsConfig) setEmptyCredentialsIfNil() {
  74. if c.Password == nil {
  75. c.Password = kms.NewEmptySecret()
  76. }
  77. if c.APIKey == nil {
  78. c.APIKey = kms.NewEmptySecret()
  79. }
  80. }
  81. func (c *HTTPFsConfig) isEqual(other HTTPFsConfig) bool {
  82. if c.Endpoint != other.Endpoint {
  83. return false
  84. }
  85. if c.Username != other.Username {
  86. return false
  87. }
  88. if c.SkipTLSVerify != other.SkipTLSVerify {
  89. return false
  90. }
  91. c.setEmptyCredentialsIfNil()
  92. other.setEmptyCredentialsIfNil()
  93. if !c.Password.IsEqual(other.Password) {
  94. return false
  95. }
  96. return c.APIKey.IsEqual(other.APIKey)
  97. }
  98. func (c *HTTPFsConfig) isSameResource(other HTTPFsConfig) bool {
  99. if c.EqualityCheckMode > 0 || other.EqualityCheckMode > 0 {
  100. if c.Username != other.Username {
  101. return false
  102. }
  103. }
  104. return c.Endpoint == other.Endpoint
  105. }
  106. // validate returns an error if the configuration is not valid
  107. func (c *HTTPFsConfig) validate() error {
  108. c.setEmptyCredentialsIfNil()
  109. if c.Endpoint == "" {
  110. return errors.New("httpfs: endpoint cannot be empty")
  111. }
  112. c.Endpoint = strings.TrimRight(c.Endpoint, "/")
  113. endpointURL, err := url.Parse(c.Endpoint)
  114. if err != nil {
  115. return fmt.Errorf("httpfs: invalid endpoint: %w", err)
  116. }
  117. if !util.IsStringPrefixInSlice(c.Endpoint, supportedEndpointSchema) {
  118. return errors.New("httpfs: invalid endpoint schema: http and https are supported")
  119. }
  120. if endpointURL.Host == "unix" {
  121. socketPath := endpointURL.Query().Get("socket_path")
  122. if !filepath.IsAbs(socketPath) {
  123. return fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
  124. }
  125. }
  126. if !isEqualityCheckModeValid(c.EqualityCheckMode) {
  127. return errors.New("invalid equality_check_mode")
  128. }
  129. if c.Password.IsEncrypted() && !c.Password.IsValid() {
  130. return errors.New("httpfs: invalid encrypted password")
  131. }
  132. if !c.Password.IsEmpty() && !c.Password.IsValidInput() {
  133. return errors.New("httpfs: invalid password")
  134. }
  135. if c.APIKey.IsEncrypted() && !c.APIKey.IsValid() {
  136. return errors.New("httpfs: invalid encrypted API key")
  137. }
  138. if !c.APIKey.IsEmpty() && !c.APIKey.IsValidInput() {
  139. return errors.New("httpfs: invalid API key")
  140. }
  141. return nil
  142. }
  143. // ValidateAndEncryptCredentials validates the config and encrypts credentials if they are in plain text
  144. func (c *HTTPFsConfig) ValidateAndEncryptCredentials(additionalData string) error {
  145. if err := c.validate(); err != nil {
  146. return util.NewValidationError(fmt.Sprintf("could not validate HTTP fs config: %v", err))
  147. }
  148. if c.Password.IsPlain() {
  149. c.Password.SetAdditionalData(additionalData)
  150. if err := c.Password.Encrypt(); err != nil {
  151. return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs password: %v", err))
  152. }
  153. }
  154. if c.APIKey.IsPlain() {
  155. c.APIKey.SetAdditionalData(additionalData)
  156. if err := c.APIKey.Encrypt(); err != nil {
  157. return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs API key: %v", err))
  158. }
  159. }
  160. return nil
  161. }
  162. // HTTPFs is a Fs implementation for the SFTPGo HTTP filesystem backend
  163. type HTTPFs struct {
  164. connectionID string
  165. localTempDir string
  166. // if not empty this fs is mouted as virtual folder in the specified path
  167. mountPath string
  168. config *HTTPFsConfig
  169. client *http.Client
  170. ctxTimeout time.Duration
  171. }
  172. // NewHTTPFs returns an HTTPFs object that allows to interact with SFTPGo HTTP filesystem backends
  173. func NewHTTPFs(connectionID, localTempDir, mountPath string, config HTTPFsConfig) (Fs, error) {
  174. if localTempDir == "" {
  175. if tempPath != "" {
  176. localTempDir = tempPath
  177. } else {
  178. localTempDir = filepath.Clean(os.TempDir())
  179. }
  180. }
  181. config.setEmptyCredentialsIfNil()
  182. if !config.Password.IsEmpty() {
  183. if err := config.Password.TryDecrypt(); err != nil {
  184. return nil, err
  185. }
  186. }
  187. if !config.APIKey.IsEmpty() {
  188. if err := config.APIKey.TryDecrypt(); err != nil {
  189. return nil, err
  190. }
  191. }
  192. fs := &HTTPFs{
  193. connectionID: connectionID,
  194. localTempDir: localTempDir,
  195. mountPath: mountPath,
  196. config: &config,
  197. ctxTimeout: 30 * time.Second,
  198. }
  199. transport := http.DefaultTransport.(*http.Transport).Clone()
  200. transport.MaxResponseHeaderBytes = 1 << 16
  201. transport.WriteBufferSize = 1 << 16
  202. transport.ReadBufferSize = 1 << 16
  203. if fs.config.isUnixDomainSocket() {
  204. endpointURL, err := url.Parse(fs.config.Endpoint)
  205. if err != nil {
  206. return nil, err
  207. }
  208. if endpointURL.Host == "unix" {
  209. socketPath := endpointURL.Query().Get("socket_path")
  210. if !filepath.IsAbs(socketPath) {
  211. return nil, fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
  212. }
  213. if endpointURL.Scheme == "https" {
  214. transport.DialTLSContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
  215. var tlsConfig *tls.Config
  216. var d tls.Dialer
  217. if config.SkipTLSVerify {
  218. tlsConfig = getInsecureTLSConfig()
  219. }
  220. d.Config = tlsConfig
  221. return d.DialContext(ctx, "unix", socketPath)
  222. }
  223. } else {
  224. transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
  225. var d net.Dialer
  226. return d.DialContext(ctx, "unix", socketPath)
  227. }
  228. }
  229. endpointURL.Path = path.Join(endpointURL.Path, endpointURL.Query().Get("api_prefix"))
  230. endpointURL.RawQuery = ""
  231. endpointURL.RawFragment = ""
  232. fs.config.Endpoint = endpointURL.String()
  233. }
  234. }
  235. if config.SkipTLSVerify {
  236. if transport.TLSClientConfig != nil {
  237. transport.TLSClientConfig.InsecureSkipVerify = true
  238. } else {
  239. transport.TLSClientConfig = getInsecureTLSConfig()
  240. }
  241. }
  242. fs.client = &http.Client{
  243. Transport: transport,
  244. }
  245. return fs, nil
  246. }
  247. // Name returns the name for the Fs implementation
  248. func (fs *HTTPFs) Name() string {
  249. return fmt.Sprintf("%v %q", httpFsName, fs.config.Endpoint)
  250. }
  251. // ConnectionID returns the connection ID associated to this Fs implementation
  252. func (fs *HTTPFs) ConnectionID() string {
  253. return fs.connectionID
  254. }
  255. // Stat returns a FileInfo describing the named file
  256. func (fs *HTTPFs) Stat(name string) (os.FileInfo, error) {
  257. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  258. defer cancelFn()
  259. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
  260. if err != nil {
  261. return nil, err
  262. }
  263. defer resp.Body.Close()
  264. var response statResponse
  265. err = json.NewDecoder(resp.Body).Decode(&response)
  266. if err != nil {
  267. return nil, err
  268. }
  269. return response.getFileInfo(), nil
  270. }
  271. // Lstat returns a FileInfo describing the named file
  272. func (fs *HTTPFs) Lstat(name string) (os.FileInfo, error) {
  273. return fs.Stat(name)
  274. }
  275. // Open opens the named file for reading
  276. func (fs *HTTPFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) {
  277. r, w, err := pipeat.PipeInDir(fs.localTempDir)
  278. if err != nil {
  279. return nil, nil, nil, err
  280. }
  281. ctx, cancelFn := context.WithCancel(context.Background())
  282. var queryString string
  283. if offset > 0 {
  284. queryString = fmt.Sprintf("?offset=%d", offset)
  285. }
  286. go func() {
  287. defer cancelFn()
  288. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "open", name, queryString, "", nil)
  289. if err != nil {
  290. fsLog(fs, logger.LevelError, "download error, path %q, err: %v", name, err)
  291. w.CloseWithError(err) //nolint:errcheck
  292. metric.HTTPFsTransferCompleted(0, 1, err)
  293. return
  294. }
  295. defer resp.Body.Close()
  296. n, err := io.Copy(w, resp.Body)
  297. w.CloseWithError(err) //nolint:errcheck
  298. fsLog(fs, logger.LevelDebug, "download completed, path %q size: %v, err: %+v", name, n, err)
  299. metric.HTTPFsTransferCompleted(n, 1, err)
  300. }()
  301. return nil, r, cancelFn, nil
  302. }
  303. // Create creates or opens the named file for writing
  304. func (fs *HTTPFs) Create(name string, flag int) (File, *PipeWriter, func(), error) {
  305. r, w, err := pipeat.PipeInDir(fs.localTempDir)
  306. if err != nil {
  307. return nil, nil, nil, err
  308. }
  309. p := NewPipeWriter(w)
  310. ctx, cancelFn := context.WithCancel(context.Background())
  311. var queryString string
  312. if flag > 0 {
  313. queryString = fmt.Sprintf("?flags=%d", flag)
  314. }
  315. go func() {
  316. defer cancelFn()
  317. contentType := mime.TypeByExtension(path.Ext(name))
  318. resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "create", name, queryString, contentType,
  319. &wrapReader{reader: r})
  320. if err != nil {
  321. fsLog(fs, logger.LevelError, "upload error, path %q, err: %v", name, err)
  322. r.CloseWithError(err) //nolint:errcheck
  323. p.Done(err)
  324. metric.HTTPFsTransferCompleted(0, 0, err)
  325. return
  326. }
  327. defer resp.Body.Close()
  328. r.CloseWithError(err) //nolint:errcheck
  329. p.Done(err)
  330. fsLog(fs, logger.LevelDebug, "upload completed, path: %q, readed bytes: %d", name, r.GetReadedBytes())
  331. metric.HTTPFsTransferCompleted(r.GetReadedBytes(), 0, err)
  332. }()
  333. return nil, p, cancelFn, nil
  334. }
  335. // Rename renames (moves) source to target.
  336. func (fs *HTTPFs) Rename(source, target string) (int, int64, error) {
  337. if source == target {
  338. return -1, -1, nil
  339. }
  340. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  341. defer cancelFn()
  342. queryString := fmt.Sprintf("?target=%s", url.QueryEscape(target))
  343. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "rename", source, queryString, "", nil)
  344. if err != nil {
  345. return -1, -1, err
  346. }
  347. defer resp.Body.Close()
  348. return -1, -1, nil
  349. }
  350. // Remove removes the named file or (empty) directory.
  351. func (fs *HTTPFs) Remove(name string, _ bool) error {
  352. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  353. defer cancelFn()
  354. resp, err := fs.sendHTTPRequest(ctx, http.MethodDelete, "remove", name, "", "", nil)
  355. if err != nil {
  356. return err
  357. }
  358. defer resp.Body.Close()
  359. return nil
  360. }
  361. // Mkdir creates a new directory with the specified name and default permissions
  362. func (fs *HTTPFs) Mkdir(name string) error {
  363. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  364. defer cancelFn()
  365. resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "mkdir", name, "", "", nil)
  366. if err != nil {
  367. return err
  368. }
  369. defer resp.Body.Close()
  370. return nil
  371. }
  372. // Symlink creates source as a symbolic link to target.
  373. func (*HTTPFs) Symlink(_, _ string) error {
  374. return ErrVfsUnsupported
  375. }
  376. // Readlink returns the destination of the named symbolic link
  377. func (*HTTPFs) Readlink(_ string) (string, error) {
  378. return "", ErrVfsUnsupported
  379. }
  380. // Chown changes the numeric uid and gid of the named file.
  381. func (fs *HTTPFs) Chown(_ string, _ int, _ int) error {
  382. return ErrVfsUnsupported
  383. }
  384. // Chmod changes the mode of the named file to mode.
  385. func (fs *HTTPFs) Chmod(name string, mode os.FileMode) error {
  386. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  387. defer cancelFn()
  388. queryString := fmt.Sprintf("?mode=%d", mode)
  389. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chmod", name, queryString, "", nil)
  390. if err != nil {
  391. return err
  392. }
  393. defer resp.Body.Close()
  394. return nil
  395. }
  396. // Chtimes changes the access and modification times of the named file.
  397. func (fs *HTTPFs) Chtimes(name string, atime, mtime time.Time, _ bool) error {
  398. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  399. defer cancelFn()
  400. queryString := fmt.Sprintf("?access_time=%s&modification_time=%s", atime.UTC().Format(time.RFC3339),
  401. mtime.UTC().Format(time.RFC3339))
  402. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chtimes", name, queryString, "", nil)
  403. if err != nil {
  404. return err
  405. }
  406. defer resp.Body.Close()
  407. return nil
  408. }
  409. // Truncate changes the size of the named file.
  410. // Truncate by path is not supported, while truncating an opened
  411. // file is handled inside base transfer
  412. func (fs *HTTPFs) Truncate(name string, size int64) error {
  413. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  414. defer cancelFn()
  415. queryString := fmt.Sprintf("?size=%d", size)
  416. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "truncate", name, queryString, "", nil)
  417. if err != nil {
  418. return err
  419. }
  420. defer resp.Body.Close()
  421. return nil
  422. }
  423. // ReadDir reads the directory named by dirname and returns
  424. // a list of directory entries.
  425. func (fs *HTTPFs) ReadDir(dirname string) ([]os.FileInfo, error) {
  426. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  427. defer cancelFn()
  428. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "readdir", dirname, "", "", nil)
  429. if err != nil {
  430. return nil, err
  431. }
  432. defer resp.Body.Close()
  433. var response []statResponse
  434. err = json.NewDecoder(resp.Body).Decode(&response)
  435. if err != nil {
  436. return nil, err
  437. }
  438. result := make([]os.FileInfo, 0, len(response))
  439. for _, stat := range response {
  440. result = append(result, stat.getFileInfo())
  441. }
  442. return result, nil
  443. }
  444. // IsUploadResumeSupported returns true if resuming uploads is supported.
  445. func (*HTTPFs) IsUploadResumeSupported() bool {
  446. return false
  447. }
  448. // IsAtomicUploadSupported returns true if atomic upload is supported.
  449. func (*HTTPFs) IsAtomicUploadSupported() bool {
  450. return false
  451. }
  452. // IsNotExist returns a boolean indicating whether the error is known to
  453. // report that a file or directory does not exist
  454. func (*HTTPFs) IsNotExist(err error) bool {
  455. return errors.Is(err, fs.ErrNotExist)
  456. }
  457. // IsPermission returns a boolean indicating whether the error is known to
  458. // report that permission is denied.
  459. func (*HTTPFs) IsPermission(err error) bool {
  460. return errors.Is(err, fs.ErrPermission)
  461. }
  462. // IsNotSupported returns true if the error indicate an unsupported operation
  463. func (*HTTPFs) IsNotSupported(err error) bool {
  464. if err == nil {
  465. return false
  466. }
  467. return err == ErrVfsUnsupported
  468. }
  469. // CheckRootPath creates the specified local root directory if it does not exists
  470. func (fs *HTTPFs) CheckRootPath(username string, uid int, gid int) bool {
  471. // we need a local directory for temporary files
  472. osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
  473. return osFs.CheckRootPath(username, uid, gid)
  474. }
  475. // ScanRootDirContents returns the number of files and their size
  476. func (fs *HTTPFs) ScanRootDirContents() (int, int64, error) {
  477. return fs.GetDirSize("/")
  478. }
  479. // CheckMetadata checks the metadata consistency
  480. func (*HTTPFs) CheckMetadata() error {
  481. return nil
  482. }
  483. // GetDirSize returns the number of files and the size for a folder
  484. // including any subfolders
  485. func (fs *HTTPFs) GetDirSize(dirname string) (int, int64, error) {
  486. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  487. defer cancelFn()
  488. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "dirsize", dirname, "", "", nil)
  489. if err != nil {
  490. return 0, 0, err
  491. }
  492. defer resp.Body.Close()
  493. var response dirSizeResponse
  494. err = json.NewDecoder(resp.Body).Decode(&response)
  495. if err != nil {
  496. return 0, 0, err
  497. }
  498. return response.Files, response.Size, nil
  499. }
  500. // GetAtomicUploadPath returns the path to use for an atomic upload.
  501. func (*HTTPFs) GetAtomicUploadPath(_ string) string {
  502. return ""
  503. }
  504. // GetRelativePath returns the path for a file relative to the user's home dir.
  505. // This is the path as seen by SFTPGo users
  506. func (fs *HTTPFs) GetRelativePath(name string) string {
  507. rel := path.Clean(name)
  508. if rel == "." {
  509. rel = ""
  510. }
  511. if !path.IsAbs(rel) {
  512. rel = "/" + rel
  513. }
  514. if fs.mountPath != "" {
  515. rel = path.Join(fs.mountPath, rel)
  516. }
  517. return rel
  518. }
  519. // Walk walks the file tree rooted at root, calling walkFn for each file or
  520. // directory in the tree, including root. The result are unordered
  521. func (fs *HTTPFs) Walk(root string, walkFn filepath.WalkFunc) error {
  522. info, err := fs.Lstat(root)
  523. if err != nil {
  524. return walkFn(root, nil, err)
  525. }
  526. return fs.walk(root, info, walkFn)
  527. }
  528. // Join joins any number of path elements into a single path
  529. func (*HTTPFs) Join(elem ...string) string {
  530. return strings.TrimPrefix(path.Join(elem...), "/")
  531. }
  532. // HasVirtualFolders returns true if folders are emulated
  533. func (*HTTPFs) HasVirtualFolders() bool {
  534. return false
  535. }
  536. // ResolvePath returns the matching filesystem path for the specified virtual path
  537. func (fs *HTTPFs) ResolvePath(virtualPath string) (string, error) {
  538. if fs.mountPath != "" {
  539. virtualPath = strings.TrimPrefix(virtualPath, fs.mountPath)
  540. }
  541. if !path.IsAbs(virtualPath) {
  542. virtualPath = path.Clean("/" + virtualPath)
  543. }
  544. return virtualPath, nil
  545. }
  546. // GetMimeType returns the content type
  547. func (fs *HTTPFs) GetMimeType(name string) (string, error) {
  548. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  549. defer cancelFn()
  550. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
  551. if err != nil {
  552. return "", err
  553. }
  554. defer resp.Body.Close()
  555. var response mimeTypeResponse
  556. err = json.NewDecoder(resp.Body).Decode(&response)
  557. if err != nil {
  558. return "", err
  559. }
  560. return response.Mime, nil
  561. }
  562. // Close closes the fs
  563. func (fs *HTTPFs) Close() error {
  564. fs.client.CloseIdleConnections()
  565. return nil
  566. }
  567. // GetAvailableDiskSize returns the available size for the specified path
  568. func (fs *HTTPFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) {
  569. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  570. defer cancelFn()
  571. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "statvfs", dirName, "", "", nil)
  572. if err != nil {
  573. return nil, err
  574. }
  575. defer resp.Body.Close()
  576. var response statVFSResponse
  577. err = json.NewDecoder(resp.Body).Decode(&response)
  578. if err != nil {
  579. return nil, err
  580. }
  581. return response.toSFTPStatVFS(), nil
  582. }
  583. func (fs *HTTPFs) sendHTTPRequest(ctx context.Context, method, base, name, queryString, contentType string,
  584. body io.Reader,
  585. ) (*http.Response, error) {
  586. url := fmt.Sprintf("%s/%s/%s%s", fs.config.Endpoint, base, url.PathEscape(name), queryString)
  587. req, err := http.NewRequest(method, url, body)
  588. if err != nil {
  589. return nil, err
  590. }
  591. if contentType != "" {
  592. req.Header.Set("Content-Type", contentType)
  593. }
  594. if fs.config.APIKey.GetPayload() != "" {
  595. req.Header.Set("X-API-KEY", fs.config.APIKey.GetPayload())
  596. }
  597. if fs.config.Username != "" || fs.config.Password.GetPayload() != "" {
  598. req.SetBasicAuth(fs.config.Username, fs.config.Password.GetPayload())
  599. }
  600. resp, err := fs.client.Do(req.WithContext(ctx))
  601. if err != nil {
  602. return nil, fmt.Errorf("unable to send HTTP request to URL %v: %w", url, err)
  603. }
  604. if err = getErrorFromResponseCode(resp.StatusCode); err != nil {
  605. resp.Body.Close()
  606. return nil, err
  607. }
  608. return resp, nil
  609. }
  610. // walk recursively descends path, calling walkFn.
  611. func (fs *HTTPFs) walk(filePath string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
  612. if !info.IsDir() {
  613. return walkFn(filePath, info, nil)
  614. }
  615. files, err := fs.ReadDir(filePath)
  616. err1 := walkFn(filePath, info, err)
  617. if err != nil || err1 != nil {
  618. return err1
  619. }
  620. for _, fi := range files {
  621. objName := path.Join(filePath, fi.Name())
  622. err = fs.walk(objName, fi, walkFn)
  623. if err != nil {
  624. return err
  625. }
  626. }
  627. return nil
  628. }
  629. func getErrorFromResponseCode(code int) error {
  630. switch code {
  631. case 401, 403:
  632. return os.ErrPermission
  633. case 404:
  634. return os.ErrNotExist
  635. case 501:
  636. return ErrVfsUnsupported
  637. case 200, 201:
  638. return nil
  639. default:
  640. return fmt.Errorf("unexpected response code: %v", code)
  641. }
  642. }
  643. func getInsecureTLSConfig() *tls.Config {
  644. return &tls.Config{
  645. NextProtos: []string{"h2", "http/1.1"},
  646. InsecureSkipVerify: true,
  647. }
  648. }
  649. type wrapReader struct {
  650. reader io.Reader
  651. }
  652. func (r *wrapReader) Read(p []byte) (n int, err error) {
  653. return r.reader.Read(p)
  654. }
  655. type statResponse struct {
  656. Name string `json:"name"`
  657. Size int64 `json:"size"`
  658. Mode uint32 `json:"mode"`
  659. LastModified time.Time `json:"last_modified"`
  660. }
  661. func (s *statResponse) getFileInfo() os.FileInfo {
  662. info := NewFileInfo(s.Name, false, s.Size, s.LastModified, false)
  663. info.SetMode(fs.FileMode(s.Mode))
  664. return info
  665. }
  666. type dirSizeResponse struct {
  667. Files int `json:"files"`
  668. Size int64 `json:"size"`
  669. }
  670. type mimeTypeResponse struct {
  671. Mime string `json:"mime"`
  672. }
  673. type statVFSResponse struct {
  674. ID uint32 `json:"-"`
  675. Bsize uint64 `json:"bsize"`
  676. Frsize uint64 `json:"frsize"`
  677. Blocks uint64 `json:"blocks"`
  678. Bfree uint64 `json:"bfree"`
  679. Bavail uint64 `json:"bavail"`
  680. Files uint64 `json:"files"`
  681. Ffree uint64 `json:"ffree"`
  682. Favail uint64 `json:"favail"`
  683. Fsid uint64 `json:"fsid"`
  684. Flag uint64 `json:"flag"`
  685. Namemax uint64 `json:"namemax"`
  686. }
  687. func (s *statVFSResponse) toSFTPStatVFS() *sftp.StatVFS {
  688. return &sftp.StatVFS{
  689. Bsize: s.Bsize,
  690. Frsize: s.Frsize,
  691. Blocks: s.Blocks,
  692. Bfree: s.Bfree,
  693. Bavail: s.Bavail,
  694. Files: s.Files,
  695. Ffree: s.Ffree,
  696. Favail: s.Ffree,
  697. Flag: s.Flag,
  698. Namemax: s.Namemax,
  699. }
  700. }