From 28d3ca0ca6b3edd226ed11f2cab6b1d4e786fd57 Mon Sep 17 00:00:00 2001 From: link Date: Thu, 2 Feb 2023 03:36:59 +0000 Subject: [PATCH] added google drive and dropbox driver --- .../usr/lib/systemd/system/rclone.service | 11 + .../sysroot/usr/share/casaos/shell/helper.sh | 6 +- drivers/all.go | 1 + drivers/dropbox/drive.go | 100 ++++ drivers/dropbox/meta.go | 31 + drivers/dropbox/types.go | 88 +++ drivers/dropbox/util.go | 102 ++++ drivers/google_drive/drive.go | 17 +- drivers/google_drive/meta.go | 16 +- drivers/google_drive/types.go | 11 + go.mod | 2 + go.sum | 11 + internal/driver/driver.go | 6 +- internal/driver/item.go | 2 +- internal/op/driver.go | 25 +- internal/op/fs.go | 545 ++++++++++++++++++ internal/sign/sign.go | 36 ++ main.go | 15 + model/args.go | 1 + model/smartctl_model.go | 69 --- pkg/fs/fs.go | 12 + pkg/sign/hmac.go | 52 ++ pkg/sign/sign.go | 15 + pkg/utils/command/command_helper.go | 19 - pkg/utils/httper/drive.go | 151 +++++ pkg/utils/time.go | 37 ++ route/init.go | 5 + route/route.go | 6 - route/v1/file.go | 206 ++++++- route/v1/recover.go | 140 ++++- route/v1/samba.go | 6 +- route/v1/storage.go | 100 ++-- service/fs.go | 18 +- service/fs_link.go | 27 + service/service.go | 8 +- service/storage.go | 131 +++-- service/storage_old.go | 73 +++ service/storage_service.go | 292 +++++----- 38 files changed, 2004 insertions(+), 389 deletions(-) create mode 100644 build/sysroot/usr/lib/systemd/system/rclone.service create mode 100644 drivers/dropbox/drive.go create mode 100644 drivers/dropbox/meta.go create mode 100644 drivers/dropbox/types.go create mode 100644 drivers/dropbox/util.go create mode 100644 internal/op/fs.go create mode 100644 internal/sign/sign.go delete mode 100644 model/smartctl_model.go create mode 100644 pkg/fs/fs.go create mode 100644 pkg/sign/hmac.go create mode 100644 pkg/sign/sign.go create mode 100644 pkg/utils/httper/drive.go create mode 100644 pkg/utils/time.go create mode 100644 service/fs_link.go create mode 100644 service/storage_old.go diff --git a/build/sysroot/usr/lib/systemd/system/rclone.service b/build/sysroot/usr/lib/systemd/system/rclone.service new file mode 100644 index 0000000..92cfcab --- /dev/null +++ b/build/sysroot/usr/lib/systemd/system/rclone.service @@ -0,0 +1,11 @@ +[Unit] +Description=rclone + +[Service] +ExecStartPre=/usr/bin/rm -f /tmp/rclone.sock +ExecStart=/usr/bin/rclone rcd --rc-addr unix:///tmp/rclone.sock --rc-no-auth +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/build/sysroot/usr/share/casaos/shell/helper.sh b/build/sysroot/usr/share/casaos/shell/helper.sh index cfb001d..d44eceb 100644 --- a/build/sysroot/usr/share/casaos/shell/helper.sh +++ b/build/sysroot/usr/share/casaos/shell/helper.sh @@ -133,11 +133,7 @@ GetPlugInDisk() { fdisk -l | grep 'Disk' | grep 'sd' | awk -F , '{print substr($1,11,3)}' } -#获取磁盘状态 -#param 磁盘路径 -GetDiskHealthState() { - smartctl -H $1 | grep "SMART Health Status" | awk -F ":" '{print$2}' -} + #获取磁盘字节数量和扇区数量 #param 磁盘路径 /dev/sda diff --git a/drivers/all.go b/drivers/all.go index d27e61a..3cdfe84 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -1,6 +1,7 @@ package drivers import ( + _ "github.com/IceWhaleTech/CasaOS/drivers/dropbox" _ "github.com/IceWhaleTech/CasaOS/drivers/google_drive" ) diff --git a/drivers/dropbox/drive.go b/drivers/dropbox/drive.go new file mode 100644 index 0000000..c2aff61 --- /dev/null +++ b/drivers/dropbox/drive.go @@ -0,0 +1,100 @@ +package dropbox + +import ( + "context" + "errors" + "net/http" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" +) + +type Dropbox struct { + model.Storage + Addition + AccessToken string +} + +func (d *Dropbox) Config() driver.Config { + return config +} + +func (d *Dropbox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Dropbox) Init(ctx context.Context) error { + if len(d.RefreshToken) == 0 { + d.getRefreshToken() + } + return d.refreshToken() +} + +func (d *Dropbox) Drop(ctx context.Context) error { + + return nil +} + +func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := "https://content.dropboxapi.com/2/files/download" + link := model.Link{ + URL: url, + Method: http.MethodPost, + Header: http.Header{ + "Authorization": []string{"Bearer " + d.AccessToken}, + "Dropbox-API-Arg": []string{`{"path": "` + file.GetPath() + `"}`}, + }, + } + return &link, nil +} +func (d *Dropbox) GetUserInfo(ctx context.Context) (string, error) { + url := "https://api.dropboxapi.com/2/users/get_current_account" + user := UserInfo{} + resp, err := d.request(url, http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "") + }, &user) + if err != nil { + return "", err + } + logger.Info("resp", zap.Any("resp", string(resp))) + return user.Email, nil +} +func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return nil +} + +func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return nil +} + +func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return nil +} + +func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errors.New("not support") +} + +func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error { + return nil +} + +func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return nil +} + +var _ driver.Driver = (*Dropbox)(nil) diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.go new file mode 100644 index 0000000..ac494b3 --- /dev/null +++ b/drivers/dropbox/meta.go @@ -0,0 +1,31 @@ +package dropbox + +import ( + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" +) + +const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png" + +type Addition struct { + driver.RootID + RefreshToken string `json:"refresh_token" required:"true" omit:"true"` + AppKey string `json:"app_key" type:"string" default:"onr2ic0c0m97mxr" omit:"true"` + AppSecret string `json:"app_secret" type:"string" default:"nd3cjtikbxyj3pz" omit:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"` + AuthUrl string `json:"auth_url" type:"string" default:"https://www.dropbox.com/oauth2/authorize?client_id=onr2ic0c0m97mxr&redirect_uri=https://test-get.casaos.io&response_type=code&token_access_type=offline&state=${HOST}%2Fv1%2Frecover%2FDropbox"` + Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"` + Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"` +} + +var config = driver.Config{ + Name: "Dropbox", + OnlyProxy: true, + DefaultRoot: "root", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Dropbox{} + }) +} diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.go new file mode 100644 index 0000000..af5bdb9 --- /dev/null +++ b/drivers/dropbox/types.go @@ -0,0 +1,88 @@ +package dropbox + +import ( + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/model" + "go.uber.org/zap" +) + +type UserInfo struct { + AccountID string `json:"account_id"` + Name struct { + GivenName string `json:"given_name"` + Surname string `json:"surname"` + FamiliarName string `json:"familiar_name"` + DisplayName string `json:"display_name"` + AbbreviatedName string `json:"abbreviated_name"` + } `json:"name"` + Email string `json:"email"` + EmailVerified bool `json:"email_verified"` + Disabled bool `json:"disabled"` + Country string `json:"country"` + Locale string `json:"locale"` + ReferralLink string `json:"referral_link"` + IsPaired bool `json:"is_paired"` + AccountType struct { + Tag string `json:".tag"` + } `json:"account_type"` + RootInfo struct { + Tag string `json:".tag"` + RootNamespaceID string `json:"root_namespace_id"` + HomeNamespaceID string `json:"home_namespace_id"` + } `json:"root_info"` +} +type TokenError struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} +type File struct { + Tag string `json:".tag"` + Name string `json:"name"` + PathLower string `json:"path_lower"` + PathDisplay string `json:"path_display"` + ID string `json:"id"` + ClientModified time.Time `json:"client_modified,omitempty"` + ServerModified time.Time `json:"server_modified,omitempty"` + Rev string `json:"rev,omitempty"` + Size int `json:"size,omitempty"` + IsDownloadable bool `json:"is_downloadable,omitempty"` + ContentHash string `json:"content_hash,omitempty"` +} + +type Files struct { + Files []File `json:"entries"` + Cursor string `json:"cursor"` + HasMore bool `json:"has_more"` +} + +type Error struct { + Error struct { + Errors []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + } + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} + +func fileToObj(f File) *model.ObjThumb { + logger.Info("dropbox file", zap.Any("file", f)) + obj := &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + Name: f.Name, + Size: int64(f.Size), + Modified: f.ClientModified, + IsFolder: f.Tag == "folder", + Path: f.PathDisplay, + }, + Thumbnail: model.Thumbnail{}, + } + return obj +} diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go new file mode 100644 index 0000000..da33e17 --- /dev/null +++ b/drivers/dropbox/util.go @@ -0,0 +1,102 @@ +package dropbox + +import ( + "fmt" + "net/http" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/go-resty/resty/v2" + "go.uber.org/zap" +) + +func (d *Dropbox) getRefreshToken() error { + url := "https://api.dropbox.com/oauth2/token" + var resp base.TokenResp + var e TokenError + + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "code": d.Code, + "grant_type": "authorization_code", + "redirect_uri": "https://test-get.casaos.io", + }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.RefreshToken = resp.RefreshToken + return nil + +} +func (d *Dropbox) refreshToken() error { + url := "https://api.dropbox.com/oauth2/token" + var resp base.TokenResp + var e TokenError + + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "refresh_token": d.RefreshToken, + "grant_type": "refresh_token", + }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.AccessToken = resp.AccessToken + return nil + +} +func (d *Dropbox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeader("Content-Type", "application/json") + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Error + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } + return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + return res.Body(), nil +} +func (d *Dropbox) getFiles(path string) ([]File, error) { + + res := make([]File, 0) + var resp Files + body := base.Json{ + "limit": 2000, + "path": path, + } + + _, err := d.request("https://api.dropboxapi.com/2/files/list_folder", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp) + if err != nil { + return nil, err + } + res = append(res, resp.Files...) + + return res, nil +} diff --git a/drivers/google_drive/drive.go b/drivers/google_drive/drive.go index 0ad5cf0..9bad0c2 100644 --- a/drivers/google_drive/drive.go +++ b/drivers/google_drive/drive.go @@ -7,11 +7,13 @@ import ( "net/http" "strconv" + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS/drivers/base" "github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/pkg/utils" "github.com/go-resty/resty/v2" + "go.uber.org/zap" ) type GoogleDrive struct { @@ -33,7 +35,7 @@ func (d *GoogleDrive) Init(ctx context.Context) error { d.ChunkSize = 5 } if len(d.RefreshToken) == 0 { - return d.getRefreshToken() + d.getRefreshToken() } return d.refreshToken() } @@ -59,13 +61,24 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA return nil, err } link := model.Link{ - URL: url + "&alt=media", + Method: http.MethodGet, + URL: url + "&alt=media", Header: http.Header{ "Authorization": []string{"Bearer " + d.AccessToken}, }, } return &link, nil } +func (d *GoogleDrive) GetUserInfo(ctx context.Context) (string, error) { + url := "https://content.googleapis.com/drive/v3/about?fields=user" + user := UserInfo{} + resp, err := d.request(url, http.MethodGet, nil, &user) + if err != nil { + return "", err + } + logger.Info("resp", zap.Any("resp", resp)) + return user.User.EmailAddress, nil +} func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ diff --git a/drivers/google_drive/meta.go b/drivers/google_drive/meta.go index a617af9..16e63fb 100644 --- a/drivers/google_drive/meta.go +++ b/drivers/google_drive/meta.go @@ -5,17 +5,19 @@ import ( "github.com/IceWhaleTech/CasaOS/internal/op" ) +const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png" + type Addition struct { driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` - ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com"` - ClientSecret string `json:"client_secret" required:"true" default:"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"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5" help:"chunk size while uploading (unit: MB)"` + RefreshToken string `json:"refresh_token" required:"true" omit:"true"` + OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime" omit:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"` + ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com" omit:"true"` + ClientSecret string `json:"client_secret" required:"true" default:"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" omit:"true"` + ChunkSize int64 `json:"chunk_size" type:"number" help:"chunk size while uploading (unit: MB)" omit:"true"` AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Ftest-get.casaos.io&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"` Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"` - Code string `json:"code" type:"string" help:"code from auth_url"` + Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"` } var config = driver.Config{ diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go index 27aaf24..4bbab52 100644 --- a/drivers/google_drive/types.go +++ b/drivers/google_drive/types.go @@ -8,6 +8,17 @@ import ( log "github.com/sirupsen/logrus" ) +type UserInfo struct { + User struct { + Kind string `json:"kind"` + DisplayName string `json:"displayName"` + PhotoLink string `json:"photoLink"` + Me bool `json:"me"` + PermissionID string `json:"permissionId"` + EmailAddress string `json:"emailAddress"` + } `json:"user"` +} + type TokenError struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` diff --git a/go.mod b/go.mod index e851839..0b36533 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 + github.com/Unknwon/goconfig v1.0.0 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/deckarep/golang-set/v2 v2.1.0 @@ -81,6 +82,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect + github.com/smartystreets/goconvey v1.7.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect diff --git a/go.sum b/go.sum index e0440dd..acc3718 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 h1:jQfIty6u06fPJCutpS+97qr8uho3RpQX+B/CwHPCv/Q= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3/go.mod h1:xcemiRsXcs1zrmQxYMyExDjZ7UHYwkJqYE71IDIV0xA= +github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= +github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= @@ -119,6 +121,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googollee/go-socket.io v1.6.2 h1:olKLLHJtHz1IkL/OrTyNriZZvVQYEORNkJAqsOwPask= github.com/googollee/go-socket.io v1.6.2/go.mod h1:0vGP8/dXR9SZUMMD4+xxaGo/lohOw3YWMh2WRiWeKxg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -134,6 +138,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -198,6 +204,10 @@ github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3 github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -313,6 +323,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 1b99c0c..b33bf80 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -9,6 +9,7 @@ import ( type Driver interface { Meta Reader + User //Writer //Other } @@ -37,7 +38,10 @@ type Reader interface { // Link get url/filepath/reader of file Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) } - +type User interface { + // GetRoot get root directory of user + GetUserInfo(ctx context.Context) (string, error) +} type Getter interface { GetRoot(ctx context.Context) (model.Obj, error) } diff --git a/internal/driver/item.go b/internal/driver/item.go index 4d3142a..6dff49d 100644 --- a/internal/driver/item.go +++ b/internal/driver/item.go @@ -40,7 +40,7 @@ type RootPath struct { } type RootID struct { - RootFolderID string `json:"root_folder_id"` + RootFolderID string `json:"root_folder_id" omit:"true"` } func (r RootPath) GetRootPath() string { diff --git a/internal/op/driver.go b/internal/op/driver.go index 305cb9e..ba75641 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -13,7 +13,7 @@ import ( type New func() driver.Driver var driverNewMap = map[string]New{} -var driverInfoMap = map[string]driver.Info{} +var driverInfoMap = map[string][]driver.Item{} //driver.Info{} func RegisterDriver(driver New) { // log.Infof("register driver: [%s]", config.Name) @@ -39,23 +39,26 @@ func GetDriverNames() []string { return driverNames } -func GetDriverInfoMap() map[string]driver.Info { +// func GetDriverInfoMap() map[string]driver.Info { +// return driverInfoMap +// } +func GetDriverInfoMap() map[string][]driver.Item { return driverInfoMap } - func registerDriverItems(config driver.Config, addition driver.Additional) { // log.Debugf("addition of %s: %+v", config.Name, addition) tAddition := reflect.TypeOf(addition) for tAddition.Kind() == reflect.Pointer { tAddition = tAddition.Elem() } - mainItems := getMainItems(config) + //mainItems := getMainItems(config) additionalItems := getAdditionalItems(tAddition, config.DefaultRoot) - driverInfoMap[config.Name] = driver.Info{ - Common: mainItems, - Additional: additionalItems, - Config: config, - } + driverInfoMap[config.Name] = additionalItems + // driver.Info{ + // Common: mainItems, + // Additional: additionalItems, + // Config: config, + // } } func getMainItems(config driver.Config) []driver.Item { @@ -128,6 +131,7 @@ func getMainItems(config driver.Config) []driver.Item { func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { var items []driver.Item for i := 0; i < t.NumField(); i++ { + field := t.Field(i) if field.Type.Kind() == reflect.Struct { items = append(items, getAdditionalItems(field.Type, defaultRoot)...) @@ -139,6 +143,9 @@ func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { if (ok1 && ignore == "true") || !ok2 { continue } + if tag.Get("omit") == "true" { + continue + } item := driver.Item{ Name: name, Type: strings.ToLower(field.Type.Name()), diff --git a/internal/op/fs.go b/internal/op/fs.go new file mode 100644 index 0000000..576a4b6 --- /dev/null +++ b/internal/op/fs.go @@ -0,0 +1,545 @@ +package op + +import ( + "context" + "os" + stdpath "path" + "time" + + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/generic_sync" + "github.com/IceWhaleTech/CasaOS/pkg/singleflight" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/Xhofe/go-cache" + "github.com/pkg/errors" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// In order to facilitate adding some other things before and after file op + +var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) +var listG singleflight.Group[[]model.Obj] + +func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, obj := range objs { + if obj.GetName() == oldObj.GetName() { + objs[i] = newObj + break + } + } + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +func delCacheObj(storage driver.Driver, path string, obj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, oldObj := range objs { + if oldObj.GetName() == obj.GetName() { + objs = append(objs[:i], objs[i+1:]...) + break + } + } + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +var addSortDebounceMap generic_sync.MapOf[string, func(func())] + +func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { + key := Key(storage, path) + objs, ok := listCache.Get(key) + if ok { + for i, obj := range objs { + if obj.GetName() == newObj.GetName() { + objs[i] = newObj + return + } + } + + // Simple separation of files and folders + if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { + objs = append(objs, newObj) + } else { + objs = append([]model.Obj{newObj}, objs...) + } + + if storage.Config().LocalSort { + debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) + log.Debug("addCacheObj: wait start sort") + debounce(func() { + log.Debug("addCacheObj: start sort") + model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + addSortDebounceMap.Delete(key) + }) + } + + listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } +} + +func ClearCache(storage driver.Driver, path string) { + listCache.Del(Key(storage, path)) +} + +func Key(storage driver.Driver, path string) string { + return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) +} + +// List files in storage, not contains virtual file +func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + log.Debugf("op.List %s", path) + key := Key(storage, path) + if !utils.IsBool(refresh...) { + if files, ok := listCache.Get(key); ok { + log.Debugf("use cache when list %s", path) + return files, nil + } + } + dir, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } + log.Debugf("list dir: %+v", dir) + if !dir.IsDir() { + return nil, errors.WithStack(errors.New("not a folder")) + } + objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + files, err := storage.List(ctx, dir, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to list objs") + } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { + s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) + // call hooks + go func(reqPath string, files []model.Obj) { + for _, hook := range ObjsUpdateHooks { + hook(args.ReqPath, files) + } + }(args.ReqPath, files) + + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + + if !storage.Config().NoCache { + if len(files) > 0 { + log.Debugf("set cache: %s => %+v", key, files) + listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } else { + log.Debugf("del cache: %s", key) + listCache.Del(key) + } + } + return files, nil + }) + return objs, err +} + +// Get object from list of files +func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + path = utils.FixAndCleanPath(path) + log.Debugf("op.Get %s", path) + + // is root folder + if utils.PathEqual(path, "/") { + var rootObj model.Obj + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + Path: path, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + default: + if storage, ok := storage.(driver.Getter); ok { + obj, err := storage.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + rootObj = obj + } + } + if rootObj == nil { + return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method") + } + return &model.ObjWrapName{ + Name: RootName, + Obj: rootObj, + }, nil + } + + // not root folder + dir, name := stdpath.Split(path) + files, err := List(ctx, storage, dir, model.ListArgs{}) + if err != nil { + return nil, errors.WithMessage(err, "failed get parent list") + } + for _, f := range files { + // TODO maybe copy obj here + if f.GetName() == name { + return f, nil + } + } + log.Debugf("cant find obj with name: %s", name) + return nil, errors.WithStack(errors.New("object not found")) +} + +func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + obj, err := Get(ctx, storage, path) + if err != nil { + return nil, err + } + return model.UnwrapObjs(obj), err +} + +var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) +var linkG singleflight.Group[*model.Link] + +// Link get link, if is an url. should have an expiry time +func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + file, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if file.IsDir() { + return nil, nil, errors.WithStack(errors.New("not a file")) + } + key := Key(storage, path) + ":" + args.IP + if link, ok := linkCache.Get(key); ok { + return link, file, nil + } + fn := func() (*model.Link, error) { + link, err := storage.Link(ctx, file, args) + if err != nil { + return nil, errors.Wrapf(err, "failed get link") + } + if link.Expiration != nil { + linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration)) + } + return link, nil + } + link, err, _ := linkG.Do(key, fn) + return link, file, err +} + +// Other api +func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { + obj, err := GetUnwrap(ctx, storage, args.Path) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get obj") + } + if o, ok := storage.(driver.Other); ok { + return o.Other(ctx, model.OtherArgs{ + Obj: obj, + Method: args.Method, + Data: args.Data, + }) + } else { + return nil, errors.New("not implement") + } +} + +var mkdirG singleflight.Group[interface{}] + +func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + key := Key(storage, path) + _, err, _ := mkdirG.Do(key, func() (interface{}, error) { + // check if dir exists + f, err := GetUnwrap(ctx, storage, path) + if err != nil { + if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { + parentPath, dirName := stdpath.Split(path) + err = MakeDir(ctx, storage, parentPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) + } + parentDir, err := GetUnwrap(ctx, storage, parentPath) + // this should not happen + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } + + switch s := storage.(type) { + case driver.MkdirResult: + var newObj model.Obj + newObj, err = s.MakeDir(ctx, parentDir, dirName) + if err == nil { + if newObj != nil { + addCacheObj(storage, parentPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, parentPath) + } + } + case driver.Mkdir: + err = s.MakeDir(ctx, parentDir, dirName) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, parentPath) + } + default: + return nil, errors.New("not implement") + } + return nil, errors.WithStack(err) + } + return nil, errors.WithMessage(err, "failed to check if dir exists") + } + // dir exists + if f.IsDir() { + return nil, nil + } + // dir to make is a file + return nil, errors.New("file exists") + }) + return err +} + +func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcRawObj, err := Get(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + srcObj := model.UnwrapObjs(srcRawObj) + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + srcDirPath := stdpath.Dir(srcPath) + + switch s := storage.(type) { + case driver.MoveResult: + var newObj model.Obj + newObj, err = s.Move(ctx, srcObj, dstDir) + if err == nil { + delCacheObj(storage, srcDirPath, srcRawObj) + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Move: + err = s.Move(ctx, srcObj, dstDir) + if err == nil { + delCacheObj(storage, srcDirPath, srcRawObj) + if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + srcRawObj, err := Get(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + srcObj := model.UnwrapObjs(srcRawObj) + srcDirPath := stdpath.Dir(srcPath) + + switch s := storage.(type) { + case driver.RenameResult: + var newObj model.Obj + newObj, err = s.Rename(ctx, srcObj, dstName) + if err == nil { + if newObj != nil { + updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, srcDirPath) + } + } + case driver.Rename: + err = s.Rename(ctx, srcObj, dstName) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, srcDirPath) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +// Copy Just copy file[s] in a storage +func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + + switch s := storage.(type) { + case driver.CopyResult: + var newObj model.Obj + newObj, err = s.Copy(ctx, srcObj, dstDir) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Copy: + err = s.Copy(ctx, srcObj, dstDir) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Remove(ctx context.Context, storage driver.Driver, path string) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + rawObj, err := Get(ctx, storage, path) + if err != nil { + // if object not found, it's ok + if errors.Is(pkgerr.Cause(err), errors.New("object not found")) { + return nil + } + return errors.WithMessage(err, "failed to get object") + } + dirPath := stdpath.Dir(path) + + switch s := storage.(type) { + case driver.Remove: + err = s.Remove(ctx, model.UnwrapObjs(rawObj)) + if err == nil { + delCacheObj(storage, dirPath, rawObj) + } + default: + return errors.New("not implement") + } + return errors.WithStack(err) +} + +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + defer func() { + if f, ok := file.GetReadCloser().(*os.File); ok { + err := os.RemoveAll(f.Name()) + if err != nil { + log.Errorf("failed to remove file [%s]", f.Name()) + } + } + }() + defer func() { + if err := file.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + // if file exist and size = 0, delete it + dstDirPath = utils.FixAndCleanPath(dstDirPath) + dstPath := stdpath.Join(dstDirPath, file.GetName()) + fi, err := GetUnwrap(ctx, storage, dstPath) + if err == nil { + if fi.GetSize() == 0 { + err = Remove(ctx, storage, dstPath) + if err != nil { + return errors.WithMessagef(err, "failed remove file that exist and have size 0") + } + } else { + file.Old = fi + } + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) + } + parentDir, err := GetUnwrap(ctx, storage, dstDirPath) + // this should not happen + if err != nil { + return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + // if up is nil, set a default to prevent panic + if up == nil { + up = func(p int) {} + } + + switch s := storage.(type) { + case driver.PutResult: + var newObj model.Obj + newObj, err = s.Put(ctx, parentDir, file, up) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.Put: + err = s.Put(ctx, parentDir, file, up) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errors.New("not implement") + } + log.Debugf("put file [%s] done", file.GetName()) + //if err == nil { + // //clear cache + // key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath) + // listCache.Del(key) + //} + return errors.WithStack(err) +} diff --git a/internal/sign/sign.go b/internal/sign/sign.go new file mode 100644 index 0000000..412bdfe --- /dev/null +++ b/internal/sign/sign.go @@ -0,0 +1,36 @@ +package sign + +import ( + "sync" + "time" + + "github.com/IceWhaleTech/CasaOS/pkg/sign" +) + +var once sync.Once +var instance sign.Sign + +func Sign(data string) string { + + return NotExpired(data) + +} + +func WithDuration(data string, d time.Duration) string { + once.Do(Instance) + return instance.Sign(data, time.Now().Add(d).Unix()) +} + +func NotExpired(data string) string { + once.Do(Instance) + return instance.Sign(data, 0) +} + +func Verify(data string, sign string) error { + once.Do(Instance) + return instance.Verify(data, sign) +} + +func Instance() { + instance = sign.NewHMACSign([]byte("token")) +} diff --git a/main.go b/main.go index c63ee45..986e392 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS/pkg/cache" "github.com/IceWhaleTech/CasaOS/pkg/config" + "github.com/IceWhaleTech/CasaOS/pkg/config/configfile" "github.com/IceWhaleTech/CasaOS/pkg/sqlite" "github.com/IceWhaleTech/CasaOS/pkg/utils/command" "github.com/IceWhaleTech/CasaOS/pkg/utils/file" @@ -69,6 +70,20 @@ func init() { service.MyService.Storages().InitStorages() route.InitFunction() + data := &configfile.Storage{} + e := data.Load() + fmt.Println(e) + fmt.Println(data.GetSectionList()) + // fmt.Println(data.HasSection("google")) + // fmt.Println(data.GetKeyList("google")) + // fmt.Println(data.GetValue("google", "token")) + // data.SetValue("google", "type", "drive") + // data.SetValue("google", "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com") + // data.SetValue("google", "client_secret", "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") + // data.SetValue("google", "scope", "drive") + // data.SetValue("google", "token", `{"access_token":"ya29.a0AVvZVsqsy3vWjpjsl87mtxirrtkHpkyEXdvlORzZeIahObdEtDE47-Hzo1bIg8vJhfYKh-cdqgrUM305hiEJssFMcpkM-0IwPyxlpynMFWS0L5356AUvbv3DUd_RbV_MbKijyTThuDkfrXdLIiEOwxMOtYSXmDUaCgYKAbgSAQASFQGbdwaI6ae1NZbJARogHtpjitLGkg0166","token_type":"Bearer","refresh_token":"1//01CoIJ-aZDrUPCgYIARAAGAESNwF-L9IrNLyzp1Xzfa_sPPMouyrTgJrVchPX6uXqMizXjohTdycCpVgVcu402ND-Ikn2hArRGXA","expiry":"2023-01-28T19:26:50.198064816+08:00"}`) + //e = data.Save() + //fmt.Println(e) } // @title casaOS API diff --git a/model/args.go b/model/args.go index b365547..4d83455 100644 --- a/model/args.go +++ b/model/args.go @@ -23,6 +23,7 @@ type Link struct { Status int // status maybe 200 or 206, etc FilePath *string // local file, return the filepath Expiration *time.Duration // url expiration time + Method string `json:"method"` // http method } type OtherArgs struct { diff --git a/model/smartctl_model.go b/model/smartctl_model.go deleted file mode 100644 index ec7f75c..0000000 --- a/model/smartctl_model.go +++ /dev/null @@ -1,69 +0,0 @@ -package model - -// -type SmartctlA struct { - Smartctl struct { - Version []int `json:"version"` - SvnRevision string `json:"svn_revision"` - PlatformInfo string `json:"platform_info"` - BuildInfo string `json:"build_info"` - Argv []string `json:"argv"` - ExitStatus int `json:"exit_status"` - } `json:"smartctl"` - Device struct { - Name string `json:"name"` - InfoName string `json:"info_name"` - Type string `json:"type"` - Protocol string `json:"protocol"` - } `json:"device"` - ModelName string `json:"model_name"` - SerialNumber string `json:"serial_number"` - FirmwareVersion string `json:"firmware_version"` - UserCapacity struct { - Blocks int `json:"blocks"` - Bytes int64 `json:"bytes"` - } `json:"user_capacity"` - SmartStatus struct { - Passed bool `json:"passed"` - } `json:"smart_status"` - AtaSmartData struct { - OfflineDataCollection struct { - Status struct { - Value int `json:"value"` - String string `json:"string"` - } `json:"status"` - CompletionSeconds int `json:"completion_seconds"` - } `json:"offline_data_collection"` - SelfTest struct { - Status struct { - Value int `json:"value"` - String string `json:"string"` - Passed bool `json:"passed"` - } `json:"status"` - PollingMinutes struct { - Short int `json:"short"` - Extended int `json:"extended"` - Conveyance int `json:"conveyance"` - } `json:"polling_minutes"` - } `json:"self_test"` - Capabilities struct { - Values []int `json:"values"` - ExecOfflineImmediateSupported bool `json:"exec_offline_immediate_supported"` - OfflineIsAbortedUponNewCmd bool `json:"offline_is_aborted_upon_new_cmd"` - OfflineSurfaceScanSupported bool `json:"offline_surface_scan_supported"` - SelfTestsSupported bool `json:"self_tests_supported"` - ConveyanceSelfTestSupported bool `json:"conveyance_self_test_supported"` - SelectiveSelfTestSupported bool `json:"selective_self_test_supported"` - AttributeAutosaveEnabled bool `json:"attribute_autosave_enabled"` - ErrorLoggingSupported bool `json:"error_logging_supported"` - GpLoggingSupported bool `json:"gp_logging_supported"` - } `json:"capabilities"` - } `json:"ata_smart_data"` - PowerOnTime struct { - Hours int `json:"hours"` - } `json:"power_on_time"` - PowerCycleCount int `json:"power_cycle_count"` - Temperature struct { - Current int `json:"current"` - } `json:"temperature"` -} diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go new file mode 100644 index 0000000..ff2dee9 --- /dev/null +++ b/pkg/fs/fs.go @@ -0,0 +1,12 @@ +package fs + +import "io" + +// CheckClose is a utility function used to check the return from +// Close in a defer statement. +func CheckClose(c io.Closer, err *error) { + cerr := c.Close() + if *err == nil { + *err = cerr + } +} diff --git a/pkg/sign/hmac.go b/pkg/sign/hmac.go new file mode 100644 index 0000000..8d7f736 --- /dev/null +++ b/pkg/sign/hmac.go @@ -0,0 +1,52 @@ +package sign + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "io" + "strconv" + "strings" + "time" +) + +type HMACSign struct { + SecretKey []byte +} + +func (s HMACSign) Sign(data string, expire int64) string { + h := hmac.New(sha256.New, s.SecretKey) + expireTimeStamp := strconv.FormatInt(expire, 10) + _, err := io.WriteString(h, data+":"+expireTimeStamp) + if err != nil { + return "" + } + + return base64.URLEncoding.EncodeToString(h.Sum(nil)) + ":" + expireTimeStamp +} + +func (s HMACSign) Verify(data, sign string) error { + signSlice := strings.Split(sign, ":") + // check whether contains expire time + if signSlice[len(signSlice)-1] == "" { + return ErrExpireMissing + } + // check whether expire time is expired + expires, err := strconv.ParseInt(signSlice[len(signSlice)-1], 10, 64) + if err != nil { + return ErrExpireInvalid + } + // if expire time is expired, return error + if expires < time.Now().Unix() && expires != 0 { + return ErrSignExpired + } + // verify sign + if s.Sign(data, expires) != sign { + return ErrSignInvalid + } + return nil +} + +func NewHMACSign(secret []byte) Sign { + return HMACSign{SecretKey: secret} +} diff --git a/pkg/sign/sign.go b/pkg/sign/sign.go new file mode 100644 index 0000000..2a28667 --- /dev/null +++ b/pkg/sign/sign.go @@ -0,0 +1,15 @@ +package sign + +import "errors" + +type Sign interface { + Sign(data string, expire int64) string + Verify(data, sign string) error +} + +var ( + ErrSignExpired = errors.New("sign expired") + ErrSignInvalid = errors.New("sign invalid") + ErrExpireInvalid = errors.New("expire invalid") + ErrExpireMissing = errors.New("expire missing") +) diff --git a/pkg/utils/command/command_helper.go b/pkg/utils/command/command_helper.go index 703a8b4..1a6649e 100644 --- a/pkg/utils/command/command_helper.go +++ b/pkg/utils/command/command_helper.go @@ -2,14 +2,12 @@ package command import ( "bufio" - "context" "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "strings" - "time" ) func OnlyExec(cmdStr string) { @@ -98,23 +96,6 @@ func ExecLSBLKByPath(path string) []byte { return output } -// exec smart -func ExecSmartCTLByPath(path string) []byte { - timeout := 3 - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) - defer cancel() - output, err := exec.CommandContext(ctx, "smartctl", "-a", path, "-j").Output() - if err != nil { - fmt.Println("smartctl", err) - return nil - } - return output -} - -func ExecEnabledSMART(path string) { - exec.Command("smartctl", "-s on", path).Output() -} - func ExecuteScripts(scriptDirectory string) { if _, err := os.Stat(scriptDirectory); os.IsNotExist(err) { fmt.Printf("No post-start scripts at %s\n", scriptDirectory) diff --git a/pkg/utils/httper/drive.go b/pkg/utils/httper/drive.go new file mode 100644 index 0000000..b64fd1b --- /dev/null +++ b/pkg/utils/httper/drive.go @@ -0,0 +1,151 @@ +package httper + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "time" + + "github.com/go-resty/resty/v2" +) + +type MountList struct { + MountPoints []struct { + MountPoint string `json:"MountPoint"` + Fs string `json:"Fs"` + Icon string `json:"Icon"` + } `json:"mountPoints"` +} +type MountResult struct { + Error string `json:"error"` + Input struct { + Fs string `json:"fs"` + MountPoint string `json:"mountPoint"` + } `json:"input"` + Path string `json:"path"` + Status int `json:"status"` +} + +type RemotesResult struct { + Remotes []string `json:"remotes"` +} + +var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" +var DefaultTimeout = time.Second * 30 + +func NewRestyClient() *resty.Client { + + unixSocket := "/tmp/rclone.sock" + + transport := http.Transport{ + Dial: func(_, _ string) (net.Conn, error) { + return net.Dial("unix", unixSocket) + }, + } + + client := resty.New() + + client.SetTransport(&transport).SetBaseURL("http://localhost") + client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent) + return client +} + +func GetMountList() (MountList, error) { + var result MountList + res, err := NewRestyClient().R().Post("/mount/listmounts") + if err != nil { + return result, err + } + if res.StatusCode() != 200 { + return result, fmt.Errorf("get mount list failed") + } + json.Unmarshal(res.Body(), &result) + for i := 0; i < len(result.MountPoints); i++ { + result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1] + } + return result, err +} +func Mount(mountPoint string, fs string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "mountPoint": mountPoint, + "fs": fs, + }).Post("/mount/mount") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("mount failed") + } + return nil +} +func Unmount(mountPoint string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "mountPoint": mountPoint, + }).Post("/mount/unmount") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("unmount failed") + } + return nil +} + +func CreateConfig(data map[string]string, name, t string) error { + data["config_is_local"] = "false" + dataStr, _ := json.Marshal(data) + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + "parameters": string(dataStr), + "type": t, + }).Post("/config/create") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("create config failed") + } + return nil +} + +func GetConfigByName(name string) (map[string]string, error) { + + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + }).Post("/config/get") + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, fmt.Errorf("create config failed") + } + var result map[string]string + json.Unmarshal(res.Body(), &result) + return result, nil +} +func GetAllConfigName() (RemotesResult, error) { + var result RemotesResult + res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes") + if err != nil { + return result, err + } + if res.StatusCode() != 200 { + return result, fmt.Errorf("get config failed") + } + + json.Unmarshal(res.Body(), &result) + return result, nil +} +func DeleteConfigByName(name string) error { + res, err := NewRestyClient().R().SetFormData(map[string]string{ + "name": name, + }).Post("/config/delete") + if err != nil { + return err + } + if res.StatusCode() != 200 { + return fmt.Errorf("delete config failed") + } + return nil +} diff --git a/pkg/utils/time.go b/pkg/utils/time.go new file mode 100644 index 0000000..41632e4 --- /dev/null +++ b/pkg/utils/time.go @@ -0,0 +1,37 @@ +package utils + +import ( + "sync" + "time" +) + +func MustParseCNTime(str string) time.Time { + lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local) + return lastOpTime +} + +func NewDebounce(interval time.Duration) func(f func()) { + var timer *time.Timer + var lock sync.Mutex + return func(f func()) { + lock.Lock() + defer lock.Unlock() + if timer != nil { + timer.Stop() + } + timer = time.AfterFunc(interval, f) + } +} + +func NewDebounce2(interval time.Duration, f func()) func() { + var timer *time.Timer + var lock sync.Mutex + return func() { + lock.Lock() + defer lock.Unlock() + if timer == nil { + timer = time.AfterFunc(interval, f) + } + (*time.Timer)(timer).Reset(interval) + } +} diff --git a/route/init.go b/route/init.go index 4c55e1a..0baa719 100644 --- a/route/init.go +++ b/route/init.go @@ -89,4 +89,9 @@ func InitNetworkMount() { connection.Directories = strings.Join(directories, ",") service.MyService.Connections().UpdateConnection(&connection) } + + err := service.MyService.Storage().CheckAndMountAll() + if err != nil { + logger.Error("mount storage err", zap.Any("err", err)) + } } diff --git a/route/route.go b/route/route.go index aa6a476..9123cfe 100644 --- a/route/route.go +++ b/route/route.go @@ -118,14 +118,8 @@ func InitRouter() *gin.Engine { v1StorageGroup.Use() { v1StorageGroup.GET("", v1.ListStorages) - v1StorageGroup.POST("", v1.CreateStorage) v1StorageGroup.DELETE("", v1.DeleteStorage) } - v1FsGroup := v1Group.Group("/fs") - v1FsGroup.Use() - { - v1FsGroup.POST("/list", v1.FsList) - } v1DriverGroup := v1Group.Group("/driver") v1DriverGroup.Use() { diff --git a/route/v1/file.go b/route/v1/file.go index 42d0ec8..2646147 100644 --- a/route/v1/file.go +++ b/route/v1/file.go @@ -1,6 +1,7 @@ package v1 import ( + "errors" "fmt" "io" "io/ioutil" @@ -16,10 +17,16 @@ import ( "sync" "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/conf" + "github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/model" + + "github.com/IceWhaleTech/CasaOS/pkg/utils" "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" "github.com/IceWhaleTech/CasaOS/pkg/utils/file" "github.com/IceWhaleTech/CasaOS/service" + + "github.com/IceWhaleTech/CasaOS/internal/sign" "github.com/gin-gonic/gin" uuid "github.com/satori/go.uuid" "go.uber.org/zap" @@ -190,6 +197,37 @@ func GetDownloadSingleFile(c *gin.Context) { }) return } + fileName := path.Base(filePath) + // c.Header("Content-Disposition", "inline") + c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName)) + + storage, _ := service.MyService.FsService().GetStorage(filePath) + if storage != nil { + if shouldProxy(storage, fileName) { + Proxy(c) + return + } else { + link, _, err := service.MyService.FsService().Link(c, filePath, model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + }) + if err != nil { + c.JSON(common_err.SERVICE_ERROR, model.Result{ + Success: common_err.SERVICE_ERROR, + Message: common_err.GetMsg(common_err.SERVICE_ERROR), + Data: err.Error(), + }) + return + + } + c.Header("Referrer-Policy", "no-referrer") + c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + c.Redirect(302, link.URL) + return + } + } + fileTmp, err := os.Open(filePath) if err != nil { c.JSON(common_err.SERVICE_ERROR, model.Result{ @@ -200,9 +238,6 @@ func GetDownloadSingleFile(c *gin.Context) { } defer fileTmp.Close() - fileName := path.Base(filePath) - // c.Header("Content-Disposition", "inline") - c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName)) c.File(filePath) } @@ -701,3 +736,168 @@ func GetSize(c *gin.Context) { } c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: size}) } +func Proxy(c *gin.Context) { + rawPath := c.Query("path") + filename := filepath.Base(rawPath) + storage, err := service.MyService.FsService().GetStorage(rawPath) + if err != nil { + c.JSON(500, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + if canProxy(storage, filename) { + downProxyUrl := storage.GetStorage().DownProxyUrl + if downProxyUrl != "" { + _, ok := c.GetQuery("d") + if !ok { + URL := fmt.Sprintf("%s%s?sign=%s", + strings.Split(downProxyUrl, "\n")[0], + utils.EncodePath(rawPath, true), + sign.Sign(rawPath)) + c.Redirect(302, URL) + return + } + } + link, file, err := service.MyService.FsService().Link(c, rawPath, model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + }) + if err != nil { + c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + + return + } + err = CommonProxy(c.Writer, c.Request, link, file) + if err != nil { + c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + } else { + c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: "proxy not allowed"}) + return + } +} + +// TODO need optimize +// when should be proxy? +// 1. config.MustProxy() +// 2. storage.WebProxy +// 3. proxy_types +func shouldProxy(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { + return true + } + return false +} + +// TODO need optimize +// when can be proxy? +// 1. text file +// 2. config.MustProxy() +// 3. storage.WebProxy +// 4. proxy_types +// solution: text_file + shouldProxy() +func canProxy(storage driver.Driver, filename string) bool { + if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { + return true + } + if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) { + return true + } + return false +} + +var HttpClient = &http.Client{} + +func CommonProxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { + // read data with native + var err error + if link.Data != nil { + defer func() { + _ = link.Data.Close() + }() + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName()))) + w.Header().Set("Content-Length", strconv.FormatInt(file.GetSize(), 10)) + if link.Header != nil { + // TODO clean header with blacklist or whitelist + link.Header.Del("set-cookie") + for h, val := range link.Header { + w.Header()[h] = val + } + } + if link.Status == 0 { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(link.Status) + } + _, err = io.Copy(w, link.Data) + if err != nil { + return err + } + return nil + } + // local file + if link.FilePath != nil && *link.FilePath != "" { + f, err := os.Open(*link.FilePath) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + fileStat, err := os.Stat(*link.FilePath) + if err != nil { + return err + } + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName()))) + http.ServeContent(w, r, file.GetName(), fileStat.ModTime(), f) + return nil + } else { + req, err := http.NewRequest(link.Method, link.URL, nil) + if err != nil { + return err + } + for h, val := range r.Header { + if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) { + continue + } + req.Header[h] = val + } + for h, val := range link.Header { + req.Header[h] = val + } + res, err := HttpClient.Do(req) + if err != nil { + return err + } + defer func() { + _ = res.Body.Close() + }() + logger.Info("proxy status", zap.Any("status", res.StatusCode)) + // TODO clean header with blacklist or whitelist + res.Header.Del("set-cookie") + for h, v := range res.Header { + w.Header()[h] = v + } + w.WriteHeader(res.StatusCode) + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName()))) + if res.StatusCode >= 400 { + all, _ := ioutil.ReadAll(res.Body) + msg := string(all) + logger.Info("msg", zap.Any("msg", msg)) + + return errors.New(msg) + } + _, err = io.Copy(w, res.Body) + if err != nil { + return err + } + return nil + } +} diff --git a/route/v1/recover.go b/route/v1/recover.go index df93493..76affcf 100644 --- a/route/v1/recover.go +++ b/route/v1/recover.go @@ -2,41 +2,34 @@ package v1 import ( "strconv" + "strings" "time" - "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/dropbox" "github.com/IceWhaleTech/CasaOS/drivers/google_drive" "github.com/IceWhaleTech/CasaOS/internal/op" - "github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/service" "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "go.uber.org/zap" ) func GetRecoverStorage(c *gin.Context) { c.Header("Content-Type", "text/html; charset=utf-8") t := c.Param("type") + currentTime := time.Now().UTC() + currentDate := time.Now().UTC().Format("2006-01-02") + // timeStr := time.Now().Format("20060102150405") if t == "GoogleDrive" { - mountPath := "google" - - mountPath += time.Now().Format("20060102150405") - gd := op.GetDriverInfoMap()[t] - var req model.Storage - req.Driver = t - req.MountPath = mountPath - req.CacheExpiration = 5 add := google_drive.Addition{} add.Code = c.Query("code") if len(add.Code) == 0 { - c.String(200, `

code不可为空

`) + c.String(200, `

code cannot be empty

`) return } add.RootFolderID = "root" - for _, v := range gd.Additional { + for _, v := range gd { if v.Name == "client_id" { add.ClientID = v.Default } @@ -52,22 +45,119 @@ func GetRecoverStorage(c *gin.Context) { } } - var json = jsoniter.ConfigCompatibleWithStandardLibrary - addStr, err := json.Marshal(add) + var google_drive google_drive.GoogleDrive + google_drive.Addition = add + err := google_drive.Init(c) if err != nil { - c.String(200, `

addition序列化失败

`) + c.String(200, `

Initialization failure:`+err.Error()+`

`) return } - req.Addition = string(addStr) - logger.Info("GetRecoverStorage", zap.Any("req", req)) - if _, err := service.MyService.Storages().CreateStorage(c, req); err != nil { - c.String(200, `

添加失败:`+err.Error()+`

`) + + username, err := google_drive.GetUserInfo(c) + if err != nil { + c.String(200, `

Failed to get user information:`+err.Error()+`

`) return } - data := make(map[string]interface{}) - data["status"] = "success" - service.MyService.Notify().SendNotify("recover_status", data) + if len(username) > 0 { + a := strings.Split(username, "@") + username = a[0] + } + username += "_drive" + dataMap, _ := service.MyService.Storage().GetConfigByName(username) + if len(dataMap) > 0 { + c.String(200, `

The same configuration has been added

`) + service.MyService.Storage().CheckAndMountByName(username) + return + } + dmap := make(map[string]string) + dmap["client_id"] = add.ClientID + dmap["client_secret"] = add.ClientSecret + dmap["scope"] = "drive" + dmap["mount_point"] = "/mnt/" + username + dmap["token"] = `{"access_token":"` + google_drive.AccessToken + `","token_type":"Bearer","refresh_token":"` + google_drive.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*1).Format("15:04:05") + `Z"}` + // data.SetValue(username, "type", "drive") + // data.SetValue(username, "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com") + // data.SetValue(username, "client_secret", "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") + // data.SetValue(username, "scope", "drive") + // data.SetValue(username, "mount_point", "/mnt/"+username) + // data.SetValue(username, "token", `{"access_token":"`+google_drive.AccessToken+`","token_type":"Bearer","refresh_token":"`+google_drive.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*1).Format("15:04:05")+`Z"}`) + // e = data.Save() + // if e != nil { + // c.String(200, `

保存配置失败:`+e.Error()+`

`) + // return + // } + service.MyService.Storage().CreateConfig(dmap, username, "drive") + service.MyService.Storage().MountStorage("/mnt/"+username, username+":") + notify := make(map[string]interface{}) + notify["status"] = "success" + service.MyService.Notify().SendNotify("recover_status", notify) + } else if t == "Dropbox" { + + //mountPath += timeStr + + db := op.GetDriverInfoMap()[t] + + add := dropbox.Addition{} + add.Code = c.Query("code") + if len(add.Code) == 0 { + c.String(200, `

code cannot be empty

`) + return + } + add.RootFolderID = "" + for _, v := range db { + if v.Name == "app_key" { + add.AppKey = v.Default + } + if v.Name == "app_secret" { + add.AppSecret = v.Default + } + } + var dropbox dropbox.Dropbox + dropbox.Addition = add + err := dropbox.Init(c) + if err != nil { + c.String(200, `

Initialization failure:`+err.Error()+`

`) + return + } + username, err := dropbox.GetUserInfo(c) + if err != nil { + c.String(200, `

Failed to get user information:`+err.Error()+`

`) + return + } + if len(username) > 0 { + a := strings.Split(username, "@") + username = a[0] + } + username += "_dropbox" + dataMap, _ := service.MyService.Storage().GetConfigByName(username) + if len(dataMap) > 0 { + c.String(200, `

The same configuration has been added

`) + service.MyService.Storage().CheckAndMountByName(username) + return + } + dmap := make(map[string]string) + dmap["client_id"] = add.AppKey + dmap["client_secret"] = add.AppSecret + dmap["token"] = `{"access_token":"` + dropbox.AccessToken + `","token_type":"bearer","refresh_token":"` + dropbox.Addition.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*3).Format("15:04:05") + `.780385354Z"}` + dmap["mount_point"] = "/mnt/" + username + // data.SetValue(username, "type", "dropbox") + // data.SetValue(username, "client_id", add.AppKey) + // data.SetValue(username, "client_secret", add.AppSecret) + // data.SetValue(username, "mount_point", "/mnt/"+username) + + // data.SetValue(username, "token", `{"access_token":"`+dropbox.AccessToken+`","token_type":"bearer","refresh_token":"`+dropbox.Addition.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*3).Format("15:04:05")+`.780385354Z"}`) + // e = data.Save() + // if e != nil { + // c.String(200, `

保存配置失败:`+e.Error()+`

`) + + // return + // } + service.MyService.Storage().CreateConfig(dmap, username, "dropbox") + service.MyService.Storage().MountStorage("/mnt/"+username, username+":") + notify := make(map[string]interface{}) + notify["status"] = "success" + service.MyService.Notify().SendNotify("recover_status", notify) } - c.String(200, `

关闭该页面即可

`) + c.String(200, `

Just close the page

`) } diff --git a/route/v1/samba.go b/route/v1/samba.go index bf3cdac..ce14f1b 100644 --- a/route/v1/samba.go +++ b/route/v1/samba.go @@ -12,6 +12,7 @@ package v1 import ( "fmt" + "io/ioutil" "os" "path/filepath" "regexp" @@ -195,7 +196,10 @@ func DeleteSambaConnections(c *gin.Context) { for _, v := range mountPointList { service.MyService.Connections().UnmountSmaba(v.Path) } - os.RemoveAll(connection.MountPoint) + dir, _ := ioutil.ReadDir(connection.MountPoint) + if len(dir) == 0 { + os.RemoveAll(connection.MountPoint) + } service.MyService.Connections().DeleteConnection(id) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: id}) } diff --git a/route/v1/storage.go b/route/v1/storage.go index 12e7d84..698a1f0 100644 --- a/route/v1/storage.go +++ b/route/v1/storage.go @@ -2,8 +2,11 @@ package v1 import ( "strconv" + "strings" "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/dropbox" + "github.com/IceWhaleTech/CasaOS/drivers/google_drive" "github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" "github.com/IceWhaleTech/CasaOS/service" @@ -12,42 +15,45 @@ import ( ) func ListStorages(c *gin.Context) { - var req model.PageReq - if err := c.ShouldBind(&req); err != nil { - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) - return - } - req.Validate() + // var req model.PageReq + // if err := c.ShouldBind(&req); err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + // return + // } + // req.Validate() + + //logger.Info("ListStorages", zap.Any("req", req)) + //storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + // return + // } + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{ + // Content: storages, + // Total: total, + // }}) + r, err := service.MyService.Storage().GetStorages() - logger.Info("ListStorages", zap.Any("req", req)) - storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage) if err != nil { c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) return } - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{ - Content: storages, - Total: total, - }}) -} -func CreateStorage(c *gin.Context) { - var req model.Storage - if err := c.ShouldBind(&req); err != nil { - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) - return + for i := 0; i < len(r.MountPoints); i++ { + dataMap, err := service.MyService.Storage().GetConfigByName(r.MountPoints[i].Fs) + if err != nil { + logger.Error("GetConfigByName", zap.Any("err", err)) + continue + } + if dataMap["type"] == "drive" { + r.MountPoints[i].Icon = google_drive.ICONURL + } + if dataMap["type"] == "dropbox" { + r.MountPoints[i].Icon = dropbox.ICONURL + } } - if id, err := service.MyService.Storages().CreateStorage(c, req); err != nil { - data := make(map[string]interface{}) - data["id"] = id - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: data}) - return - } else { - data := make(map[string]interface{}) - data["id"] = id - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: data}) - } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: r}) } func UpdateStorage(c *gin.Context) { @@ -64,16 +70,19 @@ func UpdateStorage(c *gin.Context) { } func DeleteStorage(c *gin.Context) { - idStr := c.Query("id") - id, err := strconv.Atoi(idStr) - if err != nil { - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + json := make(map[string]string) + c.ShouldBind(&json) + mountPoint := json["mount_point"] + if mountPoint == "" { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: "mount_point is empty"}) return } - if err := service.MyService.Storages().DeleteStorageById(c, uint(id)); err != nil { + err := service.MyService.Storage().UnmountStorage(mountPoint) + if err != nil { c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) return } + service.MyService.Storage().DeleteConfigByName(strings.ReplaceAll(mountPoint, "/mnt/", "")) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) } @@ -106,16 +115,17 @@ func EnableStorage(c *gin.Context) { } func GetStorage(c *gin.Context) { - idStr := c.Query("id") - id, err := strconv.Atoi(idStr) - if err != nil { - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) - return - } - storage, err := service.MyService.Storage().GetStorageById(uint(id)) - if err != nil { - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) - return - } - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage}) + + // idStr := c.Query("id") + // id, err := strconv.Atoi(idStr) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + // return + // } + // storage, err := service.MyService.Storage().GetStorageById(uint(id)) + // if err != nil { + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + // return + // } + // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage}) } diff --git a/service/fs.go b/service/fs.go index efd23e0..d4dc218 100644 --- a/service/fs.go +++ b/service/fs.go @@ -6,12 +6,14 @@ import ( "github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/model" + log "github.com/dsoprea/go-logging" "go.uber.org/zap" ) type FsService interface { FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) GetStorage(path string) (driver.Driver, error) + Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) } type fsService struct { @@ -39,14 +41,14 @@ func (f *fsService) FList(ctx context.Context, path string, refresh ...bool) ([] // return res, nil // } -// func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { -// res, file, err := link(ctx, path, args) -// if err != nil { -// log.Errorf("failed link %s: %+v", path, err) -// return nil, nil, err -// } -// return res, file, nil -// } +func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { + res, file, err := MyService.FsLinkService().Link(ctx, path, args) + if err != nil { + log.Errorf("failed link %s: %+v", path, err) + return nil, nil, err + } + return res, file, nil +} // func (f *fsService) MakeDir(ctx context.Context, path string, lazyCache ...bool) error { // err := makeDir(ctx, path, lazyCache...) diff --git a/service/fs_link.go b/service/fs_link.go new file mode 100644 index 0000000..bf4c013 --- /dev/null +++ b/service/fs_link.go @@ -0,0 +1,27 @@ +package service + +import ( + "context" + + "github.com/IceWhaleTech/CasaOS/internal/op" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/pkg/errors" +) + +type FsLinkService interface { + Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) +} + +type fsLinkService struct { +} + +func (f *fsLinkService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { + storage, actualPath, err := MyService.StoragePath().GetStorageAndActualPath(path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed get storage") + } + return op.Link(ctx, storage, actualPath, args) +} +func NewFsLinkService() FsLinkService { + return &fsLinkService{} +} diff --git a/service/service.go b/service/service.go index 2d505a4..5c08b53 100644 --- a/service/service.go +++ b/service/service.go @@ -42,6 +42,7 @@ type Repository interface { Storages() StoragesService StoragePath() StoragePathService FsListService() FsListService + FsLinkService() FsLinkService FsService() FsService } @@ -63,10 +64,11 @@ func NewService(db *gorm.DB, RuntimePath string, socket *socketio.Server) Reposi system: NewSystemService(), shares: NewSharesService(db), connections: NewConnectionsService(db), - storage: NewStorageService(db), + storage: NewStorageService(), storages: NewStoragesService(), storage_path: NewStoragePathService(), fs_list: NewFsListService(), + fs_link: NewFsLinkService(), fs: NewFsService(), } } @@ -84,9 +86,13 @@ type store struct { storages StoragesService storage_path StoragePathService fs_list FsListService + fs_link FsLinkService fs FsService } +func (c *store) FsLinkService() FsLinkService { + return c.fs_link +} func (c *store) FsService() FsService { return c.fs } diff --git a/service/storage.go b/service/storage.go index 9ef53e3..cf347f1 100644 --- a/service/storage.go +++ b/service/storage.go @@ -1,73 +1,100 @@ package service import ( - "fmt" + "io/ioutil" - "github.com/IceWhaleTech/CasaOS/model" - "github.com/pkg/errors" - "gorm.io/gorm" + "github.com/IceWhaleTech/CasaOS/pkg/utils/file" + "github.com/IceWhaleTech/CasaOS/pkg/utils/httper" ) type StorageService interface { - CreateStorage(storage *model.Storage) error - UpdateStorage(storage *model.Storage) error - DeleteStorageById(id uint) error - GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) - GetStorageById(id uint) (*model.Storage, error) - GetEnabledStorages() ([]model.Storage, error) + MountStorage(mountPoint, fs string) error + UnmountStorage(mountPoint string) error + GetStorages() (httper.MountList, error) + CreateConfig(data map[string]string, name string, t string) error + CheckAndMountByName(name string) error + CheckAndMountAll() error + GetConfigByName(name string) (map[string]string, error) + DeleteConfigByName(name string) error } type storageStruct struct { - db *gorm.DB } -// CreateStorage just insert storage to database -func (s *storageStruct) CreateStorage(storage *model.Storage) error { - return errors.WithStack(s.db.Create(storage).Error) +func (s *storageStruct) MountStorage(mountPoint, fs string) error { + file.IsNotExistMkDir(mountPoint) + httper.Mount(mountPoint, fs) + return nil } +func (s *storageStruct) UnmountStorage(mountPoint string) error { + err := httper.Unmount(mountPoint) + if err == nil { + dir, _ := ioutil.ReadDir(mountPoint) -// UpdateStorage just update storage in database -func (s *storageStruct) UpdateStorage(storage *model.Storage) error { - return errors.WithStack(s.db.Save(storage).Error) -} - -// DeleteStorageById just delete storage from database by id -func (s *storageStruct) DeleteStorageById(id uint) error { - return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error) -} - -// GetStorages Get all storages from database order by index -func (s *storageStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { - storageDB := s.db.Model(&model.Storage{}) - var count int64 - if err := storageDB.Count(&count).Error; err != nil { - return nil, 0, errors.Wrapf(err, "failed get storages count") + if len(dir) == 0 { + file.RMDir(mountPoint) + } + return nil } - var storages []model.Storage - if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { - return nil, 0, errors.WithStack(err) + return err +} +func (s *storageStruct) GetStorages() (httper.MountList, error) { + return httper.GetMountList() +} +func (s *storageStruct) CreateConfig(data map[string]string, name string, t string) error { + httper.CreateConfig(data, name, t) + return nil +} +func (s *storageStruct) CheckAndMountByName(name string) error { + storages, _ := MyService.Storage().GetStorages() + currentRemote, _ := httper.GetConfigByName(name) + mountPoint := currentRemote["mount_point"] + isMount := false + for _, v := range storages.MountPoints { + if v.MountPoint == mountPoint { + isMount = true + break + } } - return storages, count, nil -} - -// GetStorageById Get Storage by id, used to update storage usually -func (s *storageStruct) GetStorageById(id uint) (*model.Storage, error) { - var storage model.Storage - storage.ID = id - if err := s.db.First(&storage).Error; err != nil { - return nil, errors.WithStack(err) + if !isMount { + MyService.Storage().MountStorage(mountPoint, name+":") } - return &storage, nil + return nil } - -func (s *storageStruct) GetEnabledStorages() ([]model.Storage, error) { - var storages []model.Storage - if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil { - return nil, errors.WithStack(err) +func (s *storageStruct) CheckAndMountAll() error { + storages, err := MyService.Storage().GetStorages() + if err != nil { + return err } - return storages, nil + section, err := httper.GetAllConfigName() + if err != nil { + return err + } + for _, v := range section.Remotes { + currentRemote, _ := httper.GetConfigByName(v) + mountPoint := currentRemote["mount_point"] + if len(mountPoint) == 0 { + continue + } + isMount := false + for _, v := range storages.MountPoints { + if v.MountPoint == mountPoint { + isMount = true + break + } + } + if !isMount { + return MyService.Storage().MountStorage(mountPoint, v+":") + } + } + return nil } - -func NewStorageService(db *gorm.DB) StorageService { - return &storageStruct{db: db} +func (s *storageStruct) GetConfigByName(name string) (map[string]string, error) { + return httper.GetConfigByName(name) +} +func (s *storageStruct) DeleteConfigByName(name string) error { + return httper.DeleteConfigByName(name) +} +func NewStorageService() StorageService { + return &storageStruct{} } diff --git a/service/storage_old.go b/service/storage_old.go new file mode 100644 index 0000000..d5a928a --- /dev/null +++ b/service/storage_old.go @@ -0,0 +1,73 @@ +package service + +import ( + "fmt" + + "github.com/IceWhaleTech/CasaOS/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +type StorageOldService interface { + CreateStorage(storage *model.Storage) error + UpdateStorage(storage *model.Storage) error + DeleteStorageById(id uint) error + GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) + GetStorageById(id uint) (*model.Storage, error) + GetEnabledStorages() ([]model.Storage, error) +} + +type storageOldStruct struct { + db *gorm.DB +} + +// CreateStorage just insert storage to database +func (s *storageOldStruct) CreateStorage(storage *model.Storage) error { + return errors.WithStack(s.db.Create(storage).Error) +} + +// UpdateStorage just update storage in database +func (s *storageOldStruct) UpdateStorage(storage *model.Storage) error { + return errors.WithStack(s.db.Save(storage).Error) +} + +// DeleteStorageById just delete storage from database by id +func (s *storageOldStruct) DeleteStorageById(id uint) error { + return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error) +} + +// GetStorages Get all storages from database order by index +func (s *storageOldStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { + storageDB := s.db.Model(&model.Storage{}) + var count int64 + if err := storageDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get storages count") + } + var storages []model.Storage + if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return storages, count, nil +} + +// GetStorageById Get Storage by id, used to update storage usually +func (s *storageOldStruct) GetStorageById(id uint) (*model.Storage, error) { + var storage model.Storage + storage.ID = id + if err := s.db.First(&storage).Error; err != nil { + return nil, errors.WithStack(err) + } + return &storage, nil +} + +func (s *storageOldStruct) GetEnabledStorages() ([]model.Storage, error) { + var storages []model.Storage + if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil { + return nil, errors.WithStack(err) + } + return storages, nil +} + +func NewStorageOldService(db *gorm.DB) StorageOldService { + return &storageOldStruct{db: db} +} diff --git a/service/storage_service.go b/service/storage_service.go index 89ed619..30cfeed 100644 --- a/service/storage_service.go +++ b/service/storage_service.go @@ -15,7 +15,6 @@ import ( "github.com/IceWhaleTech/CasaOS/model" - "github.com/IceWhaleTech/CasaOS/internal/conf" "github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/internal/op" mapset "github.com/deckarep/golang-set/v2" @@ -30,9 +29,9 @@ type StoragesService interface { DisableStorage(ctx context.Context, id uint) error UpdateStorage(ctx context.Context, storage model.Storage) error DeleteStorageById(ctx context.Context, id uint) error - MustSaveDriverStorage(driver driver.Driver) + MustSaveDriverStorage(driver driver.Driver) error GetStorageVirtualFilesByPath(prefix string) []model.Obj - initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) + initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error) InitStorages() GetBalancedStorage(path string) driver.Driver } @@ -75,18 +74,28 @@ func (s *storagesStruct) CreateStorage(ctx context.Context, storage model.Storag return 0, errors.WithMessage(err, "failed get driver new") } storageDriver := driverNew() - // insert storage to database - err = MyService.Storage().CreateStorage(&storage) - if err != nil { - return storage.ID, errors.WithMessage(err, "failed create storage in database") - } + // // insert storage to database + // err = MyService.Storage().CreateStorage(&storage) + // if err != nil { + + // return storage.ID, errors.WithMessage(err, "failed create storage in database") + // } // already has an id - err = s.initStorage(ctx, storage, storageDriver) + err = s.initStorage(ctx, storage, storageDriver, func(d driver.Driver, ctx context.Context) string { + u, _ := d.GetUserInfo(ctx) + if len(u) > 0 { + a := strings.Split(u, "@") + u = a[0] + } + return u + }) + if err != nil { + s.DeleteStorageById(ctx, storage.ID) + return storage.ID, errors.Wrap(err, "failed init storage") + } + go op.CallStorageHooks("add", storageDriver) - if err != nil { - return storage.ID, errors.Wrap(err, "failed init storage but storage is already created") - } logger.Error("storage created", zap.Any("storage", storageDriver)) return storage.ID, nil } @@ -102,14 +111,14 @@ func (s *storagesStruct) LoadStorage(ctx context.Context, storage model.Storage) } storageDriver := driverNew() - err = s.initStorage(ctx, storage, storageDriver) + err = s.initStorage(ctx, storage, storageDriver, nil) go op.CallStorageHooks("add", storageDriver) logger.Info("storage created", zap.Any("storage", storageDriver)) return err } // initStorage initialize the driver and store to storagesMap -func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { +func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error) { storageDriver.SetStorage(storage) driverStorage := storageDriver.GetStorage() @@ -121,61 +130,72 @@ func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, if err == nil { err = storageDriver.Init(ctx) } + if setMountPath != nil { + driverStorage.MountPath += "_" + setMountPath(storageDriver, ctx) + + } + if s.HasStorage(driverStorage.MountPath) { + return errors.New("mount path already exists") + } + storageDriver.SetStorage(*driverStorage) storagesMap.Store(driverStorage.MountPath, storageDriver) + if err != nil { driverStorage.SetStatus(err.Error()) err = errors.Wrap(err, "failed init storage") } else { driverStorage.SetStatus(op.WORK) } - s.MustSaveDriverStorage(storageDriver) + + err = s.MustSaveDriverStorage(storageDriver) + return err } func (s *storagesStruct) EnableStorage(ctx context.Context, id uint) error { - storage, err := MyService.Storage().GetStorageById(id) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - if !storage.Disabled { - return errors.Errorf("this storage have enabled") - } - storage.Disabled = false - err = MyService.Storage().UpdateStorage(storage) - if err != nil { - return errors.WithMessage(err, "failed update storage in db") - } - err = s.LoadStorage(ctx, *storage) - if err != nil { - return errors.WithMessage(err, "failed load storage") - } + // storage, err := MyService.Storage().GetStorageById(id) + // if err != nil { + // return errors.WithMessage(err, "failed get storage") + // } + // if !storage.Disabled { + // return errors.Errorf("this storage have enabled") + // } + // storage.Disabled = false + // err = MyService.Storage().UpdateStorage(storage) + // if err != nil { + // return errors.WithMessage(err, "failed update storage in db") + // } + // err = s.LoadStorage(ctx, *storage) + // if err != nil { + // return errors.WithMessage(err, "failed load storage") + // } return nil } func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error { - storage, err := MyService.Storage().GetStorageById(id) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - if storage.Disabled { - return errors.Errorf("this storage have disabled") - } - storageDriver, err := GetStorageByMountPath(storage.MountPath) - if err != nil { - return errors.WithMessage(err, "failed get storage driver") - } - // drop the storage in the driver - if err := storageDriver.Drop(ctx); err != nil { - return errors.Wrap(err, "failed drop storage") - } - // delete the storage in the memory - storage.Disabled = true - err = MyService.Storage().UpdateStorage(storage) - if err != nil { - return errors.WithMessage(err, "failed update storage in db") - } - storagesMap.Delete(storage.MountPath) - go op.CallStorageHooks("del", storageDriver) + // storage, err := MyService.Storage().GetStorageById(id) + // if err != nil { + // return errors.WithMessage(err, "failed get storage") + // } + // if storage.Disabled { + // return errors.Errorf("this storage have disabled") + // } + // storageDriver, err := GetStorageByMountPath(storage.MountPath) + // if err != nil { + // return errors.WithMessage(err, "failed get storage driver") + // } + // // drop the storage in the driver + // if err := storageDriver.Drop(ctx); err != nil { + // return errors.Wrap(err, "failed drop storage") + // } + // // delete the storage in the memory + // storage.Disabled = true + // err = MyService.Storage().UpdateStorage(storage) + // if err != nil { + // return errors.WithMessage(err, "failed update storage in db") + // } + // storagesMap.Delete(storage.MountPath) + // go op.CallStorageHooks("del", storageDriver) return nil } @@ -183,90 +203,92 @@ func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error { // get old storage first // drop the storage then reinitialize func (s *storagesStruct) UpdateStorage(ctx context.Context, storage model.Storage) error { - oldStorage, err := MyService.Storage().GetStorageById(storage.ID) - if err != nil { - return errors.WithMessage(err, "failed get old storage") - } - if oldStorage.Driver != storage.Driver { - return errors.Errorf("driver cannot be changed") - } - storage.Modified = time.Now() - storage.MountPath = utils.FixAndCleanPath(storage.MountPath) - err = MyService.Storage().UpdateStorage(&storage) - if err != nil { - return errors.WithMessage(err, "failed update storage in database") - } - if storage.Disabled { - return nil - } - storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) - if oldStorage.MountPath != storage.MountPath { - // mount path renamed, need to drop the storage - storagesMap.Delete(oldStorage.MountPath) - } - if err != nil { - return errors.WithMessage(err, "failed get storage driver") - } - err = storageDriver.Drop(ctx) - if err != nil { - return errors.Wrapf(err, "failed drop storage") - } + // oldStorage, err := MyService.Storage().GetStorageById(storage.ID) + // if err != nil { + // return errors.WithMessage(err, "failed get old storage") + // } + // if oldStorage.Driver != storage.Driver { + // return errors.Errorf("driver cannot be changed") + // } + // storage.Modified = time.Now() + // storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + // err = MyService.Storage().UpdateStorage(&storage) + // if err != nil { + // return errors.WithMessage(err, "failed update storage in database") + // } + // if storage.Disabled { + // return nil + // } + // storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + // if oldStorage.MountPath != storage.MountPath { + // // mount path renamed, need to drop the storage + // storagesMap.Delete(oldStorage.MountPath) + // } + // if err != nil { + // return errors.WithMessage(err, "failed get storage driver") + // } + // err = storageDriver.Drop(ctx) + // if err != nil { + // return errors.Wrapf(err, "failed drop storage") + // } - err = s.initStorage(ctx, storage, storageDriver) - go op.CallStorageHooks("update", storageDriver) + // err = s.initStorage(ctx, storage, storageDriver, nil) + // go op.CallStorageHooks("update", storageDriver) - logger.Info("storage updated", zap.Any("storage", storageDriver)) - return err + // logger.Info("storage updated", zap.Any("storage", storageDriver)) + //return err + return nil } func (s *storagesStruct) DeleteStorageById(ctx context.Context, id uint) error { - storage, err := MyService.Storage().GetStorageById(id) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - if !storage.Disabled { - storageDriver, err := GetStorageByMountPath(storage.MountPath) - if err != nil { - return errors.WithMessage(err, "failed get storage driver") - } - // drop the storage in the driver - if err := storageDriver.Drop(ctx); err != nil { - return errors.Wrapf(err, "failed drop storage") - } - // delete the storage in the memory - storagesMap.Delete(storage.MountPath) - go op.CallStorageHooks("del", storageDriver) - } - // delete the storage in the database - if err := MyService.Storage().DeleteStorageById(id); err != nil { - return errors.WithMessage(err, "failed delete storage in database") - } + // storage, err := MyService.Storage().GetStorageById(id) + // if err != nil { + // return errors.WithMessage(err, "failed get storage") + // } + // if !storage.Disabled { + // storageDriver, err := GetStorageByMountPath(storage.MountPath) + // if err == nil { + // // drop the storage in the driver + // if err := storageDriver.Drop(ctx); err != nil { + // return errors.Wrapf(err, "failed drop storage") + // } + // // delete the storage in the memory + // storagesMap.Delete(storage.MountPath) + // } + + // go op.CallStorageHooks("del", storageDriver) + // } + // // delete the storage in the database + // if err := MyService.Storage().DeleteStorageById(id); err != nil { + // return errors.WithMessage(err, "failed delete storage in database") + // } return nil } // MustSaveDriverStorage call from specific driver -func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) { +func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) error { err := saveDriverStorage(driver) if err != nil { logger.Error("failed save driver storage", zap.Any("err", err)) } + return err } func saveDriverStorage(driver driver.Driver) error { - storage := driver.GetStorage() - addition := driver.GetAddition() + // storage := driver.GetStorage() + // addition := driver.GetAddition() - var json = jsoniter.ConfigCompatibleWithStandardLibrary + // var json = jsoniter.ConfigCompatibleWithStandardLibrary - str, err := json.MarshalToString(addition) - if err != nil { - return errors.Wrap(err, "error while marshal addition") - } - storage.Addition = str - err = MyService.Storage().UpdateStorage(storage) - if err != nil { - return errors.WithMessage(err, "failed update storage in database") - } + // str, err := json.MarshalToString(addition) + // if err != nil { + // return errors.Wrap(err, "error while marshal addition") + // } + // storage.Addition = str + // err = MyService.Storage().UpdateStorage(storage) + // if err != nil { + // return errors.WithMessage(err, "failed update storage in database") + // } return nil } @@ -354,21 +376,21 @@ func (s *storagesStruct) GetBalancedStorage(path string) driver.Driver { } } func (s *storagesStruct) InitStorages() { - storages, err := MyService.Storage().GetEnabledStorages() - if err != nil { - logger.Error("failed get enabled storages", zap.Any("err", err)) - } - go func(storages []model.Storage) { - for i := range storages { - err := s.LoadStorage(context.Background(), storages[i]) - if err != nil { - logger.Error("failed get enabled storages", zap.Any("err", err)) - } else { - logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath)) - } - } - conf.StoragesLoaded = true - }(storages) + // storages, err := MyService.Storage().GetEnabledStorages() + // if err != nil { + // logger.Error("failed get enabled storages", zap.Any("err", err)) + // } + // go func(storages []model.Storage) { + // for i := range storages { + // err := s.LoadStorage(context.Background(), storages[i]) + // if err != nil { + // logger.Error("failed get enabled storages", zap.Any("err", err)) + // } else { + // logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath)) + // } + // } + // conf.StoragesLoaded = true + // }(storages) } func NewStoragesService() StoragesService {