From 87d8be8c61ab3d44d69c59b66ccb0c3e90c5fe87 Mon Sep 17 00:00:00 2001 From: link Date: Mon, 16 Jan 2023 05:54:44 +0000 Subject: [PATCH] added google drive --- drivers/all.go | 11 + drivers/base/client.go | 30 +++ drivers/base/types.go | 12 + drivers/google_drive/drive.go | 170 +++++++++++++ drivers/google_drive/meta.go | 31 +++ drivers/google_drive/types.go | 66 +++++ drivers/google_drive/util.go | 152 ++++++++++++ go.mod | 84 ++++++- go.sum | 64 ++--- internal/conf/config.go | 43 ++++ internal/conf/const.go | 72 ++++++ internal/conf/var.go | 30 +++ internal/driver/config.go | 25 ++ internal/driver/driver.go | 127 ++++++++++ internal/driver/item.go | 56 +++++ internal/op/const.go | 6 + internal/op/driver.go | 166 +++++++++++++ internal/op/hook.go | 109 ++++++++ main.go | 4 +- model/args.go | 38 +++ model/common.go | 6 + model/obj.go | 186 ++++++++++++++ model/object.go | 90 +++++++ model/req.go | 20 ++ model/setting.go | 33 +++ model/storage.go | 54 ++++ model/stream.go | 33 +++ pkg/generic_sync/generic_sync.go | 412 +++++++++++++++++++++++++++++++ pkg/singleflight/singleflight.go | 212 ++++++++++++++++ pkg/sqlite/db.go | 3 +- pkg/utils/balance.go | 18 ++ pkg/utils/bool.go | 5 + pkg/utils/ctx.go | 14 ++ pkg/utils/path.go | 81 ++++++ pkg/utils/slice.go | 46 ++++ route/route.go | 19 ++ route/v1/driver.go | 12 + route/v1/file.go | 53 +++- route/v1/file_read.go | 93 +++++++ route/v1/recover.go | 73 ++++++ route/v1/storage.go | 121 +++++++++ service/fs.go | 152 ++++++++++++ service/fs_list.go | 198 +++++++++++++++ service/service.go | 65 +++-- service/storage.go | 73 ++++++ service/storage_path.go | 34 +++ service/storage_service.go | 376 ++++++++++++++++++++++++++++ 47 files changed, 3711 insertions(+), 67 deletions(-) create mode 100644 drivers/all.go create mode 100644 drivers/base/client.go create mode 100644 drivers/base/types.go create mode 100644 drivers/google_drive/drive.go create mode 100644 drivers/google_drive/meta.go create mode 100644 drivers/google_drive/types.go create mode 100644 drivers/google_drive/util.go create mode 100644 internal/conf/config.go create mode 100644 internal/conf/const.go create mode 100644 internal/conf/var.go create mode 100644 internal/driver/config.go create mode 100644 internal/driver/driver.go create mode 100644 internal/driver/item.go create mode 100644 internal/op/const.go create mode 100644 internal/op/driver.go create mode 100644 internal/op/hook.go create mode 100644 model/args.go create mode 100644 model/common.go create mode 100644 model/obj.go create mode 100644 model/object.go create mode 100644 model/req.go create mode 100644 model/setting.go create mode 100644 model/storage.go create mode 100644 model/stream.go create mode 100644 pkg/generic_sync/generic_sync.go create mode 100644 pkg/singleflight/singleflight.go create mode 100644 pkg/utils/balance.go create mode 100644 pkg/utils/bool.go create mode 100644 pkg/utils/ctx.go create mode 100644 pkg/utils/path.go create mode 100644 pkg/utils/slice.go create mode 100644 route/v1/driver.go create mode 100644 route/v1/file_read.go create mode 100644 route/v1/recover.go create mode 100644 route/v1/storage.go create mode 100644 service/fs.go create mode 100644 service/fs_list.go create mode 100644 service/storage.go create mode 100644 service/storage_path.go create mode 100644 service/storage_service.go diff --git a/drivers/all.go b/drivers/all.go new file mode 100644 index 0000000..d27e61a --- /dev/null +++ b/drivers/all.go @@ -0,0 +1,11 @@ +package drivers + +import ( + _ "github.com/IceWhaleTech/CasaOS/drivers/google_drive" +) + +// All do nothing,just for import +// same as _ import +func All() { + +} diff --git a/drivers/base/client.go b/drivers/base/client.go new file mode 100644 index 0000000..02e314b --- /dev/null +++ b/drivers/base/client.go @@ -0,0 +1,30 @@ +package base + +import ( + "net/http" + "time" + + "github.com/go-resty/resty/v2" +) + +var NoRedirectClient *resty.Client +var RestyClient = NewRestyClient() +var HttpClient = &http.Client{} +var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" +var DefaultTimeout = time.Second * 30 + +func init() { + NoRedirectClient = resty.New().SetRedirectPolicy( + resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }), + ) + NoRedirectClient.SetHeader("user-agent", UserAgent) +} + +func NewRestyClient() *resty.Client { + return resty.New(). + SetHeader("user-agent", UserAgent). + SetRetryCount(3). + SetTimeout(DefaultTimeout) +} diff --git a/drivers/base/types.go b/drivers/base/types.go new file mode 100644 index 0000000..e2757f2 --- /dev/null +++ b/drivers/base/types.go @@ -0,0 +1,12 @@ +package base + +import "github.com/go-resty/resty/v2" + +type Json map[string]interface{} + +type TokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` +} + +type ReqCallback func(req *resty.Request) diff --git a/drivers/google_drive/drive.go b/drivers/google_drive/drive.go new file mode 100644 index 0000000..0ad5cf0 --- /dev/null +++ b/drivers/google_drive/drive.go @@ -0,0 +1,170 @@ +package google_drive + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type GoogleDrive struct { + model.Storage + Addition + AccessToken string +} + +func (d *GoogleDrive) Config() driver.Config { + return config +} + +func (d *GoogleDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GoogleDrive) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = 5 + } + if len(d.RefreshToken) == 0 { + return d.getRefreshToken() + } + return d.refreshToken() +} + +func (d *GoogleDrive) Drop(ctx context.Context) error { + return nil +} + +func (d *GoogleDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?includeItemsFromAllDrives=true&supportsAllDrives=true", file.GetID()) + _, err := d.request(url, http.MethodGet, nil, nil) + if err != nil { + return nil, err + } + link := model.Link{ + URL: url + "&alt=media", + Header: http.Header{ + "Authorization": []string{"Bearer " + d.AccessToken}, + }, + } + return &link, nil +} + +func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + data := base.Json{ + "name": dirName, + "parents": []string{parentDir.GetID()}, + "mimeType": "application/vnd.google-apps.folder", + } + _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return err +} + +func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + query := map[string]string{ + "addParents": dstDir.GetID(), + "removeParents": "root", + } + url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() + _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + return err +} + +func (d *GoogleDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + data := base.Json{ + "name": newName, + } + url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() + _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return err +} + +func (d *GoogleDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errors.New("not support") +} + +func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { + url := "https://www.googleapis.com/drive/v3/files/" + obj.GetID() + _, err := d.request(url, http.MethodDelete, nil, nil) + return err +} + +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + obj := stream.GetOld() + var ( + e Error + url string + data base.Json + res *resty.Response + err error + ) + if obj != nil { + url = fmt.Sprintf("https://www.googleapis.com/upload/drive/v3/files/%s?uploadType=resumable&supportsAllDrives=true", obj.GetID()) + data = base.Json{} + } else { + data = base.Json{ + "name": stream.GetName(), + "parents": []string{dstDir.GetID()}, + } + url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" + } + req := base.NoRedirectClient.R(). + SetHeaders(map[string]string{ + "Authorization": "Bearer " + d.AccessToken, + "X-Upload-Content-Type": stream.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + }). + SetError(&e).SetBody(data).SetContext(ctx) + if obj != nil { + res, err = req.Patch(url) + } else { + res, err = req.Post(url) + } + if err != nil { + return err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return err + } + return d.Put(ctx, dstDir, stream, up) + } + return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + putUrl := res.Header().Get("location") + if stream.GetSize() < d.ChunkSize*1024*1024 { + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)).SetBody(stream.GetReadCloser()) + }, nil) + } else { + err = d.chunkUpload(ctx, stream, putUrl) + } + return err +} + +var _ driver.Driver = (*GoogleDrive)(nil) diff --git a/drivers/google_drive/meta.go b/drivers/google_drive/meta.go new file mode 100644 index 0000000..a617af9 --- /dev/null +++ b/drivers/google_drive/meta.go @@ -0,0 +1,31 @@ +package google_drive + +import ( + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" +) + +type Addition struct { + driver.RootID + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com"` + ClientSecret string `json:"client_secret" required:"true" default:"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"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5" help:"chunk size while uploading (unit: MB)"` + AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Ftest-get.casaos.io&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"` + Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"` + Code string `json:"code" type:"string" help:"code from auth_url"` +} + +var config = driver.Config{ + Name: "GoogleDrive", + OnlyProxy: true, + DefaultRoot: "root", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GoogleDrive{} + }) +} diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.go new file mode 100644 index 0000000..27aaf24 --- /dev/null +++ b/drivers/google_drive/types.go @@ -0,0 +1,66 @@ +package google_drive + +import ( + "strconv" + "time" + + "github.com/IceWhaleTech/CasaOS/model" + log "github.com/sirupsen/logrus" +) + +type TokenError struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +type Files struct { + NextPageToken string `json:"nextPageToken"` + Files []File `json:"files"` +} + +type File struct { + Id string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mimeType"` + ModifiedTime time.Time `json:"modifiedTime"` + Size string `json:"size"` + ThumbnailLink string `json:"thumbnailLink"` + ShortcutDetails struct { + TargetId string `json:"targetId"` + TargetMimeType string `json:"targetMimeType"` + } `json:"shortcutDetails"` +} + +func fileToObj(f File) *model.ObjThumb { + log.Debugf("google file: %+v", f) + size, _ := strconv.ParseInt(f.Size, 10, 64) + obj := &model.ObjThumb{ + Object: model.Object{ + ID: f.Id, + Name: f.Name, + Size: size, + Modified: f.ModifiedTime, + IsFolder: f.MimeType == "application/vnd.google-apps.folder", + }, + Thumbnail: model.Thumbnail{}, + } + if f.MimeType == "application/vnd.google-apps.shortcut" { + obj.ID = f.ShortcutDetails.TargetId + obj.IsFolder = f.ShortcutDetails.TargetMimeType == "application/vnd.google-apps.folder" + } + return obj +} + +type Error struct { + Error struct { + Errors []struct { + Domain string `json:"domain"` + Reason string `json:"reason"` + Message string `json:"message"` + LocationType string `json:"location_type"` + Location string `json:"location"` + } + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` +} diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go new file mode 100644 index 0000000..6c09608 --- /dev/null +++ b/drivers/google_drive/util.go @@ -0,0 +1,152 @@ +package google_drive + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/base" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "go.uber.org/zap" +) + +// do others that not defined in Driver interface + +func (d *GoogleDrive) getRefreshToken() error { + url := "https://www.googleapis.com/oauth2/v4/token" + var resp base.TokenResp + var e TokenError + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + "code": d.Code, + "grant_type": "authorization_code", + "redirect_uri": "http://test-get.casaos.io", + }).Post(url) + if err != nil { + return err + } + logger.Info("get refresh token", zap.String("res", res.String())) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.RefreshToken = resp.RefreshToken + return nil +} + +func (d *GoogleDrive) refreshToken() error { + url := "https://www.googleapis.com/oauth2/v4/token" + var resp base.TokenResp + var e TokenError + res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). + SetFormData(map[string]string{ + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + "refresh_token": d.RefreshToken, + "grant_type": "refresh_token", + }).Post(url) + if err != nil { + return err + } + log.Debug(res.String()) + if e.Error != "" { + return fmt.Errorf(e.Error) + } + d.AccessToken = resp.AccessToken + return nil +} + +func (d *GoogleDrive) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetQueryParam("includeItemsFromAllDrives", "true") + req.SetQueryParam("supportsAllDrives", "true") + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Error + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + if e.Error.Code != 0 { + if e.Error.Code == 401 { + err = d.refreshToken() + if err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } + return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) + } + return res.Body(), nil +} + +func (d *GoogleDrive) getFiles(id string) ([]File, error) { + pageToken := "first" + res := make([]File, 0) + for pageToken != "" { + if pageToken == "first" { + pageToken = "" + } + var resp Files + orderBy := "folder,name,modifiedTime desc" + if d.OrderBy != "" { + orderBy = d.OrderBy + " " + d.OrderDirection + } + query := map[string]string{ + "orderBy": orderBy, + "fields": "files(id,name,mimeType,size,modifiedTime,thumbnailLink,shortcutDetails),nextPageToken", + "pageSize": "1000", + "q": fmt.Sprintf("'%s' in parents and trashed = false", id), + //"includeItemsFromAllDrives": "true", + //"supportsAllDrives": "true", + "pageToken": pageToken, + } + _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + pageToken = resp.NextPageToken + res = append(res, resp.Files...) + } + return res, nil +} + +func (d *GoogleDrive) chunkUpload(ctx context.Context, stream model.FileStreamer, url string) error { + var defaultChunkSize = d.ChunkSize * 1024 * 1024 + var finish int64 = 0 + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + chunkSize := stream.GetSize() - finish + if chunkSize > defaultChunkSize { + chunkSize = defaultChunkSize + } + _, err := d.request(url, http.MethodPut, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Length": strconv.FormatInt(chunkSize, 10), + "Content-Range": fmt.Sprintf("bytes %d-%d/%d", finish, finish+chunkSize-1, stream.GetSize()), + }).SetBody(io.LimitReader(stream.GetReadCloser(), chunkSize)).SetContext(ctx) + }, nil) + if err != nil { + return err + } + finish += chunkSize + } + return nil +} diff --git a/go.mod b/go.mod index 556d251..e851839 100644 --- a/go.mod +++ b/go.mod @@ -1,36 +1,108 @@ module github.com/IceWhaleTech/CasaOS -go 1.16 +go 1.18 require ( github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf + github.com/deckarep/golang-set/v2 v2.1.0 github.com/disintegration/imaging v1.6.2 github.com/dsoprea/go-exif/v3 v3.0.0-20221012082141-d21ac8e2de85 + github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd github.com/gin-contrib/gzip v0.0.6 github.com/gin-gonic/gin v1.8.2 github.com/glebarez/sqlite v1.6.0 github.com/go-ini/ini v1.67.0 + github.com/go-resty/resty/v2 v2.7.0 github.com/golang/mock v1.6.0 - github.com/golang/snappy v0.0.4 // indirect github.com/gomodule/redigo v1.8.9 github.com/google/go-github/v36 v36.0.0 github.com/googollee/go-socket.io v1.6.2 github.com/gorilla/websocket v1.5.0 github.com/hirochachacha/go-smb2 v1.1.0 - github.com/klauspost/compress v1.15.13 // indirect + github.com/json-iterator/go v1.1.12 + github.com/maruel/natural v1.1.0 github.com/mholt/archiver/v3 v3.5.1 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/pkg/errors v0.9.1 github.com/robfig/cron v1.2.0 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil/v3 v3.22.11 + github.com/sirupsen/logrus v1.9.0 github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.14.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.4.0 + golang.org/x/crypto v0.5.0 golang.org/x/oauth2 v0.3.0 - gorm.io/gorm v1.24.2 + gorm.io/gorm v1.24.3 gotest.tools v2.2.0+incompatible ) + +require ( + github.com/andybalholm/brotli v1.0.1 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/glebarez/go-sqlite v1.20.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.11.1 // indirect + github.com/goccy/go-json v0.9.11 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect + github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.15.13 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.15 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/ugorji/go/codec v1.2.7 // indirect + github.com/ulikunitz/xz v0.5.9 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/image v0.3.0 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.21.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.4.0 // indirect + modernc.org/sqlite v1.20.0 // indirect +) diff --git a/go.sum b/go.sum index f343921..e0440dd 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,12 @@ -cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d h1:62lEBImTxZ83pgzywgDNIrPPuQ+j4ep9QjqrWBn1hrU= github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 h1:jQfIty6u06fPJCutpS+97qr8uho3RpQX+B/CwHPCv/Q= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3/go.mod h1:xcemiRsXcs1zrmQxYMyExDjZ7UHYwkJqYE71IDIV0xA= +github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= +github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= @@ -21,6 +20,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= @@ -37,7 +38,6 @@ github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2 github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8= github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg= github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8= -github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf h1:/w4QxepU4AHh3AuO6/g8y/YIIHH5+aKP3Bj8sg5cqhU= github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8= github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU= github.com/dsoprea/go-utility/v2 v2.0.0-20221003142440-7a1927d49d9d/go.mod h1:LVjRU0RNUuMDqkPTxcALio0LWPFPXxxFCvVGVAwEpFc= @@ -76,6 +76,8 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -83,8 +85,8 @@ github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= @@ -105,7 +107,6 @@ github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs0 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v36 v36.0.0 h1:ndCzM616/oijwufI7nBRa+5eZHLldT+4yIB68ib5ogs= @@ -153,10 +154,11 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= +github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= @@ -173,10 +175,10 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= -github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -196,7 +198,6 @@ github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3 github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -218,7 +219,6 @@ github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+Kd github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= -github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= @@ -235,10 +235,8 @@ github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -248,12 +246,11 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/image v0.3.0 h1:HTDXbdK9bjfSWkPzDJIw89W8CAtfFGduujWs33NLLsg= +golang.org/x/image v0.3.0/go.mod h1:fXd9211C/0VTlYuAcOhW8dY/RtEJqODXOWBDpmYBf+A= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -268,13 +265,12 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= @@ -289,7 +285,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -297,7 +292,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -306,26 +300,22 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -351,21 +341,19 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/gorm v1.24.2 h1:9wR6CFD+G8nOusLdvkZelOEhpJVwwHzpQOUM+REd6U0= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.3 h1:WL2ifUmzR/SLp85CSURAfybcHnGZ+yLSGSxgYXlFBHg= +gorm.io/gorm v1.24.3/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= -gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= diff --git a/internal/conf/config.go b/internal/conf/config.go new file mode 100644 index 0000000..16dabae --- /dev/null +++ b/internal/conf/config.go @@ -0,0 +1,43 @@ +package conf + +type Database struct { + Type string `json:"type" env:"DB_TYPE"` + Host string `json:"host" env:"DB_HOST"` + Port int `json:"port" env:"DB_PORT"` + User string `json:"user" env:"DB_USER"` + Password string `json:"password" env:"DB_PASS"` + Name string `json:"name" env:"DB_NAME"` + DBFile string `json:"db_file" env:"DB_FILE"` + TablePrefix string `json:"table_prefix" env:"DB_TABLE_PREFIX"` + SSLMode string `json:"ssl_mode" env:"DB_SSL_MODE"` +} + +type Scheme struct { + Https bool `json:"https" env:"HTTPS"` + CertFile string `json:"cert_file" env:"CERT_FILE"` + KeyFile string `json:"key_file" env:"KEY_FILE"` +} + +type LogConfig struct { + Enable bool `json:"enable" env:"LOG_ENABLE"` + Name string `json:"name" env:"LOG_NAME"` + MaxSize int `json:"max_size" env:"MAX_SIZE"` + MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` + MaxAge int `json:"max_age" env:"MAX_AGE"` + Compress bool `json:"compress" env:"COMPRESS"` +} + +type Config struct { + Force bool `json:"force" env:"FORCE"` + Address string `json:"address" env:"ADDR"` + Port int `json:"port" env:"PORT"` + SiteURL string `json:"site_url" env:"SITE_URL"` + Cdn string `json:"cdn" env:"CDN"` + JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` + TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` + Database Database `json:"database"` + Scheme Scheme `json:"scheme"` + TempDir string `json:"temp_dir" env:"TEMP_DIR"` + BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` + Log LogConfig `json:"log"` +} diff --git a/internal/conf/const.go b/internal/conf/const.go new file mode 100644 index 0000000..6a75634 --- /dev/null +++ b/internal/conf/const.go @@ -0,0 +1,72 @@ +package conf + +const ( + TypeString = "string" + TypeSelect = "select" + TypeBool = "bool" + TypeText = "text" + TypeNumber = "number" +) + +const ( + // site + VERSION = "version" + ApiUrl = "api_url" + BasePath = "base_path" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + + Logo = "logo" + Favicon = "favicon" + MainColor = "main_color" + + // preview + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + + // global + HideFiles = "hide_files" + CustomizeHead = "customize_head" + CustomizeBody = "customize_body" + LinkExpiration = "link_expiration" + SignAll = "sign_all" + PrivacyRegs = "privacy_regs" + OcrApi = "ocr_api" + FilenameCharMapping = "filename_char_mapping" + + // index + SearchIndex = "search_index" + AutoUpdateIndex = "auto_update_index" + IndexPaths = "index_paths" + IgnorePaths = "ignore_paths" + + // aria2 + Aria2Uri = "aria2_uri" + Aria2Secret = "aria2_secret" + + // single + Token = "token" + IndexProgress = "index_progress" + + //Github + GithubClientId = "github_client_id" + GithubClientSecrets = "github_client_secrets" + GithubLoginEnabled = "github_login_enabled" +) + +const ( + UNKNOWN = iota + FOLDER + //OFFICE + VIDEO + AUDIO + TEXT + IMAGE +) diff --git a/internal/conf/var.go b/internal/conf/var.go new file mode 100644 index 0000000..a91547d --- /dev/null +++ b/internal/conf/var.go @@ -0,0 +1,30 @@ +package conf + +import "regexp" + +var ( + BuiltAt string + GoVersion string + GitAuthor string + GitCommit string + Version string = "dev" + WebVersion string +) + +var ( + Conf *Config +) + +var SlicesMap = make(map[string][]string) +var FilenameCharMap = make(map[string]string) +var PrivacyReg []*regexp.Regexp + +var ( + // StoragesLoaded loaded success if empty + StoragesLoaded = false +) +var ( + RawIndexHtml string + ManageHtml string + IndexHtml string +) diff --git a/internal/driver/config.go b/internal/driver/config.go new file mode 100644 index 0000000..a7758ba --- /dev/null +++ b/internal/driver/config.go @@ -0,0 +1,25 @@ +/* + * @Author: a624669980@163.com a624669980@163.com + * @Date: 2022-12-13 11:05:05 + * @LastEditors: a624669980@163.com a624669980@163.com + * @LastEditTime: 2022-12-13 11:05:13 + * @FilePath: /drive/internal/driver/config.go + * @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + */ +package driver + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` // if need get message from user, such as validate code + DefaultRoot string `json:"default_root"` + CheckStatus bool +} + +func (c Config) MustProxy() bool { + return c.OnlyProxy || c.OnlyLocal +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go new file mode 100644 index 0000000..1b99c0c --- /dev/null +++ b/internal/driver/driver.go @@ -0,0 +1,127 @@ +package driver + +import ( + "context" + + "github.com/IceWhaleTech/CasaOS/model" +) + +type Driver interface { + Meta + Reader + //Writer + //Other +} + +type Meta interface { + Config() Config + // GetStorage just get raw storage, no need to implement, because model.Storage have implemented + GetStorage() *model.Storage + SetStorage(model.Storage) + // GetAddition Additional is used for unmarshal of JSON, so need return pointer + GetAddition() Additional + // Init If already initialized, drop first + Init(ctx context.Context) error + Drop(ctx context.Context) error +} + +type Other interface { + Other(ctx context.Context, args model.OtherArgs) (interface{}, error) +} + +type Reader interface { + // List files in the path + // if identify files by path, need to set ID with path,like path.Join(dir.GetID(), obj.GetName()) + // if identify files by id, need to set ID with corresponding id + List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) + // Link get url/filepath/reader of file + Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) +} + +type Getter interface { + GetRoot(ctx context.Context) (model.Obj, error) +} + +//type Writer interface { +// Mkdir +// Move +// Rename +// Copy +// Remove +// Put +//} + +type Mkdir interface { + MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error +} + +type Move interface { + Move(ctx context.Context, srcObj, dstDir model.Obj) error +} + +type Rename interface { + Rename(ctx context.Context, srcObj model.Obj, newName string) error +} + +type Copy interface { + Copy(ctx context.Context, srcObj, dstDir model.Obj) error +} + +type Remove interface { + Remove(ctx context.Context, obj model.Obj) error +} + +type Put interface { + Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error +} + +//type WriteResult interface { +// MkdirResult +// MoveResult +// RenameResult +// CopyResult +// PutResult +// Remove +//} + +type MkdirResult interface { + MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) +} + +type MoveResult interface { + Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) +} + +type RenameResult interface { + Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) +} + +type CopyResult interface { + Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) +} + +type PutResult interface { + Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) +} + +type UpdateProgress func(percentage int) + +type Progress struct { + Total int64 + Done int64 + up UpdateProgress +} + +func (p *Progress) Write(b []byte) (n int, err error) { + n = len(b) + p.Done += int64(n) + p.up(int(float64(p.Done) / float64(p.Total) * 100)) + return +} + +func NewProgress(total int64, up UpdateProgress) *Progress { + return &Progress{ + Total: total, + up: up, + } +} diff --git a/internal/driver/item.go b/internal/driver/item.go new file mode 100644 index 0000000..4d3142a --- /dev/null +++ b/internal/driver/item.go @@ -0,0 +1,56 @@ +/* + * @Author: a624669980@163.com a624669980@163.com + * @Date: 2022-12-13 11:05:47 + * @LastEditors: a624669980@163.com a624669980@163.com + * @LastEditTime: 2022-12-13 11:05:54 + * @FilePath: /drive/internal/driver/item.go + * @Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE + */ +package driver + +type Additional interface{} + +type Select string + +type Item struct { + Name string `json:"name"` + Type string `json:"type"` + Default string `json:"default"` + Options string `json:"options"` + Required bool `json:"required"` + Help string `json:"help"` +} + +type Info struct { + Common []Item `json:"common"` + Additional []Item `json:"additional"` + Config Config `json:"config"` +} + +type IRootPath interface { + GetRootPath() string +} + +type IRootId interface { + GetRootId() string +} + +type RootPath struct { + RootFolderPath string `json:"root_folder_path"` +} + +type RootID struct { + RootFolderID string `json:"root_folder_id"` +} + +func (r RootPath) GetRootPath() string { + return r.RootFolderPath +} + +func (r *RootPath) SetRootPath(path string) { + r.RootFolderPath = path +} + +func (r RootID) GetRootId() string { + return r.RootFolderID +} diff --git a/internal/op/const.go b/internal/op/const.go new file mode 100644 index 0000000..a465735 --- /dev/null +++ b/internal/op/const.go @@ -0,0 +1,6 @@ +package op + +const ( + WORK = "work" + RootName = "root" +) diff --git a/internal/op/driver.go b/internal/op/driver.go new file mode 100644 index 0000000..305cb9e --- /dev/null +++ b/internal/op/driver.go @@ -0,0 +1,166 @@ +package op + +import ( + "reflect" + "strings" + + "github.com/IceWhaleTech/CasaOS/internal/conf" + + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/pkg/errors" +) + +type New func() driver.Driver + +var driverNewMap = map[string]New{} +var driverInfoMap = map[string]driver.Info{} + +func RegisterDriver(driver New) { + // log.Infof("register driver: [%s]", config.Name) + tempDriver := driver() + tempConfig := tempDriver.Config() + registerDriverItems(tempConfig, tempDriver.GetAddition()) + driverNewMap[tempConfig.Name] = driver +} + +func GetDriverNew(name string) (New, error) { + n, ok := driverNewMap[name] + if !ok { + return nil, errors.Errorf("no driver named: %s", name) + } + return n, nil +} + +func GetDriverNames() []string { + var driverNames []string + for k := range driverInfoMap { + driverNames = append(driverNames, k) + } + return driverNames +} + +func GetDriverInfoMap() map[string]driver.Info { + return driverInfoMap +} + +func registerDriverItems(config driver.Config, addition driver.Additional) { + // log.Debugf("addition of %s: %+v", config.Name, addition) + tAddition := reflect.TypeOf(addition) + for tAddition.Kind() == reflect.Pointer { + tAddition = tAddition.Elem() + } + mainItems := getMainItems(config) + additionalItems := getAdditionalItems(tAddition, config.DefaultRoot) + driverInfoMap[config.Name] = driver.Info{ + Common: mainItems, + Additional: additionalItems, + Config: config, + } +} + +func getMainItems(config driver.Config) []driver.Item { + items := []driver.Item{{ + Name: "mount_path", + Type: conf.TypeString, + Required: true, + Help: "", + }, { + Name: "order", + Type: conf.TypeNumber, + Help: "use to sort", + }, { + Name: "remark", + Type: conf.TypeText, + }} + if !config.NoCache { + items = append(items, driver.Item{ + Name: "cache_expiration", + Type: conf.TypeNumber, + Default: "30", + Required: true, + Help: "The cache expiration time for this storage", + }) + } + if !config.OnlyProxy && !config.OnlyLocal { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "302_redirect", + Required: true, + }, + }...) + } else { + items = append(items, driver.Item{ + Name: "webdav_policy", + Type: conf.TypeSelect, + Default: "native_proxy", + Options: "use_proxy_url,native_proxy", + Required: true, + }) + } + items = append(items, driver.Item{ + Name: "down_proxy_url", + Type: conf.TypeText, + }) + if config.LocalSort { + items = append(items, []driver.Item{{ + Name: "order_by", + Type: conf.TypeSelect, + Options: "name,size,modified", + }, { + Name: "order_direction", + Type: conf.TypeSelect, + Options: "asc,desc", + }}...) + } + items = append(items, driver.Item{ + Name: "extract_folder", + Type: conf.TypeSelect, + Options: "front,back", + }) + return items +} + +func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { + var items []driver.Item + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.Type.Kind() == reflect.Struct { + items = append(items, getAdditionalItems(field.Type, defaultRoot)...) + continue + } + tag := field.Tag + ignore, ok1 := tag.Lookup("ignore") + name, ok2 := tag.Lookup("json") + if (ok1 && ignore == "true") || !ok2 { + continue + } + item := driver.Item{ + Name: name, + Type: strings.ToLower(field.Type.Name()), + Default: tag.Get("default"), + Options: tag.Get("options"), + Required: tag.Get("required") == "true", + Help: tag.Get("help"), + } + if tag.Get("type") != "" { + item.Type = tag.Get("type") + } + if item.Name == "root_folder_id" || item.Name == "root_folder_path" { + if item.Default == "" { + item.Default = defaultRoot + } + item.Required = item.Default != "" + } + // set default type to string + if item.Type == "" { + item.Type = "string" + } + items = append(items, item) + } + return items +} diff --git a/internal/op/hook.go b/internal/op/hook.go new file mode 100644 index 0000000..5c07fc4 --- /dev/null +++ b/internal/op/hook.go @@ -0,0 +1,109 @@ +package op + +import ( + "regexp" + "strings" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/conf" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +// Obj +type ObjsUpdateHook = func(parent string, objs []model.Obj) + +var ( + ObjsUpdateHooks = make([]ObjsUpdateHook, 0) +) + +func RegisterObjsUpdateHook(hook ObjsUpdateHook) { + ObjsUpdateHooks = append(ObjsUpdateHooks, hook) +} + +func HandleObjsUpdateHook(parent string, objs []model.Obj) { + for _, hook := range ObjsUpdateHooks { + hook(parent, objs) + } +} + +// Setting +type SettingItemHook func(item *model.SettingItem) error + +var settingItemHooks = map[string]SettingItemHook{ + conf.VideoTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.VideoTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.AudioTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.AudioTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ImageTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ImageTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.TextTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.TextTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ProxyTypes: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ProxyTypes] = strings.Split(item.Value, ",") + return nil + }, + conf.ProxyIgnoreHeaders: func(item *model.SettingItem) error { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = strings.Split(item.Value, ",") + return nil + }, + conf.PrivacyRegs: func(item *model.SettingItem) error { + regStrs := strings.Split(item.Value, "\n") + regs := make([]*regexp.Regexp, 0, len(regStrs)) + for _, regStr := range regStrs { + reg, err := regexp.Compile(regStr) + if err != nil { + return errors.WithStack(err) + } + regs = append(regs, reg) + } + conf.PrivacyReg = regs + return nil + }, + conf.FilenameCharMapping: func(item *model.SettingItem) error { + var json = jsoniter.ConfigCompatibleWithStandardLibrary + err := json.UnmarshalFromString(item.Value, &conf.FilenameCharMap) + if err != nil { + return err + } + logger.Info("filename char mapping", zap.Any("FilenameCharMap", conf.FilenameCharMap)) + return nil + }, +} + +func RegisterSettingItemHook(key string, hook SettingItemHook) { + settingItemHooks[key] = hook +} + +func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { + if hook, ok := settingItemHooks[item.Key]; ok { + return true, hook(item) + } + return false, nil +} + +// Storage +type StorageHook func(typ string, storage driver.Driver) + +var storageHooks = make([]StorageHook, 0) + +func CallStorageHooks(typ string, storage driver.Driver) { + for _, hook := range storageHooks { + hook(typ, storage) + } +} + +func RegisterStorageHook(hook StorageHook) { + storageHooks = append(storageHooks, hook) +} diff --git a/main.go b/main.go index 2548081..c63ee45 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( "github.com/gin-gonic/gin" "go.uber.org/zap" + _ "github.com/IceWhaleTech/CasaOS/drivers" "github.com/robfig/cron" "gorm.io/gorm" ) @@ -65,6 +66,7 @@ func init() { service.Cache = cache.Init() service.GetCPUThermalZone() + service.MyService.Storages().InitStorages() route.InitFunction() } @@ -117,7 +119,7 @@ func main() { if err != nil { panic(err) } - routers := []string{"sys", "port", "file", "folder", "batch", "image", "samba", "notify", "socketio"} + routers := []string{"sys", "port", "file", "folder", "batch", "image", "samba", "notify", "socketio", "driver", "storage", "recover", "fs"} for _, v := range routers { err = service.MyService.Gateway().CreateRoute(&model.Route{ Path: "/v1/" + v, diff --git a/model/args.go b/model/args.go new file mode 100644 index 0000000..b365547 --- /dev/null +++ b/model/args.go @@ -0,0 +1,38 @@ +package model + +import ( + "io" + "net/http" + "time" +) + +type ListArgs struct { + ReqPath string +} + +type LinkArgs struct { + IP string + Header http.Header + Type string +} + +type Link struct { + URL string `json:"url"` + Header http.Header `json:"header"` // needed header + Data io.ReadCloser // return file reader directly + Status int // status maybe 200 or 206, etc + FilePath *string // local file, return the filepath + Expiration *time.Duration // url expiration time +} + +type OtherArgs struct { + Obj Obj + Method string + Data interface{} +} + +type FsOtherArgs struct { + Path string `json:"path" form:"path"` + Method string `json:"method" form:"method"` + Data interface{} `json:"data" form:"data"` +} diff --git a/model/common.go b/model/common.go new file mode 100644 index 0000000..b99dea8 --- /dev/null +++ b/model/common.go @@ -0,0 +1,6 @@ +package model + +type PageResp struct { + Content interface{} `json:"content"` + Total int64 `json:"total"` +} diff --git a/model/obj.go b/model/obj.go new file mode 100644 index 0000000..e57b7c8 --- /dev/null +++ b/model/obj.go @@ -0,0 +1,186 @@ +package model + +import ( + "io" + "regexp" + "sort" + "strings" + "time" + + mapset "github.com/deckarep/golang-set/v2" + + "github.com/maruel/natural" +) + +type UnwrapObj interface { + Unwrap() Obj +} + +type Obj interface { + GetSize() int64 + GetName() string + ModTime() time.Time + IsDir() bool + + // The internal information of the driver. + // If you want to use it, please understand what it means + GetID() string + GetPath() string +} + +type FileStreamer interface { + io.ReadCloser + Obj + GetMimetype() string + SetReadCloser(io.ReadCloser) + NeedStore() bool + GetReadCloser() io.ReadCloser + GetOld() Obj +} + +type URL interface { + URL() string +} + +type Thumb interface { + Thumb() string +} + +type SetPath interface { + SetPath(path string) +} + +func SortFiles(objs []Obj, orderBy, orderDirection string) { + if orderBy == "" { + return + } + sort.Slice(objs, func(i, j int) bool { + switch orderBy { + case "name": + { + c := natural.Less(objs[i].GetName(), objs[j].GetName()) + if orderDirection == "desc" { + return !c + } + return c + } + case "size": + { + if orderDirection == "desc" { + return objs[i].GetSize() >= objs[j].GetSize() + } + return objs[i].GetSize() <= objs[j].GetSize() + } + case "modified": + if orderDirection == "desc" { + return objs[i].ModTime().After(objs[j].ModTime()) + } + return objs[i].ModTime().Before(objs[j].ModTime()) + } + return false + }) +} + +func ExtractFolder(objs []Obj, extractFolder string) { + if extractFolder == "" { + return + } + front := extractFolder == "front" + sort.SliceStable(objs, func(i, j int) bool { + if objs[i].IsDir() || objs[j].IsDir() { + if !objs[i].IsDir() { + return !front + } + if !objs[j].IsDir() { + return front + } + } + return false + }) +} + +// Wrap +func WrapObjName(objs Obj) Obj { + return &ObjWrapName{Obj: objs} +} + +func WrapObjsName(objs []Obj) { + for i := 0; i < len(objs); i++ { + objs[i] = &ObjWrapName{Obj: objs[i]} + } +} + +func UnwrapObjs(obj Obj) Obj { + if unwrap, ok := obj.(UnwrapObj); ok { + obj = unwrap.Unwrap() + } + return obj +} + +func GetThumb(obj Obj) (thumb string, ok bool) { + if obj, ok := obj.(Thumb); ok { + return obj.Thumb(), true + } + if unwrap, ok := obj.(UnwrapObj); ok { + return GetThumb(unwrap.Unwrap()) + } + return thumb, false +} + +func GetUrl(obj Obj) (url string, ok bool) { + if obj, ok := obj.(URL); ok { + return obj.URL(), true + } + if unwrap, ok := obj.(UnwrapObj); ok { + return GetUrl(unwrap.Unwrap()) + } + return url, false +} + +// Merge +func NewObjMerge() *ObjMerge { + return &ObjMerge{ + set: mapset.NewSet[string](), + } +} + +type ObjMerge struct { + regs []*regexp.Regexp + set mapset.Set[string] +} + +func (om *ObjMerge) Merge(objs []Obj, objs_ ...Obj) []Obj { + newObjs := make([]Obj, 0, len(objs)+len(objs_)) + newObjs = om.insertObjs(om.insertObjs(newObjs, objs...), objs_...) + return newObjs +} + +func (om *ObjMerge) insertObjs(objs []Obj, objs_ ...Obj) []Obj { + for _, obj := range objs_ { + if om.clickObj(obj) { + objs = append(objs, obj) + } + } + return objs +} + +func (om *ObjMerge) clickObj(obj Obj) bool { + for _, reg := range om.regs { + if reg.MatchString(obj.GetName()) { + return false + } + } + return om.set.Add(obj.GetName()) +} + +func (om *ObjMerge) InitHideReg(hides string) { + rs := strings.Split(hides, "\n") + om.regs = make([]*regexp.Regexp, 0, len(rs)) + for _, r := range rs { + om.regs = append(om.regs, regexp.MustCompile(r)) + } +} + +func (om *ObjMerge) Reset() { + om.set.Clear() +} diff --git a/model/object.go b/model/object.go new file mode 100644 index 0000000..b128c22 --- /dev/null +++ b/model/object.go @@ -0,0 +1,90 @@ +package model + +import ( + "time" +) + +type ObjWrapName struct { + Name string + Obj +} + +func (o *ObjWrapName) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapName) GetName() string { + if o.Name == "" { + o.Name = o.Obj.GetName() + } + return o.Name +} + +type Object struct { + ID string + Path string + Name string + Size int64 + Modified time.Time + IsFolder bool +} + +func (o *Object) GetName() string { + return o.Name +} + +func (o *Object) GetSize() int64 { + return o.Size +} + +func (o *Object) ModTime() time.Time { + return o.Modified +} + +func (o *Object) IsDir() bool { + return o.IsFolder +} + +func (o *Object) GetID() string { + return o.ID +} + +func (o *Object) GetPath() string { + return o.Path +} + +func (o *Object) SetPath(id string) { + o.Path = id +} + +type Thumbnail struct { + Thumbnail string +} + +type Url struct { + Url string +} + +func (w Url) URL() string { + return w.Url +} + +func (t Thumbnail) Thumb() string { + return t.Thumbnail +} + +type ObjThumb struct { + Object + Thumbnail +} + +type ObjectURL struct { + Object + Url +} + +type ObjThumbURL struct { + Object + Thumbnail + Url +} diff --git a/model/req.go b/model/req.go new file mode 100644 index 0000000..fe3a08b --- /dev/null +++ b/model/req.go @@ -0,0 +1,20 @@ +package model + +type PageReq struct { + Page int `json:"page" form:"page"` + PerPage int `json:"per_page" form:"per_page"` +} + +const MaxUint = ^uint(0) +const MinUint = 0 +const MaxInt = int(MaxUint >> 1) +const MinInt = -MaxInt - 1 + +func (p *PageReq) Validate() { + if p.Page < 1 { + p.Page = 1 + } + if p.PerPage < 1 { + p.PerPage = MaxInt + } +} diff --git a/model/setting.go b/model/setting.go new file mode 100644 index 0000000..883a853 --- /dev/null +++ b/model/setting.go @@ -0,0 +1,33 @@ +package model + +const ( + SINGLE = iota + SITE + STYLE + PREVIEW + GLOBAL + ARIA2 + INDEX + GITHUB +) + +const ( + PUBLIC = iota + PRIVATE + READONLY + DEPRECATED +) + +type SettingItem struct { + Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key + Value string `json:"value"` // value + Help string `json:"help"` // help message + Type string `json:"type"` // string, number, bool, select + Options string `json:"options"` // values for select + Group int `json:"group"` // use to group setting in frontend + Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. +} + +func (s SettingItem) IsDeprecated() bool { + return s.Flag == DEPRECATED +} diff --git a/model/storage.go b/model/storage.go new file mode 100644 index 0000000..2be5fa3 --- /dev/null +++ b/model/storage.go @@ -0,0 +1,54 @@ +package model + +import "time" + +type Storage struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized + Order int `json:"order"` // use to sort + Driver string `json:"driver"` // driver used + CacheExpiration int `json:"cache_expiration"` // cache expire time + Status string `json:"status"` + Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver + Remark string `json:"remark"` + Modified time.Time `json:"modified"` + Disabled bool `json:"disabled"` // if disabled + Sort + Proxy +} + +type Sort struct { + OrderBy string `json:"order_by"` + OrderDirection string `json:"order_direction"` + ExtractFolder string `json:"extract_folder"` +} + +type Proxy struct { + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + DownProxyUrl string `json:"down_proxy_url"` +} + +func (s *Storage) GetStorage() *Storage { + return s +} + +func (s *Storage) SetStorage(storage Storage) { + *s = storage +} + +func (s *Storage) SetStatus(status string) { + s.Status = status +} + +func (p Proxy) Webdav302() bool { + return p.WebdavPolicy == "302_redirect" +} + +func (p Proxy) WebdavProxy() bool { + return p.WebdavPolicy == "use_proxy_url" +} + +func (p Proxy) WebdavNative() bool { + return !p.Webdav302() && !p.WebdavProxy() +} diff --git a/model/stream.go b/model/stream.go new file mode 100644 index 0000000..cd7c336 --- /dev/null +++ b/model/stream.go @@ -0,0 +1,33 @@ +package model + +import ( + "io" +) + +type FileStream struct { + Obj + io.ReadCloser + Mimetype string + WebPutAsTask bool + Old Obj +} + +func (f *FileStream) GetMimetype() string { + return f.Mimetype +} + +func (f *FileStream) NeedStore() bool { + return f.WebPutAsTask +} + +func (f *FileStream) GetReadCloser() io.ReadCloser { + return f.ReadCloser +} + +func (f *FileStream) SetReadCloser(rc io.ReadCloser) { + f.ReadCloser = rc +} + +func (f *FileStream) GetOld() Obj { + return f.Old +} diff --git a/pkg/generic_sync/generic_sync.go b/pkg/generic_sync/generic_sync.go new file mode 100644 index 0000000..96612f0 --- /dev/null +++ b/pkg/generic_sync/generic_sync.go @@ -0,0 +1,412 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package generic_sync + +import ( + "sync" + "sync/atomic" + "unsafe" +) + +// MapOf is like a Go map[interface{}]interface{} but is safe for concurrent use +// by multiple goroutines without additional locking or coordination. +// Loads, stores, and deletes run in amortized constant time. +// +// The MapOf type is specialized. Most code should use a plain Go map instead, +// with separate locking or coordination, for better type safety and to make it +// easier to maintain other invariants along with the map content. +// +// The MapOf type is optimized for two common use cases: (1) when the entry for a given +// key is only ever written once but read many times, as in caches that only grow, +// or (2) when multiple goroutines read, write, and overwrite entries for disjoint +// sets of keys. In these two cases, use of a MapOf may significantly reduce lock +// contention compared to a Go map paired with a separate Mutex or RWMutex. +// +// The zero MapOf is empty and ready for use. A MapOf must not be copied after first use. +type MapOf[K comparable, V any] struct { + mu sync.Mutex + + // read contains the portion of the map's contents that are safe for + // concurrent access (with or without mu held). + // + // The read field itself is always safe to load, but must only be stored with + // mu held. + // + // Entries stored in read may be updated concurrently without mu, but updating + // a previously-expunged entry requires that the entry be copied to the dirty + // map and unexpunged with mu held. + read atomic.Value // readOnly + + // dirty contains the portion of the map's contents that require mu to be + // held. To ensure that the dirty map can be promoted to the read map quickly, + // it also includes all of the non-expunged entries in the read map. + // + // Expunged entries are not stored in the dirty map. An expunged entry in the + // clean map must be unexpunged and added to the dirty map before a new value + // can be stored to it. + // + // If the dirty map is nil, the next write to the map will initialize it by + // making a shallow copy of the clean map, omitting stale entries. + dirty map[K]*entry[V] + + // misses counts the number of loads since the read map was last updated that + // needed to lock mu to determine whether the key was present. + // + // Once enough misses have occurred to cover the cost of copying the dirty + // map, the dirty map will be promoted to the read map (in the unamended + // state) and the next store to the map will make a new dirty copy. + misses int +} + +// readOnly is an immutable struct stored atomically in the MapOf.read field. +type readOnly[K comparable, V any] struct { + m map[K]*entry[V] + amended bool // true if the dirty map contains some key not in m. +} + +// expunged is an arbitrary pointer that marks entries which have been deleted +// from the dirty map. +var expunged = unsafe.Pointer(new(interface{})) + +// An entry is a slot in the map corresponding to a particular key. +type entry[V any] struct { + // p points to the interface{} value stored for the entry. + // + // If p == nil, the entry has been deleted and m.dirty == nil. + // + // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry + // is missing from m.dirty. + // + // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty + // != nil, in m.dirty[key]. + // + // An entry can be deleted by atomic replacement with nil: when m.dirty is + // next created, it will atomically replace nil with expunged and leave + // m.dirty[key] unset. + // + // An entry's associated value can be updated by atomic replacement, provided + // p != expunged. If p == expunged, an entry's associated value can be updated + // only after first setting m.dirty[key] = e so that lookups using the dirty + // map find the entry. + p unsafe.Pointer // *interface{} +} + +func newEntry[V any](i V) *entry[V] { + return &entry[V]{p: unsafe.Pointer(&i)} +} + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *MapOf[K, V]) Load(key K) (value V, ok bool) { + read, _ := m.read.Load().(readOnly[K, V]) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + // Avoid reporting a spurious miss if m.dirty got promoted while we were + // blocked on m.mu. (If further loads of the same key will not miss, it's + // not worth copying the dirty map for this key.) + read, _ = m.read.Load().(readOnly[K, V]) + e, ok = read.m[key] + if !ok && read.amended { + e, ok = m.dirty[key] + // Regardless of whether the entry was present, record a miss: this key + // will take the slow path until the dirty map is promoted to the read + // map. + m.missLocked() + } + m.mu.Unlock() + } + if !ok { + return value, false + } + return e.load() +} + +func (m *MapOf[K, V]) Has(key K) bool { + _, ok := m.Load(key) + return ok +} + +func (e *entry[V]) load() (value V, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return value, false + } + return *(*V)(p), true +} + +// Store sets the value for a key. +func (m *MapOf[K, V]) Store(key K, value V) { + read, _ := m.read.Load().(readOnly[K, V]) + if e, ok := read.m[key]; ok && e.tryStore(&value) { + return + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly[K, V]) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + // The entry was previously expunged, which implies that there is a + // non-nil dirty map and this entry is not in it. + m.dirty[key] = e + } + e.storeLocked(&value) + } else if e, ok := m.dirty[key]; ok { + e.storeLocked(&value) + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly[K, V]{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + } + m.mu.Unlock() +} + +// tryStore stores a value if the entry has not been expunged. +// +// If the entry is expunged, tryStore returns false and leaves the entry +// unchanged. +func (e *entry[V]) tryStore(i *V) bool { + for { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { + return true + } + } +} + +// unexpungeLocked ensures that the entry is not marked as expunged. +// +// If the entry was previously expunged, it must be added to the dirty map +// before m.mu is unlocked. +func (e *entry[V]) unexpungeLocked() (wasExpunged bool) { + return atomic.CompareAndSwapPointer(&e.p, expunged, nil) +} + +// storeLocked unconditionally stores a value to the entry. +// +// The entry must be known not to be expunged. +func (e *entry[V]) storeLocked(i *V) { + atomic.StorePointer(&e.p, unsafe.Pointer(i)) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *MapOf[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { + // Avoid locking if it's a clean hit. + read, _ := m.read.Load().(readOnly[K, V]) + if e, ok := read.m[key]; ok { + actual, loaded, ok := e.tryLoadOrStore(value) + if ok { + return actual, loaded + } + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly[K, V]) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + m.dirty[key] = e + } + actual, loaded, _ = e.tryLoadOrStore(value) + } else if e, ok := m.dirty[key]; ok { + actual, loaded, _ = e.tryLoadOrStore(value) + m.missLocked() + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly[K, V]{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + actual, loaded = value, false + } + m.mu.Unlock() + + return actual, loaded +} + +// tryLoadOrStore atomically loads or stores a value if the entry is not +// expunged. +// +// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and +// returns with ok==false. +func (e *entry[V]) tryLoadOrStore(i V) (actual V, loaded, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return actual, false, false + } + if p != nil { + return *(*V)(p), true, true + } + + // Copy the interface after the first load to make this method more amenable + // to escape analysis: if we hit the "load" path or the entry is expunged, we + // shouldn'V bother heap-allocating. + ic := i + for { + if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) { + return i, false, true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return actual, false, false + } + if p != nil { + return *(*V)(p), true, true + } + } +} + +// Delete deletes the value for a key. +func (m *MapOf[K, V]) Delete(key K) { + read, _ := m.read.Load().(readOnly[K, V]) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + read, _ = m.read.Load().(readOnly[K, V]) + e, ok = read.m[key] + if !ok && read.amended { + delete(m.dirty, key) + } + m.mu.Unlock() + } + if ok { + e.delete() + } +} + +func (e *entry[V]) delete() (hadValue bool) { + for { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return false + } + if atomic.CompareAndSwapPointer(&e.p, p, nil) { + return true + } + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// Range does not necessarily correspond to any consistent snapshot of the MapOf's +// contents: no key will be visited more than once, but if the value for any key +// is stored or deleted concurrently, Range may reflect any mapping for that key +// from any point during the Range call. +// +// Range may be O(N) with the number of elements in the map even if f returns +// false after a constant number of calls. +func (m *MapOf[K, V]) Range(f func(key K, value V) bool) { + // We need to be able to iterate over all of the keys that were already + // present at the start of the call to Range. + // If read.amended is false, then read.m satisfies that property without + // requiring us to hold m.mu for a long time. + read, _ := m.read.Load().(readOnly[K, V]) + if read.amended { + // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) + // (assuming the caller does not break out early), so a call to Range + // amortizes an entire copy of the map: we can promote the dirty copy + // immediately! + m.mu.Lock() + read, _ = m.read.Load().(readOnly[K, V]) + if read.amended { + read = readOnly[K, V]{m: m.dirty} + m.read.Store(read) + m.dirty = nil + m.misses = 0 + } + m.mu.Unlock() + } + + for k, e := range read.m { + v, ok := e.load() + if !ok { + continue + } + if !f(k, v) { + break + } + } +} + +// Values returns a slice of the values in the map. +func (m *MapOf[K, V]) Values() []V { + var values []V + m.Range(func(key K, value V) bool { + values = append(values, value) + return true + }) + return values +} + +func (m *MapOf[K, V]) Count() int { + return len(m.dirty) +} + +func (m *MapOf[K, V]) Empty() bool { + return m.Count() == 0 +} + +func (m *MapOf[K, V]) ToMap() map[K]V { + ans := make(map[K]V) + m.Range(func(key K, value V) bool { + ans[key] = value + return true + }) + return ans +} + +func (m *MapOf[K, V]) Clear() { + m.Range(func(key K, value V) bool { + m.Delete(key) + return true + }) +} + +func (m *MapOf[K, V]) missLocked() { + m.misses++ + if m.misses < len(m.dirty) { + return + } + m.read.Store(readOnly[K, V]{m: m.dirty}) + m.dirty = nil + m.misses = 0 +} + +func (m *MapOf[K, V]) dirtyLocked() { + if m.dirty != nil { + return + } + + read, _ := m.read.Load().(readOnly[K, V]) + m.dirty = make(map[K]*entry[V], len(read.m)) + for k, e := range read.m { + if !e.tryExpungeLocked() { + m.dirty[k] = e + } + } +} + +func (e *entry[V]) tryExpungeLocked() (isExpunged bool) { + p := atomic.LoadPointer(&e.p) + for p == nil { + if atomic.CompareAndSwapPointer(&e.p, nil, expunged) { + return true + } + p = atomic.LoadPointer(&e.p) + } + return p == expunged +} diff --git a/pkg/singleflight/singleflight.go b/pkg/singleflight/singleflight.go new file mode 100644 index 0000000..dcd84a3 --- /dev/null +++ b/pkg/singleflight/singleflight.go @@ -0,0 +1,212 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value any + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v any) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call[T any] struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val T + err error + + // forgotten indicates whether Forget was called with this call's key + // while the call was still in flight. + forgotten bool + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result[T] +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group[T any] struct { + mu sync.Mutex // protects m + m map[string]*call[T] // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result[T any] struct { + Val T + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group[T]) Do(key string, fn func() (T, error)) (v T, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call[T]) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } + return c.val, c.err, true + } + c := new(call[T]) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *Group[T]) DoChan(key string, fn func() (T, error)) <-chan Result[T] { + ch := make(chan Result[T], 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call[T]) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call[T]{chans: []chan<- Result[T]{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group[T]) doCall(c *call[T], key string, fn func() (T, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + c.wg.Done() + g.mu.Lock() + defer g.mu.Unlock() + if !c.forgotten { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result[T]{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group[T]) Forget(key string) { + g.mu.Lock() + if c, ok := g.m[key]; ok { + c.forgotten = true + } + delete(g.m, key) + g.mu.Unlock() +} diff --git a/pkg/sqlite/db.go b/pkg/sqlite/db.go index be826a4..3802d3d 100644 --- a/pkg/sqlite/db.go +++ b/pkg/sqlite/db.go @@ -14,6 +14,7 @@ import ( "time" "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/pkg/utils/file" model2 "github.com/IceWhaleTech/CasaOS/service/model" "github.com/glebarez/sqlite" @@ -42,7 +43,7 @@ func GetDb(dbPath string) *gorm.DB { } gdb = db - err = db.AutoMigrate(&model2.AppNotify{}, model2.SharesDBModel{}, model2.ConnectionsDBModel{}) + err = db.AutoMigrate(&model2.AppNotify{}, model2.SharesDBModel{}, model2.ConnectionsDBModel{}, model.Storage{}) db.Exec("DROP TABLE IF EXISTS o_application") db.Exec("DROP TABLE IF EXISTS o_friend") db.Exec("DROP TABLE IF EXISTS o_person_download") diff --git a/pkg/utils/balance.go b/pkg/utils/balance.go new file mode 100644 index 0000000..e6df65c --- /dev/null +++ b/pkg/utils/balance.go @@ -0,0 +1,18 @@ +package utils + +import "strings" + +var balance = ".balance" + +func IsBalance(str string) bool { + return strings.Contains(str, balance) +} + +// GetActualMountPath remove balance suffix +func GetActualMountPath(virtualPath string) string { + bIndex := strings.LastIndex(virtualPath, ".balance") + if bIndex != -1 { + virtualPath = virtualPath[:bIndex] + } + return virtualPath +} diff --git a/pkg/utils/bool.go b/pkg/utils/bool.go new file mode 100644 index 0000000..eecf550 --- /dev/null +++ b/pkg/utils/bool.go @@ -0,0 +1,5 @@ +package utils + +func IsBool(bs ...bool) bool { + return len(bs) > 0 && bs[0] +} diff --git a/pkg/utils/ctx.go b/pkg/utils/ctx.go new file mode 100644 index 0000000..89f27f8 --- /dev/null +++ b/pkg/utils/ctx.go @@ -0,0 +1,14 @@ +package utils + +import ( + "context" +) + +func IsCanceled(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} diff --git a/pkg/utils/path.go b/pkg/utils/path.go new file mode 100644 index 0000000..cc24fc3 --- /dev/null +++ b/pkg/utils/path.go @@ -0,0 +1,81 @@ +package utils + +import ( + "errors" + "net/url" + stdpath "path" + "strings" +) + +// FixAndCleanPath +// The upper layer of the root directory is still the root directory. +// So ".." And "." will be cleared +// for example +// 1. ".." or "." => "/" +// 2. "../..." or "./..." => "/..." +// 3. "../.x." or "./.x." => "/.x." +// 4. "x//\\y" = > "/z/x" +func FixAndCleanPath(path string) string { + path = strings.ReplaceAll(path, "\\", "/") + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return stdpath.Clean(path) +} + +// PathAddSeparatorSuffix Add path '/' suffix +// for example /root => /root/ +func PathAddSeparatorSuffix(path string) string { + if !strings.HasSuffix(path, "/") { + path = path + "/" + } + return path +} + +// PathEqual judge path is equal +func PathEqual(path1, path2 string) bool { + return FixAndCleanPath(path1) == FixAndCleanPath(path2) +} + +func IsSubPath(path string, subPath string) bool { + path, subPath = FixAndCleanPath(path), FixAndCleanPath(subPath) + return path == subPath || strings.HasPrefix(subPath, PathAddSeparatorSuffix(path)) +} + +func Ext(path string) string { + ext := stdpath.Ext(path) + if strings.HasPrefix(ext, ".") { + return ext[1:] + } + return ext +} + +func EncodePath(path string, all ...bool) string { + seg := strings.Split(path, "/") + toReplace := []struct { + Src string + Dst string + }{ + {Src: "%", Dst: "%25"}, + {"%", "%25"}, + {"?", "%3F"}, + {"#", "%23"}, + } + for i := range seg { + if len(all) > 0 && all[0] { + seg[i] = url.PathEscape(seg[i]) + } else { + for j := range toReplace { + seg[i] = strings.ReplaceAll(seg[i], toReplace[j].Src, toReplace[j].Dst) + } + } + } + return strings.Join(seg, "/") +} + +func JoinBasePath(basePath, reqPath string) (string, error) { + if strings.HasSuffix(reqPath, "..") || strings.Contains(reqPath, "../") { + return "", errors.New("access using relative path is not allowed") + } + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil +} diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go new file mode 100644 index 0000000..c1d1584 --- /dev/null +++ b/pkg/utils/slice.go @@ -0,0 +1,46 @@ +package utils + +// SliceEqual check if two slices are equal +func SliceEqual[T comparable](a, b []T) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + +// SliceContains check if slice contains element +func SliceContains[T comparable](arr []T, v T) bool { + for _, vv := range arr { + if vv == v { + return true + } + } + return false +} + +// SliceConvert convert slice to another type slice +func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) { + var res []D + for i := range srcS { + dst, err := convert(srcS[i]) + if err != nil { + return nil, err + } + res = append(res, dst) + } + return res, nil +} + +func MustSliceConvert[S any, D any](srcS []S, convert func(src S) D) []D { + var res []D + for i := range srcS { + dst := convert(srcS[i]) + res = append(res, dst) + } + return res +} diff --git a/route/route.go b/route/route.go index a947d95..aa6a476 100644 --- a/route/route.go +++ b/route/route.go @@ -53,6 +53,7 @@ func InitRouter() *gin.Engine { r.GET("/ping", func(ctx *gin.Context) { ctx.String(200, "pong") }) + r.GET("/v1/recover/:type", v1.GetRecoverStorage) v1Group := r.Group("/v1") v1Group.Use(jwt.ExceptLocalhost()) @@ -111,6 +112,24 @@ func InitRouter() *gin.Engine { v1FileGroup.POST("/upload", v1.PostFileUpload) v1FileGroup.GET("/upload", v1.GetFileUpload) // v1FileGroup.GET("/download", v1.UserFileDownloadCommonService) + + } + v1StorageGroup := v1Group.Group("/storage") + v1StorageGroup.Use() + { + v1StorageGroup.GET("", v1.ListStorages) + v1StorageGroup.POST("", v1.CreateStorage) + v1StorageGroup.DELETE("", v1.DeleteStorage) + } + v1FsGroup := v1Group.Group("/fs") + v1FsGroup.Use() + { + v1FsGroup.POST("/list", v1.FsList) + } + v1DriverGroup := v1Group.Group("/driver") + v1DriverGroup.Use() + { + v1DriverGroup.GET("", v1.ListDriverInfo) } v1FolderGroup := v1Group.Group("/folder") v1FolderGroup.Use() diff --git a/route/v1/driver.go b/route/v1/driver.go new file mode 100644 index 0000000..951f702 --- /dev/null +++ b/route/v1/driver.go @@ -0,0 +1,12 @@ +package v1 + +import ( + "github.com/IceWhaleTech/CasaOS/internal/op" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" + "github.com/gin-gonic/gin" +) + +func ListDriverInfo(c *gin.Context) { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: op.GetDriverInfoMap()}) +} diff --git a/route/v1/file.go b/route/v1/file.go index 0fcdf96..42d0ec8 100644 --- a/route/v1/file.go +++ b/route/v1/file.go @@ -215,8 +215,36 @@ func GetDownloadSingleFile(c *gin.Context) { // @Success 200 {string} string "ok" // @Router /file/dirpath [get] func DirPath(c *gin.Context) { - path := c.DefaultQuery("path", "") - info := service.MyService.System().GetDirPath(path) + var req ListReq + if err := c.ShouldBind(&req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + req.Validate() + storage, _, _ := service.MyService.StoragePath().GetStorageAndActualPath(req.Path) + if storage != nil { + req.Validate() + objs, err := service.MyService.FsService().FList(c, req.Path, req.Refresh) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + total, objs := pagination(objs, &req.PageReq) + provider := "unknown" + storage, err := service.MyService.FsService().GetStorage(req.Path) + if err == nil { + provider = storage.GetStorage().Driver + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: FsListResp{ + Content: toObjsResp(objs, req.Path, false), + Total: int64(total), + Readme: "", + Write: false, + Provider: provider, + }}) + return + } + info := service.MyService.System().GetDirPath(req.Path) shares := service.MyService.Shares().GetSharesList() sharesMap := make(map[string]string) for _, v := range shares { @@ -250,17 +278,30 @@ func DirPath(c *gin.Context) { } } - pathList := []model.Path{} + pathList := []ObjResp{} for i := 0; i < len(info); i++ { if info[i].Name == ".temp" && info[i].IsDir { continue } + if _, ok := fileQueue[info[i].Path]; !ok { - pathList = append(pathList, info[i]) + t := ObjResp{} + t.IsDir = info[i].IsDir + t.Name = info[i].Name + t.Modified = info[i].Date + t.Size = info[i].Size + t.Path = info[i].Path + pathList = append(pathList, t) } } - - c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: pathList}) + flist := FsListResp{ + Content: pathList, + Total: int64(len(pathList)), + Readme: "", + Write: true, + Provider: "local", + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: flist}) } // @Summary rename file or dir diff --git a/route/v1/file_read.go b/route/v1/file_read.go new file mode 100644 index 0000000..377116c --- /dev/null +++ b/route/v1/file_read.go @@ -0,0 +1,93 @@ +package v1 + +import ( + "path/filepath" + "time" + + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" + "github.com/IceWhaleTech/CasaOS/service" + + "github.com/gin-gonic/gin" +) + +type ListReq struct { + model.PageReq + Path string `json:"path" form:"path"` + Refresh bool `json:"refresh"` +} +type ObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + Path string `json:"path"` +} +type FsListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Write bool `json:"write"` + Provider string `json:"provider"` +} + +func FsList(c *gin.Context) { + var req ListReq + if err := c.ShouldBind(&req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + req.Validate() + objs, err := service.MyService.FsService().FList(c, req.Path, req.Refresh) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + total, objs := pagination(objs, &req.PageReq) + provider := "unknown" + storage, err := service.MyService.FsService().GetStorage(req.Path) + if err == nil { + provider = storage.GetStorage().Driver + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: FsListResp{ + Content: toObjsResp(objs, req.Path, false), + Total: int64(total), + Readme: "", + Write: false, + Provider: provider, + }}) +} +func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { + pageIndex, pageSize := req.Page, req.PerPage + total := len(objs) + start := (pageIndex - 1) * pageSize + if start > total { + return total, []model.Obj{} + } + end := start + pageSize + if end > total { + end = total + } + return total, objs[start:end] +} + +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { + var resp []ObjResp + for _, obj := range objs { + thumb, _ := model.GetThumb(obj) + resp = append(resp, ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Sign: "", + Path: filepath.Join(parent, obj.GetName()), + Thumb: thumb, + Type: 0, + }) + } + return resp +} diff --git a/route/v1/recover.go b/route/v1/recover.go new file mode 100644 index 0000000..df93493 --- /dev/null +++ b/route/v1/recover.go @@ -0,0 +1,73 @@ +package v1 + +import ( + "strconv" + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/drivers/google_drive" + "github.com/IceWhaleTech/CasaOS/internal/op" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/service" + "github.com/gin-gonic/gin" + jsoniter "github.com/json-iterator/go" + "go.uber.org/zap" +) + +func GetRecoverStorage(c *gin.Context) { + c.Header("Content-Type", "text/html; charset=utf-8") + t := c.Param("type") + if t == "GoogleDrive" { + + mountPath := "google" + + mountPath += time.Now().Format("20060102150405") + + gd := op.GetDriverInfoMap()[t] + var req model.Storage + req.Driver = t + req.MountPath = mountPath + + req.CacheExpiration = 5 + add := google_drive.Addition{} + add.Code = c.Query("code") + if len(add.Code) == 0 { + c.String(200, `

code不可为空

`) + return + } + add.RootFolderID = "root" + for _, v := range gd.Additional { + if v.Name == "client_id" { + add.ClientID = v.Default + } + if v.Name == "client_secret" { + add.ClientSecret = v.Default + } + if v.Name == "chunk_size" { + cs, err := strconv.ParseInt(v.Default, 10, 64) + if err != nil { + cs = 5 + } + add.ChunkSize = cs + } + } + + var json = jsoniter.ConfigCompatibleWithStandardLibrary + addStr, err := json.Marshal(add) + if err != nil { + c.String(200, `

addition序列化失败

`) + return + } + req.Addition = string(addStr) + logger.Info("GetRecoverStorage", zap.Any("req", req)) + if _, err := service.MyService.Storages().CreateStorage(c, req); err != nil { + c.String(200, `

添加失败:`+err.Error()+`

`) + return + } + data := make(map[string]interface{}) + data["status"] = "success" + service.MyService.Notify().SendNotify("recover_status", data) + } + + c.String(200, `

关闭该页面即可

`) +} diff --git a/route/v1/storage.go b/route/v1/storage.go new file mode 100644 index 0000000..12e7d84 --- /dev/null +++ b/route/v1/storage.go @@ -0,0 +1,121 @@ +package v1 + +import ( + "strconv" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" + "github.com/IceWhaleTech/CasaOS/service" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +func ListStorages(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + req.Validate() + + logger.Info("ListStorages", zap.Any("req", req)) + storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{ + Content: storages, + Total: total, + }}) +} + +func CreateStorage(c *gin.Context) { + var req model.Storage + if err := c.ShouldBind(&req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + if id, err := service.MyService.Storages().CreateStorage(c, req); err != nil { + data := make(map[string]interface{}) + data["id"] = id + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: data}) + return + } else { + data := make(map[string]interface{}) + data["id"] = id + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: data}) + + } +} + +func UpdateStorage(c *gin.Context) { + var req model.Storage + if err := c.ShouldBind(&req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + if err := service.MyService.Storages().UpdateStorage(c, req); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + } else { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) + } +} + +func DeleteStorage(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + if err := service.MyService.Storages().DeleteStorageById(c, uint(id)); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) +} + +func DisableStorage(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + if err := service.MyService.Storages().DisableStorage(c, uint(id)); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) +} + +func EnableStorage(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + if err := service.MyService.Storages().EnableStorage(c, uint(id)); err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) +} + +func GetStorage(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) + return + } + storage, err := service.MyService.Storage().GetStorageById(uint(id)) + if err != nil { + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) + return + } + c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage}) +} diff --git a/service/fs.go b/service/fs.go new file mode 100644 index 0000000..efd23e0 --- /dev/null +++ b/service/fs.go @@ -0,0 +1,152 @@ +package service + +import ( + "context" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/model" + "go.uber.org/zap" +) + +type FsService interface { + FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) + GetStorage(path string) (driver.Driver, error) +} + +type fsService struct { +} + +// the param named path of functions in this package is a mount path +// So, the purpose of this package is to convert mount path to actual path +// then pass the actual path to the op package + +func (f *fsService) FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) { + res, err := MyService.FsListService().FsList(ctx, path, refresh...) + if err != nil { + logger.Info("failed list", zap.Any("path", path), zap.Any("err", err)) + return nil, err + } + return res, nil +} + +// func (f *fsService) Get(ctx context.Context, path string) (model.Obj, error) { +// res, err := get(ctx, path) +// if err != nil { +// log.Errorf("failed get %s: %+v", path, err) +// return nil, err +// } +// return res, nil +// } + +// func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { +// res, file, err := link(ctx, path, args) +// if err != nil { +// log.Errorf("failed link %s: %+v", path, err) +// return nil, nil, err +// } +// return res, file, nil +// } + +// func (f *fsService) MakeDir(ctx context.Context, path string, lazyCache ...bool) error { +// err := makeDir(ctx, path, lazyCache...) +// if err != nil { +// log.Errorf("failed make dir %s: %+v", path, err) +// } +// return err +// } + +// func (f *fsService) Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) error { +// err := move(ctx, srcPath, dstDirPath, lazyCache...) +// if err != nil { +// log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) +// } +// return err +// } + +// func (f *fsService) Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (bool, error) { +// res, err := _copy(ctx, srcObjPath, dstDirPath, lazyCache...) +// if err != nil { +// log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) +// } +// return res, err +// } + +// func (f *fsService) Rename(ctx context.Context, srcPath, dstName string, lazyCache ...bool) error { +// err := rename(ctx, srcPath, dstName, lazyCache...) +// if err != nil { +// log.Errorf("failed rename %s to %s: %+v", srcPath, dstName, err) +// } +// return err +// } + +// func (f *fsService) Remove(ctx context.Context, path string) error { +// err := remove(ctx, path) +// if err != nil { +// log.Errorf("failed remove %s: %+v", path, err) +// } +// return err +// } + +// func PutDirectly(ctx context.Context, dstDirPath string, file *model.FileStream, lazyCache ...bool) error { +// err := putDirectly(ctx, dstDirPath, file, lazyCache...) +// if err != nil { +// log.Errorf("failed put %s: %+v", dstDirPath, err) +// } +// return err +// } + +// func (f *fsService) PutAsTask(dstDirPath string, file *model.FileStream) error { +// err := putAsTask(dstDirPath, file) +// if err != nil { +// log.Errorf("failed put %s: %+v", dstDirPath, err) +// } +// return err +// } + +func (f *fsService) GetStorage(path string) (driver.Driver, error) { + storageDriver, _, err := MyService.StoragePath().GetStorageAndActualPath(path) + if err != nil { + return nil, err + } + return storageDriver, nil +} + +// func (f *fsService) Other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { +// res, err := other(ctx, args) +// if err != nil { +// log.Errorf("failed remove %s: %+v", args.Path, err) +// } +// return res, err +// } + +// func get(ctx context.Context, path string) (model.Obj, error) { +// path = utils.FixAndCleanPath(path) +// // maybe a virtual file +// if path != "/" { +// virtualFiles := op.GetStorageVirtualFilesByPath(stdpath.Dir(path)) +// for _, f := range virtualFiles { +// if f.GetName() == stdpath.Base(path) { +// return f, nil +// } +// } +// } +// storage, actualPath, err := op.GetStorageAndActualPath(path) +// if err != nil { +// // if there are no storage prefix with path, maybe root folder +// if path == "/" { +// return &model.Object{ +// Name: "root", +// Size: 0, +// Modified: time.Time{}, +// IsFolder: true, +// }, nil +// } +// return nil, errors.WithMessage(err, "failed get storage") +// } +// return op.Get(ctx, storage, actualPath) +// } + +func NewFsService() FsService { + return &fsService{} +} diff --git a/service/fs_list.go b/service/fs_list.go new file mode 100644 index 0000000..24219b0 --- /dev/null +++ b/service/fs_list.go @@ -0,0 +1,198 @@ +package service + +import ( + "context" + stdpath "path" + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" + "github.com/IceWhaleTech/CasaOS/model" + "github.com/IceWhaleTech/CasaOS/pkg/singleflight" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/Xhofe/go-cache" + + log "github.com/dsoprea/go-logging" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +type FsListService interface { + FsList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) + Key(storage driver.Driver, path string) string + Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) + GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) + List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) +} + +type fsListService struct { +} + +var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) +var listG singleflight.Group[[]model.Obj] + +// List files +func (fl *fsListService) FsList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) { + + virtualFiles := MyService.Storages().GetStorageVirtualFilesByPath(path) + storage, actualPath, err := MyService.StoragePath().GetStorageAndActualPath(path) + if err != nil && len(virtualFiles) == 0 { + return nil, errors.WithMessage(err, "failed get storage") + } + + var _objs []model.Obj + if storage != nil { + _objs, err = fl.List(ctx, storage, actualPath, model.ListArgs{ + ReqPath: path, + }, refresh...) + if err != nil { + log.Errorf("%+v", err) + if len(virtualFiles) == 0 { + return nil, errors.WithMessage(err, "failed get objs") + } + } + } + + om := model.NewObjMerge() + + objs := om.Merge(virtualFiles, _objs...) + return objs, nil +} + +func (fl *fsListService) Key(storage driver.Driver, path string) string { + return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) +} + +// Get object from list of files +func (fl *fsListService) Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + path = utils.FixAndCleanPath(path) + logger.Info("get", zap.String("path", path)) + + // is root folder + if utils.PathEqual(path, "/") { + var rootObj model.Obj + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: op.RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: op.RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + default: + if storage, ok := storage.(driver.Getter); ok { + obj, err := storage.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + rootObj = obj + } + } + if rootObj == nil { + return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method") + } + return &model.ObjWrapName{ + Name: op.RootName, + Obj: rootObj, + }, nil + } + + // not root folder + dir, name := stdpath.Split(path) + files, err := fl.List(ctx, storage, dir, model.ListArgs{}) + if err != nil { + return nil, errors.WithMessage(err, "failed get parent list") + } + for _, f := range files { + // TODO maybe copy obj here + if f.GetName() == name { + return f, nil + } + } + logger.Info("cant find obj with name", zap.Any("name", name)) + return nil, errors.WithStack(errors.New("object not found")) +} + +func (fl *fsListService) GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { + obj, err := fl.Get(ctx, storage, path) + if err != nil { + return nil, err + } + return model.UnwrapObjs(obj), err +} + +// List files in storage, not contains virtual file +func (fl *fsListService) List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + logger.Info("op.List", zap.Any("path", path)) + key := fl.Key(storage, path) + if !utils.IsBool(refresh...) { + if files, ok := listCache.Get(key); ok { + logger.Info("op.List", zap.Any("use cache", path)) + return files, nil + } + } + dir, err := fl.GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } + logger.Info("op.List", zap.Any("dir", dir)) + if !dir.IsDir() { + return nil, errors.WithStack(errors.New("not a folder")) + } + objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + files, err := storage.List(ctx, dir, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to list objs") + } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { + s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) + // call hooks + go func(reqPath string, files []model.Obj) { + for _, hook := range op.ObjsUpdateHooks { + hook(args.ReqPath, files) + } + }(args.ReqPath, files) + + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + + if !storage.Config().NoCache { + if len(files) > 0 { + logger.Info("set cache", zap.Any("key", key), zap.Any("files", files)) + listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } else { + logger.Info("del cache", zap.Any("key", key)) + listCache.Del(key) + } + } + return files, nil + }) + return objs, err +} + +func NewFsListService() FsListService { + return &fsListService{} +} diff --git a/service/service.go b/service/service.go index 12ee71f..2d505a4 100644 --- a/service/service.go +++ b/service/service.go @@ -38,6 +38,11 @@ type Repository interface { Shares() SharesService Connections() ConnectionsService Gateway() external.ManagementService + Storage() StorageService + Storages() StoragesService + StoragePath() StoragePathService + FsListService() FsListService + FsService() FsService } func NewService(db *gorm.DB, RuntimePath string, socket *socketio.Server) Repository { @@ -51,25 +56,55 @@ func NewService(db *gorm.DB, RuntimePath string, socket *socketio.Server) Reposi } return &store{ - gateway: gatewayManagement, - casa: NewCasaService(), - notify: NewNotifyService(db), - rely: NewRelyService(db), - system: NewSystemService(), - shares: NewSharesService(db), - connections: NewConnectionsService(db), + gateway: gatewayManagement, + casa: NewCasaService(), + notify: NewNotifyService(db), + rely: NewRelyService(db), + system: NewSystemService(), + shares: NewSharesService(db), + connections: NewConnectionsService(db), + storage: NewStorageService(db), + storages: NewStoragesService(), + storage_path: NewStoragePathService(), + fs_list: NewFsListService(), + fs: NewFsService(), } } type store struct { - db *gorm.DB - casa CasaService - notify NotifyServer - rely RelyService - system SystemService - shares SharesService - connections ConnectionsService - gateway external.ManagementService + db *gorm.DB + casa CasaService + notify NotifyServer + rely RelyService + system SystemService + shares SharesService + connections ConnectionsService + gateway external.ManagementService + storage StorageService + storages StoragesService + storage_path StoragePathService + fs_list FsListService + fs FsService +} + +func (c *store) FsService() FsService { + return c.fs +} + +func (c *store) FsListService() FsListService { + return c.fs_list +} + +func (c *store) StoragePath() StoragePathService { + return c.storage_path +} + +func (c *store) Storages() StoragesService { + return c.storages +} + +func (c *store) Storage() StorageService { + return c.storage } func (c *store) Gateway() external.ManagementService { diff --git a/service/storage.go b/service/storage.go new file mode 100644 index 0000000..9ef53e3 --- /dev/null +++ b/service/storage.go @@ -0,0 +1,73 @@ +package service + +import ( + "fmt" + + "github.com/IceWhaleTech/CasaOS/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +type StorageService interface { + CreateStorage(storage *model.Storage) error + UpdateStorage(storage *model.Storage) error + DeleteStorageById(id uint) error + GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) + GetStorageById(id uint) (*model.Storage, error) + GetEnabledStorages() ([]model.Storage, error) +} + +type storageStruct struct { + db *gorm.DB +} + +// CreateStorage just insert storage to database +func (s *storageStruct) CreateStorage(storage *model.Storage) error { + return errors.WithStack(s.db.Create(storage).Error) +} + +// UpdateStorage just update storage in database +func (s *storageStruct) UpdateStorage(storage *model.Storage) error { + return errors.WithStack(s.db.Save(storage).Error) +} + +// DeleteStorageById just delete storage from database by id +func (s *storageStruct) DeleteStorageById(id uint) error { + return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error) +} + +// GetStorages Get all storages from database order by index +func (s *storageStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { + storageDB := s.db.Model(&model.Storage{}) + var count int64 + if err := storageDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get storages count") + } + var storages []model.Storage + if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return storages, count, nil +} + +// GetStorageById Get Storage by id, used to update storage usually +func (s *storageStruct) GetStorageById(id uint) (*model.Storage, error) { + var storage model.Storage + storage.ID = id + if err := s.db.First(&storage).Error; err != nil { + return nil, errors.WithStack(err) + } + return &storage, nil +} + +func (s *storageStruct) GetEnabledStorages() ([]model.Storage, error) { + var storages []model.Storage + if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil { + return nil, errors.WithStack(err) + } + return storages, nil +} + +func NewStorageService(db *gorm.DB) StorageService { + return &storageStruct{db: db} +} diff --git a/service/storage_path.go b/service/storage_path.go new file mode 100644 index 0000000..5d93268 --- /dev/null +++ b/service/storage_path.go @@ -0,0 +1,34 @@ +package service + +import ( + "strings" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +type StoragePathService interface { + GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath string, err error) +} + +type storagePathStruct struct { +} + +func (s *storagePathStruct) GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath string, err error) { + rawPath = utils.FixAndCleanPath(rawPath) + storage = MyService.Storages().GetBalancedStorage(rawPath) + if storage == nil { + err = errors.Errorf("can't find storage with rawPath: %s", rawPath) + return + } + logger.Info("use storage", zap.Any("storage mount path", storage.GetStorage().MountPath)) + mountPath := utils.GetActualMountPath(storage.GetStorage().MountPath) + actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, mountPath)) + return +} +func NewStoragePathService() StoragePathService { + return &storagePathStruct{} +} diff --git a/service/storage_service.go b/service/storage_service.go new file mode 100644 index 0000000..89ed619 --- /dev/null +++ b/service/storage_service.go @@ -0,0 +1,376 @@ +package service + +import ( + "context" + "sort" + "strings" + "time" + + "github.com/IceWhaleTech/CasaOS-Common/utils/logger" + "github.com/IceWhaleTech/CasaOS/pkg/utils" + jsoniter "github.com/json-iterator/go" + "go.uber.org/zap" + + "github.com/IceWhaleTech/CasaOS/pkg/generic_sync" + + "github.com/IceWhaleTech/CasaOS/model" + + "github.com/IceWhaleTech/CasaOS/internal/conf" + "github.com/IceWhaleTech/CasaOS/internal/driver" + "github.com/IceWhaleTech/CasaOS/internal/op" + mapset "github.com/deckarep/golang-set/v2" + "github.com/pkg/errors" +) + +type StoragesService interface { + HasStorage(mountPath string) bool + CreateStorage(ctx context.Context, storage model.Storage) (uint, error) + LoadStorage(ctx context.Context, storage model.Storage) error + EnableStorage(ctx context.Context, id uint) error + DisableStorage(ctx context.Context, id uint) error + UpdateStorage(ctx context.Context, storage model.Storage) error + DeleteStorageById(ctx context.Context, id uint) error + MustSaveDriverStorage(driver driver.Driver) + GetStorageVirtualFilesByPath(prefix string) []model.Obj + initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) + InitStorages() + GetBalancedStorage(path string) driver.Driver +} + +type storagesStruct struct { +} + +// Although the driver type is stored, +// there is a storage in each driver, +// so it should actually be a storage, just wrapped by the driver +var storagesMap generic_sync.MapOf[string, driver.Driver] + +func GetAllStorages() []driver.Driver { + return storagesMap.Values() +} + +func (s *storagesStruct) HasStorage(mountPath string) bool { + return storagesMap.Has(utils.FixAndCleanPath(mountPath)) +} + +func GetStorageByMountPath(mountPath string) (driver.Driver, error) { + mountPath = utils.FixAndCleanPath(mountPath) + storageDriver, ok := storagesMap.Load(mountPath) + if !ok { + return nil, errors.Errorf("no mount path for an storage is: %s", mountPath) + } + return storageDriver, nil +} + +// CreateStorage Save the storage to database so storage can get an id +// then instantiate corresponding driver and save it in memory +func (s *storagesStruct) CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { + storage.Modified = time.Now() + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + var err error + // check driver first + driverName := storage.Driver + driverNew, err := op.GetDriverNew(driverName) + if err != nil { + return 0, errors.WithMessage(err, "failed get driver new") + } + storageDriver := driverNew() + // insert storage to database + err = MyService.Storage().CreateStorage(&storage) + if err != nil { + return storage.ID, errors.WithMessage(err, "failed create storage in database") + } + // already has an id + err = s.initStorage(ctx, storage, storageDriver) + go op.CallStorageHooks("add", storageDriver) + + if err != nil { + return storage.ID, errors.Wrap(err, "failed init storage but storage is already created") + } + logger.Error("storage created", zap.Any("storage", storageDriver)) + return storage.ID, nil +} + +// LoadStorage load exist storage in db to memory +func (s *storagesStruct) LoadStorage(ctx context.Context, storage model.Storage) error { + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + // check driver first + driverName := storage.Driver + driverNew, err := op.GetDriverNew(driverName) + if err != nil { + return errors.WithMessage(err, "failed get driver new") + } + storageDriver := driverNew() + + err = s.initStorage(ctx, storage, storageDriver) + go op.CallStorageHooks("add", storageDriver) + logger.Info("storage created", zap.Any("storage", storageDriver)) + return err +} + +// initStorage initialize the driver and store to storagesMap +func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { + storageDriver.SetStorage(storage) + driverStorage := storageDriver.GetStorage() + + // Unmarshal Addition + + var json = jsoniter.ConfigCompatibleWithStandardLibrary + + err = json.UnmarshalFromString(driverStorage.Addition, storageDriver.GetAddition()) + if err == nil { + err = storageDriver.Init(ctx) + } + storagesMap.Store(driverStorage.MountPath, storageDriver) + if err != nil { + driverStorage.SetStatus(err.Error()) + err = errors.Wrap(err, "failed init storage") + } else { + driverStorage.SetStatus(op.WORK) + } + s.MustSaveDriverStorage(storageDriver) + return err +} + +func (s *storagesStruct) EnableStorage(ctx context.Context, id uint) error { + storage, err := MyService.Storage().GetStorageById(id) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + if !storage.Disabled { + return errors.Errorf("this storage have enabled") + } + storage.Disabled = false + err = MyService.Storage().UpdateStorage(storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in db") + } + err = s.LoadStorage(ctx, *storage) + if err != nil { + return errors.WithMessage(err, "failed load storage") + } + return nil +} + +func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error { + storage, err := MyService.Storage().GetStorageById(id) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + if storage.Disabled { + return errors.Errorf("this storage have disabled") + } + storageDriver, err := GetStorageByMountPath(storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + // drop the storage in the driver + if err := storageDriver.Drop(ctx); err != nil { + return errors.Wrap(err, "failed drop storage") + } + // delete the storage in the memory + storage.Disabled = true + err = MyService.Storage().UpdateStorage(storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in db") + } + storagesMap.Delete(storage.MountPath) + go op.CallStorageHooks("del", storageDriver) + return nil +} + +// UpdateStorage update storage +// get old storage first +// drop the storage then reinitialize +func (s *storagesStruct) UpdateStorage(ctx context.Context, storage model.Storage) error { + oldStorage, err := MyService.Storage().GetStorageById(storage.ID) + if err != nil { + return errors.WithMessage(err, "failed get old storage") + } + if oldStorage.Driver != storage.Driver { + return errors.Errorf("driver cannot be changed") + } + storage.Modified = time.Now() + storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + err = MyService.Storage().UpdateStorage(&storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in database") + } + if storage.Disabled { + return nil + } + storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if oldStorage.MountPath != storage.MountPath { + // mount path renamed, need to drop the storage + storagesMap.Delete(oldStorage.MountPath) + } + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + err = storageDriver.Drop(ctx) + if err != nil { + return errors.Wrapf(err, "failed drop storage") + } + + err = s.initStorage(ctx, storage, storageDriver) + go op.CallStorageHooks("update", storageDriver) + + logger.Info("storage updated", zap.Any("storage", storageDriver)) + return err +} + +func (s *storagesStruct) DeleteStorageById(ctx context.Context, id uint) error { + storage, err := MyService.Storage().GetStorageById(id) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + if !storage.Disabled { + storageDriver, err := GetStorageByMountPath(storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + // drop the storage in the driver + if err := storageDriver.Drop(ctx); err != nil { + return errors.Wrapf(err, "failed drop storage") + } + // delete the storage in the memory + storagesMap.Delete(storage.MountPath) + go op.CallStorageHooks("del", storageDriver) + } + // delete the storage in the database + if err := MyService.Storage().DeleteStorageById(id); err != nil { + return errors.WithMessage(err, "failed delete storage in database") + } + return nil +} + +// MustSaveDriverStorage call from specific driver +func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) { + err := saveDriverStorage(driver) + if err != nil { + logger.Error("failed save driver storage", zap.Any("err", err)) + } +} + +func saveDriverStorage(driver driver.Driver) error { + storage := driver.GetStorage() + addition := driver.GetAddition() + + var json = jsoniter.ConfigCompatibleWithStandardLibrary + + str, err := json.MarshalToString(addition) + if err != nil { + return errors.Wrap(err, "error while marshal addition") + } + storage.Addition = str + err = MyService.Storage().UpdateStorage(storage) + if err != nil { + return errors.WithMessage(err, "failed update storage in database") + } + return nil +} + +// getStoragesByPath get storage by longest match path, contains balance storage. +// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance +// getStoragesByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance +func getStoragesByPath(path string) []driver.Driver { + storages := make([]driver.Driver, 0) + curSlashCount := 0 + storagesMap.Range(func(mountPath string, value driver.Driver) bool { + mountPath = utils.GetActualMountPath(mountPath) + // is this path + if utils.IsSubPath(mountPath, path) { + slashCount := strings.Count(utils.PathAddSeparatorSuffix(mountPath), "/") + // not the longest match + if slashCount > curSlashCount { + storages = storages[:0] + curSlashCount = slashCount + } + if slashCount == curSlashCount { + storages = append(storages, value) + } + } + return true + }) + // make sure the order is the same for same input + sort.Slice(storages, func(i, j int) bool { + return storages[i].GetStorage().MountPath < storages[j].GetStorage().MountPath + }) + return storages +} + +// GetStorageVirtualFilesByPath Obtain the virtual file generated by the storage according to the path +// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av +// GetStorageVirtualFilesByPath(/a) => b,c,d +func (s *storagesStruct) GetStorageVirtualFilesByPath(prefix string) []model.Obj { + files := make([]model.Obj, 0) + storages := storagesMap.Values() + sort.Slice(storages, func(i, j int) bool { + if storages[i].GetStorage().Order == storages[j].GetStorage().Order { + return storages[i].GetStorage().MountPath < storages[j].GetStorage().MountPath + } + return storages[i].GetStorage().Order < storages[j].GetStorage().Order + }) + + prefix = utils.FixAndCleanPath(prefix) + set := mapset.NewSet[string]() + for _, v := range storages { + mountPath := utils.GetActualMountPath(v.GetStorage().MountPath) + // Exclude prefix itself and non prefix + if len(prefix) >= len(mountPath) || !utils.IsSubPath(prefix, mountPath) { + continue + } + name := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2)[0] + if set.Add(name) { + files = append(files, &model.Object{ + Name: name, + Size: 0, + Modified: v.GetStorage().Modified, + IsFolder: true, + }) + } + } + return files +} + +var balanceMap generic_sync.MapOf[string, int] + +// GetBalancedStorage get storage by path +func (s *storagesStruct) GetBalancedStorage(path string) driver.Driver { + path = utils.FixAndCleanPath(path) + storages := getStoragesByPath(path) + storageNum := len(storages) + switch storageNum { + case 0: + return nil + case 1: + return storages[0] + default: + virtualPath := utils.GetActualMountPath(storages[0].GetStorage().MountPath) + i, _ := balanceMap.LoadOrStore(virtualPath, 0) + i = (i + 1) % storageNum + balanceMap.Store(virtualPath, i) + return storages[i] + } +} +func (s *storagesStruct) InitStorages() { + storages, err := MyService.Storage().GetEnabledStorages() + if err != nil { + logger.Error("failed get enabled storages", zap.Any("err", err)) + } + go func(storages []model.Storage) { + for i := range storages { + err := s.LoadStorage(context.Background(), storages[i]) + if err != nil { + logger.Error("failed get enabled storages", zap.Any("err", err)) + } else { + logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath)) + } + } + conf.StoragesLoaded = true + }(storages) + +} +func NewStoragesService() StoragesService { + return &storagesStruct{} +}