beenull/CasaOS
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

added google drive and dropbox driver

Browse source
This commit is contained in:
link 2023-02-02 03:36:59 +00:00
parent 87d8be8c61
commit 28d3ca0ca6
38 changed files with 2004 additions and 389 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
build/sysroot/usr/lib/systemd/system/rclone.service Normal file
View file

@ -0,0 +1,11 @@
[Unit]
Description=rclone
[Service]
ExecStartPre=/usr/bin/rm -f /tmp/rclone.sock
ExecStart=/usr/bin/rclone rcd --rc-addr unix:///tmp/rclone.sock --rc-no-auth
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target

6
build/sysroot/usr/share/casaos/shell/helper.sh
View file

@ -133,11 +133,7 @@ GetPlugInDisk() {
fdisk -l | grep 'Disk' | grep 'sd' | awk -F , '{print substr($1,11,3)}' fdisk -l | grep 'Disk' | grep 'sd' | awk -F , '{print substr($1,11,3)}'
} }
#获取磁盘状态
#param 磁盘路径
GetDiskHealthState() {
smartctl -H $1 | grep "SMART Health Status" | awk -F ":" '{print$2}'
}
#获取磁盘字节数量和扇区数量 #获取磁盘字节数量和扇区数量
#param 磁盘路径 /dev/sda #param 磁盘路径 /dev/sda

1
drivers/all.go
View file

@ -1,6 +1,7 @@
package drivers package drivers
import ( import (
_ "github.com/IceWhaleTech/CasaOS/drivers/dropbox"
_ "github.com/IceWhaleTech/CasaOS/drivers/google_drive" _ "github.com/IceWhaleTech/CasaOS/drivers/google_drive"
) )

100
drivers/dropbox/drive.go Normal file
View file

@ -0,0 +1,100 @@
package dropbox
import (
"context"
"errors"
"net/http"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
type Dropbox struct {
model.Storage
Addition
AccessToken string
}
func (d *Dropbox) Config() driver.Config {
return config
}
func (d *Dropbox) GetAddition() driver.Additional {
return &d.Addition
}
func (d *Dropbox) Init(ctx context.Context) error {
if len(d.RefreshToken) == 0 {
d.getRefreshToken()
}
return d.refreshToken()
}
func (d *Dropbox) Drop(ctx context.Context) error {
return nil
}
func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
files, err := d.getFiles(dir.GetID())
if err != nil {
return nil, err
}
return utils.SliceConvert(files, func(src File) (model.Obj, error) {
return fileToObj(src), nil
})
}
func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
url := "https://content.dropboxapi.com/2/files/download"
link := model.Link{
URL: url,
Method: http.MethodPost,
Header: http.Header{
"Authorization": []string{"Bearer " + d.AccessToken},
"Dropbox-API-Arg": []string{`{"path": "` + file.GetPath() + `"}`},
},
}
return &link, nil
}
func (d *Dropbox) GetUserInfo(ctx context.Context) (string, error) {
url := "https://api.dropboxapi.com/2/users/get_current_account"
user := UserInfo{}
resp, err := d.request(url, http.MethodPost, func(req *resty.Request) {
req.SetHeader("Content-Type", "")
}, &user)
if err != nil {
return "", err
}
logger.Info("resp", zap.Any("resp", string(resp)))
return user.Email, nil
}
func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
return nil
}
func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
return nil
}
func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
return nil
}
func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
return errors.New("not support")
}
func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error {
return nil
}
func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
return nil
}
var _ driver.Driver = (*Dropbox)(nil)

31
drivers/dropbox/meta.go Normal file
View file

@ -0,0 +1,31 @@
package dropbox
import (
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/internal/op"
)
const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"
type Addition struct {
driver.RootID
RefreshToken string `json:"refresh_token" required:"true" omit:"true"`
AppKey string `json:"app_key" type:"string" default:"onr2ic0c0m97mxr" omit:"true"`
AppSecret string `json:"app_secret" type:"string" default:"nd3cjtikbxyj3pz" omit:"true"`
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`
AuthUrl string `json:"auth_url" type:"string" default:"https://www.dropbox.com/oauth2/authorize?client_id=onr2ic0c0m97mxr&redirect_uri=https://test-get.casaos.io&response_type=code&token_access_type=offline&state=${HOST}%2Fv1%2Frecover%2FDropbox"`
Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"`
Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`
}
var config = driver.Config{
Name: "Dropbox",
OnlyProxy: true,
DefaultRoot: "root",
}
func init() {
op.RegisterDriver(func() driver.Driver {
return &Dropbox{}
})
}

88
drivers/dropbox/types.go Normal file
View file

@ -0,0 +1,88 @@
package dropbox
import (
"time"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/model"
"go.uber.org/zap"
)
type UserInfo struct {
AccountID string `json:"account_id"`
Name struct {
GivenName string `json:"given_name"`
Surname string `json:"surname"`
FamiliarName string `json:"familiar_name"`
DisplayName string `json:"display_name"`
AbbreviatedName string `json:"abbreviated_name"`
} `json:"name"`
Email string `json:"email"`
EmailVerified bool `json:"email_verified"`
Disabled bool `json:"disabled"`
Country string `json:"country"`
Locale string `json:"locale"`
ReferralLink string `json:"referral_link"`
IsPaired bool `json:"is_paired"`
AccountType struct {
Tag string `json:".tag"`
} `json:"account_type"`
RootInfo struct {
Tag string `json:".tag"`
RootNamespaceID string `json:"root_namespace_id"`
HomeNamespaceID string `json:"home_namespace_id"`
} `json:"root_info"`
}
type TokenError struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
type File struct {
Tag string `json:".tag"`
Name string `json:"name"`
PathLower string `json:"path_lower"`
PathDisplay string `json:"path_display"`
ID string `json:"id"`
ClientModified time.Time `json:"client_modified,omitempty"`
ServerModified time.Time `json:"server_modified,omitempty"`
Rev string `json:"rev,omitempty"`
Size int `json:"size,omitempty"`
IsDownloadable bool `json:"is_downloadable,omitempty"`
ContentHash string `json:"content_hash,omitempty"`
}
type Files struct {
Files []File `json:"entries"`
Cursor string `json:"cursor"`
HasMore bool `json:"has_more"`
}
type Error struct {
Error struct {
Errors []struct {
Domain string `json:"domain"`
Reason string `json:"reason"`
Message string `json:"message"`
LocationType string `json:"location_type"`
Location string `json:"location"`
}
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
func fileToObj(f File) *model.ObjThumb {
logger.Info("dropbox file", zap.Any("file", f))
obj := &model.ObjThumb{
Object: model.Object{
ID: f.ID,
Name: f.Name,
Size: int64(f.Size),
Modified: f.ClientModified,
IsFolder: f.Tag == "folder",
Path: f.PathDisplay,
},
Thumbnail: model.Thumbnail{},
}
return obj
}

102
drivers/dropbox/util.go Normal file
View file

@ -0,0 +1,102 @@
package dropbox
import (
"fmt"
"net/http"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/base"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
func (d *Dropbox) getRefreshToken() error {
url := "https://api.dropbox.com/oauth2/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"code": d.Code,
"grant_type": "authorization_code",
"redirect_uri": "https://test-get.casaos.io",
}).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)
if err != nil {
return err
}
logger.Info("get refresh token", zap.String("res", res.String()))
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.RefreshToken = resp.RefreshToken
return nil
}
func (d *Dropbox) refreshToken() error {
url := "https://api.dropbox.com/oauth2/token"
var resp base.TokenResp
var e TokenError
res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
SetFormData(map[string]string{
"refresh_token": d.RefreshToken,
"grant_type": "refresh_token",
}).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)
if err != nil {
return err
}
logger.Info("get refresh token", zap.String("res", res.String()))
if e.Error != "" {
return fmt.Errorf(e.Error)
}
d.AccessToken = resp.AccessToken
return nil
}
func (d *Dropbox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
req := base.RestyClient.R()
req.SetHeader("Authorization", "Bearer "+d.AccessToken)
req.SetHeader("Content-Type", "application/json")
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(resp)
}
var e Error
req.SetError(&e)
res, err := req.Execute(method, url)
if err != nil {
return nil, err
}
if e.Error.Code != 0 {
if e.Error.Code == 401 {
err = d.refreshToken()
if err != nil {
return nil, err
}
return d.request(url, method, callback, resp)
}
return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
}
return res.Body(), nil
}
func (d *Dropbox) getFiles(path string) ([]File, error) {
res := make([]File, 0)
var resp Files
body := base.Json{
"limit": 2000,
"path": path,
}
_, err := d.request("https://api.dropboxapi.com/2/files/list_folder", http.MethodPost, func(req *resty.Request) {
req.SetBody(body)
}, &resp)
if err != nil {
return nil, err
}
res = append(res, resp.Files...)
return res, nil
}

15
drivers/google_drive/drive.go
View file

@ -7,11 +7,13 @@ import (
"net/http" "net/http"
"strconv" "strconv"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/base" "github.com/IceWhaleTech/CasaOS/drivers/base"
"github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils" "github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/go-resty/resty/v2" "github.com/go-resty/resty/v2"
"go.uber.org/zap"
) )
type GoogleDrive struct { type GoogleDrive struct {
@ -33,7 +35,7 @@ func (d *GoogleDrive) Init(ctx context.Context) error {
d.ChunkSize = 5 d.ChunkSize = 5
} }
if len(d.RefreshToken) == 0 { if len(d.RefreshToken) == 0 {
return d.getRefreshToken() d.getRefreshToken()
} }
return d.refreshToken() return d.refreshToken()
} }
@ -59,6 +61,7 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA
return nil, err return nil, err
} }
link := model.Link{ link := model.Link{
Method: http.MethodGet,
URL: url + "&alt=media", URL: url + "&alt=media",
Header: http.Header{ Header: http.Header{
"Authorization": []string{"Bearer " + d.AccessToken}, "Authorization": []string{"Bearer " + d.AccessToken},
@ -66,6 +69,16 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA
} }
return &link, nil return &link, nil
} }
func (d *GoogleDrive) GetUserInfo(ctx context.Context) (string, error) {
url := "https://content.googleapis.com/drive/v3/about?fields=user"
user := UserInfo{}
resp, err := d.request(url, http.MethodGet, nil, &user)
if err != nil {
return "", err
}
logger.Info("resp", zap.Any("resp", resp))
return user.User.EmailAddress, nil
}
func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
data := base.Json{ data := base.Json{

16
drivers/google_drive/meta.go
View file

@ -5,17 +5,19 @@ import (
"github.com/IceWhaleTech/CasaOS/internal/op" "github.com/IceWhaleTech/CasaOS/internal/op"
) )
const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"
type Addition struct { type Addition struct {
driver.RootID driver.RootID
RefreshToken string `json:"refresh_token" required:"true"` RefreshToken string `json:"refresh_token" required:"true" omit:"true"`
OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime"` OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime" omit:"true"`
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`
ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com"` ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com" omit:"true"`
ClientSecret string `json:"client_secret" required:"true" default:"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"` ClientSecret string `json:"client_secret" required:"true" default:"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" omit:"true"`
ChunkSize int64 `json:"chunk_size" type:"number" default:"5" help:"chunk size while uploading (unit: MB)"` ChunkSize int64 `json:"chunk_size" type:"number" help:"chunk size while uploading (unit: MB)" omit:"true"`
AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Ftest-get.casaos.io&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"` AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Ftest-get.casaos.io&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"`
Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"` Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"`
Code string `json:"code" type:"string" help:"code from auth_url"` Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`
} }
var config = driver.Config{ var config = driver.Config{

11
drivers/google_drive/types.go
View file

@ -8,6 +8,17 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
type UserInfo struct {
User struct {
Kind string `json:"kind"`
DisplayName string `json:"displayName"`
PhotoLink string `json:"photoLink"`
Me bool `json:"me"`
PermissionID string `json:"permissionId"`
EmailAddress string `json:"emailAddress"`
} `json:"user"`
}
type TokenError struct { type TokenError struct {
Error string `json:"error"` Error string `json:"error"`
ErrorDescription string `json:"error_description"` ErrorDescription string `json:"error_description"`

2
go.mod
View file

@ -5,6 +5,7 @@ go 1.18
require ( require (
github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d
github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3
github.com/Unknwon/goconfig v1.0.0
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
github.com/deckarep/golang-set/v2 v2.1.0 github.com/deckarep/golang-set/v2 v2.1.0
@ -81,6 +82,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect

11
go.sum
View file

@ -3,6 +3,8 @@ github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d
github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ= github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ=
github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 h1:jQfIty6u06fPJCutpS+97qr8uho3RpQX+B/CwHPCv/Q= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 h1:jQfIty6u06fPJCutpS+97qr8uho3RpQX+B/CwHPCv/Q=
github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3/go.mod h1:xcemiRsXcs1zrmQxYMyExDjZ7UHYwkJqYE71IDIV0xA= github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3/go.mod h1:xcemiRsXcs1zrmQxYMyExDjZ7UHYwkJqYE71IDIV0xA=
github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A=
github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE=
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04=
github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
@ -119,6 +121,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googollee/go-socket.io v1.6.2 h1:olKLLHJtHz1IkL/OrTyNriZZvVQYEORNkJAqsOwPask= github.com/googollee/go-socket.io v1.6.2 h1:olKLLHJtHz1IkL/OrTyNriZZvVQYEORNkJAqsOwPask=
github.com/googollee/go-socket.io v1.6.2/go.mod h1:0vGP8/dXR9SZUMMD4+xxaGo/lohOw3YWMh2WRiWeKxg= github.com/googollee/go-socket.io v1.6.2/go.mod h1:0vGP8/dXR9SZUMMD4+xxaGo/lohOw3YWMh2WRiWeKxg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@ -134,6 +138,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@ -198,6 +204,10 @@ github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3
github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@ -313,6 +323,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

6
internal/driver/driver.go
View file

@ -9,6 +9,7 @@ import (
type Driver interface { type Driver interface {
Meta Meta
Reader Reader
User
//Writer //Writer
//Other //Other
} }
@ -37,7 +38,10 @@ type Reader interface {
// Link get url/filepath/reader of file // Link get url/filepath/reader of file
Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error)
} }
type User interface {
// GetRoot get root directory of user
GetUserInfo(ctx context.Context) (string, error)
}
type Getter interface { type Getter interface {
GetRoot(ctx context.Context) (model.Obj, error) GetRoot(ctx context.Context) (model.Obj, error)
} }

2
internal/driver/item.go
View file

@ -40,7 +40,7 @@ type RootPath struct {
} }
type RootID struct { type RootID struct {
RootFolderID string `json:"root_folder_id"` RootFolderID string `json:"root_folder_id" omit:"true"`
} }
func (r RootPath) GetRootPath() string { func (r RootPath) GetRootPath() string {

25
internal/op/driver.go
View file

@ -13,7 +13,7 @@ import (
type New func() driver.Driver type New func() driver.Driver
var driverNewMap = map[string]New{} var driverNewMap = map[string]New{}
var driverInfoMap = map[string]driver.Info{} var driverInfoMap = map[string][]driver.Item{} //driver.Info{}
func RegisterDriver(driver New) { func RegisterDriver(driver New) {
// log.Infof("register driver: [%s]", config.Name) // log.Infof("register driver: [%s]", config.Name)
@ -39,23 +39,26 @@ func GetDriverNames() []string {
return driverNames return driverNames
} }
func GetDriverInfoMap() map[string]driver.Info { // func GetDriverInfoMap() map[string]driver.Info {
// return driverInfoMap
// }
func GetDriverInfoMap() map[string][]driver.Item {
return driverInfoMap return driverInfoMap
} }
func registerDriverItems(config driver.Config, addition driver.Additional) { func registerDriverItems(config driver.Config, addition driver.Additional) {
// log.Debugf("addition of %s: %+v", config.Name, addition) // log.Debugf("addition of %s: %+v", config.Name, addition)
tAddition := reflect.TypeOf(addition) tAddition := reflect.TypeOf(addition)
for tAddition.Kind() == reflect.Pointer { for tAddition.Kind() == reflect.Pointer {
tAddition = tAddition.Elem() tAddition = tAddition.Elem()
} }
mainItems := getMainItems(config) //mainItems := getMainItems(config)
additionalItems := getAdditionalItems(tAddition, config.DefaultRoot) additionalItems := getAdditionalItems(tAddition, config.DefaultRoot)
driverInfoMap[config.Name] = driver.Info{ driverInfoMap[config.Name] = additionalItems
Common: mainItems, // driver.Info{
Additional: additionalItems, // Common: mainItems,
Config: config, // Additional: additionalItems,
} // Config: config,
// }
} }
func getMainItems(config driver.Config) []driver.Item { func getMainItems(config driver.Config) []driver.Item {
@ -128,6 +131,7 @@ func getMainItems(config driver.Config) []driver.Item {
func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {
var items []driver.Item var items []driver.Item
for i := 0; i < t.NumField(); i++ { for i := 0; i < t.NumField(); i++ {
field := t.Field(i) field := t.Field(i)
if field.Type.Kind() == reflect.Struct { if field.Type.Kind() == reflect.Struct {
items = append(items, getAdditionalItems(field.Type, defaultRoot)...) items = append(items, getAdditionalItems(field.Type, defaultRoot)...)
@ -139,6 +143,9 @@ func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {
if (ok1 && ignore == "true") || !ok2 { if (ok1 && ignore == "true") || !ok2 {
continue continue
} }
if tag.Get("omit") == "true" {
continue
}
item := driver.Item{ item := driver.Item{
Name: name, Name: name,
Type: strings.ToLower(field.Type.Name()), Type: strings.ToLower(field.Type.Name()),

545
internal/op/fs.go Normal file
View file

@ -0,0 +1,545 @@
package op
import (
"context"
"os"
stdpath "path"
"time"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/generic_sync"
"github.com/IceWhaleTech/CasaOS/pkg/singleflight"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/Xhofe/go-cache"
"github.com/pkg/errors"
pkgerr "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// In order to facilitate adding some other things before and after file op
var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
var listG singleflight.Group[[]model.Obj]
func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == oldObj.GetName() {
objs[i] = newObj
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func delCacheObj(storage driver.Driver, path string, obj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, oldObj := range objs {
if oldObj.GetName() == obj.GetName() {
objs = append(objs[:i], objs[i+1:]...)
break
}
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
var addSortDebounceMap generic_sync.MapOf[string, func(func())]
func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {
key := Key(storage, path)
objs, ok := listCache.Get(key)
if ok {
for i, obj := range objs {
if obj.GetName() == newObj.GetName() {
objs[i] = newObj
return
}
}
// Simple separation of files and folders
if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {
objs = append(objs, newObj)
} else {
objs = append([]model.Obj{newObj}, objs...)
}
if storage.Config().LocalSort {
debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))
log.Debug("addCacheObj: wait start sort")
debounce(func() {
log.Debug("addCacheObj: start sort")
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
addSortDebounceMap.Delete(key)
})
}
listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
}
}
func ClearCache(storage driver.Driver, path string) {
listCache.Del(Key(storage, path))
}
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))
}
// List files in storage, not contains virtual file
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
log.Debugf("op.List %s", path)
key := Key(storage, path)
if !utils.IsBool(refresh...) {
if files, ok := listCache.Get(key); ok {
log.Debugf("use cache when list %s", path)
return files, nil
}
}
dir, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, errors.WithMessage(err, "failed get dir")
}
log.Debugf("list dir: %+v", dir)
if !dir.IsDir() {
return nil, errors.WithStack(errors.New("not a folder"))
}
objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
files, err := storage.List(ctx, dir, args)
if err != nil {
return nil, errors.Wrapf(err, "failed to list objs")
}
// set path
for _, f := range files {
if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" {
s.SetPath(stdpath.Join(dir.GetPath(), f.GetName()))
}
}
// warp obj name
model.WrapObjsName(files)
// call hooks
go func(reqPath string, files []model.Obj) {
for _, hook := range ObjsUpdateHooks {
hook(args.ReqPath, files)
}
}(args.ReqPath, files)
// sort objs
if storage.Config().LocalSort {
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
} else {
log.Debugf("del cache: %s", key)
listCache.Del(key)
}
}
return files, nil
})
return objs, err
}
// Get object from list of files
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
path = utils.FixAndCleanPath(path)
log.Debugf("op.Get %s", path)
// is root folder
if utils.PathEqual(path, "/") {
var rootObj model.Obj
switch r := storage.GetAddition().(type) {
case driver.IRootId:
rootObj = &model.Object{
ID: r.GetRootId(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
Path: path,
}
case driver.IRootPath:
rootObj = &model.Object{
Path: r.GetRootPath(),
Name: RootName,
Size: 0,
Modified: storage.GetStorage().Modified,
IsFolder: true,
}
default:
if storage, ok := storage.(driver.Getter); ok {
obj, err := storage.GetRoot(ctx)
if err != nil {
return nil, errors.WithMessage(err, "failed get root obj")
}
rootObj = obj
}
}
if rootObj == nil {
return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method")
}
return &model.ObjWrapName{
Name: RootName,
Obj: rootObj,
}, nil
}
// not root folder
dir, name := stdpath.Split(path)
files, err := List(ctx, storage, dir, model.ListArgs{})
if err != nil {
return nil, errors.WithMessage(err, "failed get parent list")
}
for _, f := range files {
// TODO maybe copy obj here
if f.GetName() == name {
return f, nil
}
}
log.Debugf("cant find obj with name: %s", name)
return nil, errors.WithStack(errors.New("object not found"))
}
func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
obj, err := Get(ctx, storage, path)
if err != nil {
return nil, err
}
return model.UnwrapObjs(obj), err
}
var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
var linkG singleflight.Group[*model.Link]
// Link get link, if is an url. should have an expiry time
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
file, err := GetUnwrap(ctx, storage, path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed to get file")
}
if file.IsDir() {
return nil, nil, errors.WithStack(errors.New("not a file"))
}
key := Key(storage, path) + ":" + args.IP
if link, ok := linkCache.Get(key); ok {
return link, file, nil
}
fn := func() (*model.Link, error) {
link, err := storage.Link(ctx, file, args)
if err != nil {
return nil, errors.Wrapf(err, "failed get link")
}
if link.Expiration != nil {
linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))
}
return link, nil
}
link, err, _ := linkG.Do(key, fn)
return link, file, err
}
// Other api
func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) {
obj, err := GetUnwrap(ctx, storage, args.Path)
if err != nil {
return nil, errors.WithMessagef(err, "failed to get obj")
}
if o, ok := storage.(driver.Other); ok {
return o.Other(ctx, model.OtherArgs{
Obj: obj,
Method: args.Method,
Data: args.Data,
})
} else {
return nil, errors.New("not implement")
}
}
var mkdirG singleflight.Group[interface{}]
func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
key := Key(storage, path)
_, err, _ := mkdirG.Do(key, func() (interface{}, error) {
// check if dir exists
f, err := GetUnwrap(ctx, storage, path)
if err != nil {
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
parentPath, dirName := stdpath.Split(path)
err = MakeDir(ctx, storage, parentPath)
if err != nil {
return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
}
parentDir, err := GetUnwrap(ctx, storage, parentPath)
// this should not happen
if err != nil {
return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
switch s := storage.(type) {
case driver.MkdirResult:
var newObj model.Obj
newObj, err = s.MakeDir(ctx, parentDir, dirName)
if err == nil {
if newObj != nil {
addCacheObj(storage, parentPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
}
case driver.Mkdir:
err = s.MakeDir(ctx, parentDir, dirName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, parentPath)
}
default:
return nil, errors.New("not implement")
}
return nil, errors.WithStack(err)
}
return nil, errors.WithMessage(err, "failed to check if dir exists")
}
// dir exists
if f.IsDir() {
return nil, nil
}
// dir to make is a file
return nil, errors.New("file exists")
})
return err
}
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.MoveResult:
var newObj model.Obj
newObj, err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Move:
err = s.Move(ctx, srcObj, dstDir)
if err == nil {
delCacheObj(storage, srcDirPath, srcRawObj)
if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
srcRawObj, err := Get(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
srcObj := model.UnwrapObjs(srcRawObj)
srcDirPath := stdpath.Dir(srcPath)
switch s := storage.(type) {
case driver.RenameResult:
var newObj model.Obj
newObj, err = s.Rename(ctx, srcObj, dstName)
if err == nil {
if newObj != nil {
updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
}
case driver.Rename:
err = s.Rename(ctx, srcObj, dstName)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, srcDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
// Copy Just copy file[s] in a storage
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
srcPath = utils.FixAndCleanPath(srcPath)
dstDirPath = utils.FixAndCleanPath(dstDirPath)
srcObj, err := GetUnwrap(ctx, storage, srcPath)
if err != nil {
return errors.WithMessage(err, "failed to get src object")
}
dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed to get dst dir")
}
switch s := storage.(type) {
case driver.CopyResult:
var newObj model.Obj
newObj, err = s.Copy(ctx, srcObj, dstDir)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Copy:
err = s.Copy(ctx, srcObj, dstDir)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Remove(ctx context.Context, storage driver.Driver, path string) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.FixAndCleanPath(path)
rawObj, err := Get(ctx, storage, path)
if err != nil {
// if object not found, it's ok
if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {
return nil
}
return errors.WithMessage(err, "failed to get object")
}
dirPath := stdpath.Dir(path)
switch s := storage.(type) {
case driver.Remove:
err = s.Remove(ctx, model.UnwrapObjs(rawObj))
if err == nil {
delCacheObj(storage, dirPath, rawObj)
}
default:
return errors.New("not implement")
}
return errors.WithStack(err)
}
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
defer func() {
if f, ok := file.GetReadCloser().(*os.File); ok {
err := os.RemoveAll(f.Name())
if err != nil {
log.Errorf("failed to remove file [%s]", f.Name())
}
}
}()
defer func() {
if err := file.Close(); err != nil {
log.Errorf("failed to close file streamer, %v", err)
}
}()
// if file exist and size = 0, delete it
dstDirPath = utils.FixAndCleanPath(dstDirPath)
dstPath := stdpath.Join(dstDirPath, file.GetName())
fi, err := GetUnwrap(ctx, storage, dstPath)
if err == nil {
if fi.GetSize() == 0 {
err = Remove(ctx, storage, dstPath)
if err != nil {
return errors.WithMessagef(err, "failed remove file that exist and have size 0")
}
} else {
file.Old = fi
}
}
err = MakeDir(ctx, storage, dstDirPath)
if err != nil {
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
}
parentDir, err := GetUnwrap(ctx, storage, dstDirPath)
// this should not happen
if err != nil {
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
}
// if up is nil, set a default to prevent panic
if up == nil {
up = func(p int) {}
}
switch s := storage.(type) {
case driver.PutResult:
var newObj model.Obj
newObj, err = s.Put(ctx, parentDir, file, up)
if err == nil {
if newObj != nil {
addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
} else if !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
}
case driver.Put:
err = s.Put(ctx, parentDir, file, up)
if err == nil && !utils.IsBool(lazyCache...) {
ClearCache(storage, dstDirPath)
}
default:
return errors.New("not implement")
}
log.Debugf("put file [%s] done", file.GetName())
//if err == nil {
// //clear cache
// key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
// listCache.Del(key)
//}
return errors.WithStack(err)
}

36
internal/sign/sign.go Normal file
View file

@ -0,0 +1,36 @@
package sign
import (
"sync"
"time"
"github.com/IceWhaleTech/CasaOS/pkg/sign"
)
var once sync.Once
var instance sign.Sign
func Sign(data string) string {
return NotExpired(data)
}
func WithDuration(data string, d time.Duration) string {
once.Do(Instance)
return instance.Sign(data, time.Now().Add(d).Unix())
}
func NotExpired(data string) string {
once.Do(Instance)
return instance.Sign(data, 0)
}
func Verify(data string, sign string) error {
once.Do(Instance)
return instance.Verify(data, sign)
}
func Instance() {
instance = sign.NewHMACSign([]byte("token"))
}

15
main.go
View file

@ -13,6 +13,7 @@ import (
"github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/pkg/cache" "github.com/IceWhaleTech/CasaOS/pkg/cache"
"github.com/IceWhaleTech/CasaOS/pkg/config" "github.com/IceWhaleTech/CasaOS/pkg/config"
"github.com/IceWhaleTech/CasaOS/pkg/config/configfile"
"github.com/IceWhaleTech/CasaOS/pkg/sqlite" "github.com/IceWhaleTech/CasaOS/pkg/sqlite"
"github.com/IceWhaleTech/CasaOS/pkg/utils/command" "github.com/IceWhaleTech/CasaOS/pkg/utils/command"
"github.com/IceWhaleTech/CasaOS/pkg/utils/file" "github.com/IceWhaleTech/CasaOS/pkg/utils/file"
@ -69,6 +70,20 @@ func init() {
service.MyService.Storages().InitStorages() service.MyService.Storages().InitStorages()
route.InitFunction() route.InitFunction()
data := &configfile.Storage{}
e := data.Load()
fmt.Println(e)
fmt.Println(data.GetSectionList())
// fmt.Println(data.HasSection("google"))
// fmt.Println(data.GetKeyList("google"))
// fmt.Println(data.GetValue("google", "token"))
// data.SetValue("google", "type", "drive")
// data.SetValue("google", "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com")
// data.SetValue("google", "client_secret", "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")
// data.SetValue("google", "scope", "drive")
// data.SetValue("google", "token", `{"access_token":"ya29.a0AVvZVsqsy3vWjpjsl87mtxirrtkHpkyEXdvlORzZeIahObdEtDE47-Hzo1bIg8vJhfYKh-cdqgrUM305hiEJssFMcpkM-0IwPyxlpynMFWS0L5356AUvbv3DUd_RbV_MbKijyTThuDkfrXdLIiEOwxMOtYSXmDUaCgYKAbgSAQASFQGbdwaI6ae1NZbJARogHtpjitLGkg0166","token_type":"Bearer","refresh_token":"1//01CoIJ-aZDrUPCgYIARAAGAESNwF-L9IrNLyzp1Xzfa_sPPMouyrTgJrVchPX6uXqMizXjohTdycCpVgVcu402ND-Ikn2hArRGXA","expiry":"2023-01-28T19:26:50.198064816+08:00"}`)
//e = data.Save()
//fmt.Println(e)
} }
// @title casaOS API // @title casaOS API

1
model/args.go
View file

@ -23,6 +23,7 @@ type Link struct {
Status int // status maybe 200 or 206, etc Status int // status maybe 200 or 206, etc
FilePath *string // local file, return the filepath FilePath *string // local file, return the filepath
Expiration *time.Duration // url expiration time Expiration *time.Duration // url expiration time
Method string `json:"method"` // http method
} }
type OtherArgs struct { type OtherArgs struct {

69
model/smartctl_model.go
View file

@ -1,69 +0,0 @@
package model
//
type SmartctlA struct {
Smartctl struct {
Version []int `json:"version"`
SvnRevision string `json:"svn_revision"`
PlatformInfo string `json:"platform_info"`
BuildInfo string `json:"build_info"`
Argv []string `json:"argv"`
ExitStatus int `json:"exit_status"`
} `json:"smartctl"`
Device struct {
Name string `json:"name"`
InfoName string `json:"info_name"`
Type string `json:"type"`
Protocol string `json:"protocol"`
} `json:"device"`
ModelName string `json:"model_name"`
SerialNumber string `json:"serial_number"`
FirmwareVersion string `json:"firmware_version"`
UserCapacity struct {
Blocks int `json:"blocks"`
Bytes int64 `json:"bytes"`
} `json:"user_capacity"`
SmartStatus struct {
Passed bool `json:"passed"`
} `json:"smart_status"`
AtaSmartData struct {
OfflineDataCollection struct {
Status struct {
Value int `json:"value"`
String string `json:"string"`
} `json:"status"`
CompletionSeconds int `json:"completion_seconds"`
} `json:"offline_data_collection"`
SelfTest struct {
Status struct {
Value int `json:"value"`
String string `json:"string"`
Passed bool `json:"passed"`
} `json:"status"`
PollingMinutes struct {
Short int `json:"short"`
Extended int `json:"extended"`
Conveyance int `json:"conveyance"`
} `json:"polling_minutes"`
} `json:"self_test"`
Capabilities struct {
Values []int `json:"values"`
ExecOfflineImmediateSupported bool `json:"exec_offline_immediate_supported"`
OfflineIsAbortedUponNewCmd bool `json:"offline_is_aborted_upon_new_cmd"`
OfflineSurfaceScanSupported bool `json:"offline_surface_scan_supported"`
SelfTestsSupported bool `json:"self_tests_supported"`
ConveyanceSelfTestSupported bool `json:"conveyance_self_test_supported"`
SelectiveSelfTestSupported bool `json:"selective_self_test_supported"`
AttributeAutosaveEnabled bool `json:"attribute_autosave_enabled"`
ErrorLoggingSupported bool `json:"error_logging_supported"`
GpLoggingSupported bool `json:"gp_logging_supported"`
} `json:"capabilities"`
} `json:"ata_smart_data"`
PowerOnTime struct {
Hours int `json:"hours"`
} `json:"power_on_time"`
PowerCycleCount int `json:"power_cycle_count"`
Temperature struct {
Current int `json:"current"`
} `json:"temperature"`
}

12
pkg/fs/fs.go Normal file
View file

@ -0,0 +1,12 @@
package fs
import "io"
// CheckClose is a utility function used to check the return from
// Close in a defer statement.
func CheckClose(c io.Closer, err *error) {
cerr := c.Close()
if *err == nil {
*err = cerr
}
}

52
pkg/sign/hmac.go Normal file
View file

@ -0,0 +1,52 @@
package sign
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"io"
"strconv"
"strings"
"time"
)
type HMACSign struct {
SecretKey []byte
}
func (s HMACSign) Sign(data string, expire int64) string {
h := hmac.New(sha256.New, s.SecretKey)
expireTimeStamp := strconv.FormatInt(expire, 10)
_, err := io.WriteString(h, data+":"+expireTimeStamp)
if err != nil {
return ""
}
return base64.URLEncoding.EncodeToString(h.Sum(nil)) + ":" + expireTimeStamp
}
func (s HMACSign) Verify(data, sign string) error {
signSlice := strings.Split(sign, ":")
// check whether contains expire time
if signSlice[len(signSlice)-1] == "" {
return ErrExpireMissing
}
// check whether expire time is expired
expires, err := strconv.ParseInt(signSlice[len(signSlice)-1], 10, 64)
if err != nil {
return ErrExpireInvalid
}
// if expire time is expired, return error
if expires < time.Now().Unix() && expires != 0 {
return ErrSignExpired
}
// verify sign
if s.Sign(data, expires) != sign {
return ErrSignInvalid
}
return nil
}
func NewHMACSign(secret []byte) Sign {
return HMACSign{SecretKey: secret}
}

15
pkg/sign/sign.go Normal file
View file

@ -0,0 +1,15 @@
package sign
import "errors"
type Sign interface {
Sign(data string, expire int64) string
Verify(data, sign string) error
}
var (
ErrSignExpired = errors.New("sign expired")
ErrSignInvalid = errors.New("sign invalid")
ErrExpireInvalid = errors.New("expire invalid")
ErrExpireMissing = errors.New("expire missing")
)

19
pkg/utils/command/command_helper.go
View file

@ -2,14 +2,12 @@ package command
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
) )
func OnlyExec(cmdStr string) { func OnlyExec(cmdStr string) {
@ -98,23 +96,6 @@ func ExecLSBLKByPath(path string) []byte {
return output return output
} }
// exec smart
func ExecSmartCTLByPath(path string) []byte {
timeout := 3
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
defer cancel()
output, err := exec.CommandContext(ctx, "smartctl", "-a", path, "-j").Output()
if err != nil {
fmt.Println("smartctl", err)
return nil
}
return output
}
func ExecEnabledSMART(path string) {
exec.Command("smartctl", "-s on", path).Output()
}
func ExecuteScripts(scriptDirectory string) { func ExecuteScripts(scriptDirectory string) {
if _, err := os.Stat(scriptDirectory); os.IsNotExist(err) { if _, err := os.Stat(scriptDirectory); os.IsNotExist(err) {
fmt.Printf("No post-start scripts at %s\n", scriptDirectory) fmt.Printf("No post-start scripts at %s\n", scriptDirectory)

151
pkg/utils/httper/drive.go Normal file
View file

@ -0,0 +1,151 @@
package httper
import (
"encoding/json"
"fmt"
"net"
"net/http"
"time"
"github.com/go-resty/resty/v2"
)
type MountList struct {
MountPoints []struct {
MountPoint string `json:"MountPoint"`
Fs string `json:"Fs"`
Icon string `json:"Icon"`
} `json:"mountPoints"`
}
type MountResult struct {
Error string `json:"error"`
Input struct {
Fs string `json:"fs"`
MountPoint string `json:"mountPoint"`
} `json:"input"`
Path string `json:"path"`
Status int `json:"status"`
}
type RemotesResult struct {
Remotes []string `json:"remotes"`
}
var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
var DefaultTimeout = time.Second * 30
func NewRestyClient() *resty.Client {
unixSocket := "/tmp/rclone.sock"
transport := http.Transport{
Dial: func(_, _ string) (net.Conn, error) {
return net.Dial("unix", unixSocket)
},
}
client := resty.New()
client.SetTransport(&transport).SetBaseURL("http://localhost")
client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent)
return client
}
func GetMountList() (MountList, error) {
var result MountList
res, err := NewRestyClient().R().Post("/mount/listmounts")
if err != nil {
return result, err
}
if res.StatusCode() != 200 {
return result, fmt.Errorf("get mount list failed")
}
json.Unmarshal(res.Body(), &result)
for i := 0; i < len(result.MountPoints); i++ {
result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1]
}
return result, err
}
func Mount(mountPoint string, fs string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"mountPoint": mountPoint,
"fs": fs,
}).Post("/mount/mount")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("mount failed")
}
return nil
}
func Unmount(mountPoint string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"mountPoint": mountPoint,
}).Post("/mount/unmount")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("unmount failed")
}
return nil
}
func CreateConfig(data map[string]string, name, t string) error {
data["config_is_local"] = "false"
dataStr, _ := json.Marshal(data)
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
"parameters": string(dataStr),
"type": t,
}).Post("/config/create")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("create config failed")
}
return nil
}
func GetConfigByName(name string) (map[string]string, error) {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
}).Post("/config/get")
if err != nil {
return nil, err
}
if res.StatusCode() != 200 {
return nil, fmt.Errorf("create config failed")
}
var result map[string]string
json.Unmarshal(res.Body(), &result)
return result, nil
}
func GetAllConfigName() (RemotesResult, error) {
var result RemotesResult
res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes")
if err != nil {
return result, err
}
if res.StatusCode() != 200 {
return result, fmt.Errorf("get config failed")
}
json.Unmarshal(res.Body(), &result)
return result, nil
}
func DeleteConfigByName(name string) error {
res, err := NewRestyClient().R().SetFormData(map[string]string{
"name": name,
}).Post("/config/delete")
if err != nil {
return err
}
if res.StatusCode() != 200 {
return fmt.Errorf("delete config failed")
}
return nil
}

37
pkg/utils/time.go Normal file
View file

@ -0,0 +1,37 @@
package utils
import (
"sync"
"time"
)
func MustParseCNTime(str string) time.Time {
lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local)
return lastOpTime
}
func NewDebounce(interval time.Duration) func(f func()) {
var timer *time.Timer
var lock sync.Mutex
return func(f func()) {
lock.Lock()
defer lock.Unlock()
if timer != nil {
timer.Stop()
}
timer = time.AfterFunc(interval, f)
}
}
func NewDebounce2(interval time.Duration, f func()) func() {
var timer *time.Timer
var lock sync.Mutex
return func() {
lock.Lock()
defer lock.Unlock()
if timer == nil {
timer = time.AfterFunc(interval, f)
}
(*time.Timer)(timer).Reset(interval)
}
}

5
route/init.go
View file

@ -89,4 +89,9 @@ func InitNetworkMount() {
connection.Directories = strings.Join(directories, ",") connection.Directories = strings.Join(directories, ",")
service.MyService.Connections().UpdateConnection(&connection) service.MyService.Connections().UpdateConnection(&connection)
} }
err := service.MyService.Storage().CheckAndMountAll()
if err != nil {
logger.Error("mount storage err", zap.Any("err", err))
}
} }

6
route/route.go
View file

@ -118,14 +118,8 @@ func InitRouter() *gin.Engine {
v1StorageGroup.Use() v1StorageGroup.Use()
{ {
v1StorageGroup.GET("", v1.ListStorages) v1StorageGroup.GET("", v1.ListStorages)
v1StorageGroup.POST("", v1.CreateStorage)
v1StorageGroup.DELETE("", v1.DeleteStorage) v1StorageGroup.DELETE("", v1.DeleteStorage)
} }
v1FsGroup := v1Group.Group("/fs")
v1FsGroup.Use()
{
v1FsGroup.POST("/list", v1.FsList)
}
v1DriverGroup := v1Group.Group("/driver") v1DriverGroup := v1Group.Group("/driver")
v1DriverGroup.Use() v1DriverGroup.Use()
{ {

206
route/v1/file.go
View file

@ -1,6 +1,7 @@
package v1 package v1
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
@ -16,10 +17,16 @@ import (
"sync" "sync"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/internal/conf"
"github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils"
"github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err"
"github.com/IceWhaleTech/CasaOS/pkg/utils/file" "github.com/IceWhaleTech/CasaOS/pkg/utils/file"
"github.com/IceWhaleTech/CasaOS/service" "github.com/IceWhaleTech/CasaOS/service"
"github.com/IceWhaleTech/CasaOS/internal/sign"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
uuid "github.com/satori/go.uuid" uuid "github.com/satori/go.uuid"
"go.uber.org/zap" "go.uber.org/zap"
@ -190,6 +197,37 @@ func GetDownloadSingleFile(c *gin.Context) {
}) })
return return
} }
fileName := path.Base(filePath)
// c.Header("Content-Disposition", "inline")
c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName))
storage, _ := service.MyService.FsService().GetStorage(filePath)
if storage != nil {
if shouldProxy(storage, fileName) {
Proxy(c)
return
} else {
link, _, err := service.MyService.FsService().Link(c, filePath, model.LinkArgs{
IP: c.ClientIP(),
Header: c.Request.Header,
Type: c.Query("type"),
})
if err != nil {
c.JSON(common_err.SERVICE_ERROR, model.Result{
Success: common_err.SERVICE_ERROR,
Message: common_err.GetMsg(common_err.SERVICE_ERROR),
Data: err.Error(),
})
return
}
c.Header("Referrer-Policy", "no-referrer")
c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")
c.Redirect(302, link.URL)
return
}
}
fileTmp, err := os.Open(filePath) fileTmp, err := os.Open(filePath)
if err != nil { if err != nil {
c.JSON(common_err.SERVICE_ERROR, model.Result{ c.JSON(common_err.SERVICE_ERROR, model.Result{
@ -200,9 +238,6 @@ func GetDownloadSingleFile(c *gin.Context) {
} }
defer fileTmp.Close() defer fileTmp.Close()
fileName := path.Base(filePath)
// c.Header("Content-Disposition", "inline")
c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName))
c.File(filePath) c.File(filePath)
} }
@ -701,3 +736,168 @@ func GetSize(c *gin.Context) {
} }
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: size}) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: size})
} }
func Proxy(c *gin.Context) {
rawPath := c.Query("path")
filename := filepath.Base(rawPath)
storage, err := service.MyService.FsService().GetStorage(rawPath)
if err != nil {
c.JSON(500, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return
}
if canProxy(storage, filename) {
downProxyUrl := storage.GetStorage().DownProxyUrl
if downProxyUrl != "" {
_, ok := c.GetQuery("d")
if !ok {
URL := fmt.Sprintf("%s%s?sign=%s",
strings.Split(downProxyUrl, "\n")[0],
utils.EncodePath(rawPath, true),
sign.Sign(rawPath))
c.Redirect(302, URL)
return
}
}
link, file, err := service.MyService.FsService().Link(c, rawPath, model.LinkArgs{
Header: c.Request.Header,
Type: c.Query("type"),
})
if err != nil {
c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return
}
err = CommonProxy(c.Writer, c.Request, link, file)
if err != nil {
c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return
}
} else {
c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: "proxy not allowed"})
return
}
}
// TODO need optimize
// when should be proxy?
// 1. config.MustProxy()
// 2. storage.WebProxy
// 3. proxy_types
func shouldProxy(storage driver.Driver, filename string) bool {
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
return true
}
if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) {
return true
}
return false
}
// TODO need optimize
// when can be proxy?
// 1. text file
// 2. config.MustProxy()
// 3. storage.WebProxy
// 4. proxy_types
// solution: text_file + shouldProxy()
func canProxy(storage driver.Driver, filename string) bool {
if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() {
return true
}
if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) {
return true
}
if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) {
return true
}
return false
}
var HttpClient = &http.Client{}
func CommonProxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {
// read data with native
var err error
if link.Data != nil {
defer func() {
_ = link.Data.Close()
}()
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))
w.Header().Set("Content-Length", strconv.FormatInt(file.GetSize(), 10))
if link.Header != nil {
// TODO clean header with blacklist or whitelist
link.Header.Del("set-cookie")
for h, val := range link.Header {
w.Header()[h] = val
}
}
if link.Status == 0 {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(link.Status)
}
_, err = io.Copy(w, link.Data)
if err != nil {
return err
}
return nil
}
// local file
if link.FilePath != nil && *link.FilePath != "" {
f, err := os.Open(*link.FilePath)
if err != nil {
return err
}
defer func() {
_ = f.Close()
}()
fileStat, err := os.Stat(*link.FilePath)
if err != nil {
return err
}
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))
http.ServeContent(w, r, file.GetName(), fileStat.ModTime(), f)
return nil
} else {
req, err := http.NewRequest(link.Method, link.URL, nil)
if err != nil {
return err
}
for h, val := range r.Header {
if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) {
continue
}
req.Header[h] = val
}
for h, val := range link.Header {
req.Header[h] = val
}
res, err := HttpClient.Do(req)
if err != nil {
return err
}
defer func() {
_ = res.Body.Close()
}()
logger.Info("proxy status", zap.Any("status", res.StatusCode))
// TODO clean header with blacklist or whitelist
res.Header.Del("set-cookie")
for h, v := range res.Header {
w.Header()[h] = v
}
w.WriteHeader(res.StatusCode)
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))
if res.StatusCode >= 400 {
all, _ := ioutil.ReadAll(res.Body)
msg := string(all)
logger.Info("msg", zap.Any("msg", msg))
return errors.New(msg)
}
_, err = io.Copy(w, res.Body)
if err != nil {
return err
}
return nil
}
}

146
route/v1/recover.go
View file

@ -2,41 +2,34 @@ package v1
import ( import (
"strconv" "strconv"
"strings"
"time" "time"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS/drivers/dropbox"
"github.com/IceWhaleTech/CasaOS/drivers/google_drive" "github.com/IceWhaleTech/CasaOS/drivers/google_drive"
"github.com/IceWhaleTech/CasaOS/internal/op" "github.com/IceWhaleTech/CasaOS/internal/op"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/service" "github.com/IceWhaleTech/CasaOS/service"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
) )
func GetRecoverStorage(c *gin.Context) { func GetRecoverStorage(c *gin.Context) {
c.Header("Content-Type", "text/html; charset=utf-8") c.Header("Content-Type", "text/html; charset=utf-8")
t := c.Param("type") t := c.Param("type")
currentTime := time.Now().UTC()
currentDate := time.Now().UTC().Format("2006-01-02")
// timeStr := time.Now().Format("20060102150405")
if t == "GoogleDrive" { if t == "GoogleDrive" {
mountPath := "google"
mountPath += time.Now().Format("20060102150405")
gd := op.GetDriverInfoMap()[t] gd := op.GetDriverInfoMap()[t]
var req model.Storage
req.Driver = t
req.MountPath = mountPath
req.CacheExpiration = 5
add := google_drive.Addition{} add := google_drive.Addition{}
add.Code = c.Query("code") add.Code = c.Query("code")
if len(add.Code) == 0 { if len(add.Code) == 0 {
c.String(200, `<p>code不可为空</p>`) c.String(200, `<p>code cannot be empty</p>`)
return return
} }
add.RootFolderID = "root" add.RootFolderID = "root"
for _, v := range gd.Additional { for _, v := range gd {
if v.Name == "client_id" { if v.Name == "client_id" {
add.ClientID = v.Default add.ClientID = v.Default
} }
@ -52,22 +45,119 @@ func GetRecoverStorage(c *gin.Context) {
} }
} }
var json = jsoniter.ConfigCompatibleWithStandardLibrary var google_drive google_drive.GoogleDrive
addStr, err := json.Marshal(add) google_drive.Addition = add
err := google_drive.Init(c)
if err != nil { if err != nil {
c.String(200, `<p>addition序列化失败</p>`) c.String(200, `<p>Initialization failure:`+err.Error()+`</p>`)
return return
} }
req.Addition = string(addStr)
logger.Info("GetRecoverStorage", zap.Any("req", req))
if _, err := service.MyService.Storages().CreateStorage(c, req); err != nil {
c.String(200, `<p>添加失败:`+err.Error()+`</p>`)
return
}
data := make(map[string]interface{})
data["status"] = "success"
service.MyService.Notify().SendNotify("recover_status", data)
}
c.String(200, `<p>关闭该页面即可</p><script>window.close()</script>`) username, err := google_drive.GetUserInfo(c)
if err != nil {
c.String(200, `<p>Failed to get user information:`+err.Error()+`</p>`)
return
}
if len(username) > 0 {
a := strings.Split(username, "@")
username = a[0]
}
username += "_drive"
dataMap, _ := service.MyService.Storage().GetConfigByName(username)
if len(dataMap) > 0 {
c.String(200, `<p>The same configuration has been added</p>`)
service.MyService.Storage().CheckAndMountByName(username)
return
}
dmap := make(map[string]string)
dmap["client_id"] = add.ClientID
dmap["client_secret"] = add.ClientSecret
dmap["scope"] = "drive"
dmap["mount_point"] = "/mnt/" + username
dmap["token"] = `{"access_token":"` + google_drive.AccessToken + `","token_type":"Bearer","refresh_token":"` + google_drive.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*1).Format("15:04:05") + `Z"}`
// data.SetValue(username, "type", "drive")
// data.SetValue(username, "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com")
// data.SetValue(username, "client_secret", "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")
// data.SetValue(username, "scope", "drive")
// data.SetValue(username, "mount_point", "/mnt/"+username)
// data.SetValue(username, "token", `{"access_token":"`+google_drive.AccessToken+`","token_type":"Bearer","refresh_token":"`+google_drive.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*1).Format("15:04:05")+`Z"}`)
// e = data.Save()
// if e != nil {
// c.String(200, `<p>保存配置失败:`+e.Error()+`</p>`)
// return
// }
service.MyService.Storage().CreateConfig(dmap, username, "drive")
service.MyService.Storage().MountStorage("/mnt/"+username, username+":")
notify := make(map[string]interface{})
notify["status"] = "success"
service.MyService.Notify().SendNotify("recover_status", notify)
} else if t == "Dropbox" {
//mountPath += timeStr
db := op.GetDriverInfoMap()[t]
add := dropbox.Addition{}
add.Code = c.Query("code")
if len(add.Code) == 0 {
c.String(200, `<p>code cannot be empty</p>`)
return
}
add.RootFolderID = ""
for _, v := range db {
if v.Name == "app_key" {
add.AppKey = v.Default
}
if v.Name == "app_secret" {
add.AppSecret = v.Default
}
}
var dropbox dropbox.Dropbox
dropbox.Addition = add
err := dropbox.Init(c)
if err != nil {
c.String(200, `<p>Initialization failure:`+err.Error()+`</p>`)
return
}
username, err := dropbox.GetUserInfo(c)
if err != nil {
c.String(200, `<p>Failed to get user information:`+err.Error()+`</p>`)
return
}
if len(username) > 0 {
a := strings.Split(username, "@")
username = a[0]
}
username += "_dropbox"
dataMap, _ := service.MyService.Storage().GetConfigByName(username)
if len(dataMap) > 0 {
c.String(200, `<p>The same configuration has been added</p>`)
service.MyService.Storage().CheckAndMountByName(username)
return
}
dmap := make(map[string]string)
dmap["client_id"] = add.AppKey
dmap["client_secret"] = add.AppSecret
dmap["token"] = `{"access_token":"` + dropbox.AccessToken + `","token_type":"bearer","refresh_token":"` + dropbox.Addition.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*3).Format("15:04:05") + `.780385354Z"}`
dmap["mount_point"] = "/mnt/" + username
// data.SetValue(username, "type", "dropbox")
// data.SetValue(username, "client_id", add.AppKey)
// data.SetValue(username, "client_secret", add.AppSecret)
// data.SetValue(username, "mount_point", "/mnt/"+username)
// data.SetValue(username, "token", `{"access_token":"`+dropbox.AccessToken+`","token_type":"bearer","refresh_token":"`+dropbox.Addition.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*3).Format("15:04:05")+`.780385354Z"}`)
// e = data.Save()
// if e != nil {
// c.String(200, `<p>保存配置失败:`+e.Error()+`</p>`)
// return
// }
service.MyService.Storage().CreateConfig(dmap, username, "dropbox")
service.MyService.Storage().MountStorage("/mnt/"+username, username+":")
notify := make(map[string]interface{})
notify["status"] = "success"
service.MyService.Notify().SendNotify("recover_status", notify)
}
c.String(200, `<p>Just close the page</p><script>window.close()</script>`)
} }

4
route/v1/samba.go
View file

@ -12,6 +12,7 @@ package v1
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@ -195,7 +196,10 @@ func DeleteSambaConnections(c *gin.Context) {
for _, v := range mountPointList { for _, v := range mountPointList {
service.MyService.Connections().UnmountSmaba(v.Path) service.MyService.Connections().UnmountSmaba(v.Path)
} }
dir, _ := ioutil.ReadDir(connection.MountPoint)
if len(dir) == 0 {
os.RemoveAll(connection.MountPoint) os.RemoveAll(connection.MountPoint)
}
service.MyService.Connections().DeleteConnection(id) service.MyService.Connections().DeleteConnection(id)
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: id}) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: id})
} }

102
route/v1/storage.go
View file

@ -2,8 +2,11 @@ package v1
import ( import (
"strconv" "strconv"
"strings"
"github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/drivers/dropbox"
"github.com/IceWhaleTech/CasaOS/drivers/google_drive"
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/pkg/utils/common_err" "github.com/IceWhaleTech/CasaOS/pkg/utils/common_err"
"github.com/IceWhaleTech/CasaOS/service" "github.com/IceWhaleTech/CasaOS/service"
@ -12,42 +15,45 @@ import (
) )
func ListStorages(c *gin.Context) { func ListStorages(c *gin.Context) {
var req model.PageReq // var req model.PageReq
if err := c.ShouldBind(&req); err != nil { // if err := c.ShouldBind(&req); err != nil {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})
return // return
} // }
req.Validate() // req.Validate()
//logger.Info("ListStorages", zap.Any("req", req))
//storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage)
// if err != nil {
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
// return
// }
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{
// Content: storages,
// Total: total,
// }})
r, err := service.MyService.Storage().GetStorages()
logger.Info("ListStorages", zap.Any("req", req))
storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage)
if err != nil { if err != nil {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return return
} }
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{
Content: storages, for i := 0; i < len(r.MountPoints); i++ {
Total: total, dataMap, err := service.MyService.Storage().GetConfigByName(r.MountPoints[i].Fs)
}}) if err != nil {
logger.Error("GetConfigByName", zap.Any("err", err))
continue
}
if dataMap["type"] == "drive" {
r.MountPoints[i].Icon = google_drive.ICONURL
}
if dataMap["type"] == "dropbox" {
r.MountPoints[i].Icon = dropbox.ICONURL
}
} }
func CreateStorage(c *gin.Context) { c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: r})
var req model.Storage
if err := c.ShouldBind(&req); err != nil {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})
return
}
if id, err := service.MyService.Storages().CreateStorage(c, req); err != nil {
data := make(map[string]interface{})
data["id"] = id
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: data})
return
} else {
data := make(map[string]interface{})
data["id"] = id
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: data})
}
} }
func UpdateStorage(c *gin.Context) { func UpdateStorage(c *gin.Context) {
@ -64,16 +70,19 @@ func UpdateStorage(c *gin.Context) {
} }
func DeleteStorage(c *gin.Context) { func DeleteStorage(c *gin.Context) {
idStr := c.Query("id") json := make(map[string]string)
id, err := strconv.Atoi(idStr) c.ShouldBind(&json)
if err != nil { mountPoint := json["mount_point"]
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) if mountPoint == "" {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: "mount_point is empty"})
return return
} }
if err := service.MyService.Storages().DeleteStorageById(c, uint(id)); err != nil { err := service.MyService.Storage().UnmountStorage(mountPoint)
if err != nil {
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
return return
} }
service.MyService.Storage().DeleteConfigByName(strings.ReplaceAll(mountPoint, "/mnt/", ""))
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"}) c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"})
} }
@ -106,16 +115,17 @@ func EnableStorage(c *gin.Context) {
} }
func GetStorage(c *gin.Context) { func GetStorage(c *gin.Context) {
idStr := c.Query("id")
id, err := strconv.Atoi(idStr) // idStr := c.Query("id")
if err != nil { // id, err := strconv.Atoi(idStr)
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()}) // if err != nil {
return // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})
} // return
storage, err := service.MyService.Storage().GetStorageById(uint(id)) // }
if err != nil { // storage, err := service.MyService.Storage().GetStorageById(uint(id))
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()}) // if err != nil {
return // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})
} // return
c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage}) // }
// c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage})
} }

18
service/fs.go
View file

@ -6,12 +6,14 @@ import (
"github.com/IceWhaleTech/CasaOS-Common/utils/logger" "github.com/IceWhaleTech/CasaOS-Common/utils/logger"
"github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/model"
log "github.com/dsoprea/go-logging"
"go.uber.org/zap" "go.uber.org/zap"
) )
type FsService interface { type FsService interface {
FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error)
GetStorage(path string) (driver.Driver, error) GetStorage(path string) (driver.Driver, error)
Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error)
} }
type fsService struct { type fsService struct {
@ -39,14 +41,14 @@ func (f *fsService) FList(ctx context.Context, path string, refresh ...bool) ([]
// return res, nil // return res, nil
// } // }
// func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
// res, file, err := link(ctx, path, args) res, file, err := MyService.FsLinkService().Link(ctx, path, args)
// if err != nil { if err != nil {
// log.Errorf("failed link %s: %+v", path, err) log.Errorf("failed link %s: %+v", path, err)
// return nil, nil, err return nil, nil, err
// } }
// return res, file, nil return res, file, nil
// } }
// func (f *fsService) MakeDir(ctx context.Context, path string, lazyCache ...bool) error { // func (f *fsService) MakeDir(ctx context.Context, path string, lazyCache ...bool) error {
// err := makeDir(ctx, path, lazyCache...) // err := makeDir(ctx, path, lazyCache...)

27
service/fs_link.go Normal file
View file

@ -0,0 +1,27 @@
package service
import (
"context"
"github.com/IceWhaleTech/CasaOS/internal/op"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/pkg/errors"
)
type FsLinkService interface {
Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error)
}
type fsLinkService struct {
}
func (f *fsLinkService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
storage, actualPath, err := MyService.StoragePath().GetStorageAndActualPath(path)
if err != nil {
return nil, nil, errors.WithMessage(err, "failed get storage")
}
return op.Link(ctx, storage, actualPath, args)
}
func NewFsLinkService() FsLinkService {
return &fsLinkService{}
}

8
service/service.go
View file

@ -42,6 +42,7 @@ type Repository interface {
Storages() StoragesService Storages() StoragesService
StoragePath() StoragePathService StoragePath() StoragePathService
FsListService() FsListService FsListService() FsListService
FsLinkService() FsLinkService
FsService() FsService FsService() FsService
} }
@ -63,10 +64,11 @@ func NewService(db *gorm.DB, RuntimePath string, socket *socketio.Server) Reposi
system: NewSystemService(), system: NewSystemService(),
shares: NewSharesService(db), shares: NewSharesService(db),
connections: NewConnectionsService(db), connections: NewConnectionsService(db),
storage: NewStorageService(db), storage: NewStorageService(),
storages: NewStoragesService(), storages: NewStoragesService(),
storage_path: NewStoragePathService(), storage_path: NewStoragePathService(),
fs_list: NewFsListService(), fs_list: NewFsListService(),
fs_link: NewFsLinkService(),
fs: NewFsService(), fs: NewFsService(),
} }
} }
@ -84,9 +86,13 @@ type store struct {
storages StoragesService storages StoragesService
storage_path StoragePathService storage_path StoragePathService
fs_list FsListService fs_list FsListService
fs_link FsLinkService
fs FsService fs FsService
} }
func (c *store) FsLinkService() FsLinkService {
return c.fs_link
}
func (c *store) FsService() FsService { func (c *store) FsService() FsService {
return c.fs return c.fs
} }

125
service/storage.go
View file

@ -1,73 +1,100 @@
package service package service
import ( import (
"fmt" "io/ioutil"
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/pkg/utils/file"
"github.com/pkg/errors" "github.com/IceWhaleTech/CasaOS/pkg/utils/httper"
"gorm.io/gorm"
) )
type StorageService interface { type StorageService interface {
CreateStorage(storage *model.Storage) error MountStorage(mountPoint, fs string) error
UpdateStorage(storage *model.Storage) error UnmountStorage(mountPoint string) error
DeleteStorageById(id uint) error GetStorages() (httper.MountList, error)
GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) CreateConfig(data map[string]string, name string, t string) error
GetStorageById(id uint) (*model.Storage, error) CheckAndMountByName(name string) error
GetEnabledStorages() ([]model.Storage, error) CheckAndMountAll() error
GetConfigByName(name string) (map[string]string, error)
DeleteConfigByName(name string) error
} }
type storageStruct struct { type storageStruct struct {
db *gorm.DB
} }
// CreateStorage just insert storage to database func (s *storageStruct) MountStorage(mountPoint, fs string) error {
func (s *storageStruct) CreateStorage(storage *model.Storage) error { file.IsNotExistMkDir(mountPoint)
return errors.WithStack(s.db.Create(storage).Error) httper.Mount(mountPoint, fs)
return nil
} }
func (s *storageStruct) UnmountStorage(mountPoint string) error {
err := httper.Unmount(mountPoint)
if err == nil {
dir, _ := ioutil.ReadDir(mountPoint)
// UpdateStorage just update storage in database if len(dir) == 0 {
func (s *storageStruct) UpdateStorage(storage *model.Storage) error { file.RMDir(mountPoint)
return errors.WithStack(s.db.Save(storage).Error)
} }
return nil
// DeleteStorageById just delete storage from database by id
func (s *storageStruct) DeleteStorageById(id uint) error {
return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error)
} }
return err
// GetStorages Get all storages from database order by index
func (s *storageStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {
storageDB := s.db.Model(&model.Storage{})
var count int64
if err := storageDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get storages count")
} }
var storages []model.Storage func (s *storageStruct) GetStorages() (httper.MountList, error) {
if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { return httper.GetMountList()
return nil, 0, errors.WithStack(err)
} }
return storages, count, nil func (s *storageStruct) CreateConfig(data map[string]string, name string, t string) error {
httper.CreateConfig(data, name, t)
return nil
} }
func (s *storageStruct) CheckAndMountByName(name string) error {
// GetStorageById Get Storage by id, used to update storage usually storages, _ := MyService.Storage().GetStorages()
func (s *storageStruct) GetStorageById(id uint) (*model.Storage, error) { currentRemote, _ := httper.GetConfigByName(name)
var storage model.Storage mountPoint := currentRemote["mount_point"]
storage.ID = id isMount := false
if err := s.db.First(&storage).Error; err != nil { for _, v := range storages.MountPoints {
return nil, errors.WithStack(err) if v.MountPoint == mountPoint {
isMount = true
break
} }
return &storage, nil
} }
if !isMount {
func (s *storageStruct) GetEnabledStorages() ([]model.Storage, error) { MyService.Storage().MountStorage(mountPoint, name+":")
var storages []model.Storage
if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil {
return nil, errors.WithStack(err)
} }
return storages, nil return nil
} }
func (s *storageStruct) CheckAndMountAll() error {
func NewStorageService(db *gorm.DB) StorageService { storages, err := MyService.Storage().GetStorages()
return &storageStruct{db: db} if err != nil {
return err
}
section, err := httper.GetAllConfigName()
if err != nil {
return err
}
for _, v := range section.Remotes {
currentRemote, _ := httper.GetConfigByName(v)
mountPoint := currentRemote["mount_point"]
if len(mountPoint) == 0 {
continue
}
isMount := false
for _, v := range storages.MountPoints {
if v.MountPoint == mountPoint {
isMount = true
break
}
}
if !isMount {
return MyService.Storage().MountStorage(mountPoint, v+":")
}
}
return nil
}
func (s *storageStruct) GetConfigByName(name string) (map[string]string, error) {
return httper.GetConfigByName(name)
}
func (s *storageStruct) DeleteConfigByName(name string) error {
return httper.DeleteConfigByName(name)
}
func NewStorageService() StorageService {
return &storageStruct{}
} }

73
service/storage_old.go Normal file
View file

@ -0,0 +1,73 @@
package service
import (
"fmt"
"github.com/IceWhaleTech/CasaOS/model"
"github.com/pkg/errors"
"gorm.io/gorm"
)
type StorageOldService interface {
CreateStorage(storage *model.Storage) error
UpdateStorage(storage *model.Storage) error
DeleteStorageById(id uint) error
GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error)
GetStorageById(id uint) (*model.Storage, error)
GetEnabledStorages() ([]model.Storage, error)
}
type storageOldStruct struct {
db *gorm.DB
}
// CreateStorage just insert storage to database
func (s *storageOldStruct) CreateStorage(storage *model.Storage) error {
return errors.WithStack(s.db.Create(storage).Error)
}
// UpdateStorage just update storage in database
func (s *storageOldStruct) UpdateStorage(storage *model.Storage) error {
return errors.WithStack(s.db.Save(storage).Error)
}
// DeleteStorageById just delete storage from database by id
func (s *storageOldStruct) DeleteStorageById(id uint) error {
return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error)
}
// GetStorages Get all storages from database order by index
func (s *storageOldStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {
storageDB := s.db.Model(&model.Storage{})
var count int64
if err := storageDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get storages count")
}
var storages []model.Storage
if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil {
return nil, 0, errors.WithStack(err)
}
return storages, count, nil
}
// GetStorageById Get Storage by id, used to update storage usually
func (s *storageOldStruct) GetStorageById(id uint) (*model.Storage, error) {
var storage model.Storage
storage.ID = id
if err := s.db.First(&storage).Error; err != nil {
return nil, errors.WithStack(err)
}
return &storage, nil
}
func (s *storageOldStruct) GetEnabledStorages() ([]model.Storage, error) {
var storages []model.Storage
if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil {
return nil, errors.WithStack(err)
}
return storages, nil
}
func NewStorageOldService(db *gorm.DB) StorageOldService {
return &storageOldStruct{db: db}
}

294
service/storage_service.go
View file

@ -15,7 +15,6 @@ import (
"github.com/IceWhaleTech/CasaOS/model" "github.com/IceWhaleTech/CasaOS/model"
"github.com/IceWhaleTech/CasaOS/internal/conf"
"github.com/IceWhaleTech/CasaOS/internal/driver" "github.com/IceWhaleTech/CasaOS/internal/driver"
"github.com/IceWhaleTech/CasaOS/internal/op" "github.com/IceWhaleTech/CasaOS/internal/op"
mapset "github.com/deckarep/golang-set/v2" mapset "github.com/deckarep/golang-set/v2"
@ -30,9 +29,9 @@ type StoragesService interface {
DisableStorage(ctx context.Context, id uint) error DisableStorage(ctx context.Context, id uint) error
UpdateStorage(ctx context.Context, storage model.Storage) error UpdateStorage(ctx context.Context, storage model.Storage) error
DeleteStorageById(ctx context.Context, id uint) error DeleteStorageById(ctx context.Context, id uint) error
MustSaveDriverStorage(driver driver.Driver) MustSaveDriverStorage(driver driver.Driver) error
GetStorageVirtualFilesByPath(prefix string) []model.Obj GetStorageVirtualFilesByPath(prefix string) []model.Obj
initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error)
InitStorages() InitStorages()
GetBalancedStorage(path string) driver.Driver GetBalancedStorage(path string) driver.Driver
} }
@ -75,18 +74,28 @@ func (s *storagesStruct) CreateStorage(ctx context.Context, storage model.Storag
return 0, errors.WithMessage(err, "failed get driver new") return 0, errors.WithMessage(err, "failed get driver new")
} }
storageDriver := driverNew() storageDriver := driverNew()
// insert storage to database // // insert storage to database
err = MyService.Storage().CreateStorage(&storage) // err = MyService.Storage().CreateStorage(&storage)
if err != nil { // if err != nil {
return storage.ID, errors.WithMessage(err, "failed create storage in database")
} // return storage.ID, errors.WithMessage(err, "failed create storage in database")
// }
// already has an id // already has an id
err = s.initStorage(ctx, storage, storageDriver) err = s.initStorage(ctx, storage, storageDriver, func(d driver.Driver, ctx context.Context) string {
u, _ := d.GetUserInfo(ctx)
if len(u) > 0 {
a := strings.Split(u, "@")
u = a[0]
}
return u
})
if err != nil {
s.DeleteStorageById(ctx, storage.ID)
return storage.ID, errors.Wrap(err, "failed init storage")
}
go op.CallStorageHooks("add", storageDriver) go op.CallStorageHooks("add", storageDriver)
if err != nil {
return storage.ID, errors.Wrap(err, "failed init storage but storage is already created")
}
logger.Error("storage created", zap.Any("storage", storageDriver)) logger.Error("storage created", zap.Any("storage", storageDriver))
return storage.ID, nil return storage.ID, nil
} }
@ -102,14 +111,14 @@ func (s *storagesStruct) LoadStorage(ctx context.Context, storage model.Storage)
} }
storageDriver := driverNew() storageDriver := driverNew()
err = s.initStorage(ctx, storage, storageDriver) err = s.initStorage(ctx, storage, storageDriver, nil)
go op.CallStorageHooks("add", storageDriver) go op.CallStorageHooks("add", storageDriver)
logger.Info("storage created", zap.Any("storage", storageDriver)) logger.Info("storage created", zap.Any("storage", storageDriver))
return err return err
} }
// initStorage initialize the driver and store to storagesMap // initStorage initialize the driver and store to storagesMap
func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error) {
storageDriver.SetStorage(storage) storageDriver.SetStorage(storage)
driverStorage := storageDriver.GetStorage() driverStorage := storageDriver.GetStorage()
@ -121,61 +130,72 @@ func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage,
if err == nil { if err == nil {
err = storageDriver.Init(ctx) err = storageDriver.Init(ctx)
} }
if setMountPath != nil {
driverStorage.MountPath += "_" + setMountPath(storageDriver, ctx)
}
if s.HasStorage(driverStorage.MountPath) {
return errors.New("mount path already exists")
}
storageDriver.SetStorage(*driverStorage)
storagesMap.Store(driverStorage.MountPath, storageDriver) storagesMap.Store(driverStorage.MountPath, storageDriver)
if err != nil { if err != nil {
driverStorage.SetStatus(err.Error()) driverStorage.SetStatus(err.Error())
err = errors.Wrap(err, "failed init storage") err = errors.Wrap(err, "failed init storage")
} else { } else {
driverStorage.SetStatus(op.WORK) driverStorage.SetStatus(op.WORK)
} }
s.MustSaveDriverStorage(storageDriver)
err = s.MustSaveDriverStorage(storageDriver)
return err return err
} }
func (s *storagesStruct) EnableStorage(ctx context.Context, id uint) error { func (s *storagesStruct) EnableStorage(ctx context.Context, id uint) error {
storage, err := MyService.Storage().GetStorageById(id) // storage, err := MyService.Storage().GetStorageById(id)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed get storage") // return errors.WithMessage(err, "failed get storage")
} // }
if !storage.Disabled { // if !storage.Disabled {
return errors.Errorf("this storage have enabled") // return errors.Errorf("this storage have enabled")
} // }
storage.Disabled = false // storage.Disabled = false
err = MyService.Storage().UpdateStorage(storage) // err = MyService.Storage().UpdateStorage(storage)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed update storage in db") // return errors.WithMessage(err, "failed update storage in db")
} // }
err = s.LoadStorage(ctx, *storage) // err = s.LoadStorage(ctx, *storage)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed load storage") // return errors.WithMessage(err, "failed load storage")
} // }
return nil return nil
} }
func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error { func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error {
storage, err := MyService.Storage().GetStorageById(id) // storage, err := MyService.Storage().GetStorageById(id)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed get storage") // return errors.WithMessage(err, "failed get storage")
} // }
if storage.Disabled { // if storage.Disabled {
return errors.Errorf("this storage have disabled") // return errors.Errorf("this storage have disabled")
} // }
storageDriver, err := GetStorageByMountPath(storage.MountPath) // storageDriver, err := GetStorageByMountPath(storage.MountPath)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed get storage driver") // return errors.WithMessage(err, "failed get storage driver")
} // }
// drop the storage in the driver // // drop the storage in the driver
if err := storageDriver.Drop(ctx); err != nil { // if err := storageDriver.Drop(ctx); err != nil {
return errors.Wrap(err, "failed drop storage") // return errors.Wrap(err, "failed drop storage")
} // }
// delete the storage in the memory // // delete the storage in the memory
storage.Disabled = true // storage.Disabled = true
err = MyService.Storage().UpdateStorage(storage) // err = MyService.Storage().UpdateStorage(storage)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed update storage in db") // return errors.WithMessage(err, "failed update storage in db")
} // }
storagesMap.Delete(storage.MountPath) // storagesMap.Delete(storage.MountPath)
go op.CallStorageHooks("del", storageDriver) // go op.CallStorageHooks("del", storageDriver)
return nil return nil
} }
@ -183,90 +203,92 @@ func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error {
// get old storage first // get old storage first
// drop the storage then reinitialize // drop the storage then reinitialize
func (s *storagesStruct) UpdateStorage(ctx context.Context, storage model.Storage) error { func (s *storagesStruct) UpdateStorage(ctx context.Context, storage model.Storage) error {
oldStorage, err := MyService.Storage().GetStorageById(storage.ID) // oldStorage, err := MyService.Storage().GetStorageById(storage.ID)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed get old storage") // return errors.WithMessage(err, "failed get old storage")
} // }
if oldStorage.Driver != storage.Driver { // if oldStorage.Driver != storage.Driver {
return errors.Errorf("driver cannot be changed") // return errors.Errorf("driver cannot be changed")
} // }
storage.Modified = time.Now() // storage.Modified = time.Now()
storage.MountPath = utils.FixAndCleanPath(storage.MountPath) // storage.MountPath = utils.FixAndCleanPath(storage.MountPath)
err = MyService.Storage().UpdateStorage(&storage) // err = MyService.Storage().UpdateStorage(&storage)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed update storage in database") // return errors.WithMessage(err, "failed update storage in database")
} // }
if storage.Disabled { // if storage.Disabled {
// return nil
// }
// storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
// if oldStorage.MountPath != storage.MountPath {
// // mount path renamed, need to drop the storage
// storagesMap.Delete(oldStorage.MountPath)
// }
// if err != nil {
// return errors.WithMessage(err, "failed get storage driver")
// }
// err = storageDriver.Drop(ctx)
// if err != nil {
// return errors.Wrapf(err, "failed drop storage")
// }
// err = s.initStorage(ctx, storage, storageDriver, nil)
// go op.CallStorageHooks("update", storageDriver)
// logger.Info("storage updated", zap.Any("storage", storageDriver))
//return err
return nil return nil
} }
storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
if oldStorage.MountPath != storage.MountPath {
// mount path renamed, need to drop the storage
storagesMap.Delete(oldStorage.MountPath)
}
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
err = storageDriver.Drop(ctx)
if err != nil {
return errors.Wrapf(err, "failed drop storage")
}
err = s.initStorage(ctx, storage, storageDriver)
go op.CallStorageHooks("update", storageDriver)
logger.Info("storage updated", zap.Any("storage", storageDriver))
return err
}
func (s *storagesStruct) DeleteStorageById(ctx context.Context, id uint) error { func (s *storagesStruct) DeleteStorageById(ctx context.Context, id uint) error {
storage, err := MyService.Storage().GetStorageById(id) // storage, err := MyService.Storage().GetStorageById(id)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed get storage") // return errors.WithMessage(err, "failed get storage")
} // }
if !storage.Disabled { // if !storage.Disabled {
storageDriver, err := GetStorageByMountPath(storage.MountPath) // storageDriver, err := GetStorageByMountPath(storage.MountPath)
if err != nil { // if err == nil {
return errors.WithMessage(err, "failed get storage driver") // // drop the storage in the driver
} // if err := storageDriver.Drop(ctx); err != nil {
// drop the storage in the driver // return errors.Wrapf(err, "failed drop storage")
if err := storageDriver.Drop(ctx); err != nil { // }
return errors.Wrapf(err, "failed drop storage") // // delete the storage in the memory
} // storagesMap.Delete(storage.MountPath)
// delete the storage in the memory // }
storagesMap.Delete(storage.MountPath)
go op.CallStorageHooks("del", storageDriver) // go op.CallStorageHooks("del", storageDriver)
} // }
// delete the storage in the database // // delete the storage in the database
if err := MyService.Storage().DeleteStorageById(id); err != nil { // if err := MyService.Storage().DeleteStorageById(id); err != nil {
return errors.WithMessage(err, "failed delete storage in database") // return errors.WithMessage(err, "failed delete storage in database")
} // }
return nil return nil
} }
// MustSaveDriverStorage call from specific driver // MustSaveDriverStorage call from specific driver
func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) { func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) error {
err := saveDriverStorage(driver) err := saveDriverStorage(driver)
if err != nil { if err != nil {
logger.Error("failed save driver storage", zap.Any("err", err)) logger.Error("failed save driver storage", zap.Any("err", err))
} }
return err
} }
func saveDriverStorage(driver driver.Driver) error { func saveDriverStorage(driver driver.Driver) error {
storage := driver.GetStorage() // storage := driver.GetStorage()
addition := driver.GetAddition() // addition := driver.GetAddition()
var json = jsoniter.ConfigCompatibleWithStandardLibrary // var json = jsoniter.ConfigCompatibleWithStandardLibrary
str, err := json.MarshalToString(addition) // str, err := json.MarshalToString(addition)
if err != nil { // if err != nil {
return errors.Wrap(err, "error while marshal addition") // return errors.Wrap(err, "error while marshal addition")
} // }
storage.Addition = str // storage.Addition = str
err = MyService.Storage().UpdateStorage(storage) // err = MyService.Storage().UpdateStorage(storage)
if err != nil { // if err != nil {
return errors.WithMessage(err, "failed update storage in database") // return errors.WithMessage(err, "failed update storage in database")
} // }
return nil return nil
} }
@ -354,21 +376,21 @@ func (s *storagesStruct) GetBalancedStorage(path string) driver.Driver {
} }
} }
func (s *storagesStruct) InitStorages() { func (s *storagesStruct) InitStorages() {
storages, err := MyService.Storage().GetEnabledStorages() // storages, err := MyService.Storage().GetEnabledStorages()
if err != nil { // if err != nil {
logger.Error("failed get enabled storages", zap.Any("err", err)) // logger.Error("failed get enabled storages", zap.Any("err", err))
} // }
go func(storages []model.Storage) { // go func(storages []model.Storage) {
for i := range storages { // for i := range storages {
err := s.LoadStorage(context.Background(), storages[i]) // err := s.LoadStorage(context.Background(), storages[i])
if err != nil { // if err != nil {
logger.Error("failed get enabled storages", zap.Any("err", err)) // logger.Error("failed get enabled storages", zap.Any("err", err))
} else { // } else {
logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath)) // logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath))
} // }
} // }
conf.StoragesLoaded = true // conf.StoragesLoaded = true
}(storages) // }(storages)
} }
func NewStoragesService() StoragesService { func NewStoragesService() StoragesService {