From ae50a9bb174205be0e9d5277f07d75c3b7ea9ec0 Mon Sep 17 00:00:00 2001 From: CorrectRoadH Date: Wed, 27 Dec 2023 16:17:33 +0800 Subject: [PATCH] feat: add file upload implement to v2 api (#1566) Signed-off-by: CorrectRoadH --- api/casaos/openapi.yaml | 123 ++++++++++++++++++++++++++++++++++ go.sum | 9 +++ route/v2/file.go | 67 +++++++++++++++++++ route/v2/route.go | 9 ++- service/file_upload.go | 144 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 service/file_upload.go diff --git a/api/casaos/openapi.yaml b/api/casaos/openapi.yaml index 426db31..12ffdbc 100644 --- a/api/casaos/openapi.yaml +++ b/api/casaos/openapi.yaml @@ -90,6 +90,107 @@ paths: $ref: "#/components/responses/ResponseOK" "500": $ref: "#/components/responses/ResponseInternalServerError" + + /file/upload: + get: + tags: + - File + summary: Check upload chunk + parameters: + - name: path + in: query + description: File path + required: true + example: "/DATA/test.log" + schema: + type: string + - name: relativePath + in: query + description: File path + required: true + example: "/DATA/test.log" + schema: + type: string + - name: filename + in: query + description: File name + required: true + example: "test.log" + schema: + type: string + - name: chunkNumber + in: query + description: chunk number + required: true + example: 1 + schema: + type: string + - name: totalChunks + in: query + description: total chunks + example: 2 + required: true + schema: + type: integer + description: Check if the file block has been uploaded (needs to be modified later) + operationId: checkUploadChunk + responses: + "200": + $ref: "#/components/responses/ResponseStringOK" + "400": + $ref: "#/components/responses/ResponseClientError" + "500": + $ref: "#/components/responses/ResponseInternalServerError" + post: + tags: + - File + summary: Upload file + description: Upload file + operationId: postUploadFile + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + relativePath: + type: string + example: "/DATA/test.log" + filename: + type: string + example: "/DATA/test2.log" + totalChunks: + type: string + example: "2" + chunkNumber: + type: string + example: "20" + path: + type: string + example: "/DATA" + file: + type: string + format: binary + chunkSize: + type: string + example: "1024" + currentChunkSize: + type: string + example: "1024" + totalSize: + type: string + example: "1024" + identifier: + type: string + example: "test.log" + responses: + "200": + $ref: "#/components/responses/ResponseStringOK" + "400": + $ref: "#/components/responses/ResponseClientError" + "500": + $ref: "#/components/responses/ResponseInternalServerError" + /zt/info: get: tags: @@ -151,6 +252,20 @@ components: schema: $ref: "#/components/schemas/BaseResponse" + ResponseStringOK: + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/SuccessResponseString" + + ResponseClientError: + description: Client Error + content: + application/json: + schema: + $ref: "#/components/schemas/BaseResponse" + ResponseInternalServerError: description: Internal Server Error content: @@ -195,6 +310,14 @@ components: description: message returned by server side if there is any type: string example: "" + + SuccessResponseString: + allOf: + - $ref: "#/components/schemas/BaseResponse" + - properties: + data: + type: string + description: When the interface returns success, this field is the specific success information HealthServices: properties: diff --git a/go.sum b/go.sum index 0869a4a..63beb8e 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,16 @@ github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d h1:62lEBImTxZ83pgzywgDNIrPPuQ+j4ep9QjqrWBn1hrU= github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ= github.com/IceWhaleTech/CasaOS-Common v0.4.8-alpha3 h1:5E5LAqi2uXpOZqcPOgQ4m6d9MagYyfhKIFXnzd8s3W4= +github.com/IceWhaleTech/CasaOS-Common v0.4.8-alpha3/go.mod h1:2IuYyy5qW1BE6jqC6M+tOU+WtUec1K565rLATBJ9p/0= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/benbjohnson/clock v1.3.1 h1:Heo0FGXzOxUHquZbraxt+tT7UXVDhesUQH5ISbsOkCQ= github.com/benbjohnson/clock v1.3.1/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -36,6 +40,7 @@ github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b/go.mod h1:cg5SN github.com/dsoprea/go-exif/v3 v3.0.0-20221003160559-cf5cd88aa559/go.mod h1:rW6DMEv25U9zCtE5ukC7ttBRllXj7g7TAHl7tQrT5No= github.com/dsoprea/go-exif/v3 v3.0.0-20221003171958-de6cb6e380a8/go.mod h1:akyZEJZ/k5bmbC9gA612ZLQkcED8enS9vuTiuAkENr0= github.com/dsoprea/go-exif/v3 v3.0.1 h1:/IE4iW7gvY7BablV1XY0unqhMv26EYpOquVMwoBo/wc= +github.com/dsoprea/go-exif/v3 v3.0.1/go.mod h1:10HkA1Wz3h398cDP66L+Is9kKDmlqlIJGPv8pk4EWvc= github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA= github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8= github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg= @@ -164,6 +169,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -205,6 +211,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/mileusna/useragent v1.2.1 h1:p3RJWhi3LfuI6BHdddojREyK3p6qX67vIfOVMnUIVr0= @@ -259,6 +266,7 @@ github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQ github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -382,6 +390,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/route/v2/file.go b/route/v2/file.go index 1160838..a5cb6df 100644 --- a/route/v2/file.go +++ b/route/v2/file.go @@ -2,7 +2,9 @@ package v2 import ( "net/http" + "strconv" + "github.com/IceWhaleTech/CasaOS/codegen" "github.com/labstack/echo/v4" ) @@ -15,3 +17,68 @@ func (s *CasaOS) GetFileTest(ctx echo.Context) error { return ctx.String(200, "pong") } + +func (c *CasaOS) CheckUploadChunk(ctx echo.Context, params codegen.CheckUploadChunkParams) error { + identifier := ctx.QueryParam("identifier") + chunkNumber, err := strconv.ParseInt(ctx.QueryParam("chunkNumber"), 10, 64) + if err != nil { + return ctx.NoContent(http.StatusBadRequest) + } + + err = c.fileUploadService.TestChunk(ctx, identifier, chunkNumber) + if err != nil { + return ctx.NoContent(http.StatusNoContent) + } + return ctx.NoContent(http.StatusOK) +} + +func (c *CasaOS) PostUploadFile(ctx echo.Context) error { + path := ctx.FormValue("path") + + // handle the request + chunkNumber, err := strconv.ParseInt(ctx.FormValue("chunkNumber"), 10, 64) + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + chunkSize, err := strconv.ParseInt(ctx.FormValue("chunkSize"), 10, 64) + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + currentChunkSize, err := strconv.ParseInt(ctx.FormValue("currentChunkSize"), 10, 64) + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + totalChunks, err := strconv.ParseInt(ctx.FormValue("totalChunks"), 10, 64) + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + totalSize, err := strconv.ParseInt(ctx.FormValue("totalSize"), 10, 64) + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + + identifier := ctx.FormValue("identifier") + fileName := ctx.FormValue("filename") + bin, err := ctx.FormFile("file") + + if err != nil { + return ctx.JSON(http.StatusBadRequest, err) + } + + err = c.fileUploadService.UploadFile( + ctx, + path, + chunkNumber, + chunkSize, + currentChunkSize, + totalChunks, + totalSize, + identifier, + fileName, + bin, + ) + if err != nil { + return ctx.JSON(http.StatusInternalServerError, err) + } + return ctx.NoContent(http.StatusOK) +} diff --git a/route/v2/route.go b/route/v2/route.go index 5e124c1..c09be20 100644 --- a/route/v2/route.go +++ b/route/v2/route.go @@ -2,10 +2,15 @@ package v2 import ( "github.com/IceWhaleTech/CasaOS/codegen" + "github.com/IceWhaleTech/CasaOS/service" ) -type CasaOS struct{} +type CasaOS struct { + fileUploadService *service.FileUploadService +} func NewCasaOS() codegen.ServerInterface { - return &CasaOS{} + return &CasaOS{ + fileUploadService: service.NewFileUploadService(), + } } diff --git a/service/file_upload.go b/service/file_upload.go new file mode 100644 index 0000000..3e5acb6 --- /dev/null +++ b/service/file_upload.go @@ -0,0 +1,144 @@ +package service + +import ( + "fmt" + "io" + "mime/multipart" + "os" + "sync" + + "github.com/labstack/echo/v4" +) + +type FileInfo struct { + init bool + uploaded []bool + uploadedChunkNum int64 +} + +type FileUploadService struct { + uploadStatus sync.Map + lock sync.RWMutex +} + +func NewFileUploadService() *FileUploadService { + return &FileUploadService{ + uploadStatus: sync.Map{}, + lock: sync.RWMutex{}, + } +} + +func (s *FileUploadService) TestChunk( + c echo.Context, + identifier string, + chunkNumber int64, +) error { + fileInfoTemp, ok := s.uploadStatus.Load(identifier) + + if !ok { + return fmt.Errorf("file not found") + } + + fileInfo := fileInfoTemp.(*FileInfo) + + if !fileInfo.init { + return fmt.Errorf("file not init") + } + + // return StatusNoContent instead of 404 + // the is require by frontend + if !fileInfo.uploaded[chunkNumber-1] { + return fmt.Errorf("file not found") + } + + return nil +} + +func (s *FileUploadService) UploadFile( + c echo.Context, + path string, + chunkNumber int64, + chunkSize int64, + currentChunkSize int64, + totalChunks int64, + totalSize int64, + identifier string, + fileName string, + bin *multipart.FileHeader, +) error { + s.lock.Lock() + fileInfoTemp, ok := s.uploadStatus.Load(identifier) + var fileInfo *FileInfo + + file, err := os.OpenFile(path+"/"+fileName+".tmp", os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + s.lock.Unlock() + return err + } + + if !ok { + + if err != nil { + s.lock.Unlock() + return err + } + + // pre allocate file size + fmt.Println("truncate", totalSize) + if err != nil { + s.lock.Unlock() + return err + } + + // file info init + fileInfo = &FileInfo{ + init: true, + uploaded: make([]bool, totalChunks), + uploadedChunkNum: 0, + } + s.uploadStatus.Store(identifier, fileInfo) + } else { + fileInfo = fileInfoTemp.(*FileInfo) + } + + s.lock.Unlock() + + _, err = file.Seek((chunkNumber-1)*chunkSize, io.SeekStart) + if err != nil { + return err + } + + src, err := bin.Open() + if err != nil { + return err + } + defer src.Close() + + buf := make([]byte, int(currentChunkSize)) + _, err = io.CopyBuffer(file, src, buf) + + if err != nil { + fmt.Println(err) + return err + } + + s.lock.Lock() + // handle file after write a chunk + // handle single chunk upload twice + if !fileInfo.uploaded[chunkNumber-1] { + fileInfo.uploadedChunkNum++ + fileInfo.uploaded[chunkNumber-1] = true + } + + // handle file after write all chunk + if fileInfo.uploadedChunkNum == totalChunks { + file.Close() + os.Rename(path+"/"+fileName+".tmp", path+"/"+fileName) + + // remove upload status info after upload complete + s.uploadStatus.Delete(identifier) + } + s.lock.Unlock() + + return nil +}