feat: add file upload implement to v2 api (#1566)

Signed-off-by: CorrectRoadH <a778917369@gmail.com>

api/casaos/openapi.yaml

@@ -90,6 +90,107 @@ paths:
           $ref: "#/components/responses/ResponseOK"
         "500":
           $ref: "#/components/responses/ResponseInternalServerError"
+  
+  /file/upload:
+    get:
+      tags:
+        - File
+      summary: Check upload chunk
+      parameters:
+        - name: path
+          in: query
+          description: File path
+          required: true
+          example: "/DATA/test.log"
+          schema:
+            type: string
+        - name: relativePath
+          in: query
+          description: File path
+          required: true
+          example: "/DATA/test.log"
+          schema:
+            type: string
+        - name: filename
+          in: query
+          description: File name
+          required: true
+          example: "test.log"
+          schema:
+            type: string
+        - name: chunkNumber
+          in: query
+          description: chunk number
+          required: true
+          example: 1
+          schema:
+            type: string
+        - name: totalChunks
+          in: query
+          description: total chunks
+          example: 2
+          required: true
+          schema:
+            type: integer
+      description: Check if the file block has been uploaded (needs to be modified later)
+      operationId: checkUploadChunk
+      responses:
+        "200":
+          $ref: "#/components/responses/ResponseStringOK"
+        "400":
+          $ref: "#/components/responses/ResponseClientError"
+        "500":
+          $ref: "#/components/responses/ResponseInternalServerError"
+    post:
+      tags:
+        - File
+      summary: Upload file
+      description: Upload file
+      operationId: postUploadFile
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                relativePath:
+                  type: string
+                  example: "/DATA/test.log"
+                filename:
+                  type: string
+                  example: "/DATA/test2.log"
+                totalChunks:
+                  type: string
+                  example: "2"
+                chunkNumber:
+                  type: string
+                  example: "20"
+                path:
+                  type: string
+                  example: "/DATA"
+                file:
+                  type: string
+                  format: binary
+                chunkSize:
+                  type: string
+                  example: "1024"
+                currentChunkSize:
+                  type: string
+                  example: "1024"
+                totalSize:
+                  type: string
+                  example: "1024"
+                identifier:
+                  type: string
+                  example: "test.log"
+      responses:
+        "200":
+          $ref: "#/components/responses/ResponseStringOK"
+        "400":
+          $ref: "#/components/responses/ResponseClientError"
+        "500":
+          $ref: "#/components/responses/ResponseInternalServerError"
+  
   /zt/info:
     get:
       tags:
@@ -151,6 +252,20 @@ components:
           schema:
             $ref: "#/components/schemas/BaseResponse"
 
+    ResponseStringOK:
+      description: OK
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/SuccessResponseString"
+
+    ResponseClientError:
+      description: Client Error
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/BaseResponse"
+
     ResponseInternalServerError:
       description: Internal Server Error
       content:
@@ -195,6 +310,14 @@ components:
           description: message returned by server side if there is any
           type: string
           example: ""
+          
+    SuccessResponseString:
+      allOf:
+        - $ref: "#/components/schemas/BaseResponse"
+        - properties:
+            data:
+              type: string
+              description: When the interface returns success, this field is the specific success information
 
     HealthServices:
       properties:

go.sum

@@ -1,12 +1,16 @@
 github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d h1:62lEBImTxZ83pgzywgDNIrPPuQ+j4ep9QjqrWBn1hrU=
 github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ=
 github.com/IceWhaleTech/CasaOS-Common v0.4.8-alpha3 h1:5E5LAqi2uXpOZqcPOgQ4m6d9MagYyfhKIFXnzd8s3W4=
+github.com/IceWhaleTech/CasaOS-Common v0.4.8-alpha3/go.mod h1:2IuYyy5qW1BE6jqC6M+tOU+WtUec1K565rLATBJ9p/0=
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/benbjohnson/clock v1.3.1 h1:Heo0FGXzOxUHquZbraxt+tT7UXVDhesUQH5ISbsOkCQ=
 github.com/benbjohnson/clock v1.3.1/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
 github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
@@ -36,6 +40,7 @@ github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b/go.mod h1:cg5SN
 github.com/dsoprea/go-exif/v3 v3.0.0-20221003160559-cf5cd88aa559/go.mod h1:rW6DMEv25U9zCtE5ukC7ttBRllXj7g7TAHl7tQrT5No=
 github.com/dsoprea/go-exif/v3 v3.0.0-20221003171958-de6cb6e380a8/go.mod h1:akyZEJZ/k5bmbC9gA612ZLQkcED8enS9vuTiuAkENr0=
 github.com/dsoprea/go-exif/v3 v3.0.1 h1:/IE4iW7gvY7BablV1XY0unqhMv26EYpOquVMwoBo/wc=
+github.com/dsoprea/go-exif/v3 v3.0.1/go.mod h1:10HkA1Wz3h398cDP66L+Is9kKDmlqlIJGPv8pk4EWvc=
 github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA=
 github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8=
 github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg=
@@ -164,6 +169,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
@@ -205,6 +211,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
 github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
 github.com/mileusna/useragent v1.2.1 h1:p3RJWhi3LfuI6BHdddojREyK3p6qX67vIfOVMnUIVr0=
@@ -259,6 +266,7 @@ github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQ
 github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -382,6 +390,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

route/v2/file.go

@@ -2,7 +2,9 @@ package v2
 
 import (
 	"net/http"
+	"strconv"
 
+	"github.com/IceWhaleTech/CasaOS/codegen"
 	"github.com/labstack/echo/v4"
 )
 
@@ -15,3 +17,68 @@ func (s *CasaOS) GetFileTest(ctx echo.Context) error {
 
 	return ctx.String(200, "pong")
 }
+
+func (c *CasaOS) CheckUploadChunk(ctx echo.Context, params codegen.CheckUploadChunkParams) error {
+	identifier := ctx.QueryParam("identifier")
+	chunkNumber, err := strconv.ParseInt(ctx.QueryParam("chunkNumber"), 10, 64)
+	if err != nil {
+		return ctx.NoContent(http.StatusBadRequest)
+	}
+
+	err = c.fileUploadService.TestChunk(ctx, identifier, chunkNumber)
+	if err != nil {
+		return ctx.NoContent(http.StatusNoContent)
+	}
+	return ctx.NoContent(http.StatusOK)
+}
+
+func (c *CasaOS) PostUploadFile(ctx echo.Context) error {
+	path := ctx.FormValue("path")
+
+	// handle the request
+	chunkNumber, err := strconv.ParseInt(ctx.FormValue("chunkNumber"), 10, 64)
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+	chunkSize, err := strconv.ParseInt(ctx.FormValue("chunkSize"), 10, 64)
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+	currentChunkSize, err := strconv.ParseInt(ctx.FormValue("currentChunkSize"), 10, 64)
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+	totalChunks, err := strconv.ParseInt(ctx.FormValue("totalChunks"), 10, 64)
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+	totalSize, err := strconv.ParseInt(ctx.FormValue("totalSize"), 10, 64)
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+
+	identifier := ctx.FormValue("identifier")
+	fileName := ctx.FormValue("filename")
+	bin, err := ctx.FormFile("file")
+
+	if err != nil {
+		return ctx.JSON(http.StatusBadRequest, err)
+	}
+
+	err = c.fileUploadService.UploadFile(
+		ctx,
+		path,
+		chunkNumber,
+		chunkSize,
+		currentChunkSize,
+		totalChunks,
+		totalSize,
+		identifier,
+		fileName,
+		bin,
+	)
+	if err != nil {
+		return ctx.JSON(http.StatusInternalServerError, err)
+	}
+	return ctx.NoContent(http.StatusOK)
+}

route/v2/route.go

@@ -2,10 +2,15 @@ package v2
 
 import (
 	"github.com/IceWhaleTech/CasaOS/codegen"
+	"github.com/IceWhaleTech/CasaOS/service"
 )
 
-type CasaOS struct{}
+type CasaOS struct {
+	fileUploadService *service.FileUploadService
+}
 
 func NewCasaOS() codegen.ServerInterface {
-	return &CasaOS{}
+	return &CasaOS{
+		fileUploadService: service.NewFileUploadService(),
+	}
 }

service/file_upload.go

@@ -0,0 +1,144 @@
+package service
+
+import (
+	"fmt"
+	"io"
+	"mime/multipart"
+	"os"
+	"sync"
+
+	"github.com/labstack/echo/v4"
+)
+
+type FileInfo struct {
+	init             bool
+	uploaded         []bool
+	uploadedChunkNum int64
+}
+
+type FileUploadService struct {
+	uploadStatus sync.Map
+	lock         sync.RWMutex
+}
+
+func NewFileUploadService() *FileUploadService {
+	return &FileUploadService{
+		uploadStatus: sync.Map{},
+		lock:         sync.RWMutex{},
+	}
+}
+
+func (s *FileUploadService) TestChunk(
+	c echo.Context,
+	identifier string,
+	chunkNumber int64,
+) error {
+	fileInfoTemp, ok := s.uploadStatus.Load(identifier)
+
+	if !ok {
+		return fmt.Errorf("file not found")
+	}
+
+	fileInfo := fileInfoTemp.(*FileInfo)
+
+	if !fileInfo.init {
+		return fmt.Errorf("file not init")
+	}
+
+	// return StatusNoContent instead of 404
+	// the is require by frontend
+	if !fileInfo.uploaded[chunkNumber-1] {
+		return fmt.Errorf("file not found")
+	}
+
+	return nil
+}
+
+func (s *FileUploadService) UploadFile(
+	c echo.Context,
+	path string,
+	chunkNumber int64,
+	chunkSize int64,
+	currentChunkSize int64,
+	totalChunks int64,
+	totalSize int64,
+	identifier string,
+	fileName string,
+	bin *multipart.FileHeader,
+) error {
+	s.lock.Lock()
+	fileInfoTemp, ok := s.uploadStatus.Load(identifier)
+	var fileInfo *FileInfo
+
+	file, err := os.OpenFile(path+"/"+fileName+".tmp", os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		s.lock.Unlock()
+		return err
+	}
+
+	if !ok {
+
+		if err != nil {
+			s.lock.Unlock()
+			return err
+		}
+
+		// pre allocate file size
+		fmt.Println("truncate", totalSize)
+		if err != nil {
+			s.lock.Unlock()
+			return err
+		}
+
+		// file info init
+		fileInfo = &FileInfo{
+			init:             true,
+			uploaded:         make([]bool, totalChunks),
+			uploadedChunkNum: 0,
+		}
+		s.uploadStatus.Store(identifier, fileInfo)
+	} else {
+		fileInfo = fileInfoTemp.(*FileInfo)
+	}
+
+	s.lock.Unlock()
+
+	_, err = file.Seek((chunkNumber-1)*chunkSize, io.SeekStart)
+	if err != nil {
+		return err
+	}
+
+	src, err := bin.Open()
+	if err != nil {
+		return err
+	}
+	defer src.Close()
+
+	buf := make([]byte, int(currentChunkSize))
+	_, err = io.CopyBuffer(file, src, buf)
+
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+
+	s.lock.Lock()
+	// handle file after write a chunk
+	// handle single chunk upload twice
+	if !fileInfo.uploaded[chunkNumber-1] {
+		fileInfo.uploadedChunkNum++
+		fileInfo.uploaded[chunkNumber-1] = true
+	}
+
+	// handle file after write all chunk
+	if fileInfo.uploadedChunkNum == totalChunks {
+		file.Close()
+		os.Rename(path+"/"+fileName+".tmp", path+"/"+fileName)
+
+		// remove upload status info after upload complete
+		s.uploadStatus.Delete(identifier)
+	}
+	s.lock.Unlock()
+
+	return nil
+}