当前位置：首页 > news >正文

面对大文件（300G以上）如何加速上传速度

news 2025/7/9 23:32:32

解题思路

采用分片上传，同时每个分片多线程上传可以加速上传速度，上传速度提升10倍左右

在阿里云OSS Go SDK中，bucket.UploadStream 函数并没有直接提供，而是通过 bucket.UploadFile 或者 bucket.PutObject 等函数来实现文件上传。对于大文件上传，可以使用分片上传的方式，并且结合多线程来加速上传过程。

要结合多线程上传每个分片，你可以使用Go语言的并发特性，例如使用 goroutines 和 channels。以下是一个简化的示例，展示了如何使用Go语言并发上传文件的各个分片：

package mainimport ("bytes""context""fmt""io""log""os""sync""github.com/aliyun/aliyun-oss-go-sdk/oss"
)func main() {// 创建OSS客户端。client, err := oss.New("<Endpoint>", "<AccessKeyId>", "<AccessKeySecret>")if err != nil {log.Fatalf("Error creating OSS client: %v", err)}// 获取存储空间。bucket, err := client.Bucket("<BucketName>")if err != nil {log.Fatalf("Error getting bucket: %v", err)}// 打开本地文件。localFilePath := "<YourLocalFilePath>"file, err := os.Open(localFilePath)if err != nil {log.Fatalf("Error opening local file: %v", err)}defer file.Close()// 获取文件大小。fileInfo, err := file.Stat()if err != nil {log.Fatalf("Error getting file info: %v", err)}fileSize := fileInfo.Size()// 设置分片大小。partSize := int64(10 * 1024 * 1024) // 10MBuploadID, err := bucket.InitiateMultipartUpload("<YourObjectKey>")if err != nil {log.Fatalf("Error initiating multipart upload: %v", err)}// 创建一个通道来收集上传结果。partsChan := make(chan oss.UploadPart, 10)var wg sync.WaitGroup// 计算分片数量。partCount := int((fileSize+partSize-1)/(partSize))// 上传每个分片。for i := int64(0); i < int64(partCount); i++ {wg.Add(1)partNumber := i + 1offset := i * partSizevar reader io.Readerif partNumber == int64(partCount) {reader = io.LimitReader(file, fileSize-offset)} else {reader = io.MultiReader(io.LimitReader(file, partSize), bytes.NewReader(nil))}// 使用goroutine并发上传分片。go func(partNumber int64, offset int64, reader io.Reader) {defer wg.Done()var buffer bytes.Bufferbuffer.ReadFrom(reader)partData := buffer.Bytes()var err errorif partData, err = io.ReadAll(reader); err != nil {log.Printf("Error reading part %d: %v", partNumber, err)return}if partNumber*partSize+int64(len(partData)) > fileSize {partData = partData[:fileSize-(partNumber-1)*partSize]}uploadResult, err := bucket.UploadPart(uploadID, partNumber, bytes.NewReader(partData), int64(len(partData)))if err != nil {log.Printf("Error uploading part %d: %v", partNumber, err)return}partsChan <- oss.UploadPart{ETag:       uploadResult.ETag,PartNumber: partNumber,}}(partNumber, offset, reader)}// 等待所有分片上传完成。wg.Wait()close(partsChan)// 收集所有分片的结果。var parts []oss.UploadPartfor part := range partsChan {parts = append(parts, part)}// 完成分片上传。if err := bucket.CompleteMultipartUpload(uploadID, parts); err != nil {log.Fatalf("Error completing multipart upload: %v", err)}log.Println("Upload complete!")
}