多线程上传及其他问题修复

Author: cnbattle

README.md

@@ -17,53 +17,53 @@ go get -u github.com/cnbattle/upcloud
 
 ### 功能
 
-- [ ] 多线程上传
+- [x] 多线程上传
 
 ### 多平台
 - [x] 腾讯云
 - [x] 七牛云
 - [x] 阿里云
-- [ ] 华为云
-- [ ] 百度云
-- [ ] 滴滴云
-- [ ] UCloud
-- [ ] 青云 QingCloud
-- [ ] 京东智联云
-- [ ] AWS
-- [ ] Google Cloud
 
 ## 各平台`.upcloud.env`配置示例
 
+- UP_CLOUD_PLATFORM： 平台
+- UP_CLOUD_PATH： 需上传静态资源路径
+- UP_CLOUD_POOL_SIZE： 上传时的并发数，默认为10
+- UP_CLOUD_PREFETCH_URLS： 上传完成，要刷新缓存的链接，VUE React等静态站点一般刷新首页index.html即可
+
 ### 腾讯云 COS
 ```.env
 UP_CLOUD_PLATFORM=tencent
 UP_CLOUD_PATH=dist/
+UP_CLOUD_POOL_SIZE=10
+UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 
 UP_CLOUD_SECRET_ID=your id 
 UP_CLOUD_SECRET_KEY=your key
 UP_CLOUD_VISIT_NODE=your visis node
-UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 ```
 
 ### 七牛云
 ```.env
 UP_CLOUD_PLATFORM=qiniu
 UP_CLOUD_PATH=dist/
+UP_CLOUD_POOL_SIZE=10
+UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 
 UP_CLOUD_ACCESS_KEY=your access key
 UP_CLOUD_SECRET_KEY=your secret key
 UP_CLOUD_BUCKET=your bucket
-UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 ```
 
 ### 阿里云
 ```.env
 UP_CLOUD_PLATFORM=aliyun
 UP_CLOUD_PATH=dist/
+UP_CLOUD_POOL_SIZE=10
+UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 
 UP_CLOUD_ENDPOINT=your endpoint
 UP_CLOUD_ACCESS_KEY_ID=your access key id
 UP_CLOUD_ACCESS_KEY_SECRET=your access key secret
 UP_CLOUD_BUCKET=your bucket
-UP_CLOUD_PREFETCH_URLS=your prefetch urls (多个用,分割)
 ```

cloud/qiniu.go

@@ -70,6 +70,24 @@ func (q *Qiniu) DelAll(list []string) error {
 	deleteOps := make([]string, 0, len(list))
 	for _, key := range list {
 		deleteOps = append(deleteOps, storage.URIDelete(q.Bucket, key))
+		if len(deleteOps) == 900 {
+			rets, err := q.BucketManager.Batch(deleteOps)
+			if err != nil {
+				// 遇到错误
+				if _, ok := err.(*rpc.ErrorInfo); ok {
+					for _, ret := range rets {
+						// 200 为成功
+						fmt.Printf("%d\n", ret.Code)
+						if ret.Code != 200 {
+							fmt.Printf("%s\n", ret.Data.Error)
+						}
+					}
+				} else {
+					fmt.Printf("batch error, %s", err)
+				}
+			}
+			deleteOps = make([]string, 0, len(list))
+		}
 	}
 	rets, err := q.BucketManager.Batch(deleteOps)
 	if err != nil {
@@ -92,16 +110,9 @@ func (q *Qiniu) DelAll(list []string) error {
 // Upload 上传
 func (q *Qiniu) Upload(localFile, upKey string) error {
 	cfg := storage.Config{}
-	// 空间对应的机房
-	cfg.Zone = &storage.ZoneHuadong
-	// 是否使用https域名
-	cfg.UseHTTPS = false
-	// 上传是否使用CDN上传加速
-	cfg.UseCdnDomains = false
 	// 构建表单上传的对象
 	formUploader := storage.NewFormUploader(&cfg)
 	ret := storage.PutRet{}
-
 	err := formUploader.PutFile(context.Background(), &ret, q.UpToken, upKey, localFile, nil)
 	if err != nil {
 		return err

cloud/tencent.go

@@ -55,8 +55,23 @@ func (t *Tencent) GetAll() (list []string, err error) {
 
 // DelAll 批量删除
 func (t *Tencent) DelAll(list []string) error {
-	for _, item := range list {
-		_, err := t.Client.Object.Delete(context.Background(), item)
+	obs := []cos.Object{}
+	for _, v := range list {
+		obs = append(obs, cos.Object{Key: v})
+		if len(obs) == 1000 {
+			_, _, err := t.Client.Object.DeleteMulti(context.Background(), &cos.ObjectDeleteMultiOptions{
+				Objects: obs,
+			})
+			if err != nil {
+				return err
+			}
+			obs = []cos.Object{}
+		}
+	}
+	if len(obs) > 0 {
+		_, _, err := t.Client.Object.DeleteMulti(context.Background(), &cos.ObjectDeleteMultiOptions{
+			Objects: obs,
+		})
 		if err != nil {
 			return err
 		}

go.mod

@@ -3,14 +3,14 @@ module github.com/cnbattle/upcloud
 go 1.15
 
 require (
-	github.com/aliyun/alibaba-cloud-sdk-go v1.61.861
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.871
 	github.com/aliyun/aliyun-oss-go-sdk v2.1.6+incompatible
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/joho/godotenv v1.3.0
+	github.com/panjf2000/ants/v2 v2.4.3
 	github.com/qiniu/api.v7/v7 v7.8.1
 	github.com/satori/go.uuid v1.2.0 // indirect
-	github.com/tencentcloud/tencentcloud-sdk-go v1.0.86
+	github.com/tencentcloud/tencentcloud-sdk-go v1.0.90
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.18
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -1,6 +1,6 @@
 github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
-github.com/aliyun/alibaba-cloud-sdk-go v1.61.861 h1:ObMc47c1NA6NRUGbWlEdaARjGZ+imLsrH7k7dD2BGyw=
-github.com/aliyun/alibaba-cloud-sdk-go v1.61.861/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA=
+github.com/aliyun/alibaba-cloud-sdk-go v1.61.871 h1:wt03k4D/3d4jqovp0+uS1JVHtAWD8XbJh8f/sA/EQ/8=
+github.com/aliyun/alibaba-cloud-sdk-go v1.61.871/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA=
 github.com/aliyun/aliyun-oss-go-sdk v2.1.6+incompatible h1:Ft+KeWIJxFP76LqgJbvtOA1qBIoC8vGkTV3QeCOeJC4=
 github.com/aliyun/aliyun-oss-go-sdk v2.1.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
@@ -23,17 +23,14 @@ github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswD
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
 github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
+github.com/panjf2000/ants/v2 v2.4.3 h1:wHghL17YKFanB62QjPQ9o+DuM4q7WrQ7zAhoX8+eBXU=
+github.com/panjf2000/ants/v2 v2.4.3/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/qiniu/api.v7/v7 v7.8.1 h1:2+kFmiyhwiOYWkyKN8euuHe0DP3YQ3z+dAD48Tvt9gE=
@@ -47,8 +44,10 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tencentcloud/tencentcloud-sdk-go v1.0.86 h1:/kFWJ407xMEJPcavoa09kqO8xq96Je9qJ3JtdpHNvDc=
-github.com/tencentcloud/tencentcloud-sdk-go v1.0.86/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tencentcloud/tencentcloud-sdk-go v1.0.90 h1:/Ih44Gj8fVbz1zrP8z8XkeRch8X+WRyP+1hupLxI6HE=
+github.com/tencentcloud/tencentcloud-sdk-go v1.0.90/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.18 h1:hiHAC24LWVIeE5JriNUmCqe2BVLtN0RRsCjyE2glpEg=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.18/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -60,7 +59,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

upcloud.go

@@ -2,9 +2,13 @@ package main
 
 import (
 	"fmt"
+	"strings"
+	"sync"
+
 	"github.com/cnbattle/upcloud/config"
 	"github.com/cnbattle/upcloud/utils"
-	"strings"
+
+	"github.com/panjf2000/ants/v2"
 )
 
 func main() {
@@ -30,15 +34,21 @@ func main() {
 	// 上传新的文件
 	fmt.Println("Start uploading data.")
 	files := utils.Local(config.GetEnvForPanic("UP_CLOUD_PATH"))
+	pool, _ := ants.NewPool(config.GetDefaultEnvToInt("UP_CLOUD_POOL_SIZE", 10))
+	defer pool.Release()
+	var wg sync.WaitGroup
 	for _, file := range files {
-		fmt.Println(file)
-		//fmt.Print(".")
-		err := commInterface.Upload(file.Local, file.UpKey)
-		if err != nil {
-			fmt.Println("commInterface.Upload error:", err)
-			return
-		}
+		wg.Add(1)
+		_ = pool.Submit(func() {
+			err := commInterface.Upload(file.Local, file.UpKey)
+			if err != nil {
+				fmt.Println("commInterface.Upload error:", err)
+			}
+			fmt.Print(".")
+			wg.Done()
+		})
 	}
+	wg.Wait()
 	fmt.Println()
 	fmt.Println("Start refreshing CDN data.")
 	prefetch := config.GetEnvForPanic("UP_CLOUD_PREFETCH_URLS")