Skip to content

Commit fb5094f

Browse files
zzzhr1990CopilotKirCute
authored
feat(drivers): add halalcloud_open driver (#1430)
* 新增清真云Open驱动,支持最新的轻量SDK * Change Go version in go.mod Downgrade Go version from 1.24.2 to 1.23.4 Signed-off-by: zzzhr1990 <[email protected]> * Apply suggestions from code review * Removed unnecessary comments * Downgraded the Go version to 1.23.4. * Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. Co-authored-by: Copilot <[email protected]> Signed-off-by: zzzhr1990 <[email protected]> * feat(halalcloud_open): support disk usage * Set useSingleUpload to true for upload safety Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. Signed-off-by: zzzhr1990 <[email protected]> * Update meta.go Change required for RefreshToken, If using a personal API approach, the RefreshToken is not required. Signed-off-by: zzzhr1990 <[email protected]> * remove debug logs * bump halalcloud SDK version * fix unnecessary params * Update drivers/halalcloud_open/driver_init.go Co-authored-by: Copilot <[email protected]> Signed-off-by: zzzhr1990 <[email protected]> * Fixed spelling errors; changed hardcoded retry parameters to constants. * remove pointer in get link function in utils.go --------- Signed-off-by: zzzhr1990 <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: KirCute <[email protected]>
1 parent 670e0bd commit fb5094f

File tree

13 files changed

+1018
-0
lines changed

13 files changed

+1018
-0
lines changed

drivers/all.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive"
3737
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo"
3838
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud"
39+
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open"
3940
_ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou"
4041
_ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api"
4142
_ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox"

drivers/halalcloud_open/common.go

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package halalcloudopen
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user"
8+
)
9+
10+
var (
11+
slicePostErrorRetryInterval = time.Second * 120
12+
retryTimes = 5
13+
)
14+
15+
type halalCommon struct {
16+
// *AuthService // 登录信息
17+
UserInfo *sdkUser.User // 用户信息
18+
refreshTokenFunc func(token string) error
19+
// serv *AuthService
20+
configs sync.Map
21+
}
22+
23+
func (m *halalCommon) GetAccessToken() (string, error) {
24+
value, exists := m.configs.Load("access_token")
25+
if !exists {
26+
return "", nil // 如果不存在,返回空字符串
27+
}
28+
return value.(string), nil // 返回配置项的值
29+
}
30+
31+
// GetRefreshToken implements ConfigStore.
32+
func (m *halalCommon) GetRefreshToken() (string, error) {
33+
value, exists := m.configs.Load("refresh_token")
34+
if !exists {
35+
return "", nil // 如果不存在,返回空字符串
36+
}
37+
return value.(string), nil // 返回配置项的值
38+
}
39+
40+
// SetAccessToken implements ConfigStore.
41+
func (m *halalCommon) SetAccessToken(token string) error {
42+
m.configs.Store("access_token", token)
43+
return nil
44+
}
45+
46+
// SetRefreshToken implements ConfigStore.
47+
func (m *halalCommon) SetRefreshToken(token string) error {
48+
m.configs.Store("refresh_token", token)
49+
if m.refreshTokenFunc != nil {
50+
return m.refreshTokenFunc(token)
51+
}
52+
return nil
53+
}
54+
55+
// SetToken implements ConfigStore.
56+
func (m *halalCommon) SetToken(accessToken string, refreshToken string, expiresIn int64) error {
57+
m.configs.Store("access_token", accessToken)
58+
m.configs.Store("refresh_token", refreshToken)
59+
m.configs.Store("expires_in", expiresIn)
60+
if m.refreshTokenFunc != nil {
61+
return m.refreshTokenFunc(refreshToken)
62+
}
63+
return nil
64+
}
65+
66+
// ClearConfigs implements ConfigStore.
67+
func (m *halalCommon) ClearConfigs() error {
68+
m.configs = sync.Map{} // 清空map
69+
return nil
70+
}
71+
72+
// DeleteConfig implements ConfigStore.
73+
func (m *halalCommon) DeleteConfig(key string) error {
74+
_, exists := m.configs.Load(key)
75+
if !exists {
76+
return nil // 如果不存在,直接返回
77+
}
78+
m.configs.Delete(key) // 删除指定的配置项
79+
return nil
80+
}
81+
82+
// GetConfig implements ConfigStore.
83+
func (m *halalCommon) GetConfig(key string) (string, error) {
84+
value, exists := m.configs.Load(key)
85+
if !exists {
86+
return "", nil // 如果不存在,返回空字符串
87+
}
88+
return value.(string), nil // 返回配置项的值
89+
}
90+
91+
// ListConfigs implements ConfigStore.
92+
func (m *halalCommon) ListConfigs() (map[string]string, error) {
93+
configs := make(map[string]string)
94+
m.configs.Range(func(key, value interface{}) bool {
95+
configs[key.(string)] = value.(string) // 将每个配置项添加到map中
96+
return true // 继续遍历
97+
})
98+
return configs, nil // 返回所有配置项
99+
}
100+
101+
// SetConfig implements ConfigStore.
102+
func (m *halalCommon) SetConfig(key string, value string) error {
103+
m.configs.Store(key, value) // 使用Store方法设置或更新配置项
104+
return nil // 成功设置配置项后返回nil
105+
}
106+
107+
func NewHalalCommon() *halalCommon {
108+
return &halalCommon{
109+
configs: sync.Map{},
110+
}
111+
}

drivers/halalcloud_open/driver.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package halalcloudopen
2+
3+
import (
4+
"github.com/OpenListTeam/OpenList/v4/internal/driver"
5+
"github.com/OpenListTeam/OpenList/v4/internal/model"
6+
sdkClient "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient"
7+
sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user"
8+
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
9+
)
10+
11+
type HalalCloudOpen struct {
12+
*halalCommon
13+
model.Storage
14+
Addition
15+
sdkClient *sdkClient.Client
16+
sdkUserFileService *sdkUserFile.UserFileService
17+
sdkUserService *sdkUser.UserService
18+
uploadThread int
19+
}
20+
21+
func (d *HalalCloudOpen) Config() driver.Config {
22+
return config
23+
}
24+
25+
func (d *HalalCloudOpen) GetAddition() driver.Additional {
26+
return &d.Addition
27+
}
28+
29+
var _ driver.Driver = (*HalalCloudOpen)(nil)
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package halalcloudopen
2+
3+
import (
4+
"context"
5+
"strconv"
6+
7+
"github.com/OpenListTeam/OpenList/v4/internal/model"
8+
sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model"
9+
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
10+
)
11+
12+
func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) {
13+
14+
files := make([]model.Obj, 0)
15+
limit := int64(100)
16+
token := ""
17+
18+
for {
19+
result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{
20+
Parent: &sdkUserFile.File{Path: dir.GetPath()},
21+
ListInfo: &sdkModel.ScanListRequest{
22+
Limit: strconv.FormatInt(limit, 10),
23+
Token: token,
24+
},
25+
})
26+
if err != nil {
27+
return nil, err
28+
}
29+
30+
for i := 0; len(result.Files) > i; i++ {
31+
files = append(files, NewObjFile(result.Files[i]))
32+
}
33+
34+
if result.ListInfo == nil || result.ListInfo.Token == "" {
35+
break
36+
}
37+
token = result.ListInfo.Token
38+
39+
}
40+
return files, nil
41+
}
42+
43+
func (d *HalalCloudOpen) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) {
44+
_, err := d.sdkUserFileService.Create(ctx, &sdkUserFile.File{
45+
Path: dir.GetPath(),
46+
Name: name,
47+
})
48+
return nil, err
49+
}
50+
51+
func (d *HalalCloudOpen) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) {
52+
oldDir := obj.GetPath()
53+
newDir := dir.GetPath()
54+
_, err := d.sdkUserFileService.Move(ctx, &sdkUserFile.BatchOperationRequest{
55+
Source: []*sdkUserFile.File{
56+
{
57+
Path: oldDir,
58+
},
59+
},
60+
Dest: &sdkUserFile.File{
61+
Path: newDir,
62+
},
63+
})
64+
return nil, err
65+
}
66+
67+
func (d *HalalCloudOpen) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) {
68+
69+
_, err := d.sdkUserFileService.Rename(ctx, &sdkUserFile.File{
70+
Path: obj.GetPath(),
71+
Name: name,
72+
})
73+
return nil, err
74+
}
75+
76+
func (d *HalalCloudOpen) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) {
77+
id := obj.GetID()
78+
sourcePath := obj.GetPath()
79+
if len(id) > 0 {
80+
sourcePath = ""
81+
}
82+
83+
destID := dir.GetID()
84+
destPath := dir.GetPath()
85+
if len(destID) > 0 {
86+
destPath = ""
87+
}
88+
dest := &sdkUserFile.File{
89+
Path: destPath,
90+
Identity: destID,
91+
}
92+
_, err := d.sdkUserFileService.Copy(ctx, &sdkUserFile.BatchOperationRequest{
93+
Source: []*sdkUserFile.File{
94+
{
95+
Path: sourcePath,
96+
Identity: id,
97+
},
98+
},
99+
Dest: dest,
100+
})
101+
return nil, err
102+
}
103+
104+
func (d *HalalCloudOpen) remove(ctx context.Context, obj model.Obj) error {
105+
id := obj.GetID()
106+
_, err := d.sdkUserFileService.Delete(ctx, &sdkUserFile.BatchOperationRequest{
107+
Source: []*sdkUserFile.File{
108+
{
109+
Identity: id,
110+
Path: obj.GetPath(),
111+
},
112+
},
113+
})
114+
return err
115+
}
116+
117+
func (d *HalalCloudOpen) details(ctx context.Context) (*model.StorageDetails, error) {
118+
ret, err := d.sdkUserService.GetStatisticsAndQuota(ctx)
119+
if err != nil {
120+
return nil, err
121+
}
122+
total := uint64(ret.DiskStatisticsQuota.BytesQuota)
123+
124+
free := uint64(ret.DiskStatisticsQuota.BytesFree)
125+
return &model.StorageDetails{
126+
DiskUsage: model.DiskUsage{
127+
TotalSpace: total,
128+
FreeSpace: free,
129+
},
130+
}, nil
131+
}
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package halalcloudopen
2+
3+
import (
4+
"context"
5+
"crypto/sha1"
6+
"io"
7+
"strconv"
8+
"time"
9+
10+
"github.com/OpenListTeam/OpenList/v4/internal/model"
11+
"github.com/OpenListTeam/OpenList/v4/internal/stream"
12+
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
13+
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
14+
"github.com/rclone/rclone/lib/readers"
15+
)
16+
17+
func (d *HalalCloudOpen) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
18+
if args.Redirect {
19+
// return nil, model.ErrUnsupported
20+
fid := file.GetID()
21+
fpath := file.GetPath()
22+
if fid != "" {
23+
fpath = ""
24+
}
25+
fi, err := d.sdkUserFileService.GetDirectDownloadAddress(ctx, &sdkUserFile.DirectDownloadRequest{
26+
Identity: fid,
27+
Path: fpath,
28+
})
29+
if err != nil {
30+
return nil, err
31+
}
32+
expireAt := fi.ExpireAt
33+
duration := time.Until(time.UnixMilli(expireAt))
34+
return &model.Link{
35+
URL: fi.DownloadAddress,
36+
Expiration: &duration,
37+
}, nil
38+
}
39+
result, err := d.sdkUserFileService.ParseFileSlice(ctx, &sdkUserFile.File{
40+
Identity: file.GetID(),
41+
Path: file.GetPath(),
42+
})
43+
if err != nil {
44+
return nil, err
45+
}
46+
fileAddrs := []*sdkUserFile.SliceDownloadInfo{}
47+
var addressDuration int64
48+
49+
nodesNumber := len(result.RawNodes)
50+
nodesIndex := nodesNumber - 1
51+
startIndex, endIndex := 0, nodesIndex
52+
for nodesIndex >= 0 {
53+
if nodesIndex >= 200 {
54+
endIndex = 200
55+
} else {
56+
endIndex = nodesNumber
57+
}
58+
for ; endIndex <= nodesNumber; endIndex += 200 {
59+
if endIndex == 0 {
60+
endIndex = 1
61+
}
62+
sliceAddress, err := d.sdkUserFileService.GetSliceDownloadAddress(ctx, &sdkUserFile.SliceDownloadAddressRequest{
63+
Identity: result.RawNodes[startIndex:endIndex],
64+
Version: 1,
65+
})
66+
if err != nil {
67+
return nil, err
68+
}
69+
addressDuration, _ = strconv.ParseInt(sliceAddress.ExpireAt, 10, 64)
70+
fileAddrs = append(fileAddrs, sliceAddress.Addresses...)
71+
startIndex = endIndex
72+
nodesIndex -= 200
73+
}
74+
75+
}
76+
77+
size, _ := strconv.ParseInt(result.FileSize, 10, 64)
78+
chunks := getChunkSizes(result.Sizes)
79+
resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
80+
length := httpRange.Length
81+
if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size {
82+
length = size - httpRange.Start
83+
}
84+
oo := &openObject{
85+
ctx: ctx,
86+
d: fileAddrs,
87+
chunk: []byte{},
88+
chunks: chunks,
89+
skip: httpRange.Start,
90+
sha: result.Sha1,
91+
shaTemp: sha1.New(),
92+
}
93+
94+
return readers.NewLimitedReadCloser(oo, length), nil
95+
}
96+
97+
var duration time.Duration
98+
if addressDuration != 0 {
99+
duration = time.Until(time.UnixMilli(addressDuration))
100+
} else {
101+
duration = time.Until(time.Now().Add(time.Hour))
102+
}
103+
104+
return &model.Link{
105+
RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader),
106+
Expiration: &duration,
107+
}, nil
108+
}

0 commit comments

Comments
 (0)