Skip to content

Commit f154361

Browse files
warren830warren830
authored
feat:support comlexity data scope (#8593)
* feat:support comlexicy data scope * fix: scopedata * fix: unit test
1 parent 04d5607 commit f154361

File tree

10 files changed

+869
-2
lines changed

10 files changed

+869
-2
lines changed

‎backend/plugins/q_dev/api/init.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@ import (
2121
"github.com/apache/incubator-devlake/core/context"
2222
"github.com/apache/incubator-devlake/core/plugin"
2323
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
24+
"github.com/apache/incubator-devlake/helpers/srvhelper"
25+
"github.com/apache/incubator-devlake/plugins/q_dev/models"
2426
"github.com/go-playground/validator/v10"
2527
)
2628

2729
var vld *validator.Validate
2830
var connectionHelper *api.ConnectionApiHelper
2931
var basicRes context.BasicRes
32+
var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig]
3033

3134
func Init(br context.BasicRes, p plugin.PluginMeta) {
3235
basicRes = br
@@ -36,4 +39,15 @@ func Init(br context.BasicRes, p plugin.PluginMeta) {
3639
vld,
3740
p.Name(),
3841
)
42+
43+
dsHelper = api.NewDataSourceHelper[
44+
models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig,
45+
](
46+
basicRes,
47+
p.Name(),
48+
[]string{"prefix", "basePath", "name"},
49+
func(c models.QDevConnection) models.QDevConnection { return c.Sanitize() },
50+
func(s models.QDevS3Slice) models.QDevS3Slice { return s.Sanitize() },
51+
nil,
52+
)
3953
}

‎backend/plugins/q_dev/api/s3_slice_api.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package api
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/errors"
22+
"github.com/apache/incubator-devlake/core/plugin"
23+
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
24+
"github.com/apache/incubator-devlake/helpers/srvhelper"
25+
"github.com/apache/incubator-devlake/plugins/q_dev/models"
26+
)
27+
28+
type PutScopesReqBody = helper.PutScopesReqBody[models.QDevS3Slice]
29+
type ScopeDetail = srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig]
30+
31+
// PutScopes create or update Q Developer scopes (S3 prefixes)
32+
// @Summary create or update Q Developer scopes
33+
// @Description Create or update Q Developer scopes
34+
// @Tags plugins/q_dev
35+
// @Accept application/json
36+
// @Param connectionId path int true "connection ID"
37+
// @Param scope body PutScopesReqBody true "json"
38+
// @Success 200 {object} []models.QDevS3Slice
39+
// @Failure 400 {object} shared.ApiBody "Bad Request"
40+
// @Failure 500 {object} shared.ApiBody "Internal Error"
41+
// @Router /plugins/q_dev/connections/{connectionId}/scopes [PUT]
42+
func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
43+
return dsHelper.ScopeApi.PutMultiple(input)
44+
}
45+
46+
// GetScopeList returns Q Developer scopes
47+
// @Summary get Q Developer scopes
48+
// @Description get Q Developer scopes
49+
// @Tags plugins/q_dev
50+
// @Param connectionId path int true "connection ID"
51+
// @Param pageSize query int false "page size"
52+
// @Param page query int false "page number"
53+
// @Param blueprints query bool false "include blueprint references"
54+
// @Success 200 {object} []ScopeDetail
55+
// @Failure 400 {object} shared.ApiBody "Bad Request"
56+
// @Failure 500 {object} shared.ApiBody "Internal Error"
57+
// @Router /plugins/q_dev/connections/{connectionId}/scopes [GET]
58+
func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
59+
return dsHelper.ScopeApi.GetPage(input)
60+
}
61+
62+
// GetScope returns a single scope record
63+
// @Summary get a Q Developer scope
64+
// @Description get a Q Developer scope
65+
// @Tags plugins/q_dev
66+
// @Param connectionId path int true "connection ID"
67+
// @Param scopeId path string true "scope id"
68+
// @Param blueprints query bool false "include blueprint references"
69+
// @Success 200 {object} ScopeDetail
70+
// @Failure 400 {object} shared.ApiBody "Bad Request"
71+
// @Failure 500 {object} shared.ApiBody "Internal Error"
72+
// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [GET]
73+
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
74+
return dsHelper.ScopeApi.GetScopeDetail(input)
75+
}
76+
77+
// PatchScope updates a scope record
78+
// @Summary patch a Q Developer scope
79+
// @Description patch a Q Developer scope
80+
// @Tags plugins/q_dev
81+
// @Accept application/json
82+
// @Param connectionId path int true "connection ID"
83+
// @Param scopeId path string true "scope id"
84+
// @Param scope body models.QDevS3Slice true "json"
85+
// @Success 200 {object} models.QDevS3Slice
86+
// @Failure 400 {object} shared.ApiBody "Bad Request"
87+
// @Failure 500 {object} shared.ApiBody "Internal Error"
88+
// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [PATCH]
89+
func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
90+
return dsHelper.ScopeApi.Patch(input)
91+
}
92+
93+
// DeleteScope removes a scope and optionally associated data.
94+
// @Summary delete a Q Developer scope
95+
// @Description delete Q Developer scope data
96+
// @Tags plugins/q_dev
97+
// @Param connectionId path int true "connection ID"
98+
// @Param scopeId path string true "scope id"
99+
// @Param delete_data_only query bool false "Only delete scope data"
100+
// @Success 200
101+
// @Failure 400 {object} shared.ApiBody "Bad Request"
102+
// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope"
103+
// @Failure 500 {object} shared.ApiBody "Internal Error"
104+
// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [DELETE]
105+
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
106+
return dsHelper.ScopeApi.Delete(input)
107+
}
108+
109+
// GetScopeLatestSyncState returns scope sync state info
110+
// @Summary latest sync state for a Q Developer scope
111+
// @Description get latest sync state for a Q Developer scope
112+
// @Tags plugins/q_dev
113+
// @Param connectionId path int true "connection ID"
114+
// @Param scopeId path string true "scope id"
115+
// @Success 200
116+
// @Failure 400 {object} shared.ApiBody "Bad Request"
117+
// @Failure 500 {object} shared.ApiBody "Internal Error"
118+
// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET]
119+
func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
120+
return dsHelper.ScopeApi.GetScopeLatestSyncState(input)
121+
}

‎backend/plugins/q_dev/impl/impl.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ func (p QDev) GetTablesInfo() []dal.Tabler {
5454
&models.QDevConnection{},
5555
&models.QDevUserData{},
5656
&models.QDevS3FileMeta{},
57+
&models.QDevS3Slice{},
5758
}
5859
}
5960

@@ -70,7 +71,7 @@ func (p QDev) Connection() dal.Tabler {
7071
}
7172

7273
func (p QDev) Scope() plugin.ToolLayerScope {
73-
return nil
74+
return &models.QDevS3Slice{}
7475
}
7576

7677
func (p QDev) ScopeConfig() dal.Tabler {
@@ -146,6 +147,18 @@ func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
146147
"connections/:connectionId/test": {
147148
"POST": api.TestExistingConnection,
148149
},
150+
"connections/:connectionId/scopes": {
151+
"GET": api.GetScopeList,
152+
"PUT": api.PutScopes,
153+
},
154+
"connections/:connectionId/scopes/:scopeId": {
155+
"GET": api.GetScope,
156+
"PATCH": api.PatchScope,
157+
"DELETE": api.DeleteScope,
158+
},
159+
"connections/:connectionId/scopes/:scopeId/latest-sync-state": {
160+
"GET": api.GetScopeLatestSyncState,
161+
},
149162
}
150163
}
151164

‎backend/plugins/q_dev/impl/impl_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func TestQDev_BasicPluginMethods(t *testing.T) {
3434

3535
// Test table info
3636
tables := plugin.GetTablesInfo()
37-
assert.Len(t, tables, 3)
37+
assert.Len(t, tables, 4)
3838

3939
// Test subtask metas
4040
subtasks := plugin.SubTaskMetas()

‎backend/plugins/q_dev/models/migrationscripts/20250926_add_s3_slice_table.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
24+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived"
25+
)
26+
27+
type addS3SliceTable struct{}
28+
29+
func (*addS3SliceTable) Up(basicRes context.BasicRes) errors.Error {
30+
return migrationhelper.AutoMigrateTables(
31+
basicRes,
32+
&archived.QDevS3Slice{},
33+
)
34+
}
35+
36+
func (*addS3SliceTable) Version() uint64 {
37+
return 20250926
38+
}
39+
40+
func (*addS3SliceTable) Name() string {
41+
return "Add S3 slice table for QDev plugin"
42+
}

‎backend/plugins/q_dev/models/migrationscripts/archived/s3_slice.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package archived
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
22+
)
23+
24+
type QDevS3Slice struct {
25+
archived.NoPKModel
26+
ConnectionId uint64 `gorm:"primaryKey"`
27+
Id string `gorm:"primaryKey;type:varchar(512)"`
28+
Prefix string `gorm:"type:varchar(512);not null"`
29+
BasePath string `gorm:"type:varchar(512)"`
30+
Year int `gorm:"not null"`
31+
Month *int
32+
}
33+
34+
func (QDevS3Slice) TableName() string {
35+
return "_tool_q_dev_s3_slices"
36+
}

‎backend/plugins/q_dev/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,6 @@ func All() []plugin.MigrationScript {
2828
new(modifyFileMetaTable),
2929
new(addDisplayNameFields),
3030
new(addMissingMetrics),
31+
new(addS3SliceTable),
3132
}
3233
}

0 commit comments

Comments
 (0)