{{ message }}
Repository navigation
Expand file tree
/
Copy pathBuildPipelineSwitchService.go
More file actions
388 lines (347 loc) · 18 KB
/
Copy pathBuildPipelineSwitchService.go
File metadata and controls
388 lines (347 loc) · 18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
/*
* Copyright (c) 2024. Devtron Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package pipeline
import (
"github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
"github.com/devtron-labs/devtron/internal/util"
"github.com/devtron-labs/devtron/pkg/bean"
"github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common"
"github.com/devtron-labs/devtron/pkg/build/pipeline/read"
"github.com/devtron-labs/devtron/pkg/pipeline/adapter"
"github.com/devtron-labs/devtron/pkg/pipeline/history"
repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
"github.com/devtron-labs/devtron/pkg/sql"
"github.com/go-pg/pg"
"github.com/juju/errors"
"go.uber.org/zap"
"net/http"
"time"
)
type SwitchBuildPipelineValidationError string
const (
cannotConvertToSameType SwitchBuildPipelineValidationError = "cannot convert this pipeline to same type"
cannotConvertToExternalCi SwitchBuildPipelineValidationError = "current ci-pipeline cannot be converted to external-webhook type"
cannotConvertIfLinkedCiFound SwitchBuildPipelineValidationError = "cannot convert this ci-pipeline as it contains some linked ci-pipeline's"
cannotConvertIfLatestWorkflowIsInNonTerminalState SwitchBuildPipelineValidationError = "cannot convert this ci-pipeline as recent build of this ci-pipeline is in progressing state"
)
type BuildPipelineSwitchService interface {
SwitchToExternalCi(tx *pg.Tx, appWorkflowMapping *appWorkflow.AppWorkflowMapping, switchFromCiPipelineId int, userId int32) error
SwitchToCiPipelineExceptExternal(request *bean.CiPatchRequest, ciConfig *bean.CiConfigRequest) (*bean.CiConfigRequest, error)
}
type BuildPipelineSwitchServiceImpl struct {
logger *zap.SugaredLogger
ciPipelineConfigReadService read.CiPipelineConfigReadService
ciPipelineRepository pipelineConfig.CiPipelineRepository
ciCdPipelineOrchestrator CiCdPipelineOrchestrator
pipelineRepository pipelineConfig.PipelineRepository
ciWorkflowRepository pipelineConfig.CiWorkflowRepository
appWorkflowRepository appWorkflow.AppWorkflowRepository
ciPipelineHistoryService history.CiPipelineHistoryService
ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository
}
func NewBuildPipelineSwitchServiceImpl(logger *zap.SugaredLogger,
ciPipelineConfigReadService read.CiPipelineConfigReadService,
ciPipelineRepository pipelineConfig.CiPipelineRepository,
ciCdPipelineOrchestrator CiCdPipelineOrchestrator,
pipelineRepository pipelineConfig.PipelineRepository,
ciWorkflowRepository pipelineConfig.CiWorkflowRepository,
appWorkflowRepository appWorkflow.AppWorkflowRepository,
ciPipelineHistoryService history.CiPipelineHistoryService,
ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository,
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository) *BuildPipelineSwitchServiceImpl {
return &BuildPipelineSwitchServiceImpl{
logger: logger,
ciPipelineConfigReadService: ciPipelineConfigReadService,
ciPipelineRepository: ciPipelineRepository,
ciCdPipelineOrchestrator: ciCdPipelineOrchestrator,
pipelineRepository: pipelineRepository,
ciWorkflowRepository: ciWorkflowRepository,
appWorkflowRepository: appWorkflowRepository,
ciPipelineHistoryService: ciPipelineHistoryService,
ciTemplateOverrideRepository: ciTemplateOverrideRepository,
ciPipelineMaterialRepository: ciPipelineMaterialRepository,
}
}
func (impl *BuildPipelineSwitchServiceImpl) SwitchToExternalCi(tx *pg.Tx, appWorkflowMapping *appWorkflow.AppWorkflowMapping, switchFromCiPipelineId int, userId int32) error {
err := impl.validateSwitchPreConditions(switchFromCiPipelineId)
if err != nil {
return err
}
err = impl.deleteCiAndItsWorkflowMappings(tx, switchFromCiPipelineId, userId)
if err != nil {
impl.logger.Errorw("error in deleting old ci-pipeline and getting the appWorkflow mapping of that", "err", err, "userId", userId)
return err
}
oldWorkflowMapping, err := impl.deleteAndGetAppWorkflowMappings(tx, appWorkflow.CIPIPELINE, switchFromCiPipelineId, userId)
if err != nil {
impl.logger.Errorw("error in deleting workflow mapping", "switchFromCiPipelineId", switchFromCiPipelineId, "err", err)
return err
}
err = impl.updateLinkedAppWorkflowMappings(tx, oldWorkflowMapping, appWorkflowMapping)
if err != nil {
impl.logger.Errorw("error in updating linked app-workflow-mappings ", "oldAppWorkflowMappingId", oldWorkflowMapping.Id, "currentAppWorkflowMapId", appWorkflowMapping.Id, "err", err, "userId", userId)
return err
}
// setting new ci_pipeline_id to 0 because we dont store ci_pipeline_id if the ci_pipeline is external/webhook type.
err = impl.pipelineRepository.UpdateOldCiPipelineIdToNewCiPipelineId(tx, switchFromCiPipelineId, 0)
if err != nil {
impl.logger.Errorw("error in updating pipelines ci_pipeline_ids with new ci_pipelineId", "oldCiPipelineId", switchFromCiPipelineId)
return err
}
return nil
}
func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(request *bean.CiPatchRequest, ciConfig *bean.CiConfigRequest) (*bean.CiConfigRequest, error) {
if request.SwitchFromCiPipelineId != 0 && request.SwitchFromExternalCiPipelineId != 0 {
return nil, errors.New("invalid request payload, both switchFromCiPipelineId and switchFromExternalCiPipelineId cannot be set in the payload")
}
//get the ciPipeline
var switchFromType common.PipelineType
var switchFromPipelineId int
if request.SwitchFromExternalCiPipelineId != 0 {
switchFromType = common.EXTERNAL
switchFromPipelineId = request.SwitchFromExternalCiPipelineId
} else {
switchFromPipelineId = request.SwitchFromCiPipelineId
switchFromType = request.SwitchFromCiPipelineType
}
//validate switch request
err := impl.validateCiPipelineSwitch(switchFromPipelineId, request.CiPipeline.PipelineType, switchFromType)
if err != nil {
impl.logger.Errorw("validating failed for ci-pipeline switch request", "switchFromPipelineId", "switchFromType", switchFromType, "pipelineType", request.CiPipeline.PipelineType, "err", err)
return nil, err
}
// delete old pipeline and it's appworkflow mapping
return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId)
}
func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*bean.CiConfigRequest, error) {
isSelfLinkedCiPipeline := switchFromType != common.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId
if isSelfLinkedCiPipeline {
errMsg := "cannot create linked ci pipeline from the same source"
return nil, util.DefaultApiError().WithInternalMessage(errMsg).WithUserMessage(errMsg).WithHttpStatusCode(http.StatusBadRequest)
}
tx, err := impl.ciPipelineRepository.StartTx()
if err != nil {
impl.logger.Errorw("error in starting transaction", "switchFromPipelineId", switchFromPipelineId, "switchFromType", switchFromType, "userId", userId, "err", err)
return nil, err
}
defer impl.ciPipelineRepository.RollbackTx(tx)
oldAppWorkflowMapping, err := impl.deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx, switchFromPipelineId, switchFromType, userId)
if err != nil {
impl.logger.Errorw("error in deleting old ci-pipeline and getting the appWorkflow mapping of that", "err", err, "userId", userId)
return nil, err
}
ciConfig.CiPipelines = []*bean.CiPipeline{ciPipelineReq} //request.CiPipeline
res, err := impl.ciCdPipelineOrchestrator.AddPipelineToTemplate(ciConfig, true)
if err != nil {
impl.logger.Errorw("error in adding pipeline to template", "ciConf", ciConfig, "err", err)
return nil, err
}
//go and update all the app workflow mappings of old ci_pipeline with new ci_pipeline_id.
err = impl.updateLinkedAppWorkflowMappings(tx, oldAppWorkflowMapping, res.AppWorkflowMapping)
if err != nil {
impl.logger.Errorw("error in updating app workflow mappings", "err", err)
return nil, err
}
if switchFromPipelineId > 0 {
// get all the cd workflow mappings whose parent component is our old pipeline
cdwfmappings, err := impl.appWorkflowRepository.FindWFCDMappingsByWorkflowId(oldAppWorkflowMapping.AppWorkflowId)
if err != nil {
impl.logger.Errorw("error in finding parent cd workflowMappings using parent component details", "parentComponentType", oldAppWorkflowMapping.Type, "parentComponentId", oldAppWorkflowMapping.ComponentId, "err", err)
return nil, err
}
pipelineIds := make([]int, 0, len(cdwfmappings))
for _, cdwfMapping := range cdwfmappings {
pipelineIds = append(pipelineIds, cdwfMapping.ComponentId)
}
err = impl.pipelineRepository.UpdateCiPipelineId(tx, pipelineIds, res.CiPipelines[0].Id)
if err != nil {
impl.logger.Errorw("error in updating pipelines ci_pipeline_ids with new ci_pipelineId", "oldCiPipelineId", switchFromPipelineId, "newCiPipelineId", res.CiPipelines[0].Id, "err", err)
return nil, err
}
}
err = impl.ciPipelineRepository.CommitTx(tx)
if err != nil {
impl.logger.Errorw("error in committing the transaction", "switchFromPipelineId", switchFromPipelineId, "switchFromType", switchFromType, "userId", userId, "err", err)
return nil, err
}
return res, nil
}
// add switchType and remove other id
// make constants for error msgs
func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType common.PipelineType) error {
// this will only allow below conversions
// ext -> {ci_job,direct,linked}
// direct -> {ci_job,linked}
// linked -> {ci_job,direct}
// ci_job -> {direct,linked}
if switchToType == switchFromType {
return errors.New(string(cannotConvertToSameType))
}
// refer SwitchToExternalCi
if switchToType == common.EXTERNAL {
return errors.New(string(cannotConvertToExternalCi))
}
// we should not check the below logic for external_ci type as builds are not built in devtron and
// linked pipelines won't be there as per current external-ci-pipeline architecture
if switchFromCiPipelineId > 0 && switchFromType != common.EXTERNAL {
err := impl.validateSwitchPreConditions(switchFromCiPipelineId)
if err != nil {
return err
}
}
return nil
}
func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *pg.Tx, switchFromPipelineId int, userId int32) error {
ciPipeline, err := impl.ciPipelineRepository.FindById(switchFromPipelineId)
if err == pg.ErrNoRows {
impl.logger.Errorw("no ci-pipeline found for given switchFromCiPipelineId", "switchFromCiPipelineId", switchFromPipelineId)
return errors.New("requested ci pipeline doesn't exist")
}
if err != nil {
impl.logger.Errorw("error in finding ci-pipeline by id", "ciPipelineId", switchFromPipelineId, "err", err)
return err
}
err = impl.deleteBuildPipeline(tx, ciPipeline, userId)
if err != nil {
impl.logger.Errorw("error in deleting ciPipeline", "ciPipelineId", ciPipeline, "err", err)
}
return err
}
func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) {
// 1) delete build pipelines
// 2) delete app workflowMappings
var err error
pipelineId := switchFromPipelineId
pipelineType := ""
if switchFromType == common.EXTERNAL {
err = impl.deleteExternalCi(tx, switchFromPipelineId, userId)
pipelineType = appWorkflow.WEBHOOK
} else {
err = impl.deleteCiAndItsWorkflowMappings(tx, switchFromPipelineId, userId)
pipelineType = appWorkflow.CIPIPELINE
}
if err != nil {
impl.logger.Errorw("error in deleting ci pipeline before switching ", "switchFromPipelineId", switchFromPipelineId, "switchFromType", switchFromType, "err", err)
return nil, err
}
return impl.deleteAndGetAppWorkflowMappings(tx, pipelineType, pipelineId, userId)
}
func (impl *BuildPipelineSwitchServiceImpl) deleteAndGetAppWorkflowMappings(tx *pg.Tx, pipelineType string, pipelineId int, userId int32) (*appWorkflow.AppWorkflowMapping, error) {
appWorkflowMappings, err := impl.appWorkflowRepository.FindWFMappingByComponent(pipelineType, pipelineId)
if err != nil {
impl.logger.Errorw("error in getting appWorkflowMappings", "err", err, "pipelineType", pipelineType, "pipelineId", pipelineId)
return appWorkflowMappings, err
}
//deleting app workflow mapping in tx
appWorkflowMappings.UpdatedBy = userId
appWorkflowMappings.UpdatedOn = time.Now()
err = impl.appWorkflowRepository.DeleteAppWorkflowMapping(appWorkflowMappings, tx)
if err != nil {
impl.logger.Errorw("error in deleting workflow mapping", "CiPipelineType", pipelineType, "pipelineId", pipelineId, "err", err)
return appWorkflowMappings, err
}
return appWorkflowMappings, nil
}
func (impl *BuildPipelineSwitchServiceImpl) deleteBuildPipeline(tx *pg.Tx, ciPipeline *pipelineConfig.CiPipeline, userId int32) error {
err := impl.ciCdPipelineOrchestrator.DeleteCiPipelineAndCiEnvMappings(tx, ciPipeline, userId)
if err != nil {
impl.logger.Errorw("error in deleting ci pipeline and its env mappings", "pipelineId", ciPipeline.Id, "err", err)
return err
}
//not deleting ciPipeline material or template override as these can be useful for artifact built from the old ciPipeline
return err
}
func (impl *BuildPipelineSwitchServiceImpl) deleteExternalCi(tx *pg.Tx, externalCiPipelineId int, userId int32) error {
externalCiPipeline, err := impl.ciPipelineRepository.FindExternalCiById(externalCiPipelineId)
externalCiPipeline.Active = false
externalCiPipeline.AuditLog = sql.AuditLog{UpdatedBy: userId, UpdatedOn: time.Now()}
_, err = impl.ciPipelineRepository.UpdateExternalCi(externalCiPipeline, tx)
if err != nil {
impl.logger.Errorw("error in deleting workflow mapping", "externalCiPipelineId", externalCiPipelineId, "err", err)
return err
}
return nil
}
func (impl *BuildPipelineSwitchServiceImpl) DeleteCiMaterial(tx *pg.Tx, ciPipeline *pipelineConfig.CiPipeline) ([]*pipelineConfig.CiPipelineMaterial, error) {
materialDbObject, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id)
var materials []*pipelineConfig.CiPipelineMaterial
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("error in getting ci pipeline materials", "externalCiPipelineId", "ciPipelineId", ciPipeline.Id, "err", err)
return materials, err
}
if len(materialDbObject) == 0 {
return materials, nil
}
for _, material := range materialDbObject {
material.Active = false
materials = append(materials, material)
}
err = impl.ciPipelineMaterialRepository.Update(tx, materials...)
if err != nil {
impl.logger.Errorw("error in updating ci pipeline materials, DeleteCiPipeline", "err", err, "pipelineId", ciPipeline.Id)
return materials, err
}
return materials, nil
}
func (impl *BuildPipelineSwitchServiceImpl) saveHistoryOfOverriddenTemplate(ciPipeline *pipelineConfig.CiPipeline, userId int32, materials []*pipelineConfig.CiPipelineMaterial) error {
ciTemplate, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipeline.Id)
if err != nil {
impl.logger.Errorw("error in getting ciTemplate ", "err", err, "ciTemplate", ciTemplate.Id)
return err
}
buildConfig, err := adapter.ConvertDbBuildConfigToBean(ciTemplate.CiBuildConfig)
if err != nil {
impl.logger.Errorw("error in ConvertDbBuildConfigToBean ", "err", err, "buildConfigId", buildConfig.Id)
return err
}
CiTemplateBean := impl.ciCdPipelineOrchestrator.CreateCiTemplateBean(ciPipeline.Id, ciTemplate.DockerRegistry.Id, ciTemplate.DockerRepository, ciTemplate.GitMaterialId, buildConfig, userId)
err = impl.ciPipelineHistoryService.SaveHistory(ciPipeline, materials, &CiTemplateBean, repository4.TRIGGER_DELETE)
if err != nil {
impl.logger.Errorw("error in saving delete history for ci pipeline material and ci template overridden", "err", err)
}
return nil
}
func (impl *BuildPipelineSwitchServiceImpl) updateLinkedAppWorkflowMappings(tx *pg.Tx, oldAppWorkflowMapping *appWorkflow.AppWorkflowMapping, newAppWorkflowMapping *appWorkflow.AppWorkflowMapping) error {
return impl.appWorkflowRepository.UpdateParentComponentDetails(tx, oldAppWorkflowMapping.ComponentId, oldAppWorkflowMapping.Type, newAppWorkflowMapping.ComponentId, newAppWorkflowMapping.Type, nil)
}
func (impl *BuildPipelineSwitchServiceImpl) validateSwitchPreConditions(switchFromCiPipelineId int) error {
// old ci_pipeline should not contain any linked ci_pipelines.
linkedCiPipelines, err := impl.ciPipelineConfigReadService.FindLinkedCiCount(switchFromCiPipelineId)
if err != nil {
impl.logger.Errorw("error in finding the linkedCi count for the pipeline", "ciPipelineId", switchFromCiPipelineId, "err", err)
return err
}
if linkedCiPipelines > 0 {
return errors.New(string(cannotConvertIfLinkedCiFound))
}
// note: ideally we should have found any builds running on old ci_pipeline, if yes block this conversion with proper message.
// but checking only latest wf for now.
ciWorkflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(switchFromCiPipelineId)
// no build is triggered case
if err == pg.ErrNoRows {
return nil
}
if err != nil {
impl.logger.Errorw("error in finding latest ciwokflow by ciPipelineId", "ciPipelineId", switchFromCiPipelineId)
return err
}
if ciWorkflow.InProgress() {
return errors.New(string(cannotConvertIfLatestWorkflowIsInNonTerminalState))
}
return nil
}
You can’t perform that action at this time.