Skip to content

Commit c1e2897

Browse files
Ash-expAsh-exp
authored
fix: stage artifact logic (#5913)
* fix: stage artifact logic (initial commit) * fix: data converter * added migration script * updated ConvertCdWorkflowRunnerDtoToDbObj data adapter method * refactored CdWorkflowWithArtifact struct to bean * fix: SubscribeCDStageCompleteEvent status update * added: unit test file * fix: docker build * removed redundant tables ci_workflow_config and cd_workflow_config * updated ci fail notification logic * fix: FindByPipelineId build histroy list query * common constants added and removed todos * deprecated env flags * fix: panic handling * fix: data updation for CD compelete event * fix: data updation on ci compelete event * populate IsArtifactUploaded flag in triggerInfo api * updated common-lib version * updated destinationKey in request * updated logging * updated db column type * removed space * updated backward compatibility * fix: cdArtifactLocation in UpdateWorkflow method * updated migration for artifact location * updated HandleCiSuccessEvent func * removed unused wire injection * updated wire_gen file * updated common-lib version
1 parent 3419695 commit c1e2897

62 files changed

Lines changed: 1600 additions & 609 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,7 +1016,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon
10161016
return
10171017
}
10181018
//RBAC
1019-
resp, err := handler.ciHandler.GetHistoricBuildLogs(pipelineId, workflowId, nil)
1019+
resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil)
10201020
if err != nil {
10211021
handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId)
10221022
common.WriteJsonResp(w, err, resp, http.StatusInternalServerError)
@@ -1155,7 +1155,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter
11551155
return
11561156
}
11571157
}
1158-
logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(pipelineId, workflowId)
1158+
logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId)
11591159
if err != nil {
11601160
handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId)
11611161
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

‎api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,10 @@ import (
5151
)
5252

5353
type DeploymentHistoryResp struct {
54-
CdWorkflows []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"`
55-
TagsEdiatable bool `json:"tagsEditable"`
56-
AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app
57-
HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"`
54+
CdWorkflows []pipelineBean.CdWorkflowWithArtifact `json:"cdWorkflows"`
55+
TagsEdiatable bool `json:"tagsEditable"`
56+
AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app
57+
HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"`
5858
}
5959

6060
type DevtronAppDeploymentRestHandler interface {
@@ -1903,7 +1903,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW
19031903
}
19041904
//RBAC CHECK
19051905

1906-
file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(pipelineId, buildId)
1906+
file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId)
19071907
defer file.Close()
19081908

19091909
if err != nil {

‎client/events/EventBuilder.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import (
3737
type EventFactory interface {
3838
Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) Event
3939
BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event
40-
BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event
40+
BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event
4141
//BuildFinalData(event Event) *Payload
4242
}
4343

@@ -163,7 +163,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC
163163
return event
164164
}
165165

166-
func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event {
166+
func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event {
167167
if material == nil {
168168
materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId)
169169
if err != nil {

‎cmd/external-app/wire_gen.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎env_gen.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
| BUILD_LOG_TTL_VALUE_IN_SECS | 3600 | |
4141
| CACHE_LIMIT | 5000000000 | |
4242
| CASBIN_DATABASE | casbin | |
43-
| CD_ARTIFACT_LOCATION_FORMAT | %d/%d.zip | |
4443
| CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | | |
4544
| CD_DEFAULT_ADDRESS_POOL_SIZE | | |
4645
| CD_HELM_PIPELINE_STATUS_CRON_TIME | */2 * * * * | |
@@ -59,7 +58,6 @@
5958
| CD_WORKFLOW_EXECUTOR_TYPE | AWF | |
6059
| CD_WORKFLOW_SERVICE_ACCOUNT | cd-runner | |
6160
| CExpirationTime | 600 | |
62-
| CI_ARTIFACT_LOCATION_FORMAT | %d/%d.zip | |
6361
| CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | | |
6462
| CI_DEFAULT_ADDRESS_POOL_SIZE | | |
6563
| CI_IGNORE_DOCKER_CACHE | | |

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ require (
2121
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
2222
github.com/deckarep/golang-set v1.8.0
2323
github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8
24-
github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214
24+
github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f
2525
github.com/devtron-labs/go-bitbucket v0.9.60-beta
2626
github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80
2727
github.com/evanphx/json-patch v5.7.0+incompatible

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -794,8 +794,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF
794794
github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
795795
github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64=
796796
github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI=
797-
github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214 h1:QBBienyRjnNw/IwVt8O4WxIQvQJUbqvLp51mQqfwcTM=
798-
github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E=
797+
github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f h1:5PDhi7bAc9r67OpwXZDG7y1n9lhHX26Ke4Y9VE9+kd8=
798+
github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E=
799799
github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
800800
github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
801801
github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=

‎internal/sql/repository/CiArtifactRepository.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ type CiArtifact struct {
7373
ScanEnabled bool `sql:"scan_enabled,notnull"`
7474
Scanned bool `sql:"scanned,notnull"`
7575
ExternalCiPipelineId int `sql:"external_ci_pipeline_id"`
76-
IsArtifactUploaded bool `sql:"is_artifact_uploaded"`
76+
IsArtifactUploaded bool `sql:"is_artifact_uploaded"` // Deprecated; Use pipelineConfig.CiWorkflow instead.
7777
CredentialsSourceType string `sql:"credentials_source_type"`
7878
CredentialSourceValue string `sql:"credentials_source_value"`
7979
ComponentId int `sql:"component_id"`

‎internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go‎

Lines changed: 45 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ import (
2323
apiBean "github.com/devtron-labs/devtron/api/bean"
2424
"github.com/devtron-labs/devtron/client/gitSensor"
2525
"github.com/devtron-labs/devtron/internal/sql/repository"
26-
repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging"
27-
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow"
26+
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow"
27+
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow"
2828
"github.com/devtron-labs/devtron/internal/util"
2929
"github.com/devtron-labs/devtron/pkg/sql"
3030
"github.com/go-pg/pg"
@@ -43,13 +43,13 @@ type CdWorkflowRepository interface {
4343
FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType apiBean.WorkflowType, limit int, runnerStatuses []string) ([]CdWorkflowRunner, error)
4444
SaveWorkFlowRunner(wfr *CdWorkflowRunner) (*CdWorkflowRunner, error)
4545
UpdateWorkFlowRunner(wfr *CdWorkflowRunner) error
46+
UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error
4647
GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error)
4748
UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error
4849
UpdateWorkFlowRunnersWithTxn(wfrs []*CdWorkflowRunner, tx *pg.Tx) error
4950
UpdateWorkFlowRunners(wfr []*CdWorkflowRunner) error
5051
FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error)
5152
FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error)
52-
FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error)
5353
FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error)
5454
FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error)
5555
FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error)
@@ -76,6 +76,9 @@ type CdWorkflowRepository interface {
7676
FetchArtifactsByCdPipelineId(pipelineId int, runnerType apiBean.WorkflowType, offset, limit int, searchString string) ([]CdWorkflowRunner, error)
7777
GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours int) ([]*CdWorkflowRunner, error)
7878
FindLatestRunnerByPipelineIdsAndRunnerType(ctx context.Context, pipelineIds []int, runnerType apiBean.WorkflowType) ([]CdWorkflowRunner, error)
79+
80+
MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool)
81+
MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string)
7982
}
8083

8184
type CdWorkflowRepositoryImpl struct {
@@ -95,27 +98,6 @@ type CdWorkflow struct {
9598
sql.AuditLog
9699
}
97100

98-
type CdWorkflowConfig struct {
99-
tableName struct{} `sql:"cd_workflow_config" pg:",discard_unknown_columns"`
100-
Id int `sql:"id,pk"`
101-
CdTimeout int64 `sql:"cd_timeout"`
102-
MinCpu string `sql:"min_cpu"`
103-
MaxCpu string `sql:"max_cpu"`
104-
MinMem string `sql:"min_mem"`
105-
MaxMem string `sql:"max_mem"`
106-
MinStorage string `sql:"min_storage"`
107-
MaxStorage string `sql:"max_storage"`
108-
MinEphStorage string `sql:"min_eph_storage"`
109-
MaxEphStorage string `sql:"max_eph_storage"`
110-
CdCacheBucket string `sql:"cd_cache_bucket"`
111-
CdCacheRegion string `sql:"cd_cache_region"`
112-
CdImage string `sql:"cd_image"`
113-
Namespace string `sql:"wf_namespace"`
114-
CdPipelineId int `sql:"cd_pipeline_id"`
115-
LogsBucket string `sql:"logs_bucket"`
116-
CdArtifactLocationFormat string `sql:"cd_artifact_location_format"`
117-
}
118-
119101
type CdWorkflowRunnerWithExtraFields struct {
120102
CdWorkflowRunner
121103
TotalCount int
@@ -134,6 +116,8 @@ type CdWorkflowRunner struct {
134116
FinishedOn time.Time `sql:"finished_on"`
135117
Namespace string `sql:"namespace"`
136118
LogLocation string `sql:"log_file_path"`
119+
CdArtifactLocation string `sql:"cd_artifact_location"`
120+
IsArtifactUploaded workflow.ArtifactUploadedType `sql:"is_artifact_uploaded"`
137121
TriggeredBy int32 `sql:"triggered_by"`
138122
CdWorkflowId int `sql:"cd_workflow_id"`
139123
PodName string `sql:"pod_name"`
@@ -145,6 +129,15 @@ type CdWorkflowRunner struct {
145129
sql.AuditLog
146130
}
147131

132+
func (c *CdWorkflowRunner) GetIsArtifactUploaded() (isArtifactUploaded bool, isMigrationRequired bool) {
133+
return workflow.IsArtifactUploaded(c.IsArtifactUploaded)
134+
}
135+
136+
func (c *CdWorkflowRunner) WithIsArtifactUploaded(isArtifactUploaded bool) *CdWorkflowRunner {
137+
c.IsArtifactUploaded = workflow.GetArtifactUploadedType(isArtifactUploaded)
138+
return c
139+
}
140+
148141
func (c *CdWorkflowRunner) IsExternalRun() bool {
149142
var isExtCluster bool
150143
if c.WorkflowType == cdWorkflow.WorkflowTypePre {
@@ -173,34 +166,6 @@ type CiPipelineMaterialResponse struct {
173166
Regex string `json:"regex"`
174167
}
175168

176-
type CdWorkflowWithArtifact struct {
177-
Id int `json:"id"`
178-
CdWorkflowId int `json:"cd_workflow_id"`
179-
Name string `json:"name"`
180-
Status string `json:"status"`
181-
PodStatus string `json:"pod_status"`
182-
Message string `json:"message"`
183-
StartedOn time.Time `json:"started_on"`
184-
FinishedOn time.Time `json:"finished_on"`
185-
PipelineId int `json:"pipeline_id"`
186-
Namespace string `json:"namespace"`
187-
LogFilePath string `json:"log_file_path"`
188-
TriggeredBy int32 `json:"triggered_by"`
189-
EmailId string `json:"email_id"`
190-
Image string `json:"image"`
191-
MaterialInfo string `json:"material_info,omitempty"`
192-
DataSource string `json:"data_source,omitempty"`
193-
CiArtifactId int `json:"ci_artifact_id,omitempty"`
194-
WorkflowType string `json:"workflow_type,omitempty"`
195-
ExecutorType string `json:"executor_type,omitempty"`
196-
BlobStorageEnabled bool `json:"blobStorageEnabled"`
197-
GitTriggers map[int]GitCommit `json:"gitTriggers"`
198-
CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"`
199-
ImageReleaseTags []*repository2.ImageTag `json:"imageReleaseTags"`
200-
ImageComment *repository2.ImageComment `json:"imageComment"`
201-
RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"`
202-
}
203-
204169
type TriggerWorkflowStatus struct {
205170
CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"`
206171
CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"`
@@ -277,12 +242,6 @@ func (impl *CdWorkflowRepositoryImpl) FindById(wfId int) (*CdWorkflow, error) {
277242
return ddWorkflow, err
278243
}
279244

280-
func (impl *CdWorkflowRepositoryImpl) FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error) {
281-
cdWorkflowConfig := &CdWorkflowConfig{}
282-
err := impl.dbConnection.Model(cdWorkflowConfig).Where("cd_pipeline_id = ?", pipelineId).Select()
283-
return cdWorkflowConfig, err
284-
}
285-
286245
func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) {
287246
cdWorkflow := &CdWorkflow{}
288247
err := impl.dbConnection.Model(cdWorkflow).Where("pipeline_id in (?)", pg.In(pipelineIds)).Order("id DESC").Limit(1).Select()
@@ -502,6 +461,14 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunner(wfr *CdWorkflowRunner
502461
return err
503462
}
504463

464+
func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error {
465+
_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
466+
Set("is_artifact_uploaded = ?", isArtifactUploaded).
467+
Where("id = ?", wfrId).
468+
Update()
469+
return err
470+
}
471+
505472
func (impl *CdWorkflowRepositoryImpl) GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) {
506473
var cdWfrs []*CdWorkflowRunner
507474
err := impl.dbConnection.Model(&cdWfrs).
@@ -773,3 +740,23 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestRunnerByPipelineIdsAndRunnerType
773740
}
774741
return latestWfrs, err
775742
}
743+
744+
func (impl *CdWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) {
745+
_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
746+
Set("is_artifact_uploaded = ?", workflow.GetArtifactUploadedType(isArtifactUploaded)).
747+
Where("id = ?", wfrId).
748+
Update()
749+
if err != nil {
750+
impl.logger.Errorw("error in updating is artifact uploaded", "wfrId", wfrId, "err", err)
751+
}
752+
}
753+
754+
func (impl *CdWorkflowRepositoryImpl) MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string) {
755+
_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
756+
Set("cd_artifact_location = ?", cdArtifactLocation).
757+
Where("id = ?", wfrId).
758+
Update()
759+
if err != nil {
760+
impl.logger.Errorw("error in updating cd artifact location", "wfrId", wfrId, "err", err)
761+
}
762+
}

0 commit comments

Comments
 (0)