Skip to content

Commit

Permalink
support for interop rhel programatically
Browse files Browse the repository at this point in the history
* Initial re structure for rhel event handling

* Added pipeline and evet for interop-rhel

* Change interop-rhel topic for testing

* Add testing event for interop-rhel

* Redactor for handling multiple events from same destination

* Add logging for repos debug

* fix for decoding _ golang/go#40351

* bump tekton client version to V0.26.0

* Wait reconcile from informer to get results

* Fix informer waiting for results on pipelinerun

* Added helper function to inspect xunit files
  • Loading branch information
adrianriobo authored Aug 11, 2021
1 parent 05bb997 commit 7216aca
Show file tree
Hide file tree
Showing 17 changed files with 513 additions and 127 deletions.
9 changes: 6 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,25 @@ go 1.14
require (
github.com/go-stomp/stomp v2.1.4+incompatible
github.com/go-stomp/stomp/v3 v3.0.0
github.com/gobuffalo/envy v1.7.1 // indirect
github.com/golangci/golangci-lint v1.39.0 // indirect
github.com/hjson/hjson-go v3.1.0+incompatible
github.com/joshdk/go-junit v0.0.0-20210226021600-6145f504ca0d
github.com/markbates/inflect v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.1.2
github.com/openshift/api v0.0.0-20210422150128-d8a48168c81c
github.com/openshift/client-go v0.0.0-20210422153130-25c8450d1535
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1
github.com/tektoncd/pipeline v0.23.0
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
github.com/tektoncd/pipeline v0.26.0
golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc
google.golang.org/appengine v1.6.7
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
k8s.io/component-base v0.21.0
k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
knative.dev/pkg v0.0.0-20210127163530-0d31134d5f4e
knative.dev/pkg v0.0.0-20210510175900-4564797bf3b7
)
64 changes: 64 additions & 0 deletions go.sum

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions pkg/crc/pipelines/contants.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,19 @@ import (
)

const (
crcNamespace string = "codeready-container"
pipelinesDashboardBaseUrl string = "https://tekton-dashboard-openshift-pipelines.apps.ocp4.prod.psi.redhat.com"
pipelinesDashboardUrlFormat string = "%s/#/namespaces/%s/pipelineruns/%s"
Namespace string = "codeready-container"
DashboardBaseUrl string = "https://tekton-dashboard-openshift-pipelines.apps.ocp4.prod.psi.redhat.com"
DashboardUrlFormat string = "%s/#/namespaces/%s/pipelineruns/%s"
)

var (
crcWorkspace v1beta1.WorkspaceBinding = v1beta1.WorkspaceBinding{
Workspace v1beta1.WorkspaceBinding = v1beta1.WorkspaceBinding{
Name: "pipelines-data",
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "pipelines-data"},
}

defaultTimeout v1.Duration = v1.Duration{
DefaultTimeout v1.Duration = v1.Duration{
Duration: 8 * time.Hour,
}
)
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package pipelines
package interopOCP

import (
"fmt"

crcPipelines "github.com/adrianriobo/qe-eventmanager/pkg/crc/pipelines"
"github.com/adrianriobo/qe-eventmanager/pkg/services/ci/pipelines"

v1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
Expand All @@ -19,35 +20,35 @@ const (
platformsParamName string = "platforms"
)

func RunInteropOCP(ocpVersion, correlation, serversids, platforms string) (string, string, *v1beta1.PipelineRunStatus, error) {
pipelinerun, err := pipelines.CreatePipelinerun(crcNamespace, getSpecInteropOCP(ocpVersion, correlation, serversids, platforms))
func Run(ocpVersion, correlation, serversids, platforms string) (string, string, *v1beta1.PipelineRunStatus, error) {
pipelinerun, err := pipelines.CreatePipelinerun(crcPipelines.Namespace, getSpec(ocpVersion, correlation, serversids, platforms))
if err != nil {
return "", "", nil, err
}
status := make(chan *v1beta1.PipelineRunStatus)
informerStopper := make(chan struct{})
defer close(status)
defer close(informerStopper)
go pipelines.AddInformer(crcNamespace, pipelinerun.GetName(), status, informerStopper)
go pipelines.AddInformer(crcPipelines.Namespace, pipelinerun.GetName(), status, informerStopper)
return pipelinerun.GetName(), correlation, <-status, nil
}

func getSpecInteropOCP(ocpVersion, correlation, serversids, platforms string) *v1beta1.PipelineRun {
func getSpec(ocpVersion, correlation, serversids, platforms string) *v1beta1.PipelineRun {
return &v1beta1.PipelineRun{
TypeMeta: v1.TypeMeta{},
ObjectMeta: v1.ObjectMeta{GenerateName: pipelineRunName, Namespace: crcNamespace},
ObjectMeta: v1.ObjectMeta{GenerateName: pipelineRunName, Namespace: crcPipelines.Namespace},
Spec: v1beta1.PipelineRunSpec{
PipelineRef: &v1beta1.PipelineRef{Name: pipelineRefName},
Params: []v1beta1.Param{
{Name: ocpVersionParamName, Value: *v1beta1.NewArrayOrString(ocpVersion)},
{Name: correlationParamName, Value: *v1beta1.NewArrayOrString(correlation)},
{Name: serversidsParamName, Value: *v1beta1.NewArrayOrString(serversids)},
{Name: platformsParamName, Value: *v1beta1.NewArrayOrString(platforms)}},
Timeout: &defaultTimeout,
Workspaces: []v1beta1.WorkspaceBinding{crcWorkspace}},
Timeout: &crcPipelines.DefaultTimeout,
Workspaces: []v1beta1.WorkspaceBinding{crcPipelines.Workspace}},
}
}

func GetPipelinerunDashboardUrl(pipelinerunName string) string {
return fmt.Sprintf(pipelinesDashboardUrlFormat, pipelinesDashboardBaseUrl, crcNamespace, pipelinerunName)
return fmt.Sprintf(crcPipelines.DashboardUrlFormat, crcPipelines.DashboardBaseUrl, crcPipelines.Namespace, pipelinerunName)
}
97 changes: 97 additions & 0 deletions pkg/crc/pipelines/interop-rhel/pipeline.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package interopRHEL

import (
"fmt"

crcPipelines "github.com/adrianriobo/qe-eventmanager/pkg/crc/pipelines"
"github.com/adrianriobo/qe-eventmanager/pkg/services/ci/pipelines"
"github.com/adrianriobo/qe-eventmanager/pkg/util/http"
"github.com/adrianriobo/qe-eventmanager/pkg/util/logging"
"github.com/adrianriobo/qe-eventmanager/pkg/util/xunit"

v1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
pipelineRefName string = "interop-rhel"
pipelineRunName string = pipelineRefName + "-"

rhelVersionParamName string = "rhel-version"
repoBaseosParamName string = "repo-baseos-url"
repoAppStreamParamName string = "repo-appstream-url"
imageIDParamName string = "image-id"

xunitURLResultName string = "results-url"
qeDurationResultName string = "qe-duration"

resultStatusPassed string = "passed"
resultStatusFailed string = "failed"
)

func Run(rhelVersion, repoBaseos, repoAppStream, imageID string) (string, string, string, string, error) {
pipelinerun, err := pipelines.CreatePipelinerun(crcPipelines.Namespace, getSpec(rhelVersion, repoBaseos, repoAppStream, imageID))
if err != nil {
return "", "", "", "", err
}
status := make(chan *v1beta1.PipelineRunStatus)
informerStopper := make(chan struct{})
defer close(status)
defer close(informerStopper)
go pipelines.AddInformer(crcPipelines.Namespace, pipelinerun.GetName(), status, informerStopper)
runStatus := <-status
xunitURL := getResultValue(runStatus.PipelineResults, xunitURLResultName)
return pipelinerun.GetName(),
xunitURL,
getResultValue(runStatus.PipelineResults, qeDurationResultName),
getResultState(xunitURL),
nil
}

// TODO make general vailable
func getResultValue(results []v1beta1.PipelineRunResult, resultParamID string) string {
for _, result := range results {
if result.Name == resultParamID {
return result.Value
}
}
return ""
}

// TODO this should be moved to result parameter from the pipeline
func getResultState(url string) string {
file, err := http.GetFile(url)
if err != nil {
logging.Error(err)
return ""
}
count, err := xunit.CountFailures(file)
if err != nil {
logging.Error(err)
return ""
}
if count == 0 {
return resultStatusPassed
}
return resultStatusFailed
}

func getSpec(rhelVersion, repoBaseos, repoAppStream, imageID string) *v1beta1.PipelineRun {
return &v1beta1.PipelineRun{
TypeMeta: v1.TypeMeta{},
ObjectMeta: v1.ObjectMeta{GenerateName: pipelineRunName, Namespace: crcPipelines.Namespace},
Spec: v1beta1.PipelineRunSpec{
PipelineRef: &v1beta1.PipelineRef{Name: pipelineRefName},
Params: []v1beta1.Param{
{Name: rhelVersionParamName, Value: *v1beta1.NewArrayOrString(rhelVersion)},
{Name: repoBaseosParamName, Value: *v1beta1.NewArrayOrString(repoBaseos)},
{Name: repoAppStreamParamName, Value: *v1beta1.NewArrayOrString(repoAppStream)},
{Name: imageIDParamName, Value: *v1beta1.NewArrayOrString(imageID)}},
Timeout: &crcPipelines.DefaultTimeout,
Workspaces: []v1beta1.WorkspaceBinding{crcPipelines.Workspace}},
}
}

func GetPipelinerunDashboardUrl(pipelinerunName string) string {
return fmt.Sprintf(crcPipelines.DashboardUrlFormat, crcPipelines.DashboardBaseUrl, crcPipelines.Namespace, pipelinerunName)
}
6 changes: 6 additions & 0 deletions pkg/event/build-complete/build-complete.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package buildComplete

const (
// Topic string = "VirtualTopic.qe.ci.product-scenario.vipatel.build.complete"
Topic string = "VirtualTopic.qe.ci.product-scenario.build.complete"
)
Original file line number Diff line number Diff line change
@@ -1,22 +1,21 @@
package ocp
package interopOCP

import (
"fmt"
"strings"
"time"

"github.com/mitchellh/mapstructure"

"github.com/adrianriobo/qe-eventmanager/pkg/crc/pipelines"
interopPipelineOCP "github.com/adrianriobo/qe-eventmanager/pkg/crc/pipelines/interop-ocp"
buildComplete "github.com/adrianriobo/qe-eventmanager/pkg/event/build-complete"
"github.com/adrianriobo/qe-eventmanager/pkg/services/messaging/umb"
"github.com/adrianriobo/qe-eventmanager/pkg/util"
"github.com/adrianriobo/qe-eventmanager/pkg/util/logging"
"github.com/mitchellh/mapstructure"
)

const (
topicBuildComplete string = "VirtualTopic.qe.ci.product-scenario.build.complete"
topicTestComplete string = "VirtualTopic.qe.ci.product-scenario.test.complete"
// testError string = "VirtualTopic.qe.ci.product-scenario.ascerra.test.error"
topicTestComplete string = "VirtualTopic.qe.ci.product-scenario.test.complete"
// testError string = "VirtualTopic.qe.ci.product-scenario.test.error"
)

var (
Expand All @@ -34,9 +33,6 @@ func New() ProductScenarioBuild {
return ProductScenarioBuild{}
}

func (p ProductScenarioBuild) GetDestination() string {
return topicBuildComplete
}
func (p ProductScenarioBuild) Handler(event interface{}) error {
var data BuildComplete

Expand All @@ -57,7 +53,7 @@ func (p ProductScenarioBuild) Handler(event interface{}) error {
// Filtering this will be improved in future versions
if len(openshiftVersion) > 0 && codereadyContainersMessage {
name, correlation, _, err :=
pipelines.RunInteropOCP(openshiftVersion, util.GenerateCorrelation(),
interopPipelineOCP.Run(openshiftVersion, util.GenerateCorrelation(),
strings.Join(serversids[:], ","),
strings.Join(platforms[:], ","))
if err != nil {
Expand All @@ -73,10 +69,10 @@ func (p ProductScenarioBuild) Handler(event interface{}) error {
func buildResponse(name, correlation string, source *BuildComplete) *TestComplete {
return &TestComplete{
Artifact: source.Artifact,
Run: Run{
URL: pipelines.GetPipelinerunDashboardUrl(name),
Log: pipelines.GetPipelinerunDashboardUrl(name)},
Test: Test{
Run: buildComplete.Run{
URL: interopPipelineOCP.GetPipelinerunDashboardUrl(name),
Log: interopPipelineOCP.GetPipelinerunDashboardUrl(name)},
Test: buildComplete.Test{
Category: "interoperability",
Namespace: "interop",
TestType: "product-scenario",
Expand Down
58 changes: 58 additions & 0 deletions pkg/event/build-complete/interop-ocp/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package interopOCP

import (
buildComplete "github.com/adrianriobo/qe-eventmanager/pkg/event/build-complete"
)

type BuildComplete struct {
Artifact Artifact `json:"artifact"`
Contact buildComplete.Contact `json:"contact"`
GenerateAt string `json:"generated_at"`
System []buildComplete.System `json:"system"`
Version string `json:"version"`
}

type TestComplete struct {
Contact buildComplete.Contact `json:"contact"`
Run buildComplete.Run `json:"run"`
Artifact Artifact `json:"artifact"`
Test buildComplete.Test `json:"test"`
GenerateAt string `json:"generated_at"`
System []buildComplete.System `json:"system"`
Version string `json:"version"`
}

type TestError struct {
Contact buildComplete.Contact `json:"contact"`
Run buildComplete.Run `json:"run"`
Artifact Artifact `json:"artifact"`
Test buildComplete.Test `json:"test"`
Error buildComplete.Error `json:"error"`
GenerateAt string `json:"generated_at"`
System []buildComplete.System `json:"system"`
Version string `json:"version"`
}

type Artifact struct {
ArtifcatType string `json:"type"`
Id string `json:"id"`
Products []Product `json:"products"`
Email string `json:"email"`
Url string `json:"url"`
}

type Product struct {
Id string `json:"id"`
NVR string `json:"nvr"`
Name string `json:"name"`
Version string `json:"version"`
Architecture string `json:"architecture"`
Build string `json:"build"`
Internal_build_index_url string `json:"internal_build_index_url"`
External_build_index_url string `json:"external_build_index_url"`
ProductType string `json:"type"`
State string `json:"state"`
Artifacts []interface{} `json:"artifacts"`
Phase string `json:"phase"`
Release string `json:"release"`
}
Loading

0 comments on commit 7216aca

Please sign in to comment.