Skip to content

Commit

Permalink
chore(inject_time): Removing the chaos inject time from metrics labels (
Browse files Browse the repository at this point in the history
#128)

undefined
  • Loading branch information
ispeakc0de authored Mar 16, 2023
1 parent 340ad14 commit b952d02
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 181 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

# Checkout to the latest commit
# On specific directory/path
Expand Down Expand Up @@ -46,7 +46,7 @@ jobs:
- run: snyk auth ${SNYK_TOKEN}
- uses: actions/setup-go@v1
with:
go-version: '1.17'
go-version: '1.18'
- name: Snyk monitor
run: snyk test

Expand Down Expand Up @@ -102,13 +102,12 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

# Checkout to the latest commit
# On specific directory/path
- name: Checkout
uses: actions/checkout@v2


#Install and configure a kind cluster
- name: Installing Prerequisites (K3S Cluster)
Expand All @@ -127,6 +126,7 @@ jobs:
- name: Running Go BDD Test
run: |
go mod tidy
make test
security-scan:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

# Checkout to the latest commit
# On specific directory/path
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

#Install and configure a kind cluster
- name: Installing Prerequisites (KinD Cluster)
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

# Checkout to the latest commit
# On specific directory/path
Expand Down Expand Up @@ -95,7 +95,7 @@ jobs:
# Install golang
- uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.18

#Install and configure a kind cluster
- name: Installing Prerequisites (KinD Cluster)
Expand Down
13 changes: 1 addition & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ unused-package-check:
fi

.PHONY: deps
deps: build_check_docker godeps bdddeps
deps: build_check_docker godeps

.PHONY: build_check_docker
build_check_docker:
Expand All @@ -55,17 +55,6 @@ godeps:
@echo "INFO:\tverifying dependencies for chaos exporter build ..."
@go get -u -v golang.org/x/lint/golint
@go get -u -v golang.org/x/tools/cmd/goimports

.PHONY: bdddeps
bdddeps:
@echo "------------------"
@echo "bdd test dependencies"
@echo "INFO:\tverifying dependencies for bdddeps ..."
@echo "------------------"
@go get -u github.com/onsi/ginkgo
@go get -u github.com/onsi/gomega
kubectl create -f https://raw.githubusercontent.com/litmuschaos/chaos-operator/master/deploy/chaos_crds.yaml
kubectl create ns litmus

.PHONY: test
test:
Expand Down
20 changes: 4 additions & 16 deletions controller/collect-data.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,13 @@ func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosRes
resultDetails.setName(chaosResult.Name).
setUID(chaosResult.UID).
setNamespace(chaosResult.Namespace).
setProbeSuccesPercentage(probeSuccesPercentage).
setProbeSuccessPercentage(probeSuccesPercentage).
setVerdict(string(chaosResult.Status.ExperimentStatus.Verdict)).
setStartTime(events).
setEndTime(events).
setChaosInjectTime(events).
setChaosEngineName(chaosResult.Spec.EngineName).
setChaosEngineContext(engine.Labels[EngineContext]).
setChaosInjectLabel().
setWorkflowName(engine.Labels[WorkFlowName]).
setAppLabel(engine.Spec.Appinfo.Applabel).
setAppNs(engine.Spec.Appinfo.Appns).
Expand Down Expand Up @@ -140,9 +139,9 @@ func (resultDetails *ChaosResultDetails) setVerdictCount(verdict string, chaosRe
return resultDetails
}

// setProbeSuccesPercentage sets ProbeSuccesPercentage inside resultDetails struct
func (resultDetails *ChaosResultDetails) setProbeSuccesPercentage(probeSuccesPercentage float64) *ChaosResultDetails {
resultDetails.ProbeSuccesPercentage = probeSuccesPercentage
// setProbeSuccessPercentage sets ProbeSuccessPercentage inside resultDetails struct
func (resultDetails *ChaosResultDetails) setProbeSuccessPercentage(probeSuccessPercentage float64) *ChaosResultDetails {
resultDetails.ProbeSuccessPercentage = probeSuccessPercentage
return resultDetails
}

Expand Down Expand Up @@ -182,17 +181,6 @@ func (resultDetails *ChaosResultDetails) setWorkflowName(workflowName string) *C
return resultDetails
}

// setChaosInjectLabel sets the chaos inject label inside resultDetails struct
func (resultDetails *ChaosResultDetails) setChaosInjectLabel() *ChaosResultDetails {
injectTime := ""
if resultDetails.InjectionTime != 0 {
injectTime = strconv.Itoa(int(resultDetails.InjectionTime))
}
resultDetails.ChaosInjectLabel = injectTime

return resultDetails
}

// setStartTime sets start time of experiment run
func (resultDetails *ChaosResultDetails) setStartTime(events corev1.EventList) *ChaosResultDetails {
startTime := int64(0)
Expand Down
11 changes: 1 addition & 10 deletions controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ limitations under the License.
package controller

import (
"os"
"strings"
"time"

"github.com/litmuschaos/chaos-exporter/pkg/clients"
Expand Down Expand Up @@ -54,16 +52,9 @@ func Exporter(clients clients.ClientSets) {

// RegisterFixedMetrics register the prometheus metrics
func (gaugeMetrics *GaugeMetrics) RegisterFixedMetrics() {
if os.Getenv("INJECTION_TIME_FILTER") != "" {
injectionTimeFilter = os.Getenv("INJECTION_TIME_FILTER")
}
prometheus.MustRegister(gaugeMetrics.ResultPassedExperiments)
prometheus.MustRegister(gaugeMetrics.ResultFailedExperiments)
if strings.ToLower(injectionTimeFilter) == "disable" {
prometheus.MustRegister(gaugeMetrics.ResultAwaitedExperimentsWithoutInjectionTime)
} else {
prometheus.MustRegister(gaugeMetrics.ResultAwaitedExperiments)
}
prometheus.MustRegister(gaugeMetrics.ResultAwaitedExperiments)
prometheus.MustRegister(gaugeMetrics.ResultProbeSuccessPercentage)
prometheus.MustRegister(gaugeMetrics.ResultVerdict)
prometheus.MustRegister(gaugeMetrics.ExperimentStartTime)
Expand Down
25 changes: 4 additions & 21 deletions controller/handle-result-deletion.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@ func (gaugeMetrics *GaugeMetrics) unsetDeletedChaosResults(oldChaosResults, newC
resultDetails := initialiseResult().
setName(oldResult.Name).
setNamespace(oldResult.Namespace).
setProbeSuccesPercentage(probeSuccesPercentage).
setProbeSuccessPercentage(probeSuccesPercentage).
setVerdict(value.Verdict).
setAppLabel(value.AppLabel).
setAppNs(value.AppNs).
setAppKind(value.AppKind).
setChaosEngineName(oldResult.Spec.EngineName).
setChaosEngineContext(value.ChaosEngineContext).
setWorkflowName(value.WorkFlowName)
resultDetails.ChaosInjectLabel = value.ChaosInjectLabel

gaugeMetrics.unsetResultChaosMetrics(resultDetails)
}
Expand All @@ -58,15 +57,7 @@ func (gaugeMetrics *GaugeMetrics) unsetOutdatedMetrics(resultDetails ChaosResult
case result.Verdict != resultDetails.Verdict:
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, result.Verdict, fmt.Sprintf("%f", result.ProbeSuccessPercentage), resultDetails.AppLabel,
resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, result.ChaosInjectLabel)
result.Count = 1
// if chaos injectionTime is different then delete the older metrics having outdated data
case result.ChaosInjectLabel != resultDetails.ChaosInjectLabel:
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, result.Verdict, fmt.Sprintf("%f", result.ProbeSuccessPercentage), resultDetails.AppLabel,
resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, result.ChaosInjectLabel)
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, resultDetails.WorkflowName, result.ChaosInjectLabel)
resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName)
result.Count = 1
default:
// if time passed scrape time then reset the value to 0
Expand All @@ -83,8 +74,7 @@ func (gaugeMetrics *GaugeMetrics) unsetOutdatedMetrics(resultDetails ChaosResult

// update the values inside matchVerdict
matchVerdict[string(resultDetails.UID)] = result.setVerdict(resultDetails.Verdict).
setProbeSuccesPercentage(resultDetails.ProbeSuccesPercentage).
setChaosInjectLabel(resultDetails.ChaosInjectLabel)
setProbeSuccesPercentage(resultDetails.ProbeSuccessPercentage)

if reset {
return float64(0)
Expand All @@ -107,13 +97,12 @@ func (resultDetails *ChaosResultDetails) setResultData() {
resultData := initialiseResultData().
setContext(resultDetails.ChaosEngineContext).
setWorkflowName(resultDetails.WorkflowName).
setChaosInjectLabel(resultDetails.ChaosInjectLabel).
setAppKind(resultDetails.AppKind).
setNs(resultDetails.AppNs).
setAppLabel(resultDetails.AppLabel).
setVerdict(resultDetails.Verdict).
setCount(0).
setProbeSuccesPercentage(resultDetails.ProbeSuccesPercentage)
setProbeSuccesPercentage(resultDetails.ProbeSuccessPercentage)

if resultStore[string(resultDetails.UID)] != nil {
resultStore[string(resultDetails.UID)] = append(resultStore[string(resultDetails.UID)], *resultData)
Expand All @@ -139,12 +128,6 @@ func (resultData *ResultData) setWorkflowName(workflowName string) *ResultData {
return resultData
}

// setChaosInjectLabel sets the chaos injection inside resultData struct
func (resultData *ResultData) setChaosInjectLabel(injectTime string) *ResultData {
resultData.ChaosInjectLabel = injectTime
return resultData
}

// setAppKind sets the appkind inside resultData struct
func (resultData *ResultData) setAppKind(appKind string) *ResultData {
resultData.AppKind = appKind
Expand Down
31 changes: 11 additions & 20 deletions controller/scrape.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe
"PassedExperiments": resultDetails.PassedExperiments,
"FailedExperiments": resultDetails.FailedExperiments,
"AwaitedExperiments": resultDetails.AwaitedExperiments,
"ProbeSuccessPercentage": resultDetails.ProbeSuccesPercentage,
"ProbeSuccessPercentage": resultDetails.ProbeSuccessPercentage,
"StartTime": resultDetails.StartTime,
"EndTime": resultDetails.EndTime,
"ChaosInjectTime": resultDetails.InjectionTime,
Expand Down Expand Up @@ -155,22 +155,18 @@ func (gaugeMetrics *GaugeMetrics) setNamespacedChaosMetrics(namespacedScopeMetri

// setResultChaosMetrics sets metrics for the given chaosresult details
func (gaugeMetrics *GaugeMetrics) setResultChaosMetrics(resultDetails ChaosResultDetails, verdictValue float64) {
if strings.ToLower(injectionTimeFilter) == "disable" {
gaugeMetrics.ResultAwaitedExperimentsWithoutInjectionTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName).Set(resultDetails.AwaitedExperiments)
} else {
gaugeMetrics.ResultAwaitedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName,
resultDetails.ChaosInjectLabel).Set(resultDetails.AwaitedExperiments)
}

gaugeMetrics.ResultAwaitedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName).Set(resultDetails.AwaitedExperiments)
gaugeMetrics.ResultPassedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.PassedExperiments)
gaugeMetrics.ResultFailedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.FailedExperiments)
gaugeMetrics.ResultProbeSuccessPercentage.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.ProbeSuccesPercentage)
gaugeMetrics.ResultProbeSuccessPercentage.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.ProbeSuccessPercentage)
switch strings.ToLower(resultDetails.Verdict) {
case "awaited":
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel).Set(float64(0))
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccessPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName).Set(float64(0))
default:
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel).Set(verdictValue)
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccessPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName).Set(verdictValue)
}
gaugeMetrics.ExperimentStartTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.StartTime)
gaugeMetrics.ExperimentEndTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.EndTime)
Expand All @@ -180,17 +176,12 @@ func (gaugeMetrics *GaugeMetrics) setResultChaosMetrics(resultDetails ChaosResul

// unsetResultChaosMetrics unset metrics for the given chaosresult details
func (gaugeMetrics *GaugeMetrics) unsetResultChaosMetrics(resultDetails *ChaosResultDetails) {
if strings.ToLower(injectionTimeFilter) == "disable" {
gaugeMetrics.ResultAwaitedExperimentsWithoutInjectionTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName)
} else {
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel)
}
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel)
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName)
gaugeMetrics.ResultPassedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultFailedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultProbeSuccessPercentage.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict,
fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage), resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel)
fmt.Sprintf("%f", resultDetails.ProbeSuccessPercentage), resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName)
gaugeMetrics.ExperimentStartTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ExperimentEndTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ExperimentChaosInjectedTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
Expand All @@ -205,7 +196,7 @@ func (awsConfig *AWSConfig) setAwsResultChaosMetrics(resultDetails ChaosResultDe
awsConfig.putAwsMetricData(sess, "chaosresult_passed_experiments", "Count", resultDetails.PassedExperiments)
awsConfig.putAwsMetricData(sess, "chaosresult_failed_experiments", "Count", resultDetails.FailedExperiments)
awsConfig.putAwsMetricData(sess, "chaosresult_awaited_experiments", "Count", resultDetails.AwaitedExperiments)
awsConfig.putAwsMetricData(sess, "chaosresult_probe_success_percentage", "Count", resultDetails.ProbeSuccesPercentage)
awsConfig.putAwsMetricData(sess, "chaosresult_probe_success_percentage", "Count", resultDetails.ProbeSuccessPercentage)
awsConfig.putAwsMetricData(sess, "chaosresult_start_time", "Count", resultDetails.StartTime)
awsConfig.putAwsMetricData(sess, "chaosresult_end_time", "Count", resultDetails.EndTime)
awsConfig.putAwsMetricData(sess, "chaosresult_inject_time", "Count", float64(resultDetails.InjectionTime))
Expand Down
Loading

0 comments on commit b952d02

Please sign in to comment.