Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change field processingGuarantee to processingGuarantees for WindowConfig #794

Merged
merged 2 commits into from
Nov 26, 2024
Merged

Change field processingGuarantee to processingGuarantees for WindowConfig #794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
45b2d2a
Change field processingGuarantee to processingGuarantees for WindowCo…
jiangpengcheng Nov 25, 2024
04c276a
fix ci
jiangpengcheng Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .ci/helm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ function ci::install_pulsar_charts() {
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml
# the superset chart seems got some issue, so we remove it
yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml
helm dependency update charts/pulsar
${HELM} install sn-platform --set initialize=true --values charts/pulsar/mini_values.yaml charts/pulsar --debug

Expand Down
1 change: 1 addition & 0 deletions .ci/tests/integration-oauth2/e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ setup:
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml
yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml
helm dependency update charts/pulsar
helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml charts/pulsar

Expand Down
1 change: 1 addition & 0 deletions .ci/tests/integration-oauth2/e2e_with_downloader.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ setup:
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml
yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml
helm dependency update charts/pulsar
helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml charts/pulsar

Expand Down
2 changes: 1 addition & 1 deletion .ci/tests/integration/cases/logging-window-function/manifests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ spec:
windowConfig:
windowLengthCount: 10
slidingIntervalCount: 5
processingGuarantee: ATLEAST_ONCE
processingGuarantees: ATLEAST_ONCE
# the processingGuarantee should be manual for window function
# see: https://github.com/apache/pulsar/pull/16279/files#diff-c77c024ccb31c94a7aa80cb8e96d7e370709157bdc104a1be7867fb6c7aa0586R318-R319
processingGuarantee: manual
Expand Down
10 changes: 9 additions & 1 deletion .ci/tests/integration/cases/logging-window-function/verify.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ if [ $? -ne 0 ]; then
exit 1
fi

# verify the `processingGuarantees` config
verify_pg=$(kubectl logs window-function-sample-function-0 | grep processingGuarantees=ATLEAST_ONCE)
if [ $? -ne 0 ]; then
echo "$verify_pg"
kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
exit 1
fi

verify_java_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::send_test_data "persistent://public/default/window-function-input-topic" "test-message" 3 2>&1)
if [ $? -ne 0 ]; then
echo "$verify_java_result"
Expand Down Expand Up @@ -80,7 +88,7 @@ fi
#fi

verify_log_result=$(kubectl logs -l compute.functionmesh.io/name=window-function-sample --tail=-1 | grep -e "-window-log" | wc -l)
if [ $verify_log_result -ne 0 ]; then
if [ $verify_log_result -eq 15 ]; then
sub_name=$(echo $RANDOM | md5sum | head -c 20; echo;)
verify_log_topic_result=$(kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-broker-0 -- bin/pulsar-client consume -n 15 -s $sub_name --subscription-position Earliest "persistent://public/default/window-function-logs" | grep -e "-window-log" | wc -l)
if [ $verify_log_topic_result -ne 0 ]; then
Expand Down
1 change: 1 addition & 0 deletions .ci/tests/integration/e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ setup:
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml
yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml
helm dependency update charts/pulsar
helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster.yaml charts/pulsar

Expand Down
1 change: 1 addition & 0 deletions .ci/tests/integration/e2e_with_tls.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ setup:
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml
yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml
helm dependency update charts/pulsar
helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_tls.yaml charts/pulsar

Expand Down
2 changes: 1 addition & 1 deletion api/compute/v1alpha1/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ type WindowConfig struct {
MaxLagMs *int64 `json:"maxLagMs,omitempty"`
WatermarkEmitIntervalMs *int64 `json:"watermarkEmitIntervalMs,omitempty"`
TimestampExtractorClassName *string `json:"timestampExtractorClassName,omitempty"`
ProcessingGuarantee WindowProcessGuarantee `json:"processingGuarantee,omitempty"`
ProcessingGuarantees WindowProcessGuarantee `json:"processingGuarantees,omitempty"`
}

type VPASpec struct {
Expand Down
2 changes: 1 addition & 1 deletion ...erator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3786,7 +3786,7 @@ spec:
maxLagMs:
format: int64
type: integer
processingGuarantee:
processingGuarantees:
enum:
- ATLEAST_ONCE
- ATMOST_ONCE
Expand Down
2 changes: 1 addition & 1 deletion ...sh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3805,7 +3805,7 @@ spec:
maxLagMs:
format: int64
type: integer
processingGuarantee:
processingGuarantees:
enum:
- ATLEAST_ONCE
- ATMOST_ONCE
Expand Down
2 changes: 1 addition & 1 deletion config/crd/bases/compute.functionmesh.io_functionmeshes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3786,7 +3786,7 @@ spec:
maxLagMs:
format: int64
type: integer
processingGuarantee:
processingGuarantees:
enum:
- ATLEAST_ONCE
- ATMOST_ONCE
Expand Down
2 changes: 1 addition & 1 deletion config/crd/bases/compute.functionmesh.io_functions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3783,7 +3783,7 @@ spec:
maxLagMs:
format: int64
type: integer
processingGuarantee:
processingGuarantees:
enum:
- ATLEAST_ONCE
- ATMOST_ONCE
Expand Down
Loading