Merge develop to main

.buildkite/diff

@@ -8,11 +8,11 @@
 
 COMMIT=$1
 
-if [ -n $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]; then
+if [ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]; then
   HEAD_BRANCH="origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH"
   MERGE_BASE=$(git merge-base "$HEAD_BRANCH" "$COMMIT")
   echo "Checking against a base branch: $BUILDKITE_PULL_REQUEST_BASE_BRANCH with merge base at $MERGE_BASE"
-  git diff --raw "$COMMIT".."$MERGE_BASE" | awk '{print $6; if($7) {print $7}}'
+  git diff --raw "$MERGE_BASE".."$COMMIT" | awk '{print $6; if($7) {print $7}}'
 else
   echo "Checking against the head of the current branch"
   git diff --raw HEAD~1 | awk '{print $6; if($7) {print $7}}'

.buildkite/publish/dra/init_dra_publishing.sh

@@ -15,18 +15,28 @@ export RELEASE_DIR="${PROJECT_ROOT}"
 
 # Create and stage the artifact
 cd $PROJECT_ROOT
-make clean sdist
+make clean zip
 export DRA_ARTIFACTS_DIR=$RELEASE_DIR/dist/dra-artifacts
 mkdir -p $DRA_ARTIFACTS_DIR
+cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip
 
 cd -
 
 # Download previous step artifacts
 buildkite-agent artifact download '.artifacts/*.tar.gz*' $RELEASE_DIR/dist/ --step build_docker_image_amd64
 buildkite-agent artifact download '.artifacts/*.tar.gz*' $RELEASE_DIR/dist/ --step build_docker_image_arm64
+buildkite-agent artifact download 'app/connectors_service/dist/*.whl' $RELEASE_DIR/ --step test_packages
+buildkite-agent artifact download 'app/connectors_service/dist/*.tar.gz' $RELEASE_DIR/ --step test_packages
+buildkite-agent artifact download 'libs/connectors_sdk/dist/*.whl' $RELEASE_DIR/ --step test_packages
+buildkite-agent artifact download 'libs/connectors_sdk/dist/*.tar.gz' $RELEASE_DIR/ --step test_packages
+
+
+# Copy previous step artifacts to DRA dir
 cp $RELEASE_DIR/dist/.artifacts/* $DRA_ARTIFACTS_DIR
+cp $RELEASE_DIR/app/connectors_service/dist/* $DRA_ARTIFACTS_DIR
+cp $RELEASE_DIR/libs/connectors_sdk/dist/* $DRA_ARTIFACTS_DIR
 
-# Rename to match DRA expectations (<name>-<version>-<classifier>-<os>-<arch>)
+# Rename docker images to match DRA expectations (<name>-<version>-<classifier>-<os>-<arch>)
 cd $DRA_ARTIFACTS_DIR
 mv $DOCKER_ARTIFACT_KEY-$VERSION-amd64.tar.gz $PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz
 mv $DOCKER_ARTIFACT_KEY-$VERSION-arm64.tar.gz $PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz
@@ -138,9 +148,16 @@ if [[ "${PUBLISH_SNAPSHOT:-}" == "true" ]]; then
   generateDependencyReport $DEPENDENCIES_REPORTS_DIR/$dependencyReportName
 
   echo "-------- Publishing SNAPSHOT DRA Artifacts"
-  cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}-SNAPSHOT.zip
+
+  # Make *-SNAPSHOT* copies
+  cp $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/connectors-${VERSION}-SNAPSHOT.zip
+  cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-SNAPSHOT.whl
+  cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-SNAPSHOT.tar.gz
+  cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-SNAPSHOT.whl
+  cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-SNAPSHOT.tar.gz
   cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-SNAPSHOT-docker-image-linux-amd64.tar.gz
   cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-SNAPSHOT-docker-image-linux-arm64.tar.gz
+
   setDraVaultCredentials
   export WORKFLOW="snapshot"
 
@@ -155,19 +172,23 @@ fi
 if [[ "${PUBLISH_STAGING:-}" == "true" ]]; then
   if [ -n "${VERSION_QUALIFIER:-}" ]; then
     dependencyReportName="dependencies-${VERSION}-${VERSION_QUALIFIER}.csv";
-    zip_artifact_name="connectors-${VERSION}-${VERSION_QUALIFIER}.zip"
+
+    # Make *-$VERSION_QUALIFIER* copies
+    cp $DRA_ARTIFACTS_DIR/connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/$PROJECT_NAME-${VERSION}-${VERSION_QUALIFIER}.zip
     cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-amd64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-$VERSION_QUALIFIER-docker-image-linux-amd64.tar.gz
     cp $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-docker-image-linux-arm64.tar.gz $DRA_ARTIFACTS_DIR/$PROJECT_NAME-$VERSION-$VERSION_QUALIFIER-docker-image-linux-arm64.tar.gz
+    cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-$VERSION_QUALIFIER.whl
+    cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors-$VERSION-$VERSION_QUALIFIER.tar.gz
+    cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.whl $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-$VERSION_QUALIFIER.whl
+    cp $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION.tar.gz $DRA_ARTIFACTS_DIR/elasticsearch_connectors_sdk-$VERSION-$VERSION_QUALIFIER.tar.gz
   else
     dependencyReportName="dependencies-${VERSION}.csv";
-    zip_artifact_name="connectors-${VERSION}.zip"
   fi
 
   echo "-------- Generating STAGING dependency report: ${dependencyReportName}"
   generateDependencyReport $DEPENDENCIES_REPORTS_DIR/$dependencyReportName
 
   echo "-------- Publishing STAGING DRA Artifacts"
-  cp $RELEASE_DIR/dist/elasticsearch_connectors-${VERSION}.zip $DRA_ARTIFACTS_DIR/${zip_artifact_name}
   setDraVaultCredentials
   export WORKFLOW="staging"
 

.buildkite/publish/manual-release/update-release-version.sh

@@ -7,10 +7,10 @@ CURDIR="$(dirname "$REL_DIR")"
 
 source $CURDIR/publish-common.sh
 
-echo $VERSION > $PROJECT_ROOT/connectors/VERSION # adds the timestamp suffix
-UPDATED_VERSION=`cat $PROJECT_ROOT/connectors/VERSION`
+echo $VERSION > $PROJECT_ROOT/app/connectors/VERSION # adds the timestamp suffix
+UPDATED_VERSION=`cat $PROJECT_ROOT/app/connectors/VERSION`
 
-git add $PROJECT_ROOT/connectors/VERSION
+git add $PROJECT_ROOT/app/connectors/VERSION
 git commit -m "Bumping version from ${ORIG_VERSION} to ${UPDATED_VERSION}"
 git push origin ${GIT_BRANCH}
 

.buildkite/publish_docker.sh

@@ -31,4 +31,4 @@ docker login -u "${DOCKER_USER}" -p ${DOCKER_PASSWORD} docker.elastic.co
 echo "Done!"
 echo
 echo "Pushing the image to docker.elastic.co"
-make docker-push
+make docker-push

.buildkite/publish_to_pypi.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# !!! WARNING DO NOT add -x to avoid leaking vault passwords
+set -euo pipefail
+
+source .buildkite/shared.sh
+
+init_python
+cd "$PACKAGE_PATH"
+python -m pip install --upgrade build twine
+python -m build
+
+export TWINE_USERNAME="__token__"
+
+# upload to test or real PyPI based on TEST_PYPI=1 env var or arg
+if [[ "${1:-}" == "TEST_PYPI=1" ]] || [[ "${TEST_PYPI:-}" =~ ^(1|TRUE|true)$ ]]; then
+  TWINE_PASSWORD=$(vault read -field publishing-api-key secret/ci/elastic-connectors/test-pypi)
+  export TWINE_PASSWORD
+  python -m twine upload --repository testpypi dist/*
+else
+  TWINE_PASSWORD=$(vault read -field publishing-api-key secret/ci/elastic-connectors/pypi)
+  export TWINE_PASSWORD
+  python -m twine upload --repository pypi dist/*
+fi

.buildkite/pypi-publish-pipeline.yml

@@ -0,0 +1,92 @@
+## .buildkite/pypi-publish-pipeline.yml
+# Manually-triggered pipeline to build and publish Python packages to PyPI
+notify:
+  - if: 'build.branch =~ /^((main)|([0-9]+\.[0-9]+))$/ && (build.state == "failed" || pipeline.started_passing)'
+    slack:
+      channels:
+        - "#search-et-alerts"
+      message: "${BUILDKITE_MESSAGE}"
+
+# add paths for all the packages we want to build here
+# then make sure they're added to the matrix.setup.package_path lists in the steps below
+env:
+  SERVICE_PACKAGE_PATH: "app/connectors_service"
+  SDK_PACKAGE_PATH: "libs/connectors_sdk"
+
+steps:
+  - group: ":building_construction: Build Python Packages"
+    key: "build_package"
+    steps:
+      - label: ":python: Build Python {{matrix.python_version}} {{matrix.package_path}} package"
+        key: build_python_package
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          useVault: true
+          image: family/enterprise-search-ubuntu-2204-connectors-py
+        matrix:
+          setup:
+            python_version:
+              - "3.10"
+              - "3.11"
+            package_path:
+              - "${SERVICE_PACKAGE_PATH}"
+              - "${SDK_PACKAGE_PATH}"
+        env:
+          PYTHON_VERSION: "{{ matrix.python_version }}"
+          PACKAGE_PATH: "{{ matrix.package_path }}"
+        commands: ".buildkite/test_python_packages.sh"
+
+  - wait
+
+  - group: ":test_tube: Publish to Test PyPI"
+    key: "publish_to_test_pypi"
+    depends_on:
+      - build_package
+    steps:
+      - label: ":package: Publish to Test PyPI"
+        key: publish_test_pypi
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          useVault: true
+          image: family/enterprise-search-ubuntu-2204-connectors-py
+        matrix:
+          setup:
+            package_path:
+              - "${SERVICE_PACKAGE_PATH}"
+              - "${SDK_PACKAGE_PATH}"
+        env:
+          PYTHON_VERSION: "3.11"
+          PACKAGE_PATH: "{{ matrix.package_path }}"
+        commands: "TEST_PYPI=1 ./buildkite/publish_to_pypi.sh"
+
+# FOR WHEN WE'RE READY TO PUBLISH TO PRODUCTION PYPI
+#
+#  - block: ":rocket: Publish to Production PyPI?"
+#    prompt: "Test PyPI publish succeeded. Proceed to publish to Production PyPI?"
+#    key: "confirm_production_publish"
+#    depends_on:
+#      - publish_to_test_pypi
+#
+#  - group: ":truck: Publish to Production PyPI"
+#    key: "publish_to_pypi"
+#    depends_on:
+#      - confirm_production_publish
+#    steps:
+#      - label: ":package: Publish to Production PyPI"
+#        key: publish_pypi
+#        agents:
+#          provider: "gcp"
+#          machineType: "n1-standard-8"
+#          useVault: true
+#          image: family/enterprise-search-ubuntu-2204-connectors-py
+#        matrix:
+#          setup:
+#            package_path:
+#              - "${SERVICE_PACKAGE_PATH}"
+#              - "${SDK_PACKAGE_PATH}"
+#        env:
+#          PYTHON_VERSION: "3.11"
+#          PACKAGE_PATH: "{{ matrix.package_path }}"
+#        commands: ".buildkite/publish_to_pypi.sh"

.buildkite/release-pipeline.yml

@@ -15,12 +15,12 @@ steps:
     key: "release_setup"
     steps:
     # ----
-    # Set the build timestamp (for the verion suffix)
+    # Set the build timestamp (for the version suffix)
     # ---
     - label: "Set build metadata"
       commands:
         - buildkite-agent meta-data set timestamp "$(date -u +'%Y%m%d%H%M')"
-        - buildkite-agent meta-data set orig_version "$(cat connectors/VERSION)"
+        - buildkite-agent meta-data set orig_version "$(cat app/connectors/VERSION)"
       key: set_timestamp
     - wait
     - label: ":github: update version and tag"

.buildkite/run_functional_test.sh

@@ -16,7 +16,7 @@ init_python
 
 BASEDIR=$(realpath $(dirname $0))
 ROOT=$(realpath $BASEDIR/../)
-VENV_ROOT=$ROOT/.venv
+VENV_ROOT=$ROOT/app/connectors_service/.venv
 
 cd $ROOT
 

.buildkite/run_notice_check.sh

@@ -21,8 +21,9 @@ if is_pr && ! is_fork; then
     echo 'New changes to NOTICE.txt:'
     git --no-pager diff
 
-    git add NOTICE.txt
-    git commit -m"Update NOTICE.txt"
+    git status --porcelain | grep app/connectors_service/NOTICE.txt && git add app/connectors_service/NOTICE.txt
+    git status --porcelain | grep libs/connectors_sdk/NOTICE.txt && git add libs/connectors_sdk/NOTICE.txt
+    git commit -m "Update NOTICE.txt"
     git push
 
     exit 1

.buildkite/test_python_packages.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# !!! WARNING DO NOT add -x to avoid leaking vault passwords
+set -euo pipefail
+
+source .buildkite/shared.sh
+
+init_python
+
+python -m pip install --upgrade build twine
+python -m build "$PACKAGE_PATH"
+ls -lah "$PACKAGE_PATH/dist/"
+python -m twine check "$PACKAGE_PATH/dist/*"
+
+# If this is the connectors_service package, test the installation and CLI
+if [[ "$PACKAGE_PATH" == *app/connectors_service* ]]; then
+  echo "Testing connectors_service package installation and CLI..."
+
+  # Install the connectors_sdk package first
+  LIB_PATH="libs/connectors_sdk"
+  python -m build "$LIB_PATH"
+  python -m pip install "$LIB_PATH"/dist/*.whl
+
+  python -m pip install "$PACKAGE_PATH"/dist/*.whl
+  connectors --help
+  elastic-ingest --help
+#  elastic-agent-connectors --help
+  test-connectors --help
+else
+  python -m pip install "$PACKAGE_PATH"/dist/*.whl
+  python -c "import connectors_sdk; print(f'🎉 Success! connectors_sdk version: {connectors_sdk.__version__}')"
+fi

.gitignore

@@ -10,7 +10,7 @@ pyvenv.cfg
 __pycache__
 
 # real configurations
-config.yml
+app/connectors_service/config.yml
 
 # jetbrains files
 .idea

Dockerfile

@@ -8,8 +8,8 @@ COPY --chown=nonroot:nonroot . /app
 
 USER nonroot
 WORKDIR /app
-RUN make clean install
-RUN ln -s .venv/bin /app/bin
+RUN make clean install-package
+RUN ln -s app/connectors_service/.venv/bin /app/bin
 
 USER root
 RUN apk del make git

Dockerfile.wolfi

@@ -2,6 +2,6 @@ FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:c85a849c182a812950c3036b3965
 USER root
 COPY . /app
 WORKDIR /app
-RUN make clean install
-RUN ln -s .venv/bin /app/bin
+RUN make clean install-package
+RUN ln -s app/connectors_service/.venv/bin /app/bin
 ENTRYPOINT []