diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 00000000..b6738f3b --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,15 @@ +agents: + provider: "gcp" + machineType: "n1-standard-8" + +defaultTimeoutInMinutes: 45 + +steps: + - label: ":safety_vest: Connectors Tests" + commands: + - ".buildkite/scripts/run_command.sh tests" + artifact_paths: + - "coverage/index.html" + - label: ":wrench: Linter" + commands: + - ".buildkite/scripts/run_command.sh linter" diff --git a/.buildkite/scripts/run_ci_step.sh b/.buildkite/scripts/run_ci_step.sh new file mode 100755 index 00000000..6f94e6bf --- /dev/null +++ b/.buildkite/scripts/run_ci_step.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -euxo pipefail + +export PATH="$PATH:/root/.rbenv/bin:/root/.rbenv/plugins/ruby-build/bin:/ci/.rbenv/shims" + +RUBY_VERSION=$(cat .ruby-version) +echo "---- installing Ruby version $RUBY_VERSION" +rbenv install $RUBY_VERSION +rbenv global $RUBY_VERSION + +case $1 in + + tests) + echo "---- running unit tests" + make install test + ;; + + linter) + echo "---- running linter" + make install lint + ;; + + packaging) + echo "---- running packaging" + git config --global --add safe.directory '*' + git config --global --add safe.directory /ci + curl -L -o yq https://github.com/mikefarah/yq/releases/download/v4.21.1/yq_linux_amd64 + chmod +x yq + YQ=`realpath yq` make install build_service build_service_gem + gem install .gems/connectors_service-8.* + ;; + + *) + echo "Usage: run_command {tests|linter|packaging}" + exit 2 + ;; +esac diff --git a/.buildkite/scripts/run_command.sh b/.buildkite/scripts/run_command.sh new file mode 100755 index 00000000..b198caff --- /dev/null +++ b/.buildkite/scripts/run_command.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euxo pipefail + +COMMAND_TO_RUN=${1:-} + +if [[ "${COMMAND_TO_RUN:-}" == "" ]]; then + echo "Usage: run_command.sh {tests|linter|docker|packaging}" + exit 2 +fi + +function realpath { + echo "$(cd "$(dirname "$1")"; pwd)"/"$(basename "$1")"; +} + +SCRIPT_WORKING_DIR=$(realpath "$(dirname "$0")") +BUILDKITE_DIR=$(realpath "$(dirname "$SCRIPT_WORKING_DIR")") +PROJECT_ROOT=$(realpath "$(dirname "$BUILDKITE_DIR")") + +if [[ "${COMMAND_TO_RUN:-}" == "docker" ]]; then + echo "running docker build" + make build-docker +else + DOCKER_IMAGE="docker.elastic.co/ci-agent-images/enterprise-search/rbenv-buildkite-agent:latest" + SCRIPT_CMD="/ci/.buildkite/scripts/run_ci_step.sh" + + docker run --interactive --rm \ + --sig-proxy=true --init \ + --user "root" \ + --volume "$PROJECT_ROOT:/ci" \ + --workdir /ci \ + --env HOME=/ci \ + --env CI \ + --env GIT_REVISION=${BUILDKITE_COMMIT-} \ + --env BUILD_ID=${BUILDKITE_BUILD_NUMBER-} \ + --entrypoint "${SCRIPT_CMD}" \ + $DOCKER_IMAGE \ + $COMMAND_TO_RUN +fi diff --git a/Makefile b/Makefile index f9528ce3..55de2f25 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ lint: config/connectors.yml autocorrect: config/connectors.yml bundle _$(shell cat .bundler-version)_ exec rubocop lib spec -a +autocorrect_all: config/connectors.yml + bundle _$(shell cat .bundler-version)_ exec rubocop lib spec -A + api_key: config/connectors.yml ${YQ} e ".http.api_key = \"$(shell uuidgen | tr -d '-')\"" -i config/connectors.yml @@ -41,14 +44,16 @@ tag: build_gem: mkdir -p .gems - gem build connectors_sdk.gemspec - gem build connectors_stubs.gemspec + bundle _$(shell cat .bundler-version)_ exec gem build connectors_sdk.gemspec + bundle _$(shell cat .bundler-version)_ exec gem build connectors_stubs.gemspec rm -f .gems/* mv *.gem .gems/ echo "DO NOT FORGET TO UPDATE ENT-SEARCH" push_gem: - gem push .gems/* + gem push .gems/connectors_sdk* + gem push .gems/connectors_stubs* + install: rbenv install -s diff --git a/README.md b/README.md index ae43ae7d..129e8f63 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,18 @@ -# Elastic Enterprise Search Connectors +# Elastic Workplace Search Connectors -![logo](logo-enterprise-search.png) +> [!IMPORTANT] +> _**Enterprise Search will be discontinued in 9.0.**_ +> +> Starting with Elastic version 9.0, we're deprecating the standalone Enterprise Search product with its included features and functionalities (including [Workplace Search](https://www.elastic.co/guide/en/workplace-search/8.x/index.html) and [App Search](https://www.elastic.co/guide/en/app-search/8.x/index.html)). They remain supported in their current form in version 8.x and will only receive security upgrades and fixes. Workplace Search Connector Packages will continue to be supported in their current form throughout 8.x versions, according to our EOL policy: https://www.elastic.co/support/eol. +> We recommend transitioning to our actively developed [Elastic Stack](https://www.elastic.co/elastic-stack) tools for your search use cases. However, if you're still using any Enterprise Search products, we recommend using the latest stable release. +> +> Here are some useful links with more information: +> * Enterprise Search FAQ: https://www.elastic.co/resources/enterprise-search/enterprise-search-faq +> * Migrating to 9.x from Enterprise Search 8.x versions: https://www.elastic.co/guide/en/enterprise-search/current/upgrading-to-9-x.html +___ -The home of Elastic Enterprise Connector Packages. Use connector packages to + +The home of Workplace Search Connector Packages. Use connector packages to customize connectors such as Workplace Search Content Sources for advanced use cases. @@ -97,7 +107,7 @@ This repository is always growing! At the moment, the connectors currently avail Don't see the connector you're looking for? If it is in the list of [Workplace Search Content Sources](https://www.elastic.co/guide/en/workplace-search/current/workplace-search-content-sources.html), it is probably on our roadmap to move here! See our [Getting Support](./docs/SUPPORT.md) guide for ways to reach out. -Not seeing the connector you want there, either? We encourage community contirubions! See our [Contributors Guide](./docs/CONTRIBUTING.md). +Not seeing the connector you want there, either? We encourage community contributions! See our [Contributors Guide](./docs/CONTRIBUTING.md). ### Sinatra Console run `make console` diff --git a/VERSION b/VERSION index f8d1673c..a3000f83 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.4.0.0 +8.3.4.0 diff --git a/connectors_sdk.gemspec b/connectors_sdk.gemspec index 8a68c4ff..3aed7c77 100644 --- a/connectors_sdk.gemspec +++ b/connectors_sdk.gemspec @@ -3,38 +3,38 @@ require_relative 'lib/connectors_app/config' Gem::Specification.new do |s| s.name = 'connectors_sdk' s.version = ConnectorsApp::Config['version'] - s.homepage = "https://github.com/elastic/connectors" + s.homepage = 'https://github.com/elastic/connectors' s.summary = 'Gem containing apis used by Enterprise Search and implementations of Connectors' s.description = '' s.authors = ['Elastic'] s.email = 'ent-search-dev@elastic.co' s.metadata = { - "revision" => ConnectorsApp::Config['revision'], - "repository" => ConnectorsApp::Config['repository'] + 'revision' => ConnectorsApp::Config['revision'], + 'repository' => ConnectorsApp::Config['repository'] } - s.files = Dir.glob("lib/connectors_sdk/**/*", File::FNM_DOTMATCH) + - Dir.glob("lib/connectors_shared/**/*", File::FNM_DOTMATCH) + - [ - 'LICENSE', - 'NOTICE.txt', - 'lib/connectors_sdk.rb', - 'lib/connectors_shared.rb' - ] + s.files = Dir.glob('lib/connectors_sdk/**/*', File::FNM_DOTMATCH) + + Dir.glob('lib/connectors_shared/**/*', File::FNM_DOTMATCH) + + [ + 'LICENSE', + 'NOTICE.txt', + 'lib/connectors_sdk.rb', + 'lib/connectors_shared.rb' + ] s.license = 'Elastic-2.0' # TODO: figure out how to pin versions without harming ent-search repo s.add_dependency 'activesupport' s.add_dependency 'bson' s.add_dependency 'mime-types' - s.add_dependency 'tzinfo-data' s.add_dependency 'nokogiri' + s.add_dependency 'tzinfo-data' # Dependencies for the HTTP service - s.add_dependency 'forwardable' s.add_dependency 'faraday' s.add_dependency 'faraday_middleware' - s.add_dependency 'httpclient' + s.add_dependency 'forwardable' s.add_dependency 'hashie' + s.add_dependency 'httpclient' # Dependencies for oauth s.add_dependency 'signet' diff --git a/connectors_stubs.gemspec b/connectors_stubs.gemspec index 7388cabb..9cfa6e9f 100644 --- a/connectors_stubs.gemspec +++ b/connectors_stubs.gemspec @@ -3,20 +3,20 @@ require_relative 'lib/connectors_app/config' Gem::Specification.new do |s| s.name = 'connectors_stubs' s.version = ConnectorsApp::Config['version'] - s.homepage = "https://github.com/elastic/connectors" + s.homepage = 'https://github.com/elastic/connectors' s.summary = 'Gem containing utilities used by implementations of Connectors when run external to Enterprise Search' s.description = '' s.authors = ['Elastic'] s.email = 'ent-search-dev@elastic.co' s.metadata = { - "revision" => ConnectorsApp::Config['revision'], - "repository" => ConnectorsApp::Config['repository'] + 'revision' => ConnectorsApp::Config['revision'], + 'repository' => ConnectorsApp::Config['repository'] } - s.files = Dir.glob("lib/stubs/**/*", File::FNM_DOTMATCH) + - [ - 'LICENSE', - 'NOTICE.txt' - ] + s.files = Dir.glob('lib/stubs/**/*', File::FNM_DOTMATCH) + + [ + 'LICENSE', + 'NOTICE.txt' + ] s.license = 'Elastic-2.0' s.add_dependency 'activesupport' diff --git a/lib/connectors_async/job_runner.rb b/lib/connectors_async/job_runner.rb index 444a6939..de073594 100644 --- a/lib/connectors_async/job_runner.rb +++ b/lib/connectors_async/job_runner.rb @@ -30,15 +30,17 @@ def start_job(job:, connector_class:, secret_storage:, params:) init_thread connector = connector_class.new - content_source_id = params[:content_source_id] - cursors = params[:cursors] ||= {} - cursors[:modified_since] = params.delete(:modified_since) if params[:modified_since] + extractor_params = params.dup + cursors = extractor_params[:cursors] ||= {} + cursors[:modified_since] = extractor_params.delete(:modified_since) if extractor_params[:modified_since] + extractor_params[:cursors] = cursors + extractor_params[:secret_storage] = secret_storage log_with_thread_id(:info, "Running the job #{job.id}") job.update_status(ConnectorsShared::JobStatus::RUNNING) - new_cursors = connector.extract({ :content_source_id => content_source_id, :cursors => cursors, :secret_storage => secret_storage }) do |doc| + new_cursors = connector.extract(extractor_params) do |doc| with_throttling(job) do job.store(doc) end diff --git a/lib/connectors_sdk/confluence_cloud/connector.rb b/lib/connectors_sdk/confluence_cloud/connector.rb index d603b013..13247632 100644 --- a/lib/connectors_sdk/confluence_cloud/connector.rb +++ b/lib/connectors_sdk/confluence_cloud/connector.rb @@ -6,6 +6,7 @@ # frozen_string_literal: true +require 'base64' require 'connectors_sdk/atlassian/config' require 'connectors_sdk/confluence_cloud/extractor' require 'connectors_sdk/confluence_cloud/authorization' @@ -29,23 +30,19 @@ def display_name 'Confluence Cloud' end - def connection_requires_redirect - true - end - def configurable_fields [ { 'key' => 'base_url', - 'label' => 'Base URL' + 'label' => 'Confluence Cloud Base URL' }, { - 'key' => 'client_id', - 'label' => 'Client ID' + 'key' => 'confluence_user_email', + 'label' => 'Confluence user email' }, { - 'key' => 'client_secret', - 'label' => 'Client Secret' + 'key' => 'confluence_api_token', + 'label' => 'Confluence user REST API Token' }, ] end @@ -61,7 +58,10 @@ def authorization end def client(params) - ConnectorsSdk::ConfluenceCloud::CustomClient.new(:base_url => base_url(params[:cloud_id]), :access_token => params[:access_token]) + ConnectorsSdk::ConfluenceCloud::CustomClient.new( + :base_url => extract_base_url(params), + :basic_auth_token => extract_basic_auth_token(params) + ) end def custom_client_error @@ -69,7 +69,11 @@ def custom_client_error end def config(params) - ConnectorsSdk::Atlassian::Config.new(:base_url => "#{params[:external_connector_base_url]}/wiki", :cursors => params.fetch(:cursors, {}) || {}) + ConnectorsSdk::Atlassian::Config.new( + :base_url => extract_base_url(params), + :cursors => params.fetch(:cursors, {}) || {}, + :index_permissions => params[:index_permissions] || false + ) end def health_check(params) @@ -79,6 +83,28 @@ def health_check(params) def base_url(cloud_id) "https://api.atlassian.com/ex/confluence/#{cloud_id}" end + + def is_basic_auth(params) + login = params.fetch('confluence_user_email', nil) + api_token = params.fetch('confluence_api_token', nil) + login.present? && api_token.present? + end + + def extract_basic_auth_token(params) + login = params.fetch('confluence_user_email', nil) + api_token = params.fetch('confluence_api_token', nil) + nil unless login.present? && api_token.present? + Base64.strict_encode64("#{login}:#{api_token}") + end + + def extract_base_url(params) + # From Confluence API documentation: + # Requests that use OAuth 2.0 (3LO) are made via api.atlassian.com (not https://your-domain.atlassian.net). + if is_basic_auth(params) + return params[:base_url].end_with?('/wiki') ? params[:base_url] : "#{params[:base_url]}/wiki" + end + base_url(params[:cloud_id]) + end end end end diff --git a/lib/connectors_sdk/confluence_cloud/extractor.rb b/lib/connectors_sdk/confluence_cloud/extractor.rb index 9ee94b49..9fc23047 100644 --- a/lib/connectors_sdk/confluence_cloud/extractor.rb +++ b/lib/connectors_sdk/confluence_cloud/extractor.rb @@ -52,7 +52,8 @@ def yield_permissions(source_user_id) def download(item) content = item[:content] parent_id = content.dig('container', 'id') - client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body + base_url = client.base_url.end_with?('/wiki') ? client.base_url : "#{client.base_url}/wiki" + client.download("#{base_url}/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body end end end diff --git a/lib/connectors_sdk/share_point/authorization.rb b/lib/connectors_sdk/share_point/authorization.rb index b8a1160a..318dc6e0 100644 --- a/lib/connectors_sdk/share_point/authorization.rb +++ b/lib/connectors_sdk/share_point/authorization.rb @@ -35,7 +35,7 @@ def token_credential_uri end def additional_parameters - { :prompt => 'consent' } + { :prompt => 'select_account' } end end end diff --git a/spec/connectors_sdk/confluence/custom_client_spec.rb b/spec/connectors_sdk/confluence/custom_client_spec.rb index 98e33c3e..5d4aaeb6 100644 --- a/spec/connectors_sdk/confluence/custom_client_spec.rb +++ b/spec/connectors_sdk/confluence/custom_client_spec.rb @@ -10,11 +10,21 @@ describe ConnectorsSdk::Confluence::CustomClient do let(:auth_token) { 'auth_token' } + let(:basic_auth_token) { 'confluence_is_basic' } let(:base_url) { 'http://localhost' } let(:client) do described_class.new( :base_url => base_url, - :access_token => 'access_token' + :access_token => 'access_token', + :basic_auth_token => nil + ) + end + + let(:basic_client) do + described_class.new( + :base_url => base_url, + :access_token => nil, + :basic_auth_token => basic_auth_token ) end @@ -92,6 +102,36 @@ expect(content_search_request).to have_been_requested end + it 'applies correct middleware for token auth' do + client.additional_middleware.find do |item| + item.is_a?(Array) && item[0].is_a?(ConnectorsShared::Middleware::BearerAuth) + end + end + + it 'applies correct middleware for token auth' do + expect(client.additional_middleware.find do |item| + item.is_a?(Array) && item[0] == ConnectorsShared::Middleware::BearerAuth + end).to be_present + end + + it 'does not apply incorrect middleware for token auth' do + expect(client.additional_middleware.find do |item| + item.is_a?(Array) && item[0] == ConnectorsShared::Middleware::BasicAuth + end).to be_nil + end + + it 'applies correct middleware for basic auth' do + expect(basic_client.additional_middleware.find do |item| + item.is_a?(Array) && item[0] == ConnectorsShared::Middleware::BasicAuth + end).to be_present + end + + it 'does not apply incorrect middleware for basic auth' do + expect(basic_client.additional_middleware.find do |item| + item.is_a?(Array) && item[0] == ConnectorsShared::Middleware::BearerAuth + end).to be_nil + end + describe '#content' do let(:next_value) { '/a-provided-endpoint?with=a_param' } diff --git a/spec/connectors_sdk/confluence_cloud/extractor_spec.rb b/spec/connectors_sdk/confluence_cloud/extractor_spec.rb index 52a96e79..db150d31 100644 --- a/spec/connectors_sdk/confluence_cloud/extractor_spec.rb +++ b/spec/connectors_sdk/confluence_cloud/extractor_spec.rb @@ -21,12 +21,12 @@ let(:api_base_url) { 'https://api.atlassian.com/ex/confluence/abc123' } let(:content_source_id) { BSON::ObjectId.new } let(:oauth_config) { { :client_id => 'client_id', :client_secret => 'client_secret', :base_url => base_url } } - let(:access_token) { 'access_token' } + let(:basic_auth_token) { 'basic_auth_token' } let(:headers) { { 'Content-Type' => 'application/json' } } let(:authorization_data) do { - 'access_token' => access_token, + 'basic_auth_token' => basic_auth_token, 'base_url' => base_url, 'cloud_id' => cloud_id } @@ -45,7 +45,7 @@ proc do ConnectorsSdk::ConfluenceCloud::CustomClient.new( :base_url => api_base_url, - :access_token => access_token + :basic_auth_token => basic_auth_token ) end end