From 7216e929baadd5d010d156155924ca08340ecb77 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:44:22 +0200 Subject: [PATCH] :sparkles: Source Bing Ads: added TaxCertificate field to accounts schema (#35891) --- .../connectors/source-bing-ads/metadata.yaml | 2 +- .../connectors/source-bing-ads/pyproject.toml | 2 +- .../source_bing_ads/base_streams.py | 10 ++- .../source_bing_ads/schemas/accounts.json | 26 +++++++ .../integrations/suds_response_mock.py | 10 +++ .../integrations/test_accounts_stream.py | 39 ++++++++++ .../unit_tests/test_base_streams.py | 75 +++++++++++++++++++ docs/integrations/sources/bing-ads.md | 1 + 8 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/test_base_streams.py diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 3f0b131e144c..87b39e026c09 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 2.4.0 + dockerImageTag: 2.5.0 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/pyproject.toml b/airbyte-integrations/connectors/source-bing-ads/pyproject.toml index 08edbec1a699..9cec659742a1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/pyproject.toml +++ b/airbyte-integrations/connectors/source-bing-ads/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.4.0" +version = "2.5.0" name = "source-bing-ads" description = "Source implementation for Bing Ads." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py index 7294f453deb1..6993542c57e1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -206,13 +206,21 @@ def request_params( "ReturnAdditionalFields": self.additional_fields, } + def _transform_tax_fields(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + tax_certificates = record["TaxCertificate"].get("TaxCertificates", {}) if record.get("TaxCertificate") is not None else {} + if tax_certificates and not isinstance(tax_certificates, list): + tax_certificate_pairs = tax_certificates.get("KeyValuePairOfstringbase64Binary") + if tax_certificate_pairs: + record["TaxCertificate"]["TaxCertificates"] = tax_certificate_pairs + return record + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: if response is not None and hasattr(response, self.data_field): records = self.client.asdict(response)[self.data_field] for record in records: if record["Id"] not in self._unique_account_ids: self._unique_account_ids.add(record["Id"]) - yield record + yield self._transform_tax_fields(record) class Campaigns(BingAdsCampaignManagementStream): diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json index 63bf3d699add..610377700cf8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json @@ -127,6 +127,32 @@ }, "TimeStamp": { "type": ["null", "string"] + }, + "TaxCertificate": { + "type": ["null", "object"], + "properties": { + "TaxCertificateBlobContainerName": { + "type": ["null", "string"] + }, + "Status": { + "type": ["null", "string"], + "enum": ["Invalid", "Pending", "Valid"] + }, + "TaxCertificates": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py index 3db4cffdfa1c..59b3bea844ea 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py @@ -52,6 +52,16 @@ Inactive + + Test Container Name + + + test_key + test_value + + + Active + Expert diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py new file mode 100644 index 000000000000..d97762fc0bf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from typing import Any, Dict, Optional, Tuple +from unittest.mock import MagicMock, patch + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker +from base_test import BaseTest +from source_bing_ads.source import SourceBingAds +from suds.transport.https import HttpAuthenticated +from suds_response_mock import mock_http_authenticated_send + + +class TestAccountsStream(BaseTest): + stream_name = "accounts" + + def read_stream( + self, + stream_name: str, + sync_mode: SyncMode, + config: Dict[str, Any], + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False, + ) -> Tuple[EntrypointOutput, MagicMock]: + with patch.object(HttpAuthenticated, "send", mock_http_authenticated_send): + catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build() + return read(SourceBingAds(), config, catalog, state, expecting_exception) + + @HttpMocker() + def test_read_accounts_tax_certificate_data(self, http_mocker): + # Our account doesn't have configured Tax certificate. + self.auth_client(http_mocker) + output = self.read_stream(self.stream_name, SyncMode.full_refresh, self._config) + assert output.records[0].record.data["TaxCertificate"] == { + "Status": "Active", + "TaxCertificateBlobContainerName": "Test Container Name", + "TaxCertificates": [{"key": "test_key", "value": "test_value"}], + } diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_base_streams.py new file mode 100644 index 000000000000..8f7720fbc584 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_base_streams.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +import pytest +import source_bing_ads +from source_bing_ads.base_streams import Accounts + + +@patch.object(source_bing_ads.source, "Client") +@pytest.mark.parametrize( + "record, expected", + [ + ( + { + "AccountId": 16253412, + "TaxCertificate": { + "TaxCertificates": {"KeyValuePairOfstringbase64Binary": [{"key": "test key", "value": "test value"}]}, + "Status": "Active", + "TaxCertificateBlobContainerName": "Test Container Name", + }, + }, + { + "AccountId": 16253412, + "TaxCertificate": { + "TaxCertificates": [{"key": "test key", "value": "test value"}], + "Status": "Active", + "TaxCertificateBlobContainerName": "Test Container Name", + }, + }, + ), + ( + { + "AccountId": 16253412, + "TaxCertificate": { + "TaxCertificates": [{"key": "test key", "value": "test value"}], + "Status": "Active", + "TaxCertificateBlobContainerName": "Test Container Name", + }, + }, + { + "AccountId": 16253412, + "TaxCertificate": { + "TaxCertificates": [{"key": "test key", "value": "test value"}], + "Status": "Active", + "TaxCertificateBlobContainerName": "Test Container Name", + }, + }, + ), + ( + { + "AccountId": 16253412, + }, + { + "AccountId": 16253412, + }, + ), + ( + {"AccountId": 16253412, "TaxCertificate": None}, + {"AccountId": 16253412, "TaxCertificate": None}, + ), + ], + ids=[ + "record_with_KeyValuePairOfstringbase64Binary_field", + "record_without_KeyValuePairOfstringbase64Binary_field", + "record_without_TaxCertificate_field", + "record_with_TaxCertificate_is_None", + ], +) +def test_accounts_transform_tax_fields(mocked_client, config, record, expected): + stream = Accounts(mocked_client, config) + actual = stream._transform_tax_fields(record) + assert actual == expected diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 423bcf63ddce..c52b06429dcd 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -251,6 +251,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.5.0 | 2024-03-21 | [35891](https://github.com/airbytehq/airbyte/pull/35891) | Accounts stream: add TaxCertificate field to schema. | | 2.4.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` | | 2.3.0 | 2024-03-05 | [35812](https://github.com/airbytehq/airbyte/pull/35812) | New streams: Audience Performance Report, Goals And Funnels Report, Product Dimension Performance Report. | | 2.2.0 | 2024-02-13 | [35201](https://github.com/airbytehq/airbyte/pull/35201) | New streams: Budget and Product Dimension Performance. |