Skip to content

Tdl 13966 update custom fields #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Tdl 13966 update custom fields #36

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5406ba3
TDL-13966: Updated Custom fields schema
Jul 21, 2021
6d079fc
TDL-13966: Updated sync tests to verify custom fields are available o…
Jul 21, 2021
d3894ad
TDL-13966 : Updated test cases
Jul 21, 2021
584be28
Resolved merge conflicts
savan-chovatiya Aug 26, 2021
c645581
Merge branch 'master' into TDL-13966-update-custom-fields
KAllan357 Aug 30, 2021
6c33af7
Updated discovery test
savan-chovatiya Sep 13, 2021
34e5163
Update dependancy
savan-chovatiya Sep 13, 2021
8dfffdf
Tdl 13967 add profit loss report stream (#44)
dbshah1212 Oct 6, 2021
58f7ff3
Merge branch 'TDL-13964-Improve-Quickbooks-Reliability' into TDL-1396…
dbshah1212 Oct 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
python3 -mvenv /usr/local/share/virtualenvs/tap-quickbooks
source /usr/local/share/virtualenvs/tap-quickbooks/bin/activate
pip install -U pip setuptools
pip install .[dev]
pip install .[test]
- run:
name: 'JSON Validator'
command: |
Expand Down
6 changes: 4 additions & 2 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@
'requests_oauthlib==1.3.0',
],
extras_require={
'dev': [
'ipdb==0.11',
'test': [
'pylint==2.5.3',
'nose'
],
'dev': [
'ipdb'
]
},
entry_points='''
Expand Down
6 changes: 5 additions & 1 deletion tap_quickbooks/discover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,14 @@ def do_discover():
))
# Set the replication_key MetaData to automatic as well
mdata = metadata.write(mdata, ('properties', stream.replication_keys[0]), 'inclusion', 'automatic')
custom_field = singer.utils.load_json(
os.path.normpath(
os.path.join(_get_abs_path("schemas/custom_field.json"))))
refs = {"custom_field.json": custom_field}
ref_schema = singer.utils.load_json(
os.path.normpath(
os.path.join(_get_abs_path("schemas/ref_schema.json"))))
refs = {"ref_schema.json": ref_schema}
refs.update({"ref_schema.json": ref_schema})
catalog_entry = {
"stream": stream_name,
"tap_stream_id": stream_name,
Expand Down
25 changes: 1 addition & 24 deletions tap_quickbooks/schemas/credit_memos.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,30 +256,7 @@
},
"CustomField": {
"items": {
"properties": {
"Name": {
"type": [
"null",
"string"
]
},
"Type": {
"type": [
"null",
"string"
]
},
"DefinitionId": {
"type": [
"null",
"string"
]
}
},
"type": [
"null",
"object"
]
"$ref": "custom_field.json"
},
"type": [
"null",
Expand Down
32 changes: 32 additions & 0 deletions tap_quickbooks/schemas/custom_field.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{
"properties": {
"Type": {
"type": [
"null",
"string"
]
},
"Name": {
"type": [
"null",
"string"
]
},
"StringValue": {
"type": [
"null",
"string"
]
},
"DefinitionId": {
"type": [
"null",
"string"
]
}
},
"type": [
"null",
"object"
]
}
25 changes: 1 addition & 24 deletions tap_quickbooks/schemas/estimates.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,30 +391,7 @@
},
"CustomField": {
"items": {
"properties": {
"DefinitionId": {
"type": [
"null",
"string"
]
},
"Name": {
"type": [
"null",
"string"
]
},
"Type": {
"type": [
"null",
"string"
]
}
},
"type": [
"null",
"object"
]
"$ref": "custom_field.json"
},
"type": [
"null",
Expand Down
31 changes: 1 addition & 30 deletions tap_quickbooks/schemas/invoices.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,36 +176,7 @@
},
"CustomField": {
"items": {
"properties": {
"Type": {
"type": [
"null",
"string"
]
},
"Name": {
"type": [
"null",
"string"
]
},
"StringValue": {
"type": [
"null",
"string"
]
},
"DefinitionId": {
"type": [
"null",
"string"
]
}
},
"type": [
"null",
"object"
]
"$ref": "custom_field.json"
},
"type": [
"null",
Expand Down
31 changes: 4 additions & 27 deletions tap_quickbooks/schemas/purchase_orders.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,36 +202,13 @@
]
},
"CustomField": {
"items": {
"$ref": "custom_field.json"
},
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"properties": {
"Name": {
"type": [
"null",
"string"
]
},
"DefinitionId": {
"type": [
"null",
"string"
]
},
"Type": {
"type": [
"null",
"string"
]
}
}
}
]
},
"MetaData": {
"type": [
Expand Down
25 changes: 1 addition & 24 deletions tap_quickbooks/schemas/refund_receipts.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,30 +216,7 @@
},
"CustomField": {
"items": {
"properties": {
"DefinitionId": {
"type": [
"null",
"string"
]
},
"Name": {
"type": [
"null",
"string"
]
},
"Type": {
"type": [
"null",
"string"
]
}
},
"type": [
"null",
"object"
]
"$ref": "custom_field.json"
},
"type": [
"null",
Expand Down
31 changes: 4 additions & 27 deletions tap_quickbooks/schemas/sales_receipts.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,36 +100,13 @@
}
},
"CustomField": {
"items": {
"$ref": "custom_field.json"
},
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"properties": {
"Name": {
"type": [
"null",
"string"
]
},
"DefinitionId": {
"type": [
"null",
"string"
]
},
"Type": {
"type": [
"null",
"string"
]
}
}
}
]
},
"BillAddr": {
"type": [
Expand Down
2 changes: 2 additions & 0 deletions tests/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ class TestQuickbooksBase(unittest.TestCase):
INCREMENTAL = "INCREMENTAL"
FULL = "FULL_TABLE"
START_DATE_FORMAT = "%Y-%m-%dT00:00:00Z" # %H:%M:%SZ
# list of streams which supports custom field
custom_command_streams = ['invoices','estimates','credit_memos','refund_receipts','sales_receipts','purchase_orders']

def setUp(self):
missing_envs = [x for x in [
Expand Down
14 changes: 14 additions & 0 deletions tests/test_quickbooks_discovery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def test_run(self):
self.assertEqual(len(diff), 0, msg="discovered schemas do not match: {}".format(diff))
print("discovered schemas are OK")

custom_field_stream_count = 0
for stream in self.expected_streams():
with self.subTest(stream=stream):
catalog = next(iter([catalog for catalog in found_catalogs
Expand Down Expand Up @@ -146,3 +147,16 @@ def test_run(self):
and item.get("breadcrumb", ["properties", None])[1]
not in actual_automatic_fields}),
msg="Not all non key properties are set to available in metadata")

# verify custom field in metadata
if stream in self.custom_command_streams:
actual_all_fields = [item.get("breadcrumb", ["properties", None])[1]
for item in metadata
if item.get("breadcrumb", []) != []]

self.assertTrue('CustomField' in actual_all_fields,
msg="CustomField is not found in stream {}.".format(stream))

custom_field_stream_count += 1

self.assertEqual(custom_field_stream_count,6)
20 changes: 20 additions & 0 deletions tests/test_quickbooks_sync_all.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,23 @@ def test_run(self):
# Each stream should have 1 or more records returned
self.assertGreaterEqual(sync_record_count[stream], 1)

# Taking sync response
synced_records = runner.get_records_from_target_output()

# Taking all streams from the response
streams = synced_records.keys()

custom_field_stream_count = 0
for stream in streams:
if stream in self.custom_command_streams:
first_record = synced_records.get(stream).get('messages')[0].get('data')
actual_custom_field_keys = list(first_record.get("CustomField")[0].keys())
# For sand box only 3 fields are coming
expected_custom_field_keys = ['DefinitionId','Name','Type']
actual_custom_field_keys.sort()
expected_custom_field_keys.sort()
self.assertListEqual(actual_custom_field_keys,expected_custom_field_keys)
custom_field_stream_count +=1
else:
continue
self.assertEqual(custom_field_stream_count,6)