Skip to content

Commit

Permalink
v0.5.15 stream support for bulk files; yield option for folders and l…
Browse files Browse the repository at this point in the history 
…ists; fixed typos in readme
  • Loading branch information
@jepcastelein
jepcastelein committed Jan 30, 2022
1 parent 739c893 commit 7f0750f
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 15 deletions.
35 changes: 32 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,15 @@ lead = mc.execute(method='browse_folders', root=3, maxDepth=5, maxReturn=200, wo
# will throw KeyError when no folder found
```

Browse Folders (Yield)
--------------
API Ref: http://developers.marketo.com/documentation/asset-api/browse-folders
```python
for folders in mc.execute(method='browse_folders_yield', root=3, maxDepth=5, maxReturn=200, workSpace='Default',
offset=0, return_full_result=False):
print(folders)
```

Tokens
=======

Expand Down Expand Up @@ -602,9 +611,9 @@ lead = mc.execute(method='get_list_by_id', id=724)

Get Static List by Name
--------------
API Ref: http://developers.marketo.com/rest-api/assets/static-lists/#by_id
API Ref: https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Static_Lists/getStaticListByNameUsingGET
```python
lead = mc.execute(method='get_list_by_name', id='My Test List')
lead = mc.execute(method='get_list_by_name', name='My Test List')
```

Get Multiple Lists (OLD)
Expand All @@ -628,6 +637,17 @@ lead = mc.execute(method='browse_lists', folderId=8, folderType='Folder', offset
# all parameters are optional; no parameters returns all lists
```

Get Static Lists (Yield)
------------------
API Ref: http://developers.marketo.com/rest-api/assets/static-lists/#browse
```python
for lists in mc.execute(method='browse_lists_yield', folderId=8, folderType='Folder', offset=0,
maxReturn=20, earliestUpdatedAt=None, latestUpdatedAt=None, return_full_result=False):
print(lists)

# all parameters are optional; no parameters returns all lists
```

Create Static List
-----------
API Ref: http://developers.marketo.com/rest-api/assets/static-lists/#create_and_update
Expand Down Expand Up @@ -1932,7 +1952,7 @@ Add Custom Object Type Fields
API Ref: https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Custom_Objects/addCustomObjectTypeFieldsUsingPOST
```python
fields = [{"displayName": "Email", "description": "Email", "name": "email", "dataType": "link", "relatedTo": {"name": "lead", "field": "email"}},
{"displayName": "Transaction ID", "description": "Transaction ID", "name": "txid", "dataType": "integer", "isDedupeField", True},
{"displayName": "Transaction ID", "description": "Transaction ID", "name": "txid", "dataType": "integer", "isDedupeField": True},
{"displayName": "Total", "description": "Transaction total", "name": "total", "dataType": "float"}]
result = mc.execute(method='add_field_custom_object_type', apiName='transactions',fields=fields)
```
Expand Down Expand Up @@ -2274,6 +2294,15 @@ API Ref: http://developers.marketo.com/rest-api/endpoint-reference/lead-database
```python
export_file_contents = mc.execute(method='get_activities_export_job_file', job_id='284742ec-1e5a-46f2-b164-498a41fcaaf6')
```
Or use streaming:
```python
with mc.execute(method='get_activities_export_job_file', job_id='284742ec-1e5a-46f2-b164-498a41fcaaf6', stream=True) as r:
with open('filename.csv', 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# set your preferred chunk_size
```

Named Account Object
==============
Expand Down
83 changes: 76 additions & 7 deletions marketorestpython/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def execute(self, method, *args, **kargs):
'get_list_by_name': self.get_list_by_name,
'get_multiple_lists': self.get_multiple_lists,
'browse_lists': self.browse_lists,
'browse_lists_yield': self.browse_lists_yield,
'add_leads_to_list': self.add_leads_to_list,
'remove_leads_from_list': self.remove_leads_from_list,
'member_of_list': self.member_of_list,
Expand Down Expand Up @@ -118,6 +119,7 @@ def execute(self, method, *args, **kargs):
'update_folder': self.update_folder,
'delete_folder': self.delete_folder,
'browse_folders': self.browse_folders,
'browse_folders_yield': self.browse_folders_yield,
'create_token': self.create_token,
'get_tokens': self.get_tokens,
'delete_tokens': self.delete_tokens,
Expand Down Expand Up @@ -788,6 +790,38 @@ def browse_lists(self, folderId=None, folderType=None, offset=None, maxReturn=No
args['offset'] = offset
return result_list

def browse_lists_yield(self, folderId=None, folderType=None, offset=0, maxReturn=20, earliestUpdatedAt =None,
latestUpdatedAt=None, return_full_result=False):
self.authenticate()
args = {
'access_token': self.token,
'maxReturn': maxReturn,
'offset': offset
}
if folderId and folderType:
args['folder'] = json.dumps({'id': folderId, 'type': folderType})
if earliestUpdatedAt:
args['earliestUpdatedAt'] = earliestUpdatedAt
if latestUpdatedAt:
args['latestUpdatedAt'] = latestUpdatedAt
while True:
self.authenticate()
args['access_token'] = self.token
result = self._api_call('get', self.host + "/rest/asset/v1/staticLists.json", args)
if result is None:
raise Exception("Empty Response")
if 'result' in result:
if return_full_result:
yield result
else:
yield result['result']
if len(result['result']) < maxReturn:
break
else:
break
offset += maxReturn
args['offset'] = offset

def add_leads_to_list(self, listId, id):
self.authenticate()
if listId is None:
Expand Down Expand Up @@ -1820,6 +1854,40 @@ def browse_folders(self, root, maxDepth=None, maxReturn=None, workSpace=None):
args['offset'] = offset
return result_list

def browse_folders_yield(self, root, maxDepth=None, maxReturn=20, workSpace=None, offset=0,
return_full_result=False):
self.authenticate()
if root is None:
raise ValueError(
"Invalid argument: required argument root is none.")
args = {
'access_token': self.token,
'root': root,
'offset': offset
}
if maxDepth is not None:
args['maxDepth'] = maxDepth
if workSpace is not None:
args['workSpace'] = workSpace
while True:
self.authenticate()
args['access_token'] = self.token
result = self._api_call(
'get', self.host + "/rest/asset/v1/folders.json", args)
if result is None:
raise Exception("Empty Response")
if 'result' in result:
if return_full_result:
yield result
else:
yield result['result']
if len(result['result']) < maxReturn:
break
else:
break
offset += maxReturn
args['offset'] = offset

# --------- TOKENS ---------

def create_token(self, id, folderType, type, name, value):
Expand Down Expand Up @@ -5272,7 +5340,8 @@ def _get_export_jobs_list(self, entity, object_name=None):
'get', self.host + '/bulk/v1/{}/export.json'.format(entity), args)
return result['result']

def _create_bulk_export_job(self, entity, fields=None, filters=None, format='CSV', columnHeaderNames=None, object_name=None):
def _create_bulk_export_job(self, entity, fields=None, filters=None, format='CSV', columnHeaderNames=None,
object_name=None):
assert entity is not None, 'Invalid argument: required field entity is none.'
if entity in ['leads', 'program/members', 'customobjects']:
assert fields is not None, 'Invalid argument: required field fields is none.'
Expand All @@ -5291,7 +5360,7 @@ def _create_bulk_export_job(self, entity, fields=None, filters=None, format='CSV
'post', self.host + '/bulk/v1/{}/export/create.json'.format(entity), args, data)
return result['result']

def _export_job_state_machine(self, entity, state, job_id, object_name=None):
def _export_job_state_machine(self, entity, state, job_id, object_name=None, stream=False):
assert entity is not None, 'Invalid argument: required field "entity" is none.'
assert state is not None, 'Invalid argument: required field "state" is none.'
assert job_id is not None, 'Invalid argument: required field "job_id" is none.'
Expand All @@ -5305,15 +5374,15 @@ def _export_job_state_machine(self, entity, state, job_id, object_name=None):
'file': {'suffix': '/file.json', 'method': 'get', 'mode': 'nojson'}
}
self.authenticate()
url = '{}/bulk/v1/{}/export/{}{}'.format(self.host, entity, job_id, state_info[state]['suffix'])
args = {
'access_token': self.token
}
result = self._api_call(state_info[state]['method'],
self.host + '/bulk/v1/{}/export/{}{}'.format(entity, job_id,
state_info[state]['suffix']),
args, mode=state_info[state]['mode'])
result = self._api_call(state_info[state]['method'], url, args, mode=state_info[state]['mode'], stream=stream)
if state == 'file' and result.status_code == 200:
return result.content
if not stream:
return result.content
return result
return result['result']

def get_leads_export_job_file(self, *args, **kargs):
Expand Down
8 changes: 4 additions & 4 deletions marketorestpython/helper/http_lib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

def fatal_marketo_error_code(e):
# Given a MarketoException, decide whether it is fatal or retryable.
return e.code not in retryable_error_codes
return e.code not in retryable_error_codes or 'Export daily quota' in e.message


class HttpLib:
Expand Down Expand Up @@ -57,9 +57,9 @@ def rateLimitedFunction(*args,**kargs):
max_time=lookup_max_time,
giveup=fatal_marketo_error_code)
@_rate_limited(num_calls_per_second)
def get(self, endpoint, args=None, mode=None):
def get(self, endpoint, args=None, mode=None, stream=False):
headers = {'Accept-Encoding': 'gzip'}
r = requests.get(endpoint, params=args, headers=headers)
r = requests.get(endpoint, params=args, headers=headers, stream=stream)
if mode == 'nojson':
return r
else:
Expand All @@ -74,7 +74,7 @@ def get(self, endpoint, args=None, mode=None):
giveup=fatal_marketo_error_code)
@_rate_limited(num_calls_per_second)
def post(self, endpoint, args, data=None, files=None, filename=None,
mode=None):
mode=None, stream=False):
if mode == 'nojsondumps':
headers = {'Content-type': 'application/x-www-form-urlencoded; charset=utf-8'}
r = requests.post(endpoint, params=args, data=data, headers=headers)
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

setup(
name='marketorestpython',
version= '0.5.14',
version= '0.5.15',
url='https://github.com/jepcastelein/marketo-rest-python',
author='Jep Castelein',
author_email='[email protected]',
Expand Down

0 comments on commit 7f0750f

Please sign in to comment.