7 support embeds on tickets (#10)

acarter24 · web-flow · commit e70b75107bbf · 2023-08-07T09:38:13.000+01:00
* Update poetry.lock

* Update pyproject.toml

singer-sdk to 0.30

* Update client.py

Bring in params from embeds key

* Update client.py

Schema for embeds in config

* Fix mess

* Embeds into params

* Add sla_policy to tickets schema

* Extended docs

* Add sla policy stream

* Add new stream to tap
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -14,11 +14,11 @@
         "justMyCode": true,
       }
       {
-        "name": "tap-freshdesk invoke",
+        "name": "tap-freshdesk about",
         "type": "python",
         "request": "launch",
         "module": "tap_freshdesk.tap",
-        "args": ["--config", ".secrets/config.json", "--test"],
+        "args": ["--config", ".secrets/config.json", "--about"],
         "python": "${command:python.interpreterPath}",
         // Set to true to debug third-party library code
         "justMyCode": true,
diff --git a/README.md b/README.md
@@ -51,6 +51,29 @@ This Singer tap will automatically import any environment variables within the w
 `.env` if the `--config=ENV` is provided, such that config values will be considered if a matching
 environment variable is set either in the terminal context or in the `.env` file.
 
+#### Embeds
+The Freshdesk Tickets API allows for a number of 'embeds' - these are extra fields which can be returned from the API, at the cost of consuming more 'credits'.
+Turning on these embeds will return more data, but mean that the 'cooloff' happens sooner, and overall syncs may take longer.
+
+See Freshdesk docs here: https://developers.freshdesk.com/api/#view_a_ticket
+
+The available embeds for tickets are specified under the `embeds -> tickets_detail` config key:
+```
+config:
+  embeds:
+    tickets_detail:
+      - stats
+      - sla_policy
+      - company
+      - requester
+```
+
+These extra values will be returned as json objects rather than unpacked in the main schema.
+
+If you don't need these data items, then just don't include the `embeds` key in `meltano.yml` and only the base ticket fields will be returned.
+
+As Freshdesk enables more embeds, they will be added here.
+
 ### Source Authentication and Authorization
 
 <!--
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,9 +12,9 @@ license = "Apache 2.0"
 
 [tool.poetry.dependencies]
 python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.23.0" }
 fs-s3fs = { version = "^1.1.1", optional = true }
 requests = "^2.28.2"
+singer-sdk = "^0.30.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.1"
@@ -24,7 +24,6 @@ black = "^23.1.0"
 pyupgrade = "^3.3.1"
 mypy = "^1.0.0"
 isort = "^5.11.5"
-singer-sdk = { version="^0.23.0", extras = ["testing"] }
 types-requests = "^2.28.11.12"
 
 [tool.poetry.extras]
diff --git a/tap_freshdesk/client.py b/tap_freshdesk/client.py
@@ -117,6 +117,11 @@ def get_url_params(
             A dictionary of URL query parameters.
         """
         params: dict = {}
+        embeds = self.config.get('embeds')
+        if embeds:
+            embed_fields = embeds.get(self.name, [])
+            if embed_fields:   # i.e. 'stats,company,sla_policy'
+                params['include'] = ','.join(embed_fields)
         return params
 
     def prepare_request_payload(
@@ -215,4 +220,4 @@ def get_url_params(
         return params
 
     def get_new_paginator(self) -> BasePageNumberPaginator:
-        return FreshdeskPaginator(start_value=1)
+        return FreshdeskPaginator(start_value=1)
diff --git a/tap_freshdesk/schemas/sla_policies.json b/tap_freshdesk/schemas/sla_policies.json
@@ -0,0 +1,64 @@
+{
+    "type": "object",
+    "properties": {
+        "id": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "name": {
+            "type": [
+                "null",
+                "string"
+            ]
+        },
+        "description": {
+            "type": [
+                "null",
+                "string"
+            ]
+        },
+        "active": {
+            "type": [
+                "null",
+                "boolean"
+            ]
+        },
+        "sla_target": {
+            "type": [
+                "null",
+                "object"
+            ]
+        },
+        "applicable_to": {
+            "type": "object"
+        },
+        "is_default": {
+            "type": [
+                "null",
+                "boolean"
+            ]
+        },
+        "position": {
+            "type": [
+                "null",
+                "integer"
+            ]
+        },
+        "created_at": {
+            "type": [
+                "null", 
+                "string"
+            ], 
+            "format": "date-time"
+        },
+        "updated_at": {
+            "type": [
+                "null", 
+                "string"
+            ], 
+            "format": "date-time"
+        }
+    }
+}
diff --git a/tap_freshdesk/schemas/tickets.json b/tap_freshdesk/schemas/tickets.json
@@ -357,6 +357,44 @@
             "format": "date-time"
           }
         }
+      },
+      "sla_policy": {
+        "type": [
+          "null",
+          "object"
+        ],
+        "properties": {
+          "id": {
+            "type": [
+              "null",
+              "integer"
+            ]
+          },
+          "name": {
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          "description": {
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          "active": {
+            "type": [
+              "null",
+              "boolean"
+            ]
+          },
+          "sla_target": {
+            "type": [
+              "null",
+              "object"
+            ]
+          }
+        }
       }
-    }
-  }
+  }
+}
diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py
@@ -35,6 +35,9 @@ class ContactsStream(FreshdeskStream):
 class EmailConfigsStream(FreshdeskStream):
     name = "email_configs"
 
+class SlaPoliciesStream(FreshdeskStream):
+    name = "sla_policies"
+
 class TicketsAbridgedStream(PagedFreshdeskStream):
     name = "tickets_abridged"
     replication_key = 'updated_at'
diff --git a/tap_freshdesk/tap.py b/tap_freshdesk/tap.py
@@ -39,6 +39,17 @@ class Tapfreshdesk(Tap):
             th.StringType,
             description="The url for the API service",
         ),
+        th.Property(
+            "embeds",
+            th.ObjectType(
+                th.Property(
+                    'tickets_detail', th.ArrayType(
+                        th.StringType(allowed_values=['requester', 'company', 'stats', 'sla_policy'])
+                        )
+                ),
+            ),
+            description='Dictionary of embeds for each stream, see more at https://developers.freshdesk.com/api/'
+        ),
         ## these two below to enable stream mapping
         th.Property(
             "stream_maps",
@@ -68,6 +79,7 @@ def discover_streams(self) -> list[streams.FreshdeskStream]:
             streams.TicketsDetailStream(tap=self, ticket_ids=_ticket_ids),
             streams.ConversationsStream(self),
             streams.EmailConfigsStream(self),
+            streams.SlaPoliciesStream(self),
         ]
 
 

Original file line number	Diff line number	Diff line change
`@@ -14,11 +14,11 @@`
`14`	`14`	`"justMyCode": true,`
`15`	`15`	`}`
`16`	`16`	`{`
`17`		`- "name": "tap-freshdesk invoke",`
	`17`	`+ "name": "tap-freshdesk about",`
`18`	`18`	`"type": "python",`
`19`	`19`	`"request": "launch",`
`20`	`20`	`"module": "tap_freshdesk.tap",`
`21`		`- "args": ["--config", ".secrets/config.json", "--test"],`
	`21`	`+ "args": ["--config", ".secrets/config.json", "--about"],`
`22`	`22`	`"python": "${command:python.interpreterPath}",`
`23`	`23`	`// Set to true to debug third-party library code`
`24`	`24`	`"justMyCode": true,`