refactor: Use the default offset paginator (#265)

Should be possible with meltano/sdk#1918

Author: edgarrmondragon

tap_dbt/streams.py

@@ -11,35 +11,9 @@
 
 from tap_dbt.client import DBTStream
 
-if t.TYPE_CHECKING:
-    import requests
-
 SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
 
 
-class DbtPaginator(BaseOffsetPaginator):
-    """dbt API paginator."""
-
-    def has_more(self, response: requests.Response) -> bool:
-        """Returns True until there are no more pages to retrieve.
-
-        The API returns an 'extra' key with information about pagination:
-        "extra":{"filters":{"limit":100,"offset":2,"account_id":1},"order_by":"id","pagination":{"count":100,"total_count":209}}}
-        """
-        data = response.json()
-        extra = data.get("extra", {})
-        filters = extra.get("filters", {})
-        pagination = extra.get("pagination", {})
-
-        offset = filters.get("offset", 0)
-        total_count = pagination.get("total_count")
-        count = pagination.get("count")
-
-        # The pagination has more records when:
-        # total_count is still greater than count and offset combined
-        return count + offset < total_count
-
-
 class AccountBasedStream(DBTStream):
     """A stream that requires an account ID."""
 
@@ -59,7 +33,7 @@ def partitions(self) -> list[dict]:
         )
         raise ValueError(errmsg)
 
-    def get_new_paginator(self) -> DbtPaginator:
+    def get_new_paginator(self) -> BaseOffsetPaginator:
         """Return a new paginator instance for this stream."""
         page_size = self.config["page_size"]
 
@@ -68,7 +42,7 @@ def get_new_paginator(self) -> DbtPaginator:
             page_size,
         )
 
-        return DbtPaginator(start_value=0, page_size=page_size)
+        return BaseOffsetPaginator(start_value=0, page_size=page_size)
 
     def get_url_params(
         self,