[SYNPY-1551] Tables refactor (#1151)

* Implement the table model OOP, adding support for row CRUD operations including upserts

Author: BryanFauble

docs/css/custom.css

@@ -7,3 +7,7 @@ div.doc-contents {
 .md-typeset table tbody {
     font-size: .7rem
 }
+
+.md-grid {
+    max-width: 1700px;
+}

docs/reference/experimental/async/table.md

@@ -11,11 +11,39 @@ at your own risk.
         inherited_members: true
         members:
         - get_async
-        - store_schema_async
-        - store_rows_from_csv_async
-        - delete_rows_async
-        - query_async
+        - store_async
         - delete_async
-        - get_permissions_async
-        - get_acl_async
-        - set_permissions_async
+        - query_async
+        - query_part_mask_async
+        - store_rows_async
+        - upsert_rows_async
+        - delete_rows_async
+        - snapshot_async
+        - delete_column
+        - add_column
+        - reorder_column
+        - get_permissions
+        - get_acl
+        - set_permissions
+
+::: synapseclient.models.Column
+    options:
+        members:
+
+::: synapseclient.models.SchemaStorageStrategy
+::: synapseclient.models.ColumnExpansionStrategy
+
+::: synapseclient.models.FacetType
+::: synapseclient.models.ColumnType
+::: synapseclient.models.JsonSubColumn
+
+
+::: synapseclient.models.ColumnChange
+::: synapseclient.models.PartialRow
+::: synapseclient.models.PartialRowSet
+::: synapseclient.models.TableSchemaChangeRequest
+::: synapseclient.models.AppendableRowSetRequest
+::: synapseclient.models.UploadToTableRequest
+::: synapseclient.models.TableUpdateTransaction
+::: synapseclient.models.CsvTableDescriptor
+::: synapseclient.models.mixins.table_operator.csv_to_pandas_df

docs/reference/experimental/functional_interfaces.md

@@ -0,0 +1,6 @@
+
+::: synapseclient.api
+    options:
+      inherited_members: true
+      members:
+        - delete_entity

docs/reference/experimental/sync/table.md

@@ -16,16 +16,45 @@ at your own risk.
 
 ## API Reference
 
+
 ::: synapseclient.models.Table
     options:
         inherited_members: true
         members:
         - get
-        - store_schema
-        - store_rows_from_csv
-        - delete_rows
-        - query
+        - store
         - delete
+        - query
+        - query_part_mask
+        - store_rows
+        - upsert_rows
+        - delete_rows
+        - snapshot
+        - delete_column
+        - add_column
+        - reorder_column
         - get_permissions
         - get_acl
         - set_permissions
+
+::: synapseclient.models.Column
+    options:
+        members:
+
+::: synapseclient.models.SchemaStorageStrategy
+::: synapseclient.models.ColumnExpansionStrategy
+
+::: synapseclient.models.FacetType
+::: synapseclient.models.ColumnType
+::: synapseclient.models.JsonSubColumn
+
+
+::: synapseclient.models.ColumnChange
+::: synapseclient.models.PartialRow
+::: synapseclient.models.PartialRowSet
+::: synapseclient.models.TableSchemaChangeRequest
+::: synapseclient.models.AppendableRowSetRequest
+::: synapseclient.models.UploadToTableRequest
+::: synapseclient.models.TableUpdateTransaction
+::: synapseclient.models.CsvTableDescriptor
+::: synapseclient.models.mixins.table_operator.csv_to_pandas_df

docs/scripts/object_orientated_programming_poc/oop_poc_table.py

@@ -16,9 +16,10 @@
 from datetime import date, datetime, timedelta, timezone
 
 import synapseclient
-from synapseclient.models import Column, ColumnType, CsvResultFormat, Row, Table
+from synapseclient.models import Column, ColumnType, Table
 
 PROJECT_ID = "syn52948289"
+ROWS_TO_WRITE = 10
 
 syn = synapseclient.Synapse(debug=True)
 syn.login()
@@ -36,15 +37,15 @@ def write_random_csv_with_data(path: str):
     data = {}
     for name, type in randomized_data_columns.items():
         if type == int:
-            data[name] = [random.randint(0, 100) for _ in range(10)]
+            data[name] = [random.randint(0, 100) for _ in range(ROWS_TO_WRITE + 1)]
         elif type == float:
-            data[name] = [random.uniform(0, 100) for _ in range(10)]
+            data[name] = [random.uniform(0, 100) for _ in range(ROWS_TO_WRITE + 1)]
         elif type == bool:
-            data[name] = [bool(random.getrandbits(1)) for _ in range(10)]
+            data[name] = [bool(random.getrandbits(1)) for _ in range(ROWS_TO_WRITE + 1)]
         elif type == str:
             data[name] = [
                 "".join(random.choices(string.ascii_uppercase + string.digits, k=5))
-                for _ in range(10)
+                for _ in range(ROWS_TO_WRITE + 1)
             ]
 
     with open(path, "w", newline="", encoding="utf-8") as csvfile:
@@ -54,7 +55,7 @@ def write_random_csv_with_data(path: str):
         writer.writerow(data.keys())
 
         # Write data
-        for i in range(10):
+        for i in range(ROWS_TO_WRITE + 1):
             writer.writerow([values[i] for values in data.values()])
 
 
@@ -86,13 +87,13 @@ def store_table():
 
     # Creating a table ===============================================================
     table = Table(
-        name="my_first_test_table",
+        name="my_first_test_table_ksidubhgfkjsdgf",
         columns=columns,
         parent_id=PROJECT_ID,
         annotations=annotations_for_my_table,
     )
 
-    table = table.store_schema()
+    table = table.store()
 
     print("Table created:")
     print(table)
@@ -107,7 +108,7 @@ def store_table():
 
     # Updating annotations on my table ===============================================
     copy_of_table.annotations["my_key_string"] = ["new", "values", "here"]
-    stored_table = copy_of_table.store_schema()
+    stored_table = copy_of_table.store()
     print("Table updated:")
     print(stored_table)
 
@@ -116,31 +117,24 @@ def store_table():
     path_to_csv = os.path.join(os.path.expanduser("~/temp"), f"{name_of_csv}.csv")
     write_random_csv_with_data(path_to_csv)
 
-    csv_path = copy_of_table.store_rows_from_csv(csv_path=path_to_csv)
+    copy_of_table.store_rows(values=path_to_csv)
 
-    print("Stored data to table from CSV:")
-    print(csv_path)
+    print("Stored data to table from CSV")
 
     # Querying for data from a table =================================================
-    destination_csv_location = os.path.expanduser("~/temp/my_query_results")
-
     table_id_to_query = copy_of_table.id
-    Table.query(
-        query=f"SELECT * FROM {table_id_to_query}",
-        result_format=CsvResultFormat(download_location=destination_csv_location),
-    )
+    dataframe_from_query = Table.query(query=f"SELECT * FROM {table_id_to_query}")
 
-    print(f"Created results at: {destination_csv_location}")
+    print(f"Got results: {dataframe_from_query}")
 
-    # Deleting rows from a table =====================================================
-    copy_of_table.delete_rows(rows=[Row(row_id=1)])
+    # Deleting a row from the table =====================================================
+    copy_of_table.delete_rows(query=f"SELECT * from {table_id_to_query} LIMIT 1")
 
     # Deleting a table ===============================================================
     table_to_delete = Table(
         name="my_test_table_I_want_to_delete",
-        columns=columns,
         parent_id=PROJECT_ID,
-    ).store_schema()
+    ).store()
 
     table_to_delete.delete()
 

mkdocs.yml

@@ -83,6 +83,7 @@ nav:
           - Activity: reference/experimental/sync/activity.md
           - Team: reference/experimental/sync/team.md
           - UserProfile: reference/experimental/sync/user_profile.md
+          - Functional Interfaces: reference/experimental/functional_interfaces.md
           - Asynchronous:
               - Agent: reference/experimental/async/agent.md
               - Project: reference/experimental/async/project.md
@@ -140,6 +141,8 @@ theme:
     - toc.follow
     - navigation.tabs
     - navigation.tabs.sticky
+    - navigation.instant
+    - navigation.instant.progress
 
 extra_css:
   - css/custom.css

synapseclient/api/__init__.py

@@ -24,6 +24,7 @@
 from .entity_factory import get_from_entity_factory
 from .entity_services import (
     create_access_requirements_if_none,
+    delete_entity,
     delete_entity_generated_by,
     get_entities_by_md5,
     get_entity,
@@ -46,6 +47,7 @@
     put_file_multipart_add,
     put_file_multipart_complete,
 )
+from .table_services import get_columns, post_columns
 
 __all__ = [
     # annotations
@@ -72,6 +74,7 @@
     "get_entity",
     "put_entity",
     "post_entity",
+    "delete_entity",
     "get_upload_destination",
     "get_upload_destination_location",
     "create_access_requirements_if_none",
@@ -93,4 +96,7 @@
     "get_session",
     "update_session",
     "get_trace",
+    # columns
+    "get_columns",
+    "post_columns",
 ]

synapseclient/api/entity_factory.py

@@ -20,7 +20,7 @@
 
 if TYPE_CHECKING:
     from synapseclient import Synapse
-    from synapseclient.models import File, Folder, Project
+    from synapseclient.models import Dataset, File, Folder, Project, Table
 
 
 async def get_from_entity_factory(
@@ -32,7 +32,7 @@ async def get_from_entity_factory(
     download_file: bool = True,
     download_location: str = None,
     follow_link: bool = False,
-    entity_to_update: Union["Project", "File", "Folder"] = None,
+    entity_to_update: Union["Project", "File", "Folder", "Table", "Dataset"] = None,
     *,
     synapse_client: Optional["Synapse"] = None,
 ) -> Union["Project", "File", "Folder"]:
@@ -292,7 +292,7 @@ class type. This will also download the file if `download_file` is set to True.
         synapse_annotations=entity_bundle.get("annotations", None)
     )
 
-    from synapseclient.models import File, Folder, Project
+    from synapseclient.models import File, Folder, Project, Table
 
     if entity["concreteType"] == concrete_types.PROJECT_ENTITY:
         if not entity_to_update:
@@ -349,6 +349,20 @@ class type. This will also download the file if `download_file` is set to True.
                     + "!" * len(warning_message)
                     + "\n"
                 )
+    elif entity["concreteType"] == concrete_types.TABLE_ENTITY:
+        if not entity_to_update:
+            entity_to_update = Table()
+        entity = entity_to_update.fill_from_dict(
+            entity=entity_bundle["entity"], set_annotations=False
+        )
+    elif entity["concreteType"] == concrete_types.DATASET_ENTITY:
+        if not entity_to_update:
+            from synapseclient.models import Dataset
+
+            entity_to_update = Dataset()
+        entity = entity_to_update.fill_from_dict(
+            entity=entity_bundle["entity"], set_annotations=False
+        )
     else:
         raise ValueError(
             f"Attempting to retrieve an unsupported entity type of {entity['concreteType']}."