Merge pull request #7699 from freedomofpress/frozen-snowflakes

feat(`/data`): enforce and document typing and interop considerations for snowflake IDs

Author: vickiniu

API2.md

@@ -87,6 +87,8 @@ Note over Client: Global version uvwxyz
 end
 ```
 
+#### Consistency
+
 This diagram implies single-round-trip consistency. To make that expectation
 explicit:
 
@@ -98,3 +100,18 @@ E_n\}$; and
 3. $S$ accepts $BR$ as valid and successfully processes all $E_i$; then
 
 4. $C$'s index SHOULD match $S$'s index without a subsequent synchronization.
+
+#### Snowflake IDs
+
+The `Event.id` field is a "snowflake ID", which a client can generate using a
+library like [`@sapphire/snowflake`]. To avoid precision-loss problems:
+
+- A client SHOULD store its IDs as opaque strings and sort them
+  lexicographically.
+
+- A client MUST encode its IDs on the wire as JSON strings.
+
+- The server MAY convert IDs it receives to integers, but only for sorting and
+  testing equality.
+
+[`@sapphire/snowflake`]: https://www.npmjs.com/package/@sapphire/snowflake

securedrop/journalist_app/api2/__init__.py

@@ -113,7 +113,7 @@ def data() -> Response:
         )
 
         # Process events in snowflake order.
-        for event in sorted(events, key=lambda e: e.id):
+        for event in sorted(events, key=lambda e: int(e.id)):
             result = handler.process(event)
             for uuid, source in result.sources.items():
                 response.sources[uuid] = source.to_api_v2() if source is not None else None

securedrop/tests/test_journalist_api2.py

@@ -826,3 +826,46 @@ def test_api2_idempotence_period(journalist_app):
     """
 
     assert journalist_app.config["SESSION_LIFETIME"] <= api2.events.IDEMPOTENCE_PERIOD
+
+
+def test_api2_event_ordering(journalist_app, journalist_api_token, test_files):
+    """
+    If two `item_deleted` events for the same item arrive out of order, the
+    numerically later event must observe that the item is already gone by the
+    time it's processed.
+    """
+    with journalist_app.test_client() as app:
+        index = app.get(
+            url_for("api2.index"),
+            headers=get_api_headers(journalist_api_token),
+        )
+        assert index.status_code == 200
+
+        submission_uuid = test_files["submissions"][0].uuid
+        item_version = index.json["items"][submission_uuid]
+
+        # Two `item_deleted` events targeting the same item:
+        e2 = Event(
+            id="3419026047977394171",
+            target=ItemTarget(item_uuid=submission_uuid, version=item_version),
+            type=EventType.ITEM_DELETED,
+        )
+        e1 = Event(
+            id="3419026047977394170",  # client sends as string; server orders as integer
+            target=ItemTarget(item_uuid=submission_uuid, version=item_version),
+            type=EventType.ITEM_DELETED,
+        )
+
+        # Send them out of order:
+        resp = app.post(
+            url_for("api2.data"),
+            json={"events": [asdict(e2), asdict(e1)]},
+            headers=get_api_headers(journalist_api_token),
+        )
+        assert resp.status_code == 200
+
+        # Event `1` (sent second, processed first) deletes the item.
+        assert resp.json["events"]["3419026047977394170"] == [200, None]
+
+        # Event "2" (sent first, processed second) finds it missing.
+        assert resp.json["events"]["3419026047977394171"][0] == 410