Merge pull request #227 from codeforjapan/feat/update-note-rating-status

feat: add daily rating count recalculation to ETL pipeline

Author: ayuki-joto

.github/workflows/test.yml

@@ -69,7 +69,7 @@ jobs:
           cache: pip
           cache-dependency-path: etl/pyproject.toml
       - name: dependency install
-        run: pip install -e "./etl[dev]"
+        run: pip install -e "./common" -e "./etl[dev]"
       - name: copy env
         run: cp .env.example etl/.env
       - name: test

etl/seed/settings.json

@@ -3,7 +3,7 @@
     "languages": ["ja"],
     "keywords": [],
     "start_millis": 1761922800000,
-    "end_millis": 1771858740000
+    "end_millis": null
   },
-  "description": "Note filtering configuration. start_millis is required. end_millis is optional (null = no upper limit). Timestamps in milliseconds UTC. Current: 2025-11-01 00:00:00 JST ~ 2026-02-23 23:59:00 JST"
+  "description": "Note filtering configuration. start_millis is required. end_millis is optional (null = no upper limit). Timestamps in milliseconds UTC. Current: 2025-11-01 00:00:00 JST ~"
 }

etl/src/birdxplorer_etl/extract_ecs.py

@@ -115,9 +115,7 @@ def lambda_handler(event, context):
         except Exception as e:
             logger.error(f"[ERROR] Commit failed: {str(e)}")
             postgresql.rollback()
-            batch_item_failures = [
-                {"itemIdentifier": r.get("messageId", "unknown")} for r in event["Records"]
-            ]
+            batch_item_failures = [{"itemIdentifier": r.get("messageId", "unknown")} for r in event["Records"]]
 
         return {"batchItemFailures": batch_item_failures}
 

etl/src/birdxplorer_etl/lib/lambda_handler/note_status_update_lambda.py

@@ -243,9 +243,7 @@ def _process_single_tweet(
         logger.error(f"[DLQ_CAUSE:UNEXPECTED_RESPONSE] tweet_id={tweet_id} response={json.dumps(post)}")
         raise Exception(f"Unexpected API response for tweet: {tweet_id}")
 
-    created_at = datetime.strptime(post["data"]["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").replace(
-        tzinfo=timezone.utc
-    )
+    created_at = datetime.strptime(post["data"]["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
     created_at_millis = int(created_at.timestamp() * 1000)
     now_millis = int(datetime.now(timezone.utc).timestamp() * 1000)
 

etl/src/birdxplorer_etl/lib/lambda_handler/postlookup_lambda.py

@@ -137,9 +137,7 @@ def lambda_handler(event, context):
                             f"[SQS_SUCCESS] Sent tweet lookup message for tweet {post_id}, messageId={message_id}"
                         )
                     else:
-                        raise Exception(
-                            f"Failed to send tweet lookup message for tweet {post_id} (note {note_id})"
-                        )
+                        raise Exception(f"Failed to send tweet lookup message for tweet {post_id} (note {note_id})")
 
                 except Exception as e:
                     logger.error(f"[EXCEPTION] Error sending SQS message for tweet lookup: {e}")

etl/src/birdxplorer_etl/lib/lambda_handler/topic_detect_lambda.py

@@ -120,12 +120,14 @@ def init_postgresql(use_pool: bool = False):
     if not use_pool:
         engine_kwargs["poolclass"] = NullPool  # Lambda向け: コネクションをプールせず毎回接続・切断
     else:
-        engine_kwargs.update({
-            "pool_size": 5,
-            "max_overflow": 10,
-            "pool_timeout": 30,
-            "pool_recycle": 3600,
-        })
+        engine_kwargs.update(
+            {
+                "pool_size": 5,
+                "max_overflow": 10,
+                "pool_timeout": 30,
+                "pool_recycle": 3600,
+            }
+        )
     engine = create_engine(
         f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}",
         **engine_kwargs,