Merge pull request #224 from codeforjapan/fix/helpfulness-level-nullable

Fix/helpfulness level nullable

Author: ayuki-joto

.gitignore

@@ -170,3 +170,6 @@ cython_debug/
 # accounts db for BirdXplorer
 accounts.db
 .claude/settings.local.json
+*/**/CLAUDE.md
+.env.*
+*/**/.env.*

common/birdxplorer_common/storage.py

@@ -263,7 +263,7 @@ class RowNoteRatingRecord(Base):
     disagree: Mapped[BinaryBool] = mapped_column(nullable=False)
     helpful: Mapped[BinaryBool] = mapped_column(nullable=False)
     not_helpful: Mapped[BinaryBool] = mapped_column(nullable=False)
-    helpfulness_level: Mapped[String] = mapped_column(nullable=False)
+    helpfulness_level: Mapped[Optional[String]] = mapped_column(nullable=True)
     helpful_other: Mapped[BinaryBool] = mapped_column(nullable=False)
     helpful_informative: Mapped[BinaryBool] = mapped_column(nullable=False)
     helpful_clear: Mapped[BinaryBool] = mapped_column(nullable=False)

etl/src/birdxplorer_etl/extract_ecs.py

@@ -567,18 +567,28 @@ def _process_rating_rows(reader, postgresql: Session, existing_row_note_ids: set
 
         # 5000件ごとにバッチ処理（1000 → 5000に拡大）
         if len(rows_to_add) >= 5000:
-            postgresql.execute(insert(RowNoteRatingRecord).on_conflict_do_nothing(), rows_to_add)
-            postgresql.commit()
-            logging.info(
-                f"Saved {len(rows_to_add)} rating records (batch at index {index}, file {file_index:05d})"
-            )
+            try:
+                postgresql.execute(insert(RowNoteRatingRecord).on_conflict_do_nothing(), rows_to_add)
+                postgresql.commit()
+                logging.info(
+                    f"Saved {len(rows_to_add)} rating records (batch at index {index}, file {file_index:05d})"
+                )
+            except Exception as e:
+                logging.error(
+                    f"Failed to save rating batch at index {index}, file {file_index:05d}: {e}"
+                )
+                postgresql.rollback()
             rows_to_add = []
 
     # 最後のバッチを処理
     if rows_to_add:
-        postgresql.execute(insert(RowNoteRatingRecord).on_conflict_do_nothing(), rows_to_add)
-        postgresql.commit()
-        logging.info(f"Saved final batch of {len(rows_to_add)} rating records (file {file_index:05d})")
+        try:
+            postgresql.execute(insert(RowNoteRatingRecord).on_conflict_do_nothing(), rows_to_add)
+            postgresql.commit()
+            logging.info(f"Saved final batch of {len(rows_to_add)} rating records (file {file_index:05d})")
+        except Exception as e:
+            logging.error(f"Failed to save final rating batch, file {file_index:05d}: {e}")
+            postgresql.rollback()
 
 
 def extract_ratings(postgresql: Session, dateString: str, existing_row_note_ids: set):

migrate/migration/versions/make_helpfulness_level_nullable.py

@@ -0,0 +1,41 @@
+"""make helpfulness_level nullable in row_note_ratings
+
+Community Notes の 2021年1月〜6月のレーティングデータには
+helpfulness_level フィールドが存在しない（空文字列）。
+NOT NULL 制約があると INSERT 時に NotNullViolation が発生し、
+ファイル全体のデータがロストするため nullable に変更する。
+
+Revision ID: make_helpfulness_level_nullable
+Revises: add_collab_note_column
+Create Date: 2026-02-23
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "make_helpfulness_level_nullable"
+down_revision: Union[str, None] = "add_collab_note_column"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "row_note_ratings",
+        "helpfulness_level",
+        existing_type=sa.String(),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "row_note_ratings",
+        "helpfulness_level",
+        existing_type=sa.String(),
+        nullable=False,
+    )