Allow multiple test execution runs

backend/migrations/versions/2024_10_18_1134-b234def463ad_drop_review_columns_from_test_execution_.py

@@ -0,0 +1,55 @@
+"""Drop review columns from Test Execution Table
+
+Revision ID: b234def463ad
+Revises: 91e7e3f437a0
+Create Date: 2024-10-18 11:34:38.285303+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b234def463ad"
+down_revision = "91e7e3f437a0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("test_execution", "review_decision")
+    op.drop_column("test_execution", "review_comment")
+    op.execute("DROP TYPE testexecutionreviewdecision")
+
+
+def downgrade() -> None:
+    te_review_decision = sa.Enum(
+        "REJECTED",
+        "APPROVED_INCONSISTENT_TEST",
+        "APPROVED_UNSTABLE_PHYSICAL_INFRA",
+        "APPROVED_FAULTY_HARDWARE",
+        "APPROVED_CUSTOMER_PREREQUISITE_FAIL",
+        "APPROVED_ALL_TESTS_PASS",
+        name="testexecutionreviewdecision",
+    )
+    te_review_decision.create(op.get_bind())
+    op.add_column(
+        "test_execution",
+        sa.Column(
+            "review_comment",
+            sa.VARCHAR(),
+            server_default=sa.text("''::character varying"),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "test_execution",
+        sa.Column(
+            "review_decision",
+            postgresql.ARRAY(te_review_decision),
+            server_default=sa.text("'{}'::testexecutionreviewdecision[]"),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )

backend/migrations/versions/2024_10_21_1035-063e32aac8db_allow_multiple_testexecution_runs.py

@@ -0,0 +1,53 @@
+"""Allow multiple TestExecution runs
+
+Revision ID: 063e32aac8db
+Revises: b234def463ad
+Create Date: 2024-10-21 10:35:17.364462+00:00
+
+"""
+from textwrap import dedent
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "063e32aac8db"
+down_revision = "b234def463ad"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_constraint(
+        "test_execution_artefact_build_id_environment_id_key",
+        "test_execution",
+        type_="unique",
+    )
+
+
+def downgrade() -> None:
+    remove_test_execution_runs_keeping_latest()
+
+    op.create_unique_constraint(
+        "test_execution_artefact_build_id_environment_id_key",
+        "test_execution",
+        ["artefact_build_id", "environment_id"],
+    )
+
+
+def remove_test_execution_runs_keeping_latest():
+    connection = op.get_bind()
+
+    stmt = """\
+        SELECT artefact_build_id, environment_id, MAX(id), COUNT(*)
+        FROM test_execution
+        GROUP BY artefact_build_id, environment_id
+        HAVING COUNT(*) > 1
+    """
+
+    for ab_id, e_id, max_id, _ in connection.execute(sa.text(dedent(stmt))):
+        stmt = f"""\
+            DELETE FROM test_execution
+            WHERE artefact_build_id={ab_id} AND environment_id={e_id} AND id <> {max_id}
+        """
+        op.execute(dedent(stmt))

backend/scripts/seed_data.py

@@ -46,6 +46,30 @@
         environment="rpi2",
         ci_link="http://example1",
     ),
+    StartTestExecutionRequest(
+        family=FamilyName.SNAP,
+        name="core22",
+        version="20230531",
+        revision=1,
+        track="22",
+        store="ubuntu",
+        arch="armhf",
+        execution_stage="beta",
+        environment="rpi2",
+        ci_link="http://example13",
+    ),
+    StartTestExecutionRequest(
+        family=FamilyName.SNAP,
+        name="core22",
+        version="20230531",
+        revision=1,
+        track="22",
+        store="ubuntu",
+        arch="armhf",
+        execution_stage="beta",
+        environment="rpi2",
+        ci_link="http://example14",
+    ),
     StartTestExecutionRequest(
         family=FamilyName.SNAP,
         name="core22",
@@ -184,7 +208,7 @@
         test_results=[
             C3TestResult(
                 name="docker/compose-and-basic_armhf",
-                status=C3TestResultStatus.PASS,
+                status=C3TestResultStatus.FAIL,
                 category="Docker containers",
                 comment="",
                 io_log=dedent(
@@ -270,6 +294,44 @@
             ),
         ],
     ),
+    EndTestExecutionRequest(
+        ci_link="http://example13",
+        test_results=[
+            C3TestResult(
+                name="docker/compose-and-basic_armhf",
+                status=C3TestResultStatus.PASS,
+                category="Docker containers",
+                comment="",
+                io_log="",
+            ),
+            C3TestResult(
+                name="after-suspend-audio/alsa-loopback-automated",
+                status=C3TestResultStatus.FAIL,
+                category="Audio tests",
+                comment="",
+                io_log="",
+            ),
+        ],
+    ),
+    EndTestExecutionRequest(
+        ci_link="http://example10",
+        test_results=[
+            C3TestResult(
+                name="docker/compose-and-basic_armhf",
+                status=C3TestResultStatus.PASS,
+                category="Docker containers",
+                comment="",
+                io_log="",
+            ),
+            C3TestResult(
+                name="after-suspend-audio/alsa-loopback-automated",
+                status=C3TestResultStatus.FAIL,
+                category="Audio tests",
+                comment="",
+                io_log="",
+            ),
+        ],
+    ),
     EndTestExecutionRequest(
         ci_link="http://example2",
         test_results=[

backend/test_observer/controllers/test_executions/end_test.py

@@ -19,15 +19,12 @@
 from sqlalchemy.orm import Session, joinedload
 
 from test_observer.data_access.models import (
-    Artefact,
     ArtefactBuild,
-    ArtefactBuildEnvironmentReview,
     TestCase,
     TestExecution,
     TestResult,
 )
 from test_observer.data_access.models_enums import (
-    ArtefactBuildEnvironmentReviewDecision,
     TestExecutionStatus,
     TestResultStatus,
 )
@@ -56,37 +53,6 @@ def end_test_execution(request: EndTestExecutionRequest, db: Session = Depends(g
         TestExecutionStatus.FAILED if has_failures else TestExecutionStatus.PASSED
     )
 
-    prev_test_execution = _get_previous_test_execution(db, test_execution)
-    environment_review = db.scalars(
-        select(ArtefactBuildEnvironmentReview).where(
-            ArtefactBuildEnvironmentReview.environment_id
-            == test_execution.environment_id,
-            ArtefactBuildEnvironmentReview.artefact_build_id
-            == test_execution.artefact_build_id,
-        )
-    ).one()
-
-    if prev_test_execution:
-        prev_environment_review = db.scalar(
-            select(ArtefactBuildEnvironmentReview).where(
-                ArtefactBuildEnvironmentReview.environment_id
-                == test_execution.environment_id,
-                ArtefactBuildEnvironmentReview.artefact_build_id
-                == prev_test_execution.artefact_build_id,
-            )
-        )
-
-        if (
-            prev_environment_review
-            and prev_environment_review.is_approved
-            and not has_failures
-            and _ran_all_previously_run_cases(prev_test_execution, test_execution)
-            and not environment_review.review_decision
-        ):
-            environment_review.review_decision = [
-                ArtefactBuildEnvironmentReviewDecision.APPROVED_ALL_TESTS_PASS
-            ]
-
     if request.c3_link is not None:
         test_execution.c3_link = request.c3_link
 
@@ -95,67 +61,6 @@ def end_test_execution(request: EndTestExecutionRequest, db: Session = Depends(g
     db.commit()
 
 
-def _ran_all_previously_run_cases(
-    prev_test_execution: TestExecution, test_execution: TestExecution
-) -> bool:
-    prev_test_cases = {
-        tr.test_case.name
-        for tr in prev_test_execution.test_results
-        if tr.status != TestResultStatus.SKIPPED
-    }
-
-    test_cases = {
-        tr.test_case.name
-        for tr in test_execution.test_results
-        if tr.status != TestResultStatus.SKIPPED
-    }
-
-    return prev_test_cases.issubset(test_cases)
-
-
-def _get_previous_test_execution(
-    db: Session, test_execution: TestExecution
-) -> TestExecution | None:
-    artefact = test_execution.artefact_build.artefact
-    prev_artefact = _get_previous_artefact(db, artefact)
-
-    if prev_artefact is None:
-        return None
-
-    query = (
-        select(TestExecution)
-        .join(ArtefactBuild)
-        .where(
-            ArtefactBuild.artefact_id == prev_artefact.id,
-            TestExecution.environment_id == test_execution.environment_id,
-        )
-        .order_by(ArtefactBuild.revision.desc())
-        .limit(1)
-        .options(
-            joinedload(TestExecution.test_results).joinedload(TestResult.test_case)
-        )
-    )
-
-    return db.execute(query).unique().scalar_one_or_none()
-
-
-def _get_previous_artefact(db: Session, artefact: Artefact) -> Artefact | None:
-    query = (
-        select(Artefact)
-        .where(
-            Artefact.id < artefact.id,
-            Artefact.name == artefact.name,
-            Artefact.track == artefact.track,
-            Artefact.series == artefact.series,
-            Artefact.repo == artefact.repo,
-        )
-        .order_by(Artefact.id.desc())
-        .limit(1)
-    )
-
-    return db.execute(query).scalar_one_or_none()
-
-
 def _find_related_test_execution(
     request: EndTestExecutionRequest, db: Session
 ) -> TestExecution | None:

backend/test_observer/controllers/test_executions/logic.py

@@ -14,8 +14,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from sqlalchemy import delete, desc
-from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import delete, desc, select
+from sqlalchemy.orm import Session, joinedload, selectinload
 from sqlalchemy.orm.query import RowReturningQuery
 
 from test_observer.common.constants import PREVIOUS_TEST_RESULT_COUNT
@@ -26,24 +26,24 @@
     TestExecution,
     TestResult,
 )
-from test_observer.data_access.models_enums import TestExecutionStatus
 
-from .models import StartTestExecutionRequest
 
+def delete_rerun_requests(db: Session, test_execution: TestExecution):
+    related_test_execution_runs = db.scalars(
+        select(TestExecution)
+        .where(
+            TestExecution.artefact_build_id == test_execution.artefact_build_id,
+            TestExecution.environment_id == test_execution.environment_id,
+        )
+        .options(selectinload(TestExecution.rerun_request))
+    )
 
-def reset_test_execution(
-    request: StartTestExecutionRequest,
-    db: Session,
-    test_execution: TestExecution,
-):
-    test_execution.status = TestExecutionStatus.IN_PROGRESS
-    test_execution.ci_link = request.ci_link
-    test_execution.c3_link = None
-    if test_execution.rerun_request:
-        db.delete(test_execution.rerun_request)
-    db.commit()
+    for te in related_test_execution_runs:
+        rerun_request = te.rerun_request
+        if rerun_request:
+            db.delete(rerun_request)
 
-    delete_previous_results(db, test_execution)
+    db.commit()
 
 
 def delete_previous_results(