Merge branch '166-implement-endpoints-for-uploading-version-state' in…

…to 'main'

Resolve "Implement endpoints for uploading version state"

Closes #166

See merge request pub/terra/terrarun!99

requirements.txt

@@ -14,3 +14,4 @@ schedule==1.2.0
 semantic-version==2.10.0
 blinker==1.6.2
 werkzeug<3
+typing_extensions==4.12.2

terrarun/alembic/versions/376a843af267_add_status_column_to_state_version.py

@@ -0,0 +1,32 @@
+"""Add status column to state version
+
+Revision ID: 376a843af267
+Revises: 6209dfe486dc
+Create Date: 2024-08-01 05:31:11.690395
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '376a843af267'
+down_revision = '6209dfe486dc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('state_version', sa.Column('status', sa.Enum('PENDING', 'FINALIZED', 'DISCARDED', 'BACKING_DATA_SOFT_DELETED', 'BACKING_DATA_PERMANENTLY_DELETED', name='stateversionstatus'), nullable=False))
+    bind = op.get_bind()
+    bind.execute("""UPDATE state_version SET status='FINALIZED' WHERE state_json_id IS NOT NULL""")
+    # This shouldn't be required, as state data was required in API upload, but it is a safety net
+    bind.execute("""UPDATE state_version SET status='DISCARDED' WHERE state_json_id IS NULL""")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('state_version', 'status')
+    # ### end Alembic commands ###

terrarun/alembic/versions/9309cba5bed5_add_json_output_to_state_version.py

@@ -0,0 +1,30 @@
+"""Add JSON output to state version and make state_version unique
+
+Revision ID: 9309cba5bed5
+Revises: c2ecefd99159
+Create Date: 2024-08-05 06:39:12.553556
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '9309cba5bed5'
+down_revision = 'c2ecefd99159'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('state_version', sa.Column('json_state_outputs_id', sa.Integer(), nullable=True))
+    op.create_foreign_key('fk_blob_state_version_json_state_outputs', 'state_version', 'blob', ['json_state_outputs_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('fk_blob_state_version_json_state_outputs', 'state_version', type_='foreignkey')
+    op.drop_column('state_version', 'json_state_outputs_id')
+    # ### end Alembic commands ###

terrarun/alembic/versions/c2ecefd99159_add_columns_to_state_version_to_handle_.py

@@ -0,0 +1,69 @@
+"""Add columns to state version to handle new attributes and upload endpoint
+
+Revision ID: c2ecefd99159
+Revises: 376a843af267
+Create Date: 2024-08-03 05:49:29.376688
+
+"""
+import json
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = 'c2ecefd99159'
+down_revision = '376a843af267'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Rename state_json to state column
+    op.drop_constraint('state_version_ibfk_3', 'state_version', type_='foreignkey')
+    op.alter_column('state_version', 'state_json_id', existing_type=sa.Integer(), nullable=True, new_column_name='state_id')
+    op.create_foreign_key('fk_blob_state_version_state', 'state_version', 'blob', ['state_id'], ['id'])
+
+    op.add_column('state_version', sa.Column('json_state_id', sa.Integer(), nullable=True))
+    op.add_column('state_version', sa.Column('lineage', sa.String(length=128), nullable=True))
+    op.add_column('state_version', sa.Column('md5', sa.String(length=128), nullable=True))
+    op.add_column('state_version', sa.Column('serial', sa.Integer(), nullable=True))
+    op.add_column('state_version', sa.Column('state_version', sa.Integer(), nullable=True))
+    op.add_column('state_version', sa.Column('terraform_version', sa.String(length=128), nullable=True))
+    op.add_column('state_version', sa.Column('intermediate', sa.Boolean(), nullable=False))
+    op.create_foreign_key('fk_blob_state_version_json_state', 'state_version', 'blob', ['json_state_id'], ['id'])
+
+    bind = op.get_bind()
+    for row in bind.execute("""SELECT state_version.id AS state_version_id, `blob`.data AS state_version_data FROM state_version INNER JOIN `blob` ON state_version.state_id=`blob`.id""").all():
+        state = {}
+        if row['state_version_data']:
+            state = json.loads(row['state_version_data'].decode('utf-8'))
+        bind.execute(
+            sa.sql.text(
+                """UPDATE state_version SET lineage=:lineage, md5=:md5, serial=:serial, state_version=:state_version, terraform_version=:terraform_version, intermediate=:intermediate WHERE id=:state_version_id"""
+            ),
+            lineage=state.get("lineage"),
+            md5=None,
+            serial=state.get("serial"),
+            state_version=state.get("version"),
+            terraform_version=state.get("terraform_version"),
+            intermediate=False,
+            state_version_id=row['state_version_id']
+        )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('fk_blob_state_version_json_state', 'state_version', type_='foreignkey')
+    op.drop_constraint('fk_blob_state_version_state', 'state_version', type_='foreignkey')
+    op.alter_column('state_version', 'state_id', existing_type=sa.Integer(), nullable=True, new_column_name='state_json_id')
+    op.create_foreign_key('state_version_ibfk_3', 'state_version', 'blob', ['state_json_id'], ['id'])
+    op.drop_column('state_version', 'intermediate')
+    op.drop_column('state_version', 'terraform_version')
+    op.drop_column('state_version', 'state_version')
+    op.drop_column('state_version', 'serial')
+    op.drop_column('state_version', 'md5')
+    op.drop_column('state_version', 'lineage')
+    op.drop_column('state_version', 'json_state_id')
+    # ### end Alembic commands ###

terrarun/api_entities/agent_execution_details.py

@@ -4,6 +4,7 @@
 
 from typing import Union
 
+import terrarun.auth_context
 import terrarun.models.organisation
 import terrarun.models.user
 import terrarun.workspace_execution_mode
@@ -31,7 +32,7 @@ def _get_attributes(cls):
         )
 
     @classmethod
-    def _from_object(cls, obj: Union['terrarun.models.plan.Plan', 'terrarun.models.apply.Apply'], effective_user: 'terrarun.models.user.User'):
+    def _from_object(cls, obj: Union['terrarun.models.plan.Plan', 'terrarun.models.apply.Apply'], auth_context: 'terrarun.auth_context.AuthContext'):
         """Convert plan object agent details entity"""
         attributes = {
             "mode": obj.execution_mode

terrarun/api_entities/agent_pool.py

@@ -8,6 +8,7 @@
 import terrarun.models.user
 import terrarun.workspace_execution_mode
 import terrarun.models.agent_pool
+import terrarun.auth_context
 
 from .base_entity import (
     BaseEntity,
@@ -32,7 +33,7 @@ def _get_attributes(cls):
         )
 
     @classmethod
-    def _from_object(cls, obj: 'terrarun.models.agent_pool.AgentPool', effective_user: 'terrarun.models.user.User'):
+    def _from_object(cls, obj: 'terrarun.models.agent_pool.AgentPool', auth_context: 'terrarun.auth_context.AuthContext'):
         """Convert object to agent pool entity"""
         return cls(
             id=obj.api_id,

terrarun/api_entities/apply.py

@@ -11,6 +11,7 @@
 import terrarun.models.apply
 import terrarun.terraform_command
 import terrarun.config
+import terrarun.auth_context
 
 from .base_entity import (
     BaseEntity,
@@ -71,7 +72,7 @@ def _get_attributes(cls):
         )
 
     @classmethod
-    def _from_object(cls, obj: 'terrarun.models.apply.Apply', effective_user: 'terrarun.models.user.User'):
+    def _from_object(cls, obj: 'terrarun.models.apply.Apply', auth_context: 'terrarun.auth_context.AuthContext'):
         """Convert object to apply entity"""
         return cls(
             id=obj.api_id,
@@ -81,8 +82,8 @@ def _from_object(cls, obj: 'terrarun.models.apply.Apply', effective_user: 'terra
                 "resource_destructions": 0,
                 "status": obj.status,
                 "log_read_url": f"{terrarun.config.Config().BASE_URL}/api/v2/applies/{obj.api_id}/log",
-                "execution_details": AgentExecutionDetailsEntity.from_object(obj=obj, effective_user=effective_user),
-                "status_timestamps": ExecutionStatusTimestampsEntity.from_object(obj=obj, effective_user=effective_user),
+                "execution_details": AgentExecutionDetailsEntity.from_object(obj=obj, auth_context=auth_context),
+                "status_timestamps": ExecutionStatusTimestampsEntity.from_object(obj=obj, auth_context=auth_context),
             }
         )