Merge branch '75-generate-random-ids-for-each-entity' into 'main'

Resolve "Generate random Ids for each entity"

Closes #75

See merge request pub/terra/terrarun!42

Author: MatthewJohn

terrarun/__init__.py

@@ -38,4 +38,5 @@
 from terrarun.models.oauth_client import OauthClient
 from terrarun.models.oauth_token import OauthToken
 from terrarun.models.github_app_oauth_token import GithubAppOauthToken
-from terrarun.models.authorised_repo import AuthorisedRepo
+from terrarun.models.authorised_repo import AuthorisedRepo
+from terrarun.models.api_id import ApiId

terrarun/alembic/versions/39e8a680df30_add_indexes_to_api_id_table.py

@@ -0,0 +1,30 @@
+"""Add indexes to API ID table
+
+Revision ID: 39e8a680df30
+Revises: 614792b159a0
+Create Date: 2023-05-08 08:05:56.092727
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '39e8a680df30'
+down_revision = '614792b159a0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index('_api_id_suffix_index', 'api_id', ['api_id_suffix'], unique=False)
+    op.create_index('_object_class_object_id_in', 'api_id', ['object_class', 'object_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('_object_class_object_id_in', table_name='api_id')
+    op.drop_index('_api_id_suffix_index', table_name='api_id')
+    # ### end Alembic commands ###

terrarun/alembic/versions/614792b159a0_add_table_for_assignment_of_api_ids.py

@@ -0,0 +1,36 @@
+"""Add table for assignment of API IDs
+
+Revision ID: 614792b159a0
+Revises: 46fb6be21022
+Create Date: 2023-05-08 07:32:15.266859
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '614792b159a0'
+down_revision = '46fb6be21022'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('api_id',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('api_id_suffix', sa.String(length=128), nullable=True),
+    sa.Column('object_class', sa.String(length=128), nullable=True),
+    sa.Column('object_id', sa.Integer(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('api_id_suffix'),
+    sa.UniqueConstraint('object_class', 'object_id', name='_object_class_object_id_uc')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('api_id')
+    # ### end Alembic commands ###

terrarun/alembic/versions/9afeb4e61715_add_foreign_key_to_each_object_table_to_.py

@@ -0,0 +1,152 @@
+"""Add foreign key to each object table to link to api id row
+
+Revision ID: 9afeb4e61715
+Revises: 39e8a680df30
+Create Date: 2023-05-08 12:24:21.494334
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '9afeb4e61715'
+down_revision = '39e8a680df30'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('agent', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'agent', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('agent_pool', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'agent_pool', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('agent_token', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'agent_token', 'api_id', ['api_id_fk'], ['id'])
+    op.drop_index('_api_id_suffix_index', table_name='api_id')
+    op.drop_index('_object_class_object_id_in', table_name='api_id')
+    op.drop_index('_object_class_object_id_uc', table_name='api_id')
+    op.drop_column('api_id', 'object_class')
+    op.drop_column('api_id', 'object_id')
+    op.add_column('apply', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'apply', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('audit_event', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'audit_event', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('authorised_repo', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'authorised_repo', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('blob', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'blob', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('configuration_version', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'configuration_version', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('environment', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'environment', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('github_app_oauth_token', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'github_app_oauth_token', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('lifecycle', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'lifecycle', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('oauth_client', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'oauth_client', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('oauth_token', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'oauth_token', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('organisation', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'organisation', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('plan', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'plan', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('project', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'project', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('run', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'run', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('run_queue', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'run_queue', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('state_version', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'state_version', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('tag', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'tag', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('task', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'task', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('task_result', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'task_result', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('task_stage', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'task_stage', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('team', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'team', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('team_workspace_access', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'team_workspace_access', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('user', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'user', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('user_token', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'user_token', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('workspace', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'workspace', 'api_id', ['api_id_fk'], ['id'])
+    op.add_column('workspace_task', sa.Column('api_id_fk', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'workspace_task', 'api_id', ['api_id_fk'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'workspace_task', type_='foreignkey')
+    op.drop_column('workspace_task', 'api_id_fk')
+    op.drop_constraint(None, 'workspace', type_='foreignkey')
+    op.drop_column('workspace', 'api_id_fk')
+    op.drop_constraint(None, 'user_token', type_='foreignkey')
+    op.drop_column('user_token', 'api_id_fk')
+    op.drop_constraint(None, 'user', type_='foreignkey')
+    op.drop_column('user', 'api_id_fk')
+    op.drop_constraint(None, 'team_workspace_access', type_='foreignkey')
+    op.drop_column('team_workspace_access', 'api_id_fk')
+    op.drop_constraint(None, 'team', type_='foreignkey')
+    op.drop_column('team', 'api_id_fk')
+    op.drop_constraint(None, 'task_stage', type_='foreignkey')
+    op.drop_column('task_stage', 'api_id_fk')
+    op.drop_constraint(None, 'task_result', type_='foreignkey')
+    op.drop_column('task_result', 'api_id_fk')
+    op.drop_constraint(None, 'task', type_='foreignkey')
+    op.drop_column('task', 'api_id_fk')
+    op.drop_constraint(None, 'tag', type_='foreignkey')
+    op.drop_column('tag', 'api_id_fk')
+    op.drop_constraint(None, 'state_version', type_='foreignkey')
+    op.drop_column('state_version', 'api_id_fk')
+    op.drop_constraint(None, 'run_queue', type_='foreignkey')
+    op.drop_column('run_queue', 'api_id_fk')
+    op.drop_constraint(None, 'run', type_='foreignkey')
+    op.drop_column('run', 'api_id_fk')
+    op.drop_constraint(None, 'project', type_='foreignkey')
+    op.drop_column('project', 'api_id_fk')
+    op.drop_constraint(None, 'plan', type_='foreignkey')
+    op.drop_column('plan', 'api_id_fk')
+    op.drop_constraint(None, 'organisation', type_='foreignkey')
+    op.drop_column('organisation', 'api_id_fk')
+    op.drop_constraint(None, 'oauth_token', type_='foreignkey')
+    op.drop_column('oauth_token', 'api_id_fk')
+    op.drop_constraint(None, 'oauth_client', type_='foreignkey')
+    op.drop_column('oauth_client', 'api_id_fk')
+    op.drop_constraint(None, 'lifecycle', type_='foreignkey')
+    op.drop_column('lifecycle', 'api_id_fk')
+    op.drop_constraint(None, 'github_app_oauth_token', type_='foreignkey')
+    op.drop_column('github_app_oauth_token', 'api_id_fk')
+    op.drop_constraint(None, 'environment', type_='foreignkey')
+    op.drop_column('environment', 'api_id_fk')
+    op.drop_constraint(None, 'configuration_version', type_='foreignkey')
+    op.drop_column('configuration_version', 'api_id_fk')
+    op.drop_constraint(None, 'blob', type_='foreignkey')
+    op.drop_column('blob', 'api_id_fk')
+    op.drop_constraint(None, 'authorised_repo', type_='foreignkey')
+    op.drop_column('authorised_repo', 'api_id_fk')
+    op.drop_constraint(None, 'audit_event', type_='foreignkey')
+    op.drop_column('audit_event', 'api_id_fk')
+    op.drop_constraint(None, 'apply', type_='foreignkey')
+    op.drop_column('apply', 'api_id_fk')
+    op.add_column('api_id', sa.Column('object_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True))
+    op.add_column('api_id', sa.Column('object_class', mysql.VARCHAR(length=128), nullable=True))
+    op.create_index('_object_class_object_id_uc', 'api_id', ['object_class', 'object_id'], unique=False)
+    op.create_index('_object_class_object_id_in', 'api_id', ['object_class', 'object_id'], unique=False)
+    op.create_index('_api_id_suffix_index', 'api_id', ['api_id_suffix'], unique=False)
+    op.drop_constraint(None, 'agent_token', type_='foreignkey')
+    op.drop_column('agent_token', 'api_id_fk')
+    op.drop_constraint(None, 'agent_pool', type_='foreignkey')
+    op.drop_column('agent_pool', 'api_id_fk')
+    op.drop_constraint(None, 'agent', type_='foreignkey')
+    op.drop_column('agent', 'api_id_fk')
+    # ### end Alembic commands ###

terrarun/models/agent.py

@@ -33,6 +33,9 @@ class Agent(Base, BaseObject):
 
     __tablename__ = "agent"
     id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    api_id_fk = sqlalchemy.Column(sqlalchemy.ForeignKey("api_id.id"), nullable=True)
+    api_id_obj = sqlalchemy.orm.relation("ApiId", foreign_keys=[api_id_fk])
+
     name = sqlalchemy.Column(terrarun.database.Database.GeneralString, default=None)
     created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.sql.func.now())
 

terrarun/models/agent_pool.py

@@ -24,6 +24,9 @@ class AgentPool(Base, BaseObject):
     name = sqlalchemy.Column(terrarun.database.Database.GeneralString, default=None)
     created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.sql.func.now())
 
+    api_id_fk = sqlalchemy.Column(sqlalchemy.ForeignKey("api_id.id"), nullable=True)
+    api_id_obj = sqlalchemy.orm.relation("ApiId", foreign_keys=[api_id_fk])
+
     organisation_id = sqlalchemy.Column(sqlalchemy.ForeignKey("organisation.id"), nullable=True)
     organisation = sqlalchemy.orm.relationship("Organisation")
 

terrarun/models/agent_token.py

@@ -20,6 +20,9 @@ class AgentToken(Base, BaseObject):
 
     __tablename__ = "agent_token"
     id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    api_id_fk = sqlalchemy.Column(sqlalchemy.ForeignKey("api_id.id"), nullable=True)
+    api_id_obj = sqlalchemy.orm.relation("ApiId", foreign_keys=[api_id_fk])
+
     description = sqlalchemy.Column(terrarun.database.Database.GeneralString, default=None)
     created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.sql.func.now())
     last_used_at = sqlalchemy.Column(sqlalchemy.DateTime, default=None)

terrarun/models/api_id.py

@@ -0,0 +1,62 @@
+
+import secrets
+import string
+import sqlalchemy
+
+from terrarun.database import Base, Database
+import terrarun.database
+
+
+class ApiId(Base):
+    """DB model for associating a random API ID to an object"""
+
+    __tablename__ = "api_id"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    api_id_suffix = sqlalchemy.Column(terrarun.database.Database.GeneralString, unique=True)
+
+    @classmethod
+    def _generate_api_id(cls):
+        """Generate random ID for object"""
+        alphabet = string.ascii_letters + string.digits
+        return ''.join(secrets.choice(alphabet) for i in range(16))
+
+    @classmethod
+    def get_db_id_from_api_id(cls, target_class, api_id):
+        """Get DB ID from api id"""
+        if len(api_id.split('-')) != 2:
+            return None
+
+        stripped_id = api_id.split('-')[1]
+        if len(stripped_id) != 16:
+            return None
+
+        session = Database.get_session()
+        res = session.query(target_class).join(cls).filter(
+            cls.api_id_suffix==stripped_id
+        ).first()
+        if not res:
+            return None
+
+        return res.id
+
+    @classmethod
+    def get_api_id(cls, obj):
+        """Return api ID for given object"""
+        if not 'ID_PREFIX' in dir(obj) or not obj.ID_PREFIX:
+            raise Exception("Object does not have an ID prefix")
+
+        session = Database.get_session()
+
+        if obj.api_id_obj is None:
+            api_id_object = cls(
+                api_id_suffix=cls._generate_api_id()
+            )
+            session.add(api_id_object)
+            obj.api_id_obj = api_id_object
+            session.add(api_id_object)
+            session.commit()
+
+        return f"{obj.ID_PREFIX}-{obj.api_id_obj.api_id_suffix}"
+
+

terrarun/models/apply.py

@@ -20,6 +20,9 @@ class Apply(TerraformCommand, Base):
     __tablename__ = 'apply'
     id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
 
+    api_id_fk = sqlalchemy.Column(sqlalchemy.ForeignKey("api_id.id"), nullable=True)
+    api_id_obj = sqlalchemy.orm.relation("ApiId", foreign_keys=[api_id_fk])
+
     plan_id = sqlalchemy.Column(sqlalchemy.ForeignKey("plan.id"), nullable=False)
     plan = sqlalchemy.orm.relationship("Plan", back_populates="applies")
 

terrarun/models/audit_event.py

@@ -30,6 +30,9 @@ class AuditEvent(Base, BaseObject):
     __tablename__ = 'audit_event'
 
     id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    api_id_fk = sqlalchemy.Column(sqlalchemy.ForeignKey("api_id.id"), nullable=True)
+    api_id_obj = sqlalchemy.orm.relation("ApiId", foreign_keys=[api_id_fk])
+
     timestamp = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.sql.func.now())
 
     organisation_id = sqlalchemy.Column(sqlalchemy.ForeignKey("organisation.id"), nullable=False)
@@ -71,7 +74,7 @@ def get_api_details(self):
             },
             "relationships": {
                 "user": {
-                    "data": { "id": terrarun.models.user.User.api_id_from_db_id(self.user_id), "type": "users" } if self.user_id else {}
+                    "data": { "id": self.api_id, "type": "users" } if self.user_id else {}
                 }
             },
             "type": "audit-events"