Add initial cron task to check VCS providers for new commits on branch.

Add git commit to configuration version DB table.

Issue #80

Author: MatthewJohn

cron_tasks.py

@@ -0,0 +1,6 @@
+
+from terrarun.cron_tasks import CronTasks
+
+
+cron_tasks = CronTasks()
+cron_tasks.start()

docker-compose.yml

@@ -85,16 +85,14 @@ services:
     volumes:
       - ./:/app
 
-  agent:
+  cron-tasks:
     restart: unless-stopped
     build: .
     networks:
       - web
-    depends_on:
-      - traefik
     env_file:
       - ./.env
-    entrypoint: ["python", "-u", "./agent.py", "--address", "$BASE_URL", "--token", "$AGENT_TOKEN"]
+    entrypoint: ["python", "-u", "./cron_tasks.py"]
 
     # Only for local development
     volumes:
@@ -142,6 +140,22 @@ services:
       exit 0;
       "
 
+  # Example custom agent
+  agent:
+    restart: unless-stopped
+    build: .
+    networks:
+      - web
+    depends_on:
+      - traefik
+    env_file:
+      - ./.env
+    entrypoint: ["python", "-u", "./agent.py", "--address", "$BASE_URL", "--token", "$AGENT_TOKEN"]
+
+    # Only for local development
+    volumes:
+      - ./:/app
+
   # Example run task application
   tfsec-task:
     build: https://gitlab.dockstudios.co.uk/pub/terra/tfcloud-tfsec.git

requirements.txt

@@ -10,3 +10,4 @@ mysql-connector-python==8.0.29
 boto3==1.26.70
 botocore==1.29.70
 cryptography==40.0.2
+schedule==1.2.0

terrarun/alembic/versions/90149a6955df_add_git_commit_sha_field_to_.py

@@ -0,0 +1,28 @@
+"""Add git commit sha field to configuration version
+
+Revision ID: 90149a6955df
+Revises: 9afeb4e61715
+Create Date: 2023-05-08 15:31:37.174335
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '90149a6955df'
+down_revision = '9afeb4e61715'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('configuration_version', sa.Column('git_commit_sha', sa.String(length=128), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('configuration_version', 'git_commit_sha')
+    # ### end Alembic commands ###

terrarun/cron_tasks.py

@@ -0,0 +1,103 @@
+
+import signal
+import time
+
+import schedule
+from terrarun.database import Database
+
+from terrarun.models.authorised_repo import AuthorisedRepo
+from terrarun.models.configuration import ConfigurationVersion
+from terrarun.models.run import Run
+
+
+class CronTasks:
+    """Interface to start cron tasks."""
+
+    def __init__(self):
+        """Store member variables"""
+        self._running = True
+        schedule.every(60).seconds.do(self.check_for_vcs_commits)
+
+    def stop(self):
+        """Mark as stopped, stopping any further jobs from executing"""
+        self._running = False
+
+    def start(self):
+        """Start scheduler"""
+        signal.signal(signal.SIGINT, self.stop)
+        #signal.pause()
+        while self._running:
+            schedule.run_pending()
+            time.sleep(1)
+
+    def _process_authorised_repo_workspace(self, authorised_repo, workspace, branch_shas):
+        """Handle checking workspace for new commits to create run for"""
+        print(f'Handling workspace: {workspace.name}')
+        service_provider = authorised_repo.oauth_token.oauth_client.service_provider_instance
+        workspace_branch = workspace.get_branch()
+
+        # Obtain latest sha for branch, if not already cached
+        if workspace_branch not in branch_shas:
+            branch_shas[workspace_branch] =  service_provider.get_latest_commit_ref(
+                authorised_repo=workspace.authorised_repo, branch=workspace_branch
+            )
+        if not branch_shas[workspace_branch]:
+            print(f'Could not find latest commit for branch: {workspace_branch}')
+            return branch_shas
+
+        if not ConfigurationVersion.get_configuration_version_by_git_commit_sha(
+                workspace=workspace,
+                git_commit_sha=branch_shas[workspace_branch]):
+            # If there is not a configuration version for the git commit,
+            # create one
+            cv = ConfigurationVersion.generate_from_vcs(
+                workspace=workspace,
+                commit_ref=branch_shas[workspace_branch],
+                # Allow all runs to be queued to be applied
+                speculative=False
+            )
+            if not cv:
+                print('Unable to create configuration version')
+                return branch_shas
+
+            # Create run
+            Run.create(
+                configuration_version=cv,
+                created_by=None,
+                is_destroy=False,
+                refresh=True,
+                refresh_only=False,
+                auto_apply=workspace.auto_apply,
+                plan_only=False
+            )
+        return branch_shas
+
+    def check_for_vcs_commits(self):
+        """Check for new commits on VCS repositories"""
+        # Iterate over all authorised repos that have one workspace or project defined
+        for authorised_repo in AuthorisedRepo.get_all_utilised_repos():
+            print(f'Handling repo: {authorised_repo.name}')
+            branch_shas = {}
+
+            for project in authorised_repo.projects:
+                print(f'Handling project: {project.name}')
+
+                for workspace in project.workspaces:
+                    branch_shas = self._process_authorised_repo_workspace(
+                        authorised_repo=authorised_repo,
+                        workspace=workspace,
+                        branch_shas=branch_shas
+                    )
+
+            # Handle direct member workspaces
+            for workspace in authorised_repo.workspaces:
+                branch_shas = self._process_authorised_repo_workspace(
+                    authorised_repo=authorised_repo,
+                    workspace=workspace,
+                    branch_shas=branch_shas
+                )
+
+        # Clear database session to avoid cached queries
+        Database.get_session().remove()
+
+        print("Checking for VCS commits")

terrarun/models/authorised_repo.py

@@ -83,6 +83,14 @@ def get_by_external_id(cls, oauth_token, external_id):
         session = Database.get_session()
         return session.query(cls).filter(cls.oauth_token==oauth_token, cls.external_id==external_id).first()
 
+    @classmethod
+    def get_all_utilised_repos(cls):
+        """Get all repos that are used by a workspace or project"""
+        session = Database.get_session()
+        return session.query(cls).filter(
+            sqlalchemy.or_(cls.projects.any(), cls.workspaces.any())
+        ).all()
+
     @property
     def vendor_configuration(self):
         """Return vendor configuration"""

terrarun/models/configuration.py

@@ -49,13 +49,16 @@ class ConfigurationVersion(Base, BaseObject):
     auto_queue_runs = sqlalchemy.Column(sqlalchemy.Boolean)
     status = sqlalchemy.Column(sqlalchemy.Enum(ConfigurationVersionStatus))
 
+    git_commit_sha = sqlalchemy.Column(Database.GeneralString, nullable=True)
+
     @classmethod
-    def create(cls, workspace, auto_queue_runs=True, speculative=False):
+    def create(cls, workspace, auto_queue_runs=True, speculative=False, git_commit_sha=None):
         """Create configuration and return instance."""
         cv = ConfigurationVersion(
             workspace=workspace,
             speculative=speculative,
             auto_queue_runs=auto_queue_runs,
+            git_commit_sha=git_commit_sha,
             status=ConfigurationVersionStatus.PENDING
         )
         session = Database.get_session()
@@ -67,22 +70,19 @@ def create(cls, workspace, auto_queue_runs=True, speculative=False):
         return cv
 
     @classmethod
-    def generate_from_vcs(cls, workspace, speculative):
+    def generate_from_vcs(cls, workspace, speculative, commit_ref=None):
         """Create configuration version from VCS"""
         service_provider = workspace.authorised_repo.oauth_token.oauth_client.service_provider_instance
 
-        # Obtain branch from workspace
-        branch = workspace.vcs_repo_branch
-        # If it doesn't exist, obtain default branch from repository
-        if not branch:
-            branch = service_provider.get_default_branch(authorised_repo=workspace.authorised_repo)
-
-        if not branch:
-            return None
-
-        commit_ref = service_provider.get_latest_commit_ref(
-            authorised_repo=workspace.authorised_repo, branch=branch
-        )
+        if commit_ref is None:
+            # Obtain branch from workspace
+            branch = workspace.get_branch()
+            if not branch:
+                return None
+
+            commit_ref = service_provider.get_latest_commit_ref(
+                authorised_repo=workspace.authorised_repo, branch=branch
+            )
         if commit_ref is None:
             return None
 
@@ -94,12 +94,22 @@ def generate_from_vcs(cls, workspace, speculative):
 
         configuration_version = cls.create(
             workspace=workspace,
-            auto_queue_runs=False,
-            speculative=speculative
+            auto_queue_runs=True,
+            speculative=speculative,
+            git_commit_sha=commit_ref
         )
         configuration_version.process_upload(archive_data)
         return configuration_version
 
+    @classmethod
+    def get_configuration_version_by_git_commit_sha(cls, workspace, git_commit_sha):
+        """Return configuration versions by workspace and git commit sha"""
+        session = Database.get_session()
+        return session.query(cls).filter(
+            cls.workspace==workspace,
+            cls.git_commit_sha==git_commit_sha
+        ).all()
+
     @property
     def plan_only(self):
         """Return whether only a plan."""

terrarun/models/run.py

@@ -107,6 +107,8 @@ class Run(Base, BaseObject):
 
     @property
     def replace_addrs(self):
+        if not self._replace_addrs:
+            return []
         return json.loads(self._replace_addrs)
 
     @replace_addrs.setter
@@ -115,6 +117,8 @@ def replace_addrs(self, value):
 
     @property
     def target_addrs(self):
+        if not self._target_addrs:
+            return []
         return json.loads(self._target_addrs)
 
     @target_addrs.setter

terrarun/models/task_result.py

@@ -184,7 +184,7 @@ def generate_payload(self):
             "run_id": self.task_stage.run.api_id,
             "run_message": self.task_stage.run.message,
             "run_created_at": datetime_to_json(self.task_stage.run.created_at),
-            "run_created_by": self.task_stage.run.created_by.username,
+            "run_created_by": self.task_stage.run.created_by.username if self.task_stage.run.created_by else None,
             "workspace_id": self.task_stage.run.configuration_version.workspace.api_id,
             "workspace_name": self.task_stage.run.configuration_version.workspace.name,
             "workspace_app_url": f"{config.BASE_URL}/app/{self.task_stage.run.configuration_version.workspace.organisation.name}/{self.task_stage.run.configuration_version.workspace.name}",

terrarun/models/workspace.py

@@ -394,6 +394,18 @@ def assessments_enabled(self, value):
         """Set assessments_enabled"""
         self._assessments_enabled = value
 
+    def get_branch(self):
+        """Get branch, either defined in workspace, project or default VCS branch"""
+        # Obtain branch from workspace
+        branch = self.vcs_repo_branch
+
+        # If it doesn't exist, obtain default branch from repository
+        if not branch and self.authorised_repo:
+            branch = self.authorised_repo.oauth_token.oauth_client.service_provider_instance.get_default_branch(
+                authorised_repo=self.authorised_repo
+            )
+        return branch
+
     def check_vcs_repo_update_from_request(self, vcs_repo_attributes):
         """Update VCS repo from request"""
         # Check if VCS is defined in project