Create initial module

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+# This workflow releases a new version of the Terraform module.
+
+name: Release
+
+# Only trigger when a pull request into main branch is merged.
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Get module version
+        id: get-version
+        run: echo "version=$(cat VERSION.txt)" >> $GITHUB_OUTPUT
+
+      - name: Create Release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own.
+        with:
+          tag_name: ${{ steps.get-version.outputs.version }}
+          release_name: ${{ github.event.pull_request.title }}
+          body: ${{ github.event.pull_request.body }}
+          draft: false
+          prerelease: false

.github/workflows/update-pull-request.yml

@@ -0,0 +1,18 @@
+# This workflow updates the pull request description with an auto-generated section containing the categorised commit
+# message headers of the pull request's commits. The auto generated section is enveloped between two comments:
+# "<!--- START AUTOGENERATED NOTES --->" and "<!--- END AUTOGENERATED NOTES --->". Anything outside these in the
+# description is left untouched. Auto-generated updates can be skipped for a commit if
+# "<!--- SKIP AUTOGENERATED NOTES --->" is added to the pull request description.
+
+name: update-pull-request
+
+on: [pull_request]
+
+jobs:
+  description:
+    uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      contents: read
+      pull-requests: write

.gitignore

@@ -35,3 +35,5 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+.idea

README.md

@@ -1,2 +1,149 @@
-# terraform-octue-twined-static
-A partner terraform module to terraform-octue-twined providing static resources.
+# terraform-octue-twined-core
+A Terraform module for deploying the core storage and IAM resources for an Octue Twined services network to google cloud.  
+
+
+# Resources
+These resources are automatically deployed:
+- An artifact registry repository for storing Octue Twined service revision docker images
+- A BigQuery table acting as an event store for Twined service events
+- IAM service accounts and roles for:
+  - Any number of maintainers to use with Twined services
+  - GitHub Actions to a) build and push Twined service images to the artifact registry; b) test services
+- A workload identity pool and provider allowing GitHub actions to authenticate with google cloud
+- A cloud storage bucket to store input, output, and diagnostics data for Twined services
+
+
+# Installation and usage
+
+> [!IMPORTANT]
+> Deploying this Terraform module is a prerequisite to deploying the [terraform-octue-twined-cluster](https://github.com/octue/terraform-octue-twined-cluster). 
+> module. You must deploy both to have a cloud-based Octue Twined services network. See [a live example here](https://github.com/octue/twined-infrastructure).
+
+> [!TIP]
+> Deploy this module in a separate Terraform configuration (directory/workspace) to the [terraform-octue-twined-cluster](https://github.com/octue/terraform-octue-twined-cluster)
+> module. This allows the option to spin down the Kubernetes cluster provided by the other module while keeping the core
+> resources that contain all data produced by your Twined services. Spinning the cluster down entirely can save on 
+> running costs in periods of extended non-use while keeping all data available.
+
+Add the below blocks to your Terraform configuration and run:
+```shell
+terraform plan
+```
+
+If you're happy with the plan, run:
+```shell
+terraform apply
+```
+and approve the run.
+
+## Minimal configuration
+
+```terraform
+# main.tf
+
+terraform {
+  required_version = ">= 1.8.0"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~>6.12"
+    }
+  }
+}
+
+
+provider "google" {
+  project = var.google_cloud_project_id
+  region  = var.google_cloud_region
+}
+
+
+module "octue_twined_core" {
+  source = "git::github.com/octue/terraform-octue-twined-core.git?ref=0.1.0"
+  google_cloud_project_id          = var.google_cloud_project_id
+  google_cloud_region              = var.google_cloud_region
+  github_account                   = var.github_account
+  maintainer_service_account_names = var.maintainer_service_account_names
+}
+```
+
+```terraform
+# variables.tf
+
+variable "google_cloud_project_id" {
+  type    = string
+  default = "<google-cloud-project-id>"
+}
+
+variable "google_cloud_region" {
+  type    = string
+  default = "<google-cloud-region>"
+}
+
+variable "github_account" {
+  type    = string
+  default = "<your-github-account>"
+}
+
+variable "maintainer_service_account_names" {
+  type    = set(string)
+  default = ["person1", "person2"]
+}
+```
+
+## Dependencies
+- Terraform: `>= 1.8.0, <2`
+- Providers:
+  - `hashicorp/google`: `~>6.12`
+- Google cloud APIs:
+  - The Cloud Resource Manager API must be [enabled manually](https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com) 
+    before using the module
+  - All other required google cloud APIs are enabled automatically by the module 
+
+## Authentication
+The module needs to authenticate with google cloud before it can be used:
+
+1. Create a service account for Terraform and assign it the `editor` and `owner` basic IAM permissions
+2. Download a JSON key file for the service account
+3. If using Terraform Cloud, follow [these instructions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#using-terraform-cloud).
+   before deleting the key file from your computer 
+4. If not using Terraform Cloud, follow [these instructions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#authentication-configuration)
+   or use another [authentication method](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#authentication).
+
+
+## Destruction
+> [!WARNING]
+> If the `deletion_protection` input is set to `true`, it must first be set to `false` and `terraform apply` run before 
+> running `terraform destroy` or any other operation that would result in the destruction or replacement of the cloud 
+> storage bucket or event store BigQuery table. Not doing this can lead to a state needing targeted Terraform commands 
+> and/or manual configuration changes to recover from.
+
+Disable `deletion_protection` and run:
+```shell
+terraform destroy
+```
+
+
+# Input reference
+
+| Name                                | Type          | Required | Default   |
+|-------------------------------------|---------------|----------|-----------| 
+| `google_cloud_project_id`           | `string`      | Yes      | N/A       |  
+| `google_cloud_region`               | `string`      | Yes      | N/A       | 
+| `github_account`                    | `string`      | Yes      | N/A       |                 
+| `maintainer_service_account_names`  | `set(string)` | Yes      | N/A       | 
+| `deletion_protection`               | `bool`        | No       | `true`    | 
+
+See [`variables.tf`](/variables.tf) for descriptions.
+
+
+# Output reference
+
+| Name                                | Type     |
+|-------------------------------------|----------|
+| `artifact_registry_repository_name` | `string` | 
+| `storage_bucket_url`                | `string` | 
+| `event_store_id`                    | `string` | 
+
+See [`outputs.tf`](/outputs.tf) for descriptions.

VERSION.txt

@@ -0,0 +1 @@
+0.1.0

artifact_registry.tf

@@ -0,0 +1,7 @@
+resource "google_artifact_registry_repository" "service_docker_images" {
+  repository_id = "octue-twined-services"
+  description   = "Docker image repository for Octue Twined services"
+  format        = "DOCKER"
+  location      = var.google_cloud_region
+  depends_on    = [time_sleep.wait_for_google_apis_to_enable]
+}

bigquery.tf

@@ -0,0 +1,99 @@
+resource "google_bigquery_dataset" "service_event_dataset" {
+  dataset_id  = "octue_twined"
+  description = "A dataset for storing Octue Twined service events."
+  location    = var.google_cloud_region
+  depends_on  = [time_sleep.wait_for_google_apis_to_enable]
+}
+
+resource "google_bigquery_table" "service_event_table" {
+  table_id            = "service-events"
+  dataset_id          = google_bigquery_dataset.service_event_dataset.dataset_id
+  clustering          = ["sender", "question_uuid"]
+  depends_on          = [time_sleep.wait_for_google_apis_to_enable]
+  deletion_protection = var.deletion_protection
+
+  schema = <<EOF
+[
+  {
+    "name": "datetime",
+    "type": "TIMESTAMP",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "uuid",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "kind",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "event",
+    "type": "JSON",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "other_attributes",
+    "type": "JSON",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "originator",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "parent",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "sender",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "sender_type",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "sender_sdk_version",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "recipient",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "originator_question_uuid",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "parent_question_uuid",
+    "type": "STRING",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "question_uuid",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "backend",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "backend_metadata",
+    "type": "JSON",
+    "mode": "REQUIRED"
+  }
+]
+EOF
+}

google_apis.tf

@@ -0,0 +1,23 @@
+locals {
+  apis = toset(
+    [
+      "artifactregistry.googleapis.com", # Artifact Registry is used to store docker images of the Twined services.
+      "bigquery.googleapis.com",         # BigQuery provides the event store for Twined service events (questions, results, log messages etc.).
+      "iam.googleapis.com",              # IAM provides fine-grained authentication and authorisation to use and access the Twined services and input/output data.
+    ]
+  )
+}
+
+
+resource "google_project_service" "google_apis" {
+  for_each                   = local.apis
+  service                    = each.key
+  disable_dependent_services = true
+  project                    = var.google_cloud_project_id
+}
+
+
+resource "time_sleep" "wait_for_google_apis_to_enable" {
+  depends_on      = [google_project_service.google_apis]
+  create_duration = "1m"
+}

iam_roles_github_actions.tf

@@ -0,0 +1,19 @@
+resource "google_project_iam_member" "github_actions__roles" {
+  for_each = toset(
+    [
+      "roles/iam.serviceAccountUser",
+      "roles/pubsub.editor",
+      "roles/errorreporting.writer",
+      "roles/artifactregistry.writer",
+      "roles/storage.objectAdmin",
+      # Allows the GHA to call "namespaces get" for Cloud Run to determine the resulting run URLs of the services.
+      # This should also allow a service to get its own name by using:
+      #   https://stackoverflow.com/questions/65628822/google-cloud-run-can-a-service-know-its-own-url/65634104#65634104
+      "roles/run.developer",
+    ]
+  )
+  project    = var.google_cloud_project_id
+  role       = each.value
+  member     = "serviceAccount:${google_service_account.github_actions_service_account.email}"
+  depends_on = [time_sleep.wait_for_google_apis_to_enable]
+}