enhancement: support private networking (#87)

<!-- Thank you for submitting a Pull Request. Please fill out the
template below.-->
## Overview/Summary

This PR adds the capabilities for self-hosted agent and private
networking for GitHub and Azure DevOps.

The user can choose between:

- Microsoft-hosted agents, public endpoint (no VNET)
- self-hosted agents, public endpoint (no VNET)
- self-hosted agents, private endpoint (yes VNET)

## This PR fixes/adds/changes/removes

1. #14 
2. #38 
3. #28

### Breaking Changes

This will be a major version and various variables have been updated and
removed.

## Testing Evidence

e2e tests run. These tests have now been extended to run the pipelines
and actions, so now provide full coverage of all functionality.

## As part of this Pull Request I have

- [x] Checked for duplicate [Pull
Requests](https://github.com/Azure/alz-terraform-accelerator/pulls)
- [x] Associated it with relevant
[issues](https://github.com/Azure/alz-terraform-accelerator/issues), for
tracking and closure.
- [x] Ensured my code/branch is up-to-date with the latest changes in
the `main`
[branch](https://github.com/Azure/alz-terraform-accelerator/tree/main)
- [x] Performed testing and provided evidence.
- [x] Updated relevant and associated documentation.

.github/tests/cleanup-scripts/cleanup_github-repositories.ps1

@@ -0,0 +1,15 @@
+# This file can be used to clean up GitHub repositories if there has been an issue with the End to End tests.
+# CAUTION: Make sure you are connected to the correct organization before running this script!
+$repos = gh repo list microsoft-azure-landing-zones-cd-tests --json name,owner | ConvertFrom-Json
+
+$repos | ForEach-Object -Parallel {
+    $match = "*229*"
+    $repoName = "$($_.owner.login)/$($_.name)"
+
+    if($repoName -like $match)
+    {
+        Write-Host "Deleting repo: $repoName"
+        gh repo delete $repoName --yes
+
+    }
+} -ThrottleLimit 10

.github/tests/cleanup-scripts/cleanup_resouce_groups.ps1

@@ -0,0 +1,9 @@
+# This file can be used to clean up Resource Groups if there has been an issue with the End to End tests.
+# CAUTION: Make sure you are connected to the correct subscription before running this script!
+az account show
+$resourceGroups = az group list --query "[?contains(name, '254-')]" | ConvertFrom-Json
+
+$resourceGroups | ForEach-Object -Parallel {
+    Write-Host "Deleting resource group: $($_.name)"
+    az group delete --name $_.name --yes
+} -ThrottleLimit 10

.github/tests/scripts/azuredevops-pipeline-run.ps1

@@ -0,0 +1,159 @@
+param (
+    [string]$organizationName,
+    [string]$projectName,
+    [string]$personalAccessToken,
+    [string]$pipelineNamePart = "Continuous Delivery",
+    [switch]$skipDestroy = $false,
+    [int]$maximumRetries = 10,
+    [int]$retryCount = 0,
+    [int]$retryDelay = 10000
+)
+
+function Invoke-Pipeline {
+    param (
+        [string]$organizationName,
+        [string]$projectName,
+        [int]$pipelineId,
+        [string]$pipelineAction = "",
+        [hashtable]$headers
+    )
+    $pipelineDispatchUrl = "https://dev.azure.com/$organizationName/$projectName/_apis/pipelines/$pipelineId/runs?api-version=7.2-preview.1"
+    Write-Host "Pipeline Dispatch URL: $pipelineDispatchUrl"
+
+    $pipelineDispatchBody = @{}
+    if($pipelineAction -eq "") {
+        $pipelineDispatchBody = @{
+            "resources" = @{
+                "repositories" = @{
+                    "self" = @{
+                        "refName" = "refs/heads/main"
+                    }
+                }
+            }
+        } | ConvertTo-Json -Depth 100
+    } else {
+        $pipelineDispatchBody = @{
+            "resources" = @{
+                "repositories" = @{
+                    "self" = @{
+                        "refName" = "refs/heads/main"
+                    }
+                }
+            }
+            "templateParameters" = @{
+                "terraform_action" = $pipelineAction
+            }
+        } | ConvertTo-Json -Depth 100
+    }
+
+    $result = Invoke-RestMethod -Method POST -Uri $pipelineDispatchUrl -Headers $headers -Body $pipelineDispatchBody -StatusCodeVariable statusCode -ContentType "application/json"
+    if ($statusCode -ne 200) {
+        throw "Failed to dispatch the pipeline."
+    }
+
+    # Get the pipeline run id
+    $pipelineRunId = $result.id
+    Write-Host "Pipeline Run ID: $pipelineRunId"
+    return [int]$pipelineRunId
+}
+
+function Wait-ForPipelineRunToComplete {
+    param (
+        [string]$organizationName,
+        [string]$projectName,
+        [int]$pipelineId,
+        [int]$pipelineRunId,
+        [hashtable]$headers
+    )
+
+    $pipelineRunUrl = "https://dev.azure.com/$organizationName/$projectName/_apis/pipelines/$pipelineId/runs/$($pipelineRunId)?api-version=7.2-preview.1"
+    Write-Host "Pipeline Run URL: $pipelineRunUrl"
+
+    $pipelineRunStatus = ""
+    $pipelineRunResult = ""
+    while($pipelineRunStatus -ne "completed") {
+        Start-Sleep -Seconds 10
+        $pipelineRun = Invoke-RestMethod -Method GET -Uri $pipelineRunUrl -Headers $headers -StatusCodeVariable statusCode
+        if ($statusCode -lt 300) {
+            $pipelineRunStatus = $pipelineRun.state
+            $pipelineRunResult = $pipelineRun.result
+            Write-Host "Pipeline Run Status: $pipelineRunStatus - Conclusion: $pipelineRunResult"
+        } else {
+            Write-Host "Failed to find the pipeline run. Status Code: $statusCode"
+            throw "Failed to find the pipeline run."
+        }
+    }
+
+    if($pipelineRunResult -ne "succeeded") {
+        throw "The pipeline run did not complete successfully. Conclusion: $pipelineRunResult"
+    }
+}
+
+# Setup Variables
+$token = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$personalAccessToken"))
+$headers = @{
+    "Authorization" = "Basic $token"
+    "Accept" = "application/json"
+}
+
+# Run the Module in a retry loop
+$success = $false
+
+do {
+$retryCount++
+try {
+    # Get the pipeline id
+    Write-Host "Getting the pipeline id"
+    $pipelinesUrl = "https://dev.azure.com/$organizationName/$projectName/_apis/pipelines?api-version=7.2-preview.1"
+    Write-Host "Pipelines URL: $pipelinesUrl"
+    $pipelines = Invoke-RestMethod -Method GET -Uri $pipelinesUrl -Headers $headers -StatusCodeVariable statusCode
+    if ($statusCode -ne 200) {
+        throw "Failed to find the pipelines."
+    }
+    $pipeline = $pipelines.value | Where-Object { $_.name -like "*$pipelineNamePart*" }
+
+    $pipelineId = $pipeline.id
+    Write-Host "Pipeline ID: $pipelineId"
+
+    $pipelineAction = ""
+    if(!($skipDestroy)) {
+        $pipelineAction = "apply"
+    }
+
+    # Trigger the apply pipeline
+    Write-Host "Triggering the $pipelineAction pipeline"
+    $pipelineRunId = Invoke-Pipeline -organizationName $organizationName -projectName $projectName -pipelineId $pipelineId -pipelineAction $pipelineAction -headers $headers
+    Write-Host "$pipelineAction pipeline triggered successfully"
+
+    # Wait for the apply pipeline to complete
+    Write-Host "Waiting for the $pipelineAction pipeline to complete"
+    Wait-ForPipelineRunToComplete -organizationName $organizationName -projectName $projectName -pipelineId $pipelineId -pipelineRunId $pipelineRunId -headers $headers
+    Write-Host "$pipelineAction pipeline completed successfully"
+
+    if($skipDestroy) {
+        $success = $true
+        break
+    }
+
+    $pipelineAction = "destroy"
+
+    # Trigger the destroy pipeline
+    Write-Host "Triggering the $pipelineAction pipeline"
+    $pipelineRunId = Invoke-Pipeline -organizationName $organizationName -projectName $projectName -pipelineId $pipelineId -pipelineAction "destroy" -headers $headers
+    Write-Host "$pipelineAction pipeline triggered successfully"
+
+    # Wait for the apply pipeline to complete
+    Write-Host "Waiting for the $pipelineAction pipeline to complete"
+    Wait-ForPipelineRunToComplete -organizationName $organizationName -projectName $projectName -pipelineId $pipelineId -pipelineRunId $pipelineRunId -headers $headers
+    Write-Host "$pipelineAction pipeline completed successfully"
+
+    $success = $true
+} catch {
+    Write-Host $_
+    Write-Host "Failed to trigger the pipeline successfully, trying again..."
+}
+} while ($success -eq $false -and $retryCount -lt $maximumRetries)
+
+if ($success -eq $false) {
+    throw "Failed to trigger the pipeline after $maximumRetries attempts."
+}

tests/scripts/destroy.ps1 → .github/tests/scripts/destroy.ps1

@@ -31,6 +31,9 @@ $success = $false
 do {
     $retryCount++
     try {
+        $myIp = Invoke-RestMethod -Uri http://ipinfo.io/json | Select-Object -ExpandProperty ip
+        Write-Host "Runner IP Address: $myIp"
+
         Write-Host "Running Terraform Destroy"
         terraform -chdir="$bootstrapDirectoryPath" destroy -auto-approve -var-file="override.tfvars"
         if ($LastExitCode -eq 0) {

.github/tests/scripts/github-action-run.ps1

@@ -0,0 +1,140 @@
+param (
+    [string]$organizationName,
+    [string]$repositoryName,
+    [string]$personalAccessToken,
+    [string]$workflowFileName = "cd.yaml",
+    [switch]$skipDestroy = $false,
+    [int]$maximumRetries = 10,
+    [int]$retryCount = 0,
+    [int]$retryDelay = 10000
+)
+
+function Invoke-Workflow {
+    param (
+        [string]$organizationName,
+        [string]$repositoryName,
+        [string]$workflowId,
+        [string]$workflowAction = "",
+        [hashtable]$headers
+    )
+    $workflowDispatchUrl = "https://api.github.com/repos/$organizationName/$repositoryName/actions/workflows/$workflowId/dispatches"
+    Write-Host "Workflow Dispatch URL: $workflowDispatchUrl"
+
+    $workflowDispatchBody = @{}
+    if($workflowAction -eq "") {
+        $workflowDispatchBody = @{
+            ref = "main"
+        } | ConvertTo-Json -Depth 100
+    } else {
+        $workflowDispatchBody = @{
+            ref = "main"
+            inputs = @{
+                terraform_action = $workflowAction
+            }
+        } | ConvertTo-Json -Depth 100
+    }
+
+    $result = Invoke-RestMethod -Method POST -Uri $workflowDispatchUrl -Headers $headers -Body $workflowDispatchBody -StatusCodeVariable statusCode
+    if ($statusCode -ne 204) {
+        throw "Failed to dispatch the workflow. $result"
+    }
+}
+
+function Wait-ForWorkflowRunToComplete {
+    param (
+        [string]$organizationName,
+        [string]$repositoryName,
+        [hashtable]$headers
+    )
+
+    $workflowRunUrl = "https://api.github.com/repos/$organizationName/$repositoryName/actions/runs"
+    Write-Host "Workflow Run URL: $workflowRunUrl"
+
+    $workflowRunStatus = ""
+    $workflowRunConclusion = ""
+    while($workflowRunStatus -ne "completed") {
+        Start-Sleep -Seconds 10
+
+        $workflowRun = Invoke-RestMethod -Method GET -Uri $workflowRunUrl -Headers $headers -StatusCodeVariable statusCode
+        if ($statusCode -lt 300) {
+            $workflowRunStatus = $workflowRun.workflow_runs[0].status
+            $workflowRunConclusion = $workflowRun.workflow_runs[0].conclusion
+            Write-Host "Workflow Run Status: $workflowRunStatus - Conclusion: $workflowRunConclusion"
+        } else {
+            Write-Host "Failed to find the workflow run. Status Code: $statusCode"
+            throw "Failed to find the workflow run."
+        }
+    }
+
+    if($workflowRunConclusion -ne "success") {
+        throw "The workflow run did not complete successfully. Conclusion: $workflowRunConclusion"
+    }
+}
+
+# Setup Variables
+$token = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$personalAccessToken"))
+$headers = @{
+    "Authorization" = "Basic $token"
+    "Accept" = "application/vnd.github+json"
+}
+
+# Run the Module in a retry loop
+$success = $false
+
+do {
+$retryCount++
+try {
+    # Get the workflow id
+    Write-Host "Getting the workflow id"
+    $workflowUrl = "https://api.github.com/repos/$organizationName/$repositoryName/actions/workflows/$workflowFileName"
+    Write-Host "Workflow URL: $workflowUrl"
+    $workflow = Invoke-RestMethod -Method GET -Uri $workflowUrl -Headers $headers -StatusCodeVariable statusCode
+    if ($statusCode -ne 200) {
+        throw "Failed to find the workflow."
+    }
+    $workflowId = $workflow.id
+    Write-Host "Workflow ID: $workflowId"
+
+    $workflowAction = ""
+
+    if(!($skipDestroy)) {
+        $workflowAction = "apply"
+    }
+
+    # Trigger the apply workflow
+    Write-Host "Triggering the $workflowAction workflow"
+    Invoke-Workflow -organizationName $organizationName -repositoryName $repositoryName -workflowId $workflowId -workflowAction $workflowAction -headers $headers
+    Write-Host "$workflowAction workflow triggered successfully"
+
+    # Wait for the apply workflow to complete
+    Write-Host "Waiting for the $workflowAction workflow to complete"
+    Wait-ForWorkflowRunToComplete -organizationName $organizationName -repositoryName $repositoryName -headers $headers
+    Write-Host "$workflowAction workflow completed successfully"
+
+    if($skipDestroy) {
+        $success = $true
+        break
+    }
+
+    $workflowAction = "destroy"
+
+    # Trigger the destroy workflow
+    Write-Host "Triggering the $workflowAction workflow"
+    Invoke-Workflow -organizationName $organizationName -repositoryName $repositoryName -workflowId $workflowId -workflowAction $workflowAction -headers $headers
+    Write-Host "$workflowAction workflow triggered successfully"
+
+    # Wait for the apply workflow to complete
+    Write-Host "Waiting for the $workflowAction workflow to complete"
+    Wait-ForWorkflowRunToComplete -organizationName $organizationName -repositoryName $repositoryName -headers $headers
+    Write-Host "$workflowAction workflow completed successfully"
+
+    $success = $true
+} catch {
+    Write-Host $_
+    Write-Host "Failed to trigger the workflow successfully, trying again..."
+}
+} while ($success -eq $false -and $retryCount -lt $maximumRetries)
+
+if ($success -eq $false) {
+    throw "Failed to trigger the workflow after $maximumRetries attempts."
+}