Add some tests, refactor queries to be faster/more useable

Author: janboll

test/cmd/aro-hcp-tests/custom-link-tools/artifacts/custom-link-tools.tmpl

@@ -72,7 +72,7 @@
       {{range $index, $element := .Elements}}
       <tr class="status-{{$element.Status}}">
         <td>{{$element.TestName}}</td>
-        <td>tbd</td>
+        <td>{{$element.Status}}</td>
         <td>{{$element.ResourceGroupName}}</td>
         <td>
           {{range $element.Links}}

test/cmd/aro-hcp-tests/custom-link-tools/artifacts/hosted-controlplane.kql.tmpl

@@ -1,27 +1,18 @@
-// Recommended is to first get the cluster ID from the service logs, then hard code it in the below query.
-// Alternative query to get the cluster ID
+let _startTime = datetime("{{.StartTime}}");
+let _endTime = datetime("{{.EndTime}}");
+let _resource_group = '{{.ResourceGroupName}}';
 database('ServiceLogs').table('containerLogs')
-| where log has '{{.ResourceGroupName}}'
+| where timestamp between (_startTime .. _endTime)
+| where log has _resource_group
 | extend cid=extract(@"cid='([a-v0-9]{32})'", 1, tostring(log)) 
 | where strlen( cid) > 0
 | distinct cid
 
 // Copy the cid from the above query to the below query to get the hosted control plane logs
 let cid = 'paste value here';
+let _startTime = datetime("{{.StartTime}}");
+let _endTime = datetime("{{.EndTime}}");
 database('HostedControlPlaneLogs').table('containerLogs')
+| where timestamp between (_startTime .. _endTime)
 | where namespace_name has cid
 | project timestamp, log
-
-
-// Alternative query to get the cluster ID, slower, but only one query.
-set notruncation;
- database('ServiceLogs').table('containerLogs')
-    | where log has '{{.ResourceGroupName}}'
-    | extend cid=extract(@"cid='([a-v0-9]{32})'", 1, tostring(log)) 
-    | where strlen( cid) > 0
-    | distinct cid
-    | join  kind=inner  (
-        database('HostedControlPlaneLogs').table('containerLogs')
-    | extend cid=extract(@"([a-v0-9]{32})", 1, tostring(namespace_name))
-    )
-    on $left.cid ==   $right.cid  

test/cmd/aro-hcp-tests/custom-link-tools/artifacts/service-logs.kql.tmpl

@@ -1,18 +1,37 @@
-// Get the service logs for the given resource group
-database('ServiceLogs').table('containerLogs')
-| where log has '{{.ResourceGroupName}}'
-
-
-// Get Frontend logs for the given resource group
-database('ServiceLogs').table('frontendLogs')
-| where log has '{{.ResourceGroupName}}'
-
-
-// Get Backend logs for the given resource group
-database('ServiceLogs').table('backendLogs')
-| where log has '{{.ResourceGroupName}}'
-
-
-// Get ClustersService logs for the given resource group
-database('ServiceLogs').table('clustersServiceLogs')
-| where log has '{{.ResourceGroupName}}'
+let _startTime = datetime("{{.StartTime}}");
+let _endTime = datetime("{{.EndTime}}");
+let _resource_group = '{{.ResourceGroupName}}';
+let cluster_resource_id = (database('ServiceLogs').table('frontendLogs')
+| where timestamp between (_startTime .. _endTime)
+| where resource_group == _resource_group
+| where msg has 'creating resource'
+| extend resource_id=extract(@"creating resource ([A-Za-z0-9\-\.\/]+)", 1, tostring(log)) 
+| distinct tostring(resource_id)
+| limit 1);
+union
+    (
+        database('ServiceLogs').table('backendLogs')
+        | where timestamp between (_startTime .. _endTime)
+        | where resource_id in (cluster_resource_id)
+        | project timestamp, container_name, msg
+    ),
+    (
+        database('ServiceLogs').table('frontendLogs')
+        | where timestamp between (_startTime .. _endTime)
+        | where resource_group == _resource_group
+        | project timestamp, container_name, msg
+    ),
+    ( // for now, we can parse
+        database('ServiceLogs').table('containerLogs')
+        | where timestamp between (_startTime .. _endTime)
+        | where namespace_name == 'clusters-service'
+        | extend resource_id=extract(@"resource_id='([^\']+)'", 1, tostring(log)) 
+        | where resource_id in (cluster_resource_id)
+        | project timestamp, container_name, log
+    ),
+    ( // in the future, we query
+        database('ServiceLogs').table('clustersServiceLogs')
+        | where timestamp between (_startTime .. _endTime)
+        | where resource_id in (cluster_resource_id)
+        | project timestamp, container_name, msg
+    )

test/cmd/aro-hcp-tests/custom-link-tools/cmd_test.go

@@ -0,0 +1,41 @@
+// Copyright 2025 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package customlinktools
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/testr"
+
+	"github.com/Azure/ARO-HCP/test/util/testutil"
+)
+
+func TestGeneratedHTML(t *testing.T) {
+	ctx := logr.NewContext(t.Context(), testr.New(t))
+	tmpdir := t.TempDir()
+	opts := Options{
+		completedOptions: &completedOptions{
+			TimingInputDir: "../testdata/output",
+			OutputDir:      tmpdir,
+		},
+	}
+	err := opts.Run(ctx)
+	if err != nil {
+		t.Fatalf("failed to run custom link tools: %v", err)
+	}
+	testutil.CompareFileWithFixture(t, filepath.Join(tmpdir, "custom-link-tools.html"))
+}

test/cmd/aro-hcp-tests/custom-link-tools/options.go

@@ -35,7 +35,6 @@ import (
 
 	"github.com/Azure/ARO-HCP/internal/utils"
 	"github.com/Azure/ARO-HCP/test/util/timing"
-	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
 //go:embed artifacts/*.tmpl
@@ -78,7 +77,6 @@ type ValidatedOptions struct {
 
 // completedOptions is a private wrapper that enforces a call of Complete() before config generation can be invoked.
 type completedOptions struct {
-	Steps          []pipeline.NodeInfo
 	TimingInputDir string
 	OutputDir      string
 }
@@ -124,9 +122,17 @@ type LinkDetails struct {
 
 type QueryInfo struct {
 	ResourceGroupName string
+	StartTime         string
+	EndTime           string
 	Database          string
 }
 
+type TimingInfo struct {
+	StartTime          string
+	EndTime            string
+	ResourceGroupNames []string
+}
+
 func createQueryURL(templatePath string, info QueryInfo) string {
 	currURL := url.URL{
 		Scheme: "https",
@@ -178,20 +184,8 @@ func encodeKustoQuery(query string) string {
 }
 
 func (o *ValidatedOptions) Complete(logger logr.Logger) (*Options, error) {
-	// we consume steps.yaml (output of templatize and stored for us by the visualization) to determine the cluster name
-	stepsYamlBytes, err := os.ReadFile(path.Join(o.TimingInputDir, "steps.yaml"))
-	if err != nil {
-		return nil, utils.TrackError(err)
-	}
-
-	var steps []pipeline.NodeInfo
-	if err := yaml.Unmarshal(stepsYamlBytes, &steps); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal timing input file: %w", err)
-	}
-
 	return &Options{
 		completedOptions: &completedOptions{
-			Steps:          steps,
 			OutputDir:      o.OutputDir,
 			TimingInputDir: o.TimingInputDir,
 		},
@@ -201,27 +195,32 @@ func (o *ValidatedOptions) Complete(logger logr.Logger) (*Options, error) {
 func (o Options) Run(ctx context.Context) error {
 	allTestRows := []TestRow{}
 
-	deploymentResourceGroups, err := gatherResourceGroups(o.TimingInputDir)
+	timingInfo, err := gatherTimingInfo(o.TimingInputDir)
 	if err != nil {
 		return utils.TrackError(err)
 	}
 
-	for testName, rgs := range deploymentResourceGroups {
-		for _, rg := range rgs {
+	for testName, timing := range timingInfo {
+		for _, rg := range timing.ResourceGroupNames {
 			allTestRows = append(allTestRows, TestRow{
 				TestName:          testName,
 				ResourceGroupName: rg,
 				Links: []LinkDetails{
 					createLinkForTest("Hosted Control Plane Logs", "hosted-controlplane.kql.tmpl", QueryInfo{
 						ResourceGroupName: rg,
 						Database:          "HostedControlPlaneLogs",
+						StartTime:         timing.StartTime,
+						EndTime:           timing.EndTime,
 					}),
 					createLinkForTest("Service Logs", "service-logs.kql.tmpl", QueryInfo{
 						ResourceGroupName: rg,
 						Database:          "ServiceLogs",
+						StartTime:         timing.StartTime,
+						EndTime:           timing.EndTime,
 					}),
 				},
 				Database: "HostedControlPlaneLogs",
+				Status:   "tbd",
 			})
 		}
 	}
@@ -259,7 +258,7 @@ func (o Options) Run(ctx context.Context) error {
 	return nil
 }
 
-func gatherResourceGroups(timingInputDir string) (map[string][]string, error) {
+func gatherTimingInfo(timingInputDir string) (map[string]TimingInfo, error) {
 	timingDir, err := os.Stat(path.Join(timingInputDir, "test-timing/"))
 	if err != nil {
 		return nil, err
@@ -269,7 +268,7 @@ func gatherResourceGroups(timingInputDir string) (map[string][]string, error) {
 	}
 
 	var allTimingFiles []string
-	filepath.Walk(path.Join(timingInputDir, "test-timing/"), func(path string, info os.FileInfo, err error) error {
+	err = filepath.Walk(path.Join(timingInputDir, "test-timing/"), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -278,8 +277,11 @@ func gatherResourceGroups(timingInputDir string) (map[string][]string, error) {
 		}
 		return nil
 	})
+	if err != nil {
+		return nil, err
+	}
 
-	var allResourceGroups map[string][]string = make(map[string][]string)
+	var allTimingInfo = make(map[string]TimingInfo)
 
 	for _, timingFile := range allTimingFiles {
 		timingFileBytes, err := os.ReadFile(timingFile)
@@ -293,18 +295,24 @@ func gatherResourceGroups(timingInputDir string) (map[string][]string, error) {
 		}
 		deployment := strings.Join(timing.Identifier, " ")
 
-		var rgNames map[string]bool = make(map[string]bool)
+		var rgNames = make(map[string]bool)
 		for resourceGroup := range timing.Deployments {
 			rgNames[resourceGroup] = true
 		}
 
+		rgNameList := make([]string, 0)
 		for rgName := range rgNames {
 			if rgName == "" {
 				continue
 			}
-			allResourceGroups[deployment] = append(allResourceGroups[deployment], rgName)
+			rgNameList = append(rgNameList, rgName)
+		}
+		allTimingInfo[deployment] = TimingInfo{
+			StartTime:          timing.StartedAt,
+			EndTime:            timing.FinishedAt,
+			ResourceGroupNames: rgNameList,
 		}
 	}
 
-	return allResourceGroups, nil
+	return allTimingInfo, nil
 }