add check to access agent readiness endpoint

This is intended as minimal change to fix

signadot/signadot#5172

tested with warp, it causes restarts and we are able
to connect whereas previously it hangs

still to consider:

- do we need to make this visible in status?
- do we need to make check period configurable?

add check to agent-metrics endpoints in tp monitor

internal/locald/rootmanager/tp_monitor.go

@@ -2,6 +2,7 @@ package rootmanager
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	"log/slog"
@@ -128,16 +129,34 @@ func (mon *tpMonitor) checkTunnelProxyAccess(ctx context.Context) bool {
 			restartSvcs = true
 		}
 	}
+	if !restartSvcs {
+		// the grpc check for connecting to the tunnel proxy does not suffice
+		// because it has built-in retries and may re-use a connection while
+		// we are unable to establish a new connection.  So, we also check
+		// the controller manager health endpoint
+		cli := &http.Client{
+			Transport: &http.Transport{},
+			Timeout:   10 * time.Second,
+		}
+		resp, err := cli.Get("http://agent-metrics.signadot.svc:9090/metrics")
+		if err != nil {
+			mon.log.Error("unable to reach agent-metrics, restarting services", "error", err)
+			restartSvcs = true
+		} else {
+			resp.Body.Close()
+		}
+	}
+	if !restartSvcs {
+		mon.starting = false
+		return true
+	}
 
-	if restartSvcs {
-		// Restart localnet
-		mon.root.stopLocalnetService()
-		mon.root.runLocalnetService(ctx, mon.tpLocalAddr, mon.ipMap)
+	// Restart localnet
+	mon.root.stopLocalnetService()
+	mon.root.runLocalnetService(ctx, mon.tpLocalAddr, mon.ipMap)
 
-		// Restart etc hosts
-		mon.root.stopEtcHostsService()
-		mon.root.runEtcHostsService(ctx, mon.tpLocalAddr, mon.ipMap)
-	}
-	mon.starting = false
-	return true
+	// Restart etc hosts
+	mon.root.stopEtcHostsService()
+	mon.root.runEtcHostsService(ctx, mon.tpLocalAddr, mon.ipMap)
+	return false
 }

internal/locald/sandboxmanager/sdk.go

@@ -17,7 +17,7 @@ import (
 
 var (
 	ErrSandboxManagerUnavailable = errors.New(
-		"sandboxmanager is not running, start it with \"signadot local connect\"")
+		`sandboxmanager is not running, start it with "signadot local connect"`)
 )
 
 func GetStatus() (*sbmapi.StatusResponse, error) {