feat(core): add otelto opentdf services

docker-compose.yaml

@@ -85,3 +85,16 @@ services:
       retries: 10
     ports:
       - "5432:5432"
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    environment:
+      COLLECTOR_OTLP_ENABLED: "true"
+    ports:
+      - "16686:16686"   # Web UI
+      - "4317:4317"     # OTLP gRPC
+      - "4318:4318"     # OTLP HTTP
+      - "14250:14250"   # Model/collector gRPC
+    profiles:
+      - tracing
+    restart: always

docs/Configuring.md

@@ -334,3 +334,27 @@ server:
 
 Admins can manage the authorization policy directly in the YAML configuration file. For detailed configuration options, refer to the [Casbin documentation](https://casbin.org/docs/en/syntax-for-models).
 
+## Tracing Configuration
+
+The tracing configuration controls OpenTelemetry tracing behavior.
+
+Root level key `trace`
+
+| Field           | Description                                                | Default  | Environment Variable |
+|-----------------|------------------------------------------------------------| ---------|---------------------|
+| `enabled`       | Enable OpenTelemetry tracing                               | `false`  | OPENTDF_TRACE_ENABLED |
+| `folder`        | Directory where trace logs will be stored                  | `traces` | OPENTDF_TRACE_FOLDER |
+| `exportToJaeger`| Export traces to Jaeger instead of local file             | `false`  | OPENTDF_TRACE_EXPORT_TO_JAEGER |
+
+Example:
+```yaml
+trace:
+  enabled: true
+  folder: "traces"  # Traces will be written to traces/traces.log
+  exportToJaeger: false  # Set to true to export to Jaeger at localhost:4317
+```
+
+When enabled, traces are either:
+- Written to a local file with automatic log rotation (when `exportToJaeger: false`)
+- Exported to a Jaeger instance at localhost:4317 (when `exportToJaeger: true`)
+

docs/Contributing.md

@@ -25,6 +25,30 @@ For end-users/consumers, see [here](./Consuming.md).
 Note: support was added to provision a set of fixture data into the database.
 Run `go run github.com/opentdf/platform/service provision fixtures -h` for more information.
 
+## Running with Tracing
+
+To enable distributed tracing with Jaeger:
+
+1. Start the development stack with the tracing profile:
+   ```bash
+   docker compose --profile tracing up
+   ```
+   This will start Jaeger alongside the other services.
+
+2. Configure tracing in your `opentdf.yaml`:
+   ```yaml
+   trace:
+     enabled: true
+     exportToJaeger: true  # This will export traces to Jaeger instead of local files
+   ```
+
+3. Access the Jaeger UI at http://localhost:16686 to view traces.
+   - Search for traces by service name "opentdf-service"
+   - View detailed spans and timing information
+   - Analyze request flows across services
+
+Note: When `exportToJaeger` is false, traces will be written to local files instead of being sent to Jaeger.
+
 ## Advice for Code Contributors
 
 * Follow our [Error Guidelines](./Contributing-errors.md)

opentdf-dev.yaml

@@ -2,6 +2,10 @@ logger:
   level: debug
   type: text
   output: stdout
+trace:
+  enabled: true
+  folder: "traces"
+  exportToJaeger: false
 # DB and Server configurations are defaulted for local development
 # db:
 #   host: localhost

service/authorization/authorization.go

@@ -29,6 +29,9 @@ import (
 	"github.com/opentdf/platform/service/pkg/db"
 	"github.com/opentdf/platform/service/pkg/serviceregistry"
 	"github.com/opentdf/platform/service/policies"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -42,6 +45,7 @@ type AuthorizationService struct { //nolint:revive // AuthorizationService is a
 	config Config
 	logger *logger.Logger
 	eval   rego.PreparedEvalQuery
+	trace.Tracer
 }
 
 type Config struct {
@@ -137,6 +141,7 @@ func NewRegistration() *serviceregistry.Service[authorizationconnect.Authorizati
 				}
 
 				as.config = *authZCfg
+				as.Tracer = srp.Tracer
 
 				return as, nil
 			},
@@ -151,7 +156,15 @@ func (as AuthorizationService) IsReady(ctx context.Context) error {
 }
 
 func (as *AuthorizationService) GetDecisionsByToken(ctx context.Context, req *connect.Request[authorization.GetDecisionsByTokenRequest]) (*connect.Response[authorization.GetDecisionsByTokenResponse], error) {
+	// Extract trace context from the incoming request
+	propagator := otel.GetTextMapPropagator()
+	ctx = propagator.Extract(ctx, propagation.HeaderCarrier(req.Header()))
+
+	ctx, span := as.Tracer.Start(ctx, "GetDecisionsByToken")
+	defer span.End()
+
 	decisionsRequests := []*authorization.DecisionRequest{}
+
 	// for each token decision request
 	for _, tdr := range req.Msg.GetDecisionRequests() {
 		ecResp, err := as.sdk.EntityResoution.CreateEntityChainFromJwt(ctx, &entityresolution.CreateEntityChainFromJwtRequest{Tokens: tdr.GetTokens()})
@@ -184,6 +197,9 @@ func (as *AuthorizationService) GetDecisionsByToken(ctx context.Context, req *co
 func (as *AuthorizationService) GetDecisions(ctx context.Context, req *connect.Request[authorization.GetDecisionsRequest]) (*connect.Response[authorization.GetDecisionsResponse], error) {
 	as.logger.DebugContext(ctx, "getting decisions")
 
+	ctx, span := as.Tracer.Start(ctx, "GetDecisions")
+	defer span.End()
+
 	// Temporary canned echo response with permit decision for all requested decision/entity/ra combos
 	rsp := &authorization.GetDecisionsResponse{
 		DecisionResponses: make([]*authorization.DecisionResponse, 0),
@@ -494,6 +510,9 @@ func makeScopeMap(scope *authorization.ResourceAttribute) map[string]bool {
 func (as *AuthorizationService) GetEntitlements(ctx context.Context, req *connect.Request[authorization.GetEntitlementsRequest]) (*connect.Response[authorization.GetEntitlementsResponse], error) {
 	as.logger.DebugContext(ctx, "getting entitlements")
 
+	ctx, span := as.Tracer.Start(ctx, "GetEntitlements")
+	defer span.End()
+
 	var nextOffset int32
 	attrsList := make([]*policy.Attribute, 0)
 	subjectMappingsList := make([]*policy.SubjectMapping, 0)

service/authorization/authorization_test.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"testing"
 
+	"go.opentelemetry.io/otel/trace/noop"
+
 	"connectrpc.com/connect"
 	"github.com/open-policy-agent/opa/rego"
 	"github.com/opentdf/platform/protocol/go/authorization"
@@ -282,11 +284,14 @@ func Test_GetDecisionsAllOf_Pass(t *testing.T) {
 	}
 
 	as := AuthorizationService{
-		logger: logger, sdk: &otdf.SDK{
-			SubjectMapping: &mySubjectMappingClient{},
-			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
+		logger: logger,
+		sdk: &otdf.SDK{
+			SubjectMapping:  &mySubjectMappingClient{},
+			Attributes:      &myAttributesClient{},
+			EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	resp, err := as.GetDecisions(ctxb, &req)
@@ -451,7 +456,8 @@ func Test_GetDecisions_AllOf_Fail(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	resp, err := as.GetDecisions(ctxb, &req)
@@ -550,7 +556,8 @@ func Test_GetDecisionsAllOfWithEnvironmental_Pass(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	resp, err := as.GetDecisions(ctxb, &req)
@@ -646,7 +653,8 @@ func Test_GetDecisionsAllOfWithEnvironmental_Fail(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	resp, err := as.GetDecisions(ctxb, &req)
@@ -720,7 +728,8 @@ func Test_GetEntitlementsSimple(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	req := connect.Request[authorization.GetEntitlementsRequest]{
@@ -793,7 +802,8 @@ func Test_GetEntitlementsFqnCasing(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	req := connect.Request[authorization.GetEntitlementsRequest]{
@@ -872,7 +882,8 @@ func Test_GetEntitlements_HandlesPagination(t *testing.T) {
 			Attributes:      &paginatedMockAttributesClient{},
 			EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	req := connect.Request[authorization.GetEntitlementsRequest]{
@@ -963,7 +974,8 @@ func Test_GetEntitlementsWithComprehensiveHierarchy(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	withHierarchy := true
@@ -1204,7 +1216,8 @@ func Test_GetDecisions_RA_FQN_Edge_Cases(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	///////////// TEST1: Only empty string /////////////
@@ -1411,7 +1424,8 @@ func Test_GetDecisionsAllOf_Pass_EC_RA_Length_Mismatch(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	resp, err := as.GetDecisions(ctxb, &req)
@@ -1689,7 +1703,8 @@ func Test_GetDecisions_Empty_EC_RA(t *testing.T) {
 			SubjectMapping: &mySubjectMappingClient{},
 			Attributes:     &myAttributesClient{}, EntityResoution: &myERSClient{},
 		},
-		eval: prepared,
+		eval:   prepared,
+		Tracer: noop.NewTracerProvider().Tracer(""),
 	}
 
 	///////////// Test Cases /////////////////////

service/cmd/migrate.go

@@ -125,7 +125,7 @@ func migrateDBClient(cmd *cobra.Command, opts ...db.OptsFunc) (*db.Client, error
 	if err != nil {
 		panic(fmt.Errorf("could not load config: %w", err))
 	}
-	return db.New(context.Background(), conf.DB, conf.Logger, opts...)
+	return db.New(context.Background(), conf.DB, conf.Logger, nil, opts...)
 }
 
 func init() {

service/cmd/policy.go

@@ -76,7 +76,7 @@ func policyDBClient(conf *config.Config) (policydb.PolicyDBClient, error) {
 	if !strings.HasSuffix(conf.DB.Schema, "_policy") {
 		conf.DB.Schema += "_policy"
 	}
-	dbClient, err := db.New(context.Background(), conf.DB, conf.Logger, db.WithMigrations(policy.Migrations))
+	dbClient, err := db.New(context.Background(), conf.DB, conf.Logger, nil, db.WithMigrations(policy.Migrations))
 	if err != nil {
 		//nolint:wrapcheck // we want to return the error as is. the start command will wrap it
 		return policydb.PolicyDBClient{}, err

service/cmd/provisionFixtures.go

@@ -43,7 +43,7 @@ You can clear/recycle your database with 'docker compose down' and 'docker compo
 				panic(fmt.Errorf("could not load config: %w", err))
 			}
 
-			dbClient, err := db.New(context.Background(), cfg.DB, cfg.Logger)
+			dbClient, err := db.New(context.Background(), cfg.DB, cfg.Logger, nil)
 			if err != nil {
 				panic(fmt.Errorf("issue creating database client: %w", err))
 			}

service/entityresolution/claims/claims_entity_resolution.go

@@ -12,6 +12,7 @@ import (
 	auth "github.com/opentdf/platform/service/authorization"
 	"github.com/opentdf/platform/service/logger"
 	"github.com/opentdf/platform/service/pkg/serviceregistry"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -20,6 +21,7 @@ import (
 type ClaimsEntityResolutionService struct {
 	entityresolution.UnimplementedEntityResolutionServiceServer
 	logger *logger.Logger
+	trace.Tracer
 }
 
 func RegisterClaimsERS(_ serviceregistry.ServiceConfig, logger *logger.Logger) (ClaimsEntityResolutionService, serviceregistry.HandlerServer) {
@@ -33,6 +35,9 @@ func (s ClaimsEntityResolutionService) ResolveEntities(ctx context.Context, req
 }
 
 func (s ClaimsEntityResolutionService) CreateEntityChainFromJwt(ctx context.Context, req *connect.Request[entityresolution.CreateEntityChainFromJwtRequest]) (*connect.Response[entityresolution.CreateEntityChainFromJwtResponse], error) {
+	ctx, span := s.Tracer.Start(ctx, "CreateEntityChainFromJwt")
+	defer span.End()
+
 	resp, err := CreateEntityChainFromJwt(ctx, req.Msg, s.logger)
 	return connect.NewResponse(&resp), err
 }

service/entityresolution/entityresolution.go

@@ -7,6 +7,7 @@ import (
 	claims "github.com/opentdf/platform/service/entityresolution/claims"
 	keycloak "github.com/opentdf/platform/service/entityresolution/keycloak"
 	"github.com/opentdf/platform/service/pkg/serviceregistry"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type ERSConfig struct {
@@ -20,6 +21,7 @@ const (
 
 type EntityResolution struct {
 	entityresolutionconnect.EntityResolutionServiceHandler
+	trace.Tracer
 }
 
 func NewRegistration() *serviceregistry.Service[entityresolutionconnect.EntityResolutionServiceHandler] {
@@ -37,12 +39,15 @@ func NewRegistration() *serviceregistry.Service[entityresolutionconnect.EntityRe
 				}
 				if inputConfig.Mode == ClaimsMode {
 					claimsSVC, claimsHandler := claims.RegisterClaimsERS(srp.Config, srp.Logger)
+					claimsSVC.Tracer = srp.Tracer
 					return EntityResolution{EntityResolutionServiceHandler: claimsSVC}, claimsHandler
 				}
 
 				// Default to keycloak ERS
 				kcSVC, kcHandler := keycloak.RegisterKeycloakERS(srp.Config, srp.Logger)
-				return EntityResolution{EntityResolutionServiceHandler: kcSVC}, kcHandler
+				kcSVC.Tracer = srp.Tracer
+
+				return EntityResolution{EntityResolutionServiceHandler: kcSVC, Tracer: srp.Tracer}, kcHandler
 			},
 		},
 	}

service/entityresolution/keycloak/keycloak_entity_resolution.go

@@ -17,6 +17,7 @@ import (
 	auth "github.com/opentdf/platform/service/authorization"
 	"github.com/opentdf/platform/service/logger"
 	"github.com/opentdf/platform/service/pkg/serviceregistry"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -39,6 +40,7 @@ type KeycloakEntityResolutionService struct {
 	entityresolution.UnimplementedEntityResolutionServiceServer
 	idpConfig KeycloakConfig
 	logger    *logger.Logger
+	trace.Tracer
 }
 
 type KeycloakConfig struct {
@@ -62,11 +64,17 @@ func RegisterKeycloakERS(config serviceregistry.ServiceConfig, logger *logger.Lo
 }
 
 func (s KeycloakEntityResolutionService) ResolveEntities(ctx context.Context, req *connect.Request[entityresolution.ResolveEntitiesRequest]) (*connect.Response[entityresolution.ResolveEntitiesResponse], error) {
+	ctx, span := s.Tracer.Start(ctx, "ResolveEntities")
+	defer span.End()
+
 	resp, err := EntityResolution(ctx, req.Msg, s.idpConfig, s.logger)
 	return connect.NewResponse(&resp), err
 }
 
 func (s KeycloakEntityResolutionService) CreateEntityChainFromJwt(ctx context.Context, req *connect.Request[entityresolution.CreateEntityChainFromJwtRequest]) (*connect.Response[entityresolution.CreateEntityChainFromJwtResponse], error) {
+	ctx, span := s.Tracer.Start(ctx, "CreateEntityChainFromJwt")
+	defer span.End()
+
 	resp, err := CreateEntityChainFromJwt(ctx, req.Msg, s.idpConfig, s.logger)
 	return connect.NewResponse(&resp), err
 }