diff --git a/README.md b/README.md index 25876d3..2f29f41 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The external scaler calls Cosmos DB APIs to estimate the amount of changes pendi Create `ScaledObject` resource that contains the information about your application (the scale target), the external scaler service, Cosmos DB containers, and other scaling configuration values. Check [`ScaledObject` specification](https://keda.sh/docs/concepts/scaling-deployments/) and [`External` trigger specification](https://keda.sh/docs/scalers/external/) for information on different properties supported for `ScaledObject` and their allowed values. -You can use file `deploy/deploy-scaledobject.yaml` as a template for creating the `ScaledObject`. The trigger metadata properties required to use the external scaler for Cosmos DB are described in [Trigger Specification](#trigger-specification) section below. +You can use the `deploy/deploy-scaledobject.yaml` file as a template for creating the `ScaledObject` when connecting to Cosmos DB with a managed identity. If you are using a connection string for the connection, use the `deploy/deploy-scaledobject-cs.yaml` file as your template instead. The trigger metadata properties required to use the external scaler for Cosmos DB are described in [Trigger Specification](#trigger-specification) section below. > **Note:** If you are having trouble setting up the external scaler or the listener application, the step-by-step instructions for [deploying the sample application](./src/Scaler.Demo/README.md) might help. @@ -68,10 +68,10 @@ The specification below describes the `trigger` metadata in `ScaledObject` resou - type: external metadata: scalerAddress: external-scaler-azure-cosmos-db.keda:4050 # Mandatory. Address of the external scaler service. - connection: # Mandatory. Connection string of Cosmos DB account with monitored container. + endpoint: # Mandatory. Endpoint URL of Cosmos DB account with monitored container. databaseId: # Mandatory. ID of Cosmos DB database containing monitored container. containerId: # Mandatory. ID of monitored container. - leaseConnection: # Mandatory. Connection string of Cosmos DB account with lease container. + leaseEndpoint: # Mandatory. Endpoint URL of Cosmos DB account with lease container. leaseDatabaseId: # Mandatory. ID of Cosmos DB database containing lease container. leaseContainerId: # Mandatory. ID of lease container. processorName: # Mandatory. Name of change-feed processor used by listener application. @@ -81,13 +81,13 @@ The specification below describes the `trigger` metadata in `ScaledObject` resou - **`scalerAddress`** - Address of the external scaler service. This would be in format `.:`. If you installed Azure Cosmos DB external scaler Helm chart in `keda` namespace and did not specify custom values, the metadata value would be `external-scaler-azure-cosmos-db.keda:4050`. -- **`connection`** - Connection string of the Cosmos DB account that contains the monitored container. +- **`endpoint`** - Endpoint URL of the Cosmos DB account that contains the monitored container. - **`databaseId`** - ID of Cosmos DB database that contains the monitored container. - **`containerId`** - ID of the monitored container. -- **`leaseConnection`** - Connection string of the Cosmos DB account that contains the lease container. This can be same or different from the value of `connection` metadata. +- **`leaseEndpoint`** - Endpoint URL of the Cosmos DB account that contains the lease container. This can be same or different from the value of `endpoint` metadata. - **`leaseDatabaseId`** - ID of Cosmos DB database that contains the lease container. This can be same or different from the value of `databaseId` metadata. diff --git a/deploy/deploy-scaledobject-cs.yaml b/deploy/deploy-scaledobject-cs.yaml new file mode 100644 index 0000000..e9c4ffe --- /dev/null +++ b/deploy/deploy-scaledobject-cs.yaml @@ -0,0 +1,22 @@ +# Template scaled-object for using KEDA external scaler for Azure Cosmos DB. + +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: + namespace: default +spec: + pollingInterval: 20 + scaleTargetRef: + name: + triggers: + - type: external + metadata: + scalerAddress: external-scaler-azure-cosmos-db.keda:4050 + connection: + databaseId: + containerId: + leaseConnection: + leaseDatabaseId: + leaseContainerId: + processorName: diff --git a/deploy/deploy-scaledobject.yaml b/deploy/deploy-scaledobject.yaml index e9c4ffe..5ebbff9 100644 --- a/deploy/deploy-scaledobject.yaml +++ b/deploy/deploy-scaledobject.yaml @@ -13,10 +13,10 @@ spec: - type: external metadata: scalerAddress: external-scaler-azure-cosmos-db.keda:4050 - connection: + endpoint: databaseId: containerId: - leaseConnection: + leaseEndpoint: leaseDatabaseId: leaseContainerId: processorName: diff --git a/images/architecture.pptx b/images/architecture.pptx index a75e24b..0e5ed3f 100644 Binary files a/images/architecture.pptx and b/images/architecture.pptx differ diff --git a/src/Scaler.Demo/OrderGenerator/Dockerfile b/src/Scaler.Demo/OrderGenerator/Dockerfile index 250855e..6f26736 100644 --- a/src/Scaler.Demo/OrderGenerator/Dockerfile +++ b/src/Scaler.Demo/OrderGenerator/Dockerfile @@ -1,16 +1,23 @@ -# https://hub.docker.com/_/microsoft-dotnet +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +WORKDIR /app -# Restore, build and publish project. FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR / -COPY src/Scaler.Demo/OrderGenerator/ src/Scaler.Demo/OrderGenerator/ -COPY src/Scaler.Demo/Shared/ src/Scaler.Demo/Shared/ +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj", "Scaler.Demo/OrderGenerator/"] +COPY ["Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj", "Scaler.Demo/Shared/"] +RUN dotnet restore "./Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj" +COPY . . +WORKDIR "/src/Scaler.Demo/OrderGenerator" +RUN dotnet build "./Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj" -c $BUILD_CONFIGURATION -o /app/build -WORKDIR /src/Scaler.Demo/OrderGenerator -RUN dotnet publish --configuration Release --output /app +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false -# Stage application. -FROM mcr.microsoft.com/dotnet/runtime:6.0 +FROM base AS final WORKDIR /app -COPY --from=build /app . -ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.Demo.OrderGenerator.dll"] +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.Demo.OrderGenerator.dll"] \ No newline at end of file diff --git a/src/Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj b/src/Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj index 0cbfce7..8a56bd9 100644 --- a/src/Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj +++ b/src/Scaler.Demo/OrderGenerator/Keda.CosmosDb.Scaler.Demo.OrderGenerator.csproj @@ -1,21 +1,20 @@ - - + Exe net6.0 + Linux + ..\.. - - + - + + - - - + \ No newline at end of file diff --git a/src/Scaler.Demo/OrderGenerator/appsettings.json b/src/Scaler.Demo/OrderGenerator/appsettings.json index cf68f86..1c89f94 100644 --- a/src/Scaler.Demo/OrderGenerator/appsettings.json +++ b/src/Scaler.Demo/OrderGenerator/appsettings.json @@ -1,6 +1,7 @@ { "CosmosDbConfig": { - "Connection": "", + "Endpoint": "https://{Cosmos Account Name}.documents.azure.com:443/", + "Connection": "", "DatabaseId": "StoreDatabase", "ContainerId": "OrderContainer", "ContainerThroughput": 11000 diff --git a/src/Scaler.Demo/OrderProcessor/Dockerfile b/src/Scaler.Demo/OrderProcessor/Dockerfile index 0e8a0d6..d7a7337 100644 --- a/src/Scaler.Demo/OrderProcessor/Dockerfile +++ b/src/Scaler.Demo/OrderProcessor/Dockerfile @@ -1,16 +1,23 @@ -# https://hub.docker.com/_/microsoft-dotnet +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/runtime:6.0 AS base +WORKDIR /app -# Restore, build and publish project. FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR / -COPY src/Scaler.Demo/OrderProcessor/ src/Scaler.Demo/OrderProcessor/ -COPY src/Scaler.Demo/Shared/ src/Scaler.Demo/Shared/ +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj", "Scaler.Demo/OrderProcessor/"] +COPY ["Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj", "Scaler.Demo/Shared/"] +RUN dotnet restore "./Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj" +COPY . . +WORKDIR "/src/Scaler.Demo/OrderProcessor" +RUN dotnet build "./Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj" -c $BUILD_CONFIGURATION -o /app/build -WORKDIR /src/Scaler.Demo/OrderProcessor -RUN dotnet publish --configuration Release --output /app +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false -# Stage application. -FROM mcr.microsoft.com/dotnet/runtime:6.0 +FROM base AS final WORKDIR /app -COPY --from=build /app . -ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.Demo.OrderProcessor.dll"] +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.Demo.OrderProcessor.dll"] \ No newline at end of file diff --git a/src/Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj b/src/Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj index 2d07aa2..6ad917f 100644 --- a/src/Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj +++ b/src/Scaler.Demo/OrderProcessor/Keda.CosmosDb.Scaler.Demo.OrderProcessor.csproj @@ -2,12 +2,16 @@ net6.0 + Linux + ..\.. + - - + + + diff --git a/src/Scaler.Demo/OrderProcessor/Worker.cs b/src/Scaler.Demo/OrderProcessor/Worker.cs index 949a69d..683f527 100644 --- a/src/Scaler.Demo/OrderProcessor/Worker.cs +++ b/src/Scaler.Demo/OrderProcessor/Worker.cs @@ -7,6 +7,8 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Azure.Identity; +using static Azure.Core.HttpHeader; namespace Keda.CosmosDb.Scaler.Demo.OrderProcessor { @@ -25,8 +27,49 @@ public Worker(CosmosDbConfig cosmosDbConfig, ILogger logger) public override async Task StartAsync(CancellationToken cancellationToken) { - Database leaseDatabase = await new CosmosClient(_cosmosDbConfig.LeaseConnection) - .CreateDatabaseIfNotExistsAsync(_cosmosDbConfig.LeaseDatabaseId, cancellationToken: cancellationToken); + Database leaseDatabase; + CosmosClient cosmosClient; + + if (string.IsNullOrEmpty(_cosmosDbConfig.Connection)) + { + var credential = new DefaultAzureCredential(); + + cosmosClient = new Microsoft.Azure.Cosmos.CosmosClient(_cosmosDbConfig.Endpoint, credential); + } + else + { + cosmosClient = new Microsoft.Azure.Cosmos.CosmosClient(_cosmosDbConfig.Connection); + } + + //use connection string or credentials + if (string.IsNullOrEmpty(_cosmosDbConfig.LeaseConnection)) + { + + // maintain a single instance of CosmosClient per lifetime of the application. + if (_cosmosDbConfig.LeaseEndpoint == _cosmosDbConfig.Endpoint) + { + leaseDatabase = await cosmosClient.CreateDatabaseIfNotExistsAsync(_cosmosDbConfig.LeaseDatabaseId); + } + else + { + var credential = new DefaultAzureCredential(); + leaseDatabase = await new Microsoft.Azure.Cosmos.CosmosClient(_cosmosDbConfig.LeaseEndpoint, credential) + .CreateDatabaseIfNotExistsAsync(_cosmosDbConfig.LeaseDatabaseId); + } + } + else + { + // maintain a single instance of CosmosClient per lifetime of the application. + if (_cosmosDbConfig.LeaseConnection == _cosmosDbConfig.Connection) + { + leaseDatabase = await cosmosClient.CreateDatabaseIfNotExistsAsync(_cosmosDbConfig.LeaseDatabaseId); + } + else + { + leaseDatabase = await new Microsoft.Azure.Cosmos.CosmosClient(_cosmosDbConfig.LeaseConnection) + .CreateDatabaseIfNotExistsAsync(_cosmosDbConfig.LeaseDatabaseId); + } + } Container leaseContainer = await leaseDatabase .CreateContainerIfNotExistsAsync( @@ -37,7 +80,8 @@ public override async Task StartAsync(CancellationToken cancellationToken) // Change feed processor instance name should be unique for each container application. string instanceName = $"Instance-{Dns.GetHostName()}"; - _processor = new CosmosClient(_cosmosDbConfig.Connection) + + _processor = cosmosClient .GetContainer(_cosmosDbConfig.DatabaseId, _cosmosDbConfig.ContainerId) .GetChangeFeedProcessorBuilder(_cosmosDbConfig.ProcessorName, ProcessOrdersAsync) .WithInstanceName(instanceName) diff --git a/src/Scaler.Demo/OrderProcessor/appsettings.json b/src/Scaler.Demo/OrderProcessor/appsettings.json index b4881a3..1bc4f7b 100644 --- a/src/Scaler.Demo/OrderProcessor/appsettings.json +++ b/src/Scaler.Demo/OrderProcessor/appsettings.json @@ -7,10 +7,8 @@ } }, "CosmosDbConfig": { - "Connection": "", "DatabaseId": "StoreDatabase", "ContainerId": "OrderContainer", - "LeaseConnection": "", "LeaseDatabaseId": "StoreDatabase", "LeaseContainerId": "OrderProcessorLeases", "ProcessorName": "OrderProcessor" diff --git a/src/Scaler.Demo/OrderProcessor/deploy-cs.yaml b/src/Scaler.Demo/OrderProcessor/deploy-cs.yaml new file mode 100644 index 0000000..9ae5b38 --- /dev/null +++ b/src/Scaler.Demo/OrderProcessor/deploy-cs.yaml @@ -0,0 +1,26 @@ +# Deploy order processor application. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cosmosdb-order-processor + namespace: default +spec: + replicas: 1 # A replica is required to be up momentarily to initialize the change-feed. + selector: + matchLabels: + app: cosmosdb-order-processor + template: + metadata: + labels: + app: cosmosdb-order-processor + spec: + containers: + - name: cosmosdb-order-processor + image: /cosmosdb-order-processor:latest + imagePullPolicy: Always + env: + - name: CosmosDbConfig__Connection + value: + - name: CosmosDbConfig__LeaseConnection + value: diff --git a/src/Scaler.Demo/OrderProcessor/deploy-scaledobject-cs.yaml b/src/Scaler.Demo/OrderProcessor/deploy-scaledobject-cs.yaml new file mode 100644 index 0000000..2857a4a --- /dev/null +++ b/src/Scaler.Demo/OrderProcessor/deploy-scaledobject-cs.yaml @@ -0,0 +1,22 @@ +# Create KEDA scaled object to scale order processor application. + +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cosmosdb-order-processor-scaledobject + namespace: default +spec: + pollingInterval: 20 + scaleTargetRef: + name: cosmosdb-order-processor + triggers: + - type: external + metadata: + scalerAddress: cosmosdb-scaler.default:4050 + connection: + databaseId: StoreDatabase + containerId: OrderContainer + leaseConnection: + leaseDatabaseId: StoreDatabase + leaseContainerId: OrderProcessorLeases + processorName: OrderProcessor diff --git a/src/Scaler.Demo/OrderProcessor/deploy-scaledobject.yaml b/src/Scaler.Demo/OrderProcessor/deploy-scaledobject.yaml index 2857a4a..6559ccb 100644 --- a/src/Scaler.Demo/OrderProcessor/deploy-scaledobject.yaml +++ b/src/Scaler.Demo/OrderProcessor/deploy-scaledobject.yaml @@ -13,10 +13,10 @@ spec: - type: external metadata: scalerAddress: cosmosdb-scaler.default:4050 - connection: + endpoint: databaseId: StoreDatabase containerId: OrderContainer - leaseConnection: + leaseEndpoint: leaseDatabaseId: StoreDatabase leaseContainerId: OrderProcessorLeases processorName: OrderProcessor diff --git a/src/Scaler.Demo/OrderProcessor/deploy.yaml b/src/Scaler.Demo/OrderProcessor/deploy.yaml index 9ae5b38..359e935 100644 --- a/src/Scaler.Demo/OrderProcessor/deploy.yaml +++ b/src/Scaler.Demo/OrderProcessor/deploy.yaml @@ -5,6 +5,9 @@ kind: Deployment metadata: name: cosmosdb-order-processor namespace: default + labels: + aadpodidbinding: "my-pod-identity" # refer to https://learn.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity#create-a-pod-identity + app: cosmosdb-order-processor spec: replicas: 1 # A replica is required to be up momentarily to initialize the change-feed. selector: @@ -14,13 +17,14 @@ spec: metadata: labels: app: cosmosdb-order-processor + aadpodidbinding: "my-pod-identity" # refer to https://learn.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity#create-a-pod-identity spec: containers: - name: cosmosdb-order-processor image: /cosmosdb-order-processor:latest imagePullPolicy: Always env: - - name: CosmosDbConfig__Connection - value: - - name: CosmosDbConfig__LeaseConnection - value: + - name: CosmosDbConfig__Endpoint + value: + - name: CosmosDbConfig__LeaseEndpoint + value: diff --git a/src/Scaler.Demo/README.md b/src/Scaler.Demo/README.md index 3cbde1e..22efcd7 100644 --- a/src/Scaler.Demo/README.md +++ b/src/Scaler.Demo/README.md @@ -12,17 +12,20 @@ We will later deploy the order-processor application to Kubernetes cluster and u - [Azure Cosmos DB account](https://azure.microsoft.com/free/cosmos-db/) - [Docker Hub account](https://hub.docker.com/signup) - Kubernetes cluster +- [Use pod-managed identities in Azure Kubernetes Service](https://learn.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity) - [Docker Desktop](https://www.docker.com/products/docker-desktop) ## Testing sample application locally on Docker +**Note** For simplicity, we will use the connection string method to connect locally (using docker) to Azure Cosmos DB. Once deployed to the AKS cluster the applications will use Managed Identity for Connection. Managed Identity is more secure as there is no risk of accidentally leaking the connection string. + 1. Open command prompt or shell and change to the root directory of the cloned repo. 1. Run the below commands to build the Docker container images for order-generator and order-processor applications. ```text - # docker build --file .\src\Scaler.Demo\OrderGenerator\Dockerfile --force-rm --tag cosmosdb-order-generator . - # docker build --file .\src\Scaler.Demo\OrderProcessor\Dockerfile --force-rm --tag cosmosdb-order-processor . + # docker build --file .\src\Scaler.Demo\OrderGenerator\Dockerfile --force-rm --tag cosmosdb-order-generator .\src + # docker build --file .\src\Scaler.Demo\OrderProcessor\Dockerfile --force-rm --tag cosmosdb-order-processor .\src ``` 1. Create test-database and test-container within the database in Cosmos DB account by running the order-generator application inside the container with `setup` option. Make sure to put the connection string of Cosmos DB account in the command below. @@ -95,6 +98,10 @@ We will later deploy the order-processor application to Kubernetes cluster and u 1. Follow one of the steps on [Deploying KEDA](https://keda.sh/docs/deploy/) documentation page to deploy KEDA on your Kubernetes cluster. +1. Cosmos DB supports key-based and managed identity-based authentication, both of which are supported by KEDA. Since pod managed identity is more secure, we will use it as the default. Steps for connection string-based authentication are also provided. To set up managed identity-based authentication in Cosmos DB and AKS, follow the steps below: + 1. [Configure role-based access control with Microsoft Entra ID for your Azure Cosmos DB account](https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-setup-rbac) + 1. [Use pod-managed identities in Azure Kubernetes Service](https://learn.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity) + 1. Open command prompt or shell and change to the root directory of the cloned repo. 1. Build container image for the external scaler and push the image to Docker Hub. Make sure to replace `` in below commands with your Docker ID. @@ -111,6 +118,7 @@ We will later deploy the order-processor application to Kubernetes cluster and u ```text kubectl apply --filename=src/Scaler/deploy.yaml ``` + > **Note:** In case you are not using managed identity-based authentication in Cosmos DB then use `src/Scaler/deploy-cs.yaml` in the above step. ## Deploying sample application to cluster @@ -127,6 +135,8 @@ We will later deploy the order-processor application to Kubernetes cluster and u ```text kubectl apply --filename=src/Scaler.Demo/OrderProcessor/deploy.yaml ``` + + > **Note:** In case you are not using managed identity-based authentication in Cosmos DB then use `src/Scaler/deploy-cs.yaml` in the above step. 1. Ensure that the order-processor application is running correctly on the cluster by checking application logs. The application will create lease database and container if they do not exist, hence it is needed to run for a few seconds before we enable auto-scaling for it, as that would immediately bring replicas to 0 if there are no orders pending to be processed. @@ -147,6 +157,8 @@ We will later deploy the order-processor application to Kubernetes cluster and u kubectl apply --filename=src/Scaler.Demo/OrderProcessor/deploy-scaledobject.yaml ``` + > **Note:** In case you are not using identity-based authentication in Cosmos DB then use `src/Scaler/deploy-scaledobject-cs.yaml` in the above step. + > **Note** Ideally, we would have created `TriggerAuthentication` resource that would enable sharing of the connection strings as secrets between the scaled object and the target application. However, this is not possible since at the moment, the triggers of `external` type do not support referencing a `TriggerAuthentication` resource ([link](https://keda.sh/docs/scalers/external/#authentication-parameters)). ## Testing auto-scaling for sample application diff --git a/src/Scaler.Demo/Shared/CosmosDbConfig.cs b/src/Scaler.Demo/Shared/CosmosDbConfig.cs index 4e78f08..a87278c 100644 --- a/src/Scaler.Demo/Shared/CosmosDbConfig.cs +++ b/src/Scaler.Demo/Shared/CosmosDbConfig.cs @@ -5,11 +5,13 @@ namespace Keda.CosmosDb.Scaler.Demo.Shared public class CosmosDbConfig { public string Connection { get; set; } + public string Endpoint { get; set; } public string DatabaseId { get; set; } public string ContainerId { get; set; } public int ContainerThroughput { get; set; } public string LeaseConnection { get; set; } + public string LeaseEndpoint { get; set; } public string LeaseDatabaseId { get; set; } public string LeaseContainerId { get; set; } public string ProcessorName { get; set; } diff --git a/src/Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj b/src/Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj index 6b5fb47..cbd93f4 100644 --- a/src/Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj +++ b/src/Scaler.Demo/Shared/Keda.CosmosDb.Scaler.Demo.Shared.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -6,7 +6,7 @@ - + diff --git a/src/Scaler.Tests/CosmosDbScalerServiceTests.cs b/src/Scaler.Tests/CosmosDbScalerServiceTests.cs index 4e2dc47..b800c30 100644 --- a/src/Scaler.Tests/CosmosDbScalerServiceTests.cs +++ b/src/Scaler.Tests/CosmosDbScalerServiceTests.cs @@ -18,9 +18,11 @@ public CosmosDbScalerServiceTests() } [Theory] + [InlineData("endpoint")] [InlineData("connection")] [InlineData("databaseId")] [InlineData("containerId")] + [InlineData("leaseEndpoint")] [InlineData("leaseConnection")] [InlineData("leaseDatabaseId")] [InlineData("leaseContainerId")] @@ -50,9 +52,11 @@ public async Task IsActive_ReturnsFalseOnNonZeroPartitions(long partitionCount) } [Theory] + [InlineData("endpoint")] [InlineData("connection")] [InlineData("databaseId")] [InlineData("containerId")] + [InlineData("leaseEndpoint")] [InlineData("leaseConnection")] [InlineData("leaseDatabaseId")] [InlineData("leaseContainerId")] @@ -99,9 +103,11 @@ public async Task GetMetrics_ReturnsSameMetricNameIfPassed(string requestMetricN } [Theory] + [InlineData("endpoint")] [InlineData("connection")] [InlineData("databaseId")] [InlineData("containerId")] + [InlineData("leaseEndpoint")] [InlineData("leaseConnection")] [InlineData("leaseDatabaseId")] [InlineData("leaseContainerId")] @@ -144,7 +150,8 @@ public async Task GetMetricSpec_ReturnsSameMetricNameIfPassed(string requestMetr public async Task GetMetricSpec_ReturnsNormalizedMetricName() { ScaledObjectRef request = GetScaledObjectRef(); - request.ScalerMetadata["leaseConnection"] = "AccountEndpoint=https://example.com:443/;AccountKey=ZHVtbXky"; + request.ScalerMetadata["leaseEndpoint"] = "https://example.com:443/"; + request.ScalerMetadata["leaseConnection"] = "https://example2.com:443/;AccountKey=ZHVtbXkx\\"; request.ScalerMetadata["leaseDatabaseId"] = "Dummy.Lease.Database.Id"; request.ScalerMetadata["leaseContainerId"] = "Dummy:Lease:Container:Id"; request.ScalerMetadata["processorName"] = "Dummy%Processor%Name"; @@ -194,10 +201,12 @@ private static ScaledObjectRef GetScaledObjectRef() MapField scalerMetadata = scaledObjectRef.ScalerMetadata; - scalerMetadata["connection"] = "AccountEndpoint=https://example1.com:443/;AccountKey=ZHVtbXkx"; + scalerMetadata["endpoint"] = "https://example1.com:443/"; + scalerMetadata["connection"] = "https://example1.com:443/;AccountKey=ZHVtbXkx\\"; scalerMetadata["databaseId"] = "dummy-database-id"; scalerMetadata["containerId"] = "dummy-container-id"; - scalerMetadata["leaseConnection"] = "AccountEndpoint=https://example2.com:443/;AccountKey=ZHVtbXky"; + scalerMetadata["leaseEndpoint"] = "https://example2.com:443/"; + scalerMetadata["leaseConnection"] = "https://example2.com:443/;AccountKey=ZHVtbXkx\\"; scalerMetadata["leaseDatabaseId"] = "dummy-lease-database-id"; scalerMetadata["leaseContainerId"] = "dummy-lease-container-id"; scalerMetadata["processorName"] = "dummy-processor-name"; diff --git a/src/Scaler.Tests/Keda.CosmosDb.Scaler.Tests.csproj b/src/Scaler.Tests/Keda.CosmosDb.Scaler.Tests.csproj index 5633867..949b77b 100644 --- a/src/Scaler.Tests/Keda.CosmosDb.Scaler.Tests.csproj +++ b/src/Scaler.Tests/Keda.CosmosDb.Scaler.Tests.csproj @@ -14,7 +14,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/Scaler/Dockerfile b/src/Scaler/Dockerfile index 8dc7f6f..2badb05 100644 --- a/src/Scaler/Dockerfile +++ b/src/Scaler/Dockerfile @@ -1,15 +1,23 @@ -# https://hub.docker.com/_/microsoft-dotnet +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base +WORKDIR /app +EXPOSE 80 -# Restore, build and publish project. FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR / -COPY src/Scaler/ src/Scaler/ +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["Scaler/Keda.CosmosDb.Scaler.csproj", "Scaler/"] +RUN dotnet restore "./Scaler/Keda.CosmosDb.Scaler.csproj" +COPY . . +WORKDIR "/src/Scaler" +RUN dotnet build "./Keda.CosmosDb.Scaler.csproj" -c $BUILD_CONFIGURATION -o /app/build -WORKDIR /src/Scaler -RUN dotnet publish --configuration Release --output /app +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./Keda.CosmosDb.Scaler.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false -# Stage application. -FROM mcr.microsoft.com/dotnet/aspnet:6.0 +FROM base AS final WORKDIR /app -COPY --from=build /app . -ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.dll"] +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Keda.CosmosDb.Scaler.dll"] \ No newline at end of file diff --git a/src/Scaler/Keda.CosmosDb.Scaler.csproj b/src/Scaler/Keda.CosmosDb.Scaler.csproj index 9c9c6e8..48dcbbb 100644 --- a/src/Scaler/Keda.CosmosDb.Scaler.csproj +++ b/src/Scaler/Keda.CosmosDb.Scaler.csproj @@ -2,6 +2,7 @@ net6.0 + Linux @@ -9,10 +10,13 @@ + - + + + diff --git a/src/Scaler/Services/CosmosDbFactory.cs b/src/Scaler/Services/CosmosDbFactory.cs index 25cf364..58095eb 100644 --- a/src/Scaler/Services/CosmosDbFactory.cs +++ b/src/Scaler/Services/CosmosDbFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using Microsoft.Azure.Cosmos; +using Azure.Identity; namespace Keda.CosmosDb.Scaler { @@ -9,14 +10,26 @@ internal sealed class CosmosDbFactory // maintain a single instance of CosmosClient per lifetime of the application. private readonly ConcurrentDictionary _cosmosClientCache = new(); - public CosmosClient GetCosmosClient(string connection) + public CosmosClient GetCosmosClient(string endpoint, bool useCredetials) { - return _cosmosClientCache.GetOrAdd(connection, CreateCosmosClient); + return _cosmosClientCache.GetOrAdd(endpoint, ep => CreateCosmosClient(ep, useCredetials)); } - private CosmosClient CreateCosmosClient(string connection) + //private CosmosClient CreateCosmosClient(string connection) + private CosmosClient CreateCosmosClient(string endpoint_OR_connection, bool useCredentials) { - return new CosmosClient(connection, new CosmosClientOptions { ConnectionMode = ConnectionMode.Gateway }); + //use connection string or credentials + if (useCredentials) + { + var credential = new DefaultAzureCredential(); + return new Microsoft.Azure.Cosmos.CosmosClient(endpoint_OR_connection, credential, new CosmosClientOptions { ConnectionMode = ConnectionMode.Gateway, ApplicationName = "keda-external-azure-cosmos-db" }); + } + else + { + return new Microsoft.Azure.Cosmos.CosmosClient(endpoint_OR_connection, new CosmosClientOptions { ConnectionMode = ConnectionMode.Gateway, ApplicationName = "keda-external-azure-cosmos-db" }); + } + + } } } diff --git a/src/Scaler/Services/CosmosDbMetricProvider.cs b/src/Scaler/Services/CosmosDbMetricProvider.cs index 61242dc..4c3f3fc 100644 --- a/src/Scaler/Services/CosmosDbMetricProvider.cs +++ b/src/Scaler/Services/CosmosDbMetricProvider.cs @@ -3,9 +3,16 @@ using System.Net; using System.Net.Http; using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.AspNetCore.Http; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; - +using Microsoft.Identity.Client.Platforms.Features.DesktopOs.Kerberos; +using Container = Microsoft.Azure.Cosmos.Container; +using System.Diagnostics.Metrics; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using Microsoft.Identity.Client; namespace Keda.CosmosDb.Scaler { internal sealed class CosmosDbMetricProvider : ICosmosDbMetricProvider @@ -13,37 +20,84 @@ internal sealed class CosmosDbMetricProvider : ICosmosDbMetricProvider private readonly CosmosDbFactory _factory; private readonly ILogger _logger; + private static Meter s_meter = new Meter("OrderProcessor.CFStore", "1.0.0"); + private static Counter s_CFRecordsReceived = s_meter.CreateCounter("RecordsReceived"); + private static Counter s_CFProcessorCount = s_meter.CreateCounter("ProcessorCount"); + public CosmosDbMetricProvider(CosmosDbFactory factory, ILogger logger) { _factory = factory ?? throw new ArgumentNullException(nameof(factory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + using MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("OrderProcessor.CFStore") + .Build(); } public async Task GetPartitionCountAsync(ScalerMetadata scalerMetadata) { try { + + bool useCredentials_lease = false; + bool useCredentials = false; + + string endpoint_or_connection_lease; + string endpoint_or_connection; + + //use connection string or credentials for Lease + if (!String.IsNullOrEmpty(scalerMetadata.LeaseConnection)) + { + endpoint_or_connection_lease = scalerMetadata.LeaseConnection; + useCredentials_lease = false; + } + else + { + endpoint_or_connection_lease = scalerMetadata.LeaseEndpoint; + useCredentials_lease = true; + } + Container leaseContainer = _factory - .GetCosmosClient(scalerMetadata.LeaseConnection) + .GetCosmosClient(endpoint_or_connection_lease, useCredentials_lease) .GetContainer(scalerMetadata.LeaseDatabaseId, scalerMetadata.LeaseContainerId); + //use connection string or credentials for Monitored + if (!String.IsNullOrEmpty(scalerMetadata.Connection)) + { + endpoint_or_connection = scalerMetadata.Connection; + useCredentials = false; + } + else + { + endpoint_or_connection = scalerMetadata.Endpoint; + useCredentials = true; + } ChangeFeedEstimator estimator = _factory - .GetCosmosClient(scalerMetadata.Connection) + .GetCosmosClient(endpoint_or_connection, useCredentials) .GetContainer(scalerMetadata.DatabaseId, scalerMetadata.ContainerId) .GetChangeFeedEstimator(scalerMetadata.ProcessorName, leaseContainer); - // It does not help by creating more change-feed processing instances than the number of partitions. + using FeedIterator estimatorIterator = estimator.GetCurrentStateIterator(); int partitionCount = 0; - - using (FeedIterator iterator = estimator.GetCurrentStateIterator()) + long lagCount=0; + while (estimatorIterator.HasMoreResults) { - while (iterator.HasMoreResults) + FeedResponse states = await estimatorIterator.ReadNextAsync(); + + foreach (ChangeFeedProcessorState leaseState in states) { - FeedResponse states = await iterator.ReadNextAsync(); - partitionCount += states.Where(state => state.EstimatedLag > 0).Count(); + string host = leaseState.InstanceName == null ? $"not owned by any host currently" : $"owned by host {leaseState.InstanceName}"; + _logger.LogInformation($"Lease [{leaseState.LeaseToken}] {host} reports {leaseState.EstimatedLag} as estimated lag."); + lagCount = lagCount + leaseState.EstimatedLag; + } + partitionCount += states.Where(state => state.EstimatedLag > 0).Count(); } + s_CFRecordsReceived.Add(lagCount); + s_CFProcessorCount.Add(partitionCount); + _logger.LogInformation($"Count of Partitions with lag:{partitionCount}"); + return partitionCount; } catch (CosmosException exception) diff --git a/src/Scaler/Services/ScalerMetadata.cs b/src/Scaler/Services/ScalerMetadata.cs index a2b6d74..759a9cd 100644 --- a/src/Scaler/Services/ScalerMetadata.cs +++ b/src/Scaler/Services/ScalerMetadata.cs @@ -10,9 +10,11 @@ internal sealed class ScalerMetadata private string _metricName; public string Connection { get; set; } + public string Endpoint { get; set; } public string DatabaseId { get; set; } public string ContainerId { get; set; } public string LeaseConnection { get; set; } + public string LeaseEndpoint { get; set; } public string LeaseDatabaseId { get; set; } public string LeaseContainerId { get; set; } public string ProcessorName { get; set; } @@ -39,8 +41,16 @@ private string LeaseAccountHost { get { - var builder = new DbConnectionStringBuilder { ConnectionString = this.LeaseConnection }; - return new Uri((string)builder["AccountEndpoint"]).Host; + if(!string.IsNullOrEmpty(this.LeaseConnection)) + { + var builder = new DbConnectionStringBuilder { ConnectionString = this.LeaseConnection }; + return new Uri((string)builder["AccountEndpoint"]).Host; + } + else + { + return new Uri(this.LeaseEndpoint).Host; + } + } } diff --git a/src/Scaler/deploy-cs.yaml b/src/Scaler/deploy-cs.yaml new file mode 100644 index 0000000..9e2bb6a --- /dev/null +++ b/src/Scaler/deploy-cs.yaml @@ -0,0 +1,38 @@ +# Deploy KEDA external scaler for Azure Cosmos DB. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cosmosdb-scaler + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: cosmosdb-scaler + template: + metadata: + labels: + app: cosmosdb-scaler + spec: + containers: + - image: /cosmosdb-scaler:latest + imagePullPolicy: Always + name: cosmosdb-scaler + ports: + - containerPort: 4050 + +--- +# Assign hostname to the scaler application. + +apiVersion: v1 +kind: Service +metadata: + name: cosmosdb-scaler + namespace: default +spec: + ports: + - port: 4050 + targetPort: 4050 + selector: + app: cosmosdb-scaler diff --git a/src/Scaler/deploy.yaml b/src/Scaler/deploy.yaml index 9e2bb6a..02c3840 100644 --- a/src/Scaler/deploy.yaml +++ b/src/Scaler/deploy.yaml @@ -13,6 +13,7 @@ spec: template: metadata: labels: + aadpodidbinding: "my-pod-identity" # refer to https://learn.microsoft.com/en-us/azure/aks/use-azure-ad-pod-identity#create-a-pod-identity app: cosmosdb-scaler spec: containers: