Merge pull request #162 from ts-amz/fix_fleet_sim

ts-amz · web-flow · commit 211f51ff726c · 2023-08-28T23:51:47.000-04:00
fix(simulation-manager): applied change - fixed, several minor issues…
diff --git a/source/common/changes/@cdf/installer/fix-fleet-simulator_2022-08-25-02-55.json b/source/common/changes/@cdf/installer/fix-fleet-simulator_2022-08-25-02-55.json
@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@cdf/installer",
+      "comment": "updated jmeter version",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@cdf/installer",
+  "email": "bahavar@amazon.com"
+}
diff --git a/source/common/changes/@cdf/simulation-launcher/fix-fleet-simulator_2022-08-25-02-55.json b/source/common/changes/@cdf/simulation-launcher/fix-fleet-simulator_2022-08-25-02-55.json
@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@cdf/simulation-launcher",
+      "comment": "updated java version and jmeter version",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@cdf/simulation-launcher",
+  "email": "bahavar@amazon.com"
+}
diff --git a/source/common/changes/@cdf/simulation-manager/fix-fleet-simulator_2022-08-25-02-55.json b/source/common/changes/@cdf/simulation-manager/fix-fleet-simulator_2022-08-25-02-55.json
@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@cdf/simulation-manager",
+      "comment": "fixed minor issues in the runService and updated documentation",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@cdf/simulation-manager",
+  "email": "bahavar@amazon.com"
+}
diff --git a/source/infrastructure/cloudformation/cfn-networking.yaml b/source/infrastructure/cloudformation/cfn-networking.yaml
@@ -265,6 +265,19 @@ Resources:
         From: 1024
         To: 65535
 
+  PrivateSubnetMqttOutboundRule:
+    Condition: Deploy
+    Type: AWS::EC2::NetworkAclEntry
+    Properties:
+      NetworkAclId: !Ref PrivateSubnetNacl
+      RuleNumber: 103
+      Protocol: 6
+      Egress: true
+      RuleAction: allow
+      CidrBlock: 0.0.0.0/0
+      PortRange:
+        From: 8883
+        To: 8883
 
   PublicSubnetOne:
     Condition: Deploy
diff --git a/source/packages/services/installer/src/commands/modules/service/fleetSimulator.ts b/source/packages/services/installer/src/commands/modules/service/fleetSimulator.ts
@@ -96,7 +96,7 @@ export class FleetSimulatorInstaller implements RestModule {
                         propertyName: 's3Prefix',
                     },
                     {
-                        defaultConfiguration: '/opt/apache-jmeter-5.1.1/bin/cdf',
+                        defaultConfiguration: '/opt/apache-jmeter-5.4.3/bin/cdf',
                         question: 'Local location where to store jmeter execution results',
                         propertyName: 'runnerDataDir',
                     },
diff --git a/source/packages/services/simulation-launcher/infrastructure/cfn-simulation-launcher.yaml b/source/packages/services/simulation-launcher/infrastructure/cfn-simulation-launcher.yaml
@@ -53,20 +53,6 @@ Parameters:
     Description: The KMS key ID used to encrypt the SNS topic
     Type: String
 
-  LoggingLevel:
-    Description: Application logging level
-    Type: String
-    Default: info
-    AllowedValues:
-      - error
-      - warn
-      - info
-      - debug
-      - silly
-
-Conditions:
-  LoggingLevelProvided: !Not [!Equals [!Ref LoggingLevel, '']]
-
 Resources:
   ECSCluster:
     Type: AWS::ECS::Cluster
@@ -198,7 +184,7 @@ Resources:
                   - 's3:Get*'
                   - 's3:List*'
                   - 's3:Put*'
-                Resource: 'arn:aws:s3:::${BucketName}/*'
+                Resource: !Sub 'arn:aws:s3:::${BucketName}/*'
               - Sid: lambda
                 Action:
                   - lambda:InvokeFunction
@@ -308,7 +294,6 @@ Resources:
           AWS_ECS_CLUSTERID: !Ref ECSCluster
           AWS_ECS_SECURITYGROUPID: !Ref FargateContainerSecurityGroup
           AWS_ECS_TASKDEFINITIONARN: !Ref JMeterTaskDefinition
-          LOGGING_LEVEL: !If [LoggingLevelProvided, !Ref LoggingLevel, !Ref AWS::NoValue]
       Tracing: Active
     DependsOn:
       - LambdaExecutionRole
diff --git a/source/packages/services/simulation-launcher/src/containers/jmeter/Dockerfile b/source/packages/services/simulation-launcher/src/containers/jmeter/Dockerfile
@@ -20,7 +20,7 @@ ARG AWSCLI_VERSION="1.16.278"
 ARG JMETER_PLUGINS_MANAGER_VERSION="1.3"
 ARG CMDRUNNER_VERSION="2.2"
 ARG JSON_LIB_VERSION="2.4"
-ARG JSON_LIB_FULL_VERSION="${JSON_LIB_VERSION}-jdk15"
+ARG JSON_LIB_FULL_VERSION="${JSON_LIB_VERSION}-jdk11"
 
 ENV JMETER_HOME /opt/apache-jmeter-${JMETER_VERSION}
 ENV	JMETER_BIN	${JMETER_HOME}/bin
@@ -34,17 +34,20 @@ RUN yum install -y \
     python \
     python-devel \
     python-pip \
-    java-1.8.0-openjdk \
+    openssl.x86_64 \
     && pip install awscli==$AWSCLI_VERSION --upgrade --user \
     && yum remove -y python-pip \
     && yum clean all
 
+# install openJDK11
+RUN amazon-linux-extras install java-openjdk11
+
 # Install jmeter
 RUN    yum update -y \
 	&& yum upgrade -y \
 	&& yum install -y ca-certificates \
 	&& update-ca-trust \
-	&& yum install -y openjdk8-jre tzdata curl unzip bash nss gzip tar\
+	&& yum install -y tzdata curl unzip bash nss gzip tar zip\
 	&& mkdir -p /tmp/dependencies  \
 	&& curl -L --silent ${JMETER_DOWNLOAD_URL} >  /tmp/dependencies/apache-jmeter-${JMETER_VERSION}.tgz  \
 	&& mkdir -p /opt  \
diff --git a/source/packages/services/simulation-manager/docs/configuration.md b/source/packages/services/simulation-manager/docs/configuration.md
@@ -16,7 +16,7 @@ CORS_ORIGIN=*
 # Application logging level. Set to (in order) error, warn, info, verbose, debug or silly.
 LOGGING_LEVEL=info
 # local location where to store jmeter execution results
-RUNNERS_DATADIR=/opt/apache-jmeter-5.1.1/bin/cdf
+RUNNERS_DATADIR=/opt/apache-jmeter-5.4.3/bin/cdf
 # Container allocated memory size. Tune for best performance.
 RUNNERS_MEMORY=2048
 # Container allocated cpu. Tune for best performance.
diff --git a/source/packages/services/simulation-manager/docs/swagger.yml b/source/packages/services/simulation-manager/docs/swagger.yml
@@ -49,6 +49,21 @@ paths:
             location:
               schema:
                 type: 'string'
+  /simulations/{simulation-id}/runs:
+    post:
+      tags:
+        - 'Runs'
+      summary: 'Create a run for the simulation'
+      operationId: 'CreateRun'
+      requestBody:
+        $ref: '#/components/requestBodies/Run'
+      response:
+        204:
+          description: 'success'
+          headers:
+            location:
+              schema:
+                type: 'string'
 
 components:
   requestBodies:
@@ -57,6 +72,11 @@ components:
         application/vnd.aws-cdf-v1.0+json:
           schema:
             $ref: '#/components/schemas/Simulation'
+    Run:
+      content:
+        application/vnd.aws-cdf-v1.0+json:
+          schema:
+            $ref: '#/components/schemas/Run'
 
   schemas:
     Simulation:
@@ -96,6 +116,27 @@ components:
               description: 'A list of different task that will be executed by the simulator. the Task names are dynamic and can be modified as needed'
               additionalProperties:
                 $ref: '#/components/schemas/Task'
+    Run:
+      type: 'object'
+      properties:
+        deviceCount:
+          type: 'integer'
+          format: 'int32'
+          default: 100
+          required: true
+          description: 'the number of devices to simulate'
+        simulationId:
+          type: 'string'
+          description: 'the id of the simulation you are creating runs against'
+          required: true
+        id:
+          type: 'string'
+          description: 'the id of the run'
+          required: false
+        status:
+          type: 'string'
+          description: 'the status of the run'
+          required: false
 
     Task:
       type: 'object'
diff --git a/source/packages/services/simulation-manager/infrastructure/cfn-simulation-manager.yml b/source/packages/services/simulation-manager/infrastructure/cfn-simulation-manager.yml
@@ -114,30 +114,12 @@ Parameters:
       - 'false'
     MinLength: 1
 
-  CorsOrigin:
-    Description: Sets the Access-Control-Allow-Origin response header to indicate whether the response can be shared with requesting code from the given origin.
-    Type: String
-    Default: ''
-
-  LoggingLevel:
-    Description: Application logging level
-    Type: String
-    Default: info
-    AllowedValues:
-      - error
-      - warn
-      - info
-      - debug
-      - silly
-
 Conditions:
   DeployInVPC: !Not [!Equals [!Ref VpcId, 'N/A']]
   DeployWithLambdaAuth:
     !Or [!Equals [!Ref AuthType, 'LambdaRequest'], !Equals [!Ref AuthType, 'LambdaToken']]
   KmsKeyIdProvided: !Not [!Equals [!Ref KmsKeyId, '']]
   EnableApiGatewayAccessLogs: !Equals [!Ref EnableApiGatewayAccessLogs, 'true']
-  CorsOriginProvided: !Not [!Equals [!Ref CorsOrigin, '']]
-  LoggingLevelProvided: !Not [!Equals [!Ref LoggingLevel, '']]
 
 Resources:
   ApiGatewayApi:
@@ -253,8 +235,6 @@ Resources:
           AWS_SNS_TOPICS_LAUNCH: !Ref SimulationLauncherSnsTopic
           AWS_IOT_HOST: !GetAtt IotEndpoint.address
           ASSETLIBRARY_API_FUNCTION_NAME: !Ref AssetLibraryFunctionName
-          CORS_ORIGIN: !If [CorsOriginProvided, !Ref CorsOrigin, !Ref AWS::NoValue]
-          LOGGING_LEVEL: !If [LoggingLevelProvided, !Ref LoggingLevel, !Ref AWS::NoValue]
       Tracing: Active
 
       VpcConfig:
diff --git a/source/packages/services/simulation-manager/src/config/.env.defaults b/source/packages/services/simulation-manager/src/config/.env.defaults
@@ -2,7 +2,7 @@ AWS_S3_PREFIX=simulations/
 CORS_ORIGIN=*
 
 # local working directory
-RUNNERS_DATADIR=/opt/apache-jmeter-5.1.1/bin/cdf
+RUNNERS_DATADIR=/opt/apache-jmeter-5.4.3/bin/cdf
 # task memory (MiB) per task instance
 RUNNERS_MEMORY=2048
 # task cpu (units) per task instance
diff --git a/source/packages/services/simulation-manager/src/config/README.md b/source/packages/services/simulation-manager/src/config/README.md
@@ -19,7 +19,7 @@ ASSETLIBRARY_API_FUNCTION_NAME=
 AWS_S3_PREFIX=simulations/
 CORS_ORIGIN=*
 LOGGING_LEVEL=info
-RUNNERS_DATADIR=/opt/apache-jmeter-5.1.1/bin/cdf
+RUNNERS_DATADIR=/opt/apache-jmeter-5.4.3/bin/cdf
 RUNNERS_MEMORY=2048
 RUNNERS_CPU=1024
 RUNNERS_THREADS=20
diff --git a/source/packages/services/simulation-manager/src/runs/runs.service.ts b/source/packages/services/simulation-manager/src/runs/runs.service.ts
@@ -67,11 +67,14 @@ export class RunsService {
         // TODO: validation - no. devices < total available?
 
         // data generation - retrieve last known location
-        let simulationDevices = await this._runsDao.listDeviceState(run.simulationId);
+        const simulationDevices = await this._runsDao.listDeviceState(run.simulationId);
 
         // data generation - choose devices from pool
-        const runDevices = ps.pickSome(run.deviceCount, { unique: true }, simulationDevices);
-        simulationDevices = null;
+        let runDevices = [];
+        //Validate deviceList is not empty
+        if (simulationDevices.length > 0) {
+            runDevices = ps.pickSome(run.deviceCount, { unique: true }, simulationDevices);
+        }
 
         // chunk the devices into how many instances we need to run the simulation
         const task = simulation.tasks.simulation;
@@ -140,15 +143,17 @@ export class RunsService {
                 .promise();
 
             // for each instance, prepare and upload the last known device state for the devices to simulate
-            const fields = Object.keys(runDevices[0]);
-            const startIndex = (instanceId - 1) * devicesPerInstance;
-            const endIndex = startIndex + devicesPerInstance;
-            const instanceDevices = runDevices.splice(startIndex, endIndex);
-            const dataAsCsv = await parseAsync(instanceDevices, { fields });
-            s3Key = `${s3RootKey}instances/${instanceId}/deviceState.csv`;
-            await this._s3
-                .putObject({ Bucket: this._s3Bucket, Key: s3Key, Body: dataAsCsv })
-                .promise();
+            if (simulationDevices.length > 0) {
+                const fields = Object.keys(runDevices[0]);
+                const startIndex = (instanceId - 1) * devicesPerInstance;
+                const endIndex = startIndex + devicesPerInstance;
+                const instanceDevices = runDevices.splice(startIndex, endIndex);
+                const dataAsCsv = await parseAsync(instanceDevices, { fields });
+                s3Key = `${s3RootKey}instances/${instanceId}/deviceState.csv`;
+                await this._s3
+                    .putObject({ Bucket: this._s3Bucket, Key: s3Key, Body: dataAsCsv })
+                    .promise();
+            }
         }
 
         // launch the simulation
diff --git a/source/packages/services/simulation-manager/src/templates/simulation.task.properties b/source/packages/services/simulation-manager/src/templates/simulation.task.properties
@@ -13,7 +13,7 @@ threads.number={{instance.threads}}
 threads.rampup={{simulation.tasks.simulation.threads.rampUpSecs}}
 
 # simulation specific
-{{#each simulation.tasks.provisioning.attributes}}
+{{#each simulation.tasks.simulation.attributes}}
 attributes.{{@key}}={{this}}
 {{/each}}
 

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ export class FleetSimulatorInstaller implements RestModule {`
`96`	`96`	`propertyName: 's3Prefix',`
`97`	`97`	`},`
`98`	`98`	`{`
`99`		`- defaultConfiguration: '/opt/apache-jmeter-5.1.1/bin/cdf',`
	`99`	`+ defaultConfiguration: '/opt/apache-jmeter-5.4.3/bin/cdf',`
`100`	`100`	`question: 'Local location where to store jmeter execution results',`
`101`	`101`	`propertyName: 'runnerDataDir',`
`102`	`102`	`},`