VPC Flowlogs feature implementation

apis/v1alpha1/ack-generate-metadata.yaml

@@ -1,13 +1,13 @@
 ack_generate_info:
-  build_date: "2023-05-15T23:06:36Z"
-  build_hash: 8f3ba427974fd6e769926778d54834eaee3b81a3
-  go_version: go1.19
-  version: v0.26.1
-api_directory_checksum: 6e86aa4f26729ce014485b1096286db654459af7
+  build_date: "2023-06-14T17:05:27Z"
+  build_hash: e04fca8e1fa32e988ff4dd9425d758ecc8a2a178
+  go_version: go1.20.3
+  version: v0.26.1-6-ge04fca8
+api_directory_checksum: a930caa16911411d04c5e7c14b27e077993d93a1
 api_version: v1alpha1
 aws_sdk_go_version: v1.44.93
 generator_config_info:
-  file_checksum: cc2c6590c6e77a6125d5eec82ff5f693109d4f99
+  file_checksum: dbba229402fe462b5d6b0b57027b3cfafd6bad61
   original_file_name: generator.yaml
 last_modification:
   reason: API generation

apis/v1alpha1/generator.yaml

@@ -7,6 +7,11 @@ ignore:
     - AllocateAddressInput.TagSpecifications
     - CreateDhcpOptionsInput.DryRun
     - CreateDhcpOptionsInput.TagSpecifications
+    - CreateFlowLogsInput.DryRun
+    - CreateFlowLogsInput.ClientToken
+    - CreateFlowLogsInput.TagSpecifications
+    - CreateFlowLogsInput.ResourceIds
+    - CreateFlowLogsOutput.FlowLogIds
     - CreateInternetGatewayInput.DryRun
     - CreateInternetGatewayInput.TagSpecifications
     - CreateNatGatewayInput.ClientToken
@@ -141,6 +146,14 @@ operations:
     operation_type:
       - Delete
     resource_name: VpcEndpoint
+  CreateFlowLogs:
+    operation_type:
+    - Create
+    resource_name: FlowLog
+  DeleteFlowLogs:
+    operation_type:
+    - Delete 
+    resource_name: FlowLog
   RunInstances:
     #ouput shape: Reservation
     output_wrapper_field_path: Instances
@@ -268,6 +281,42 @@ resources:
         template_path: hooks/elastic_ip_address/sdk_file_end.go.tpl
     update_operation:
       custom_method_name: customUpdateElasticIP
+  FlowLog:
+    exceptions:
+      terminal_codes:
+      - InvalidParameter
+      - InvalidParameterValue
+    fields:
+      ResourceID:
+        type: string
+        is_required: true
+      FlowLogID:
+        type: string
+        is_read_only: true
+        is_primary_key: true
+      Tags:
+        from:
+          operation: CreateTags
+          path: Tags
+        compare:
+          is_ignored: True
+    hooks:
+      delta_pre_compare:
+        code: compareTags(delta, a, b)
+      sdk_create_post_build_request:
+        template_path: hooks/flow_log/sdk_create_post_build_request.go.tpl
+      sdk_create_post_set_output:
+        template_path: hooks/flow_log/sdk_create_post_set_output.go.tpl
+      sdk_delete_post_build_request:
+        template_path: hooks/flow_log/sdk_delete_post_build_request.go.tpl
+      sdk_read_many_pre_build_request:
+        template_path: hooks/flow_log/sdk_read_many_pre_build_request.go.tpl
+      sdk_read_many_post_build_request:
+        template_path: hooks/flow_log/sdk_read_many_post_build_request.go.tpl
+      sdk_file_end:
+        template_path: hooks/flow_log/sdk_file_end.go.tpl
+    update_operation:
+      custom_method_name: customUpdateFlowLog
   InternetGateway:
     fields:
       Tags:
@@ -649,4 +698,4 @@ resources:
         template_path: hooks/vpc_endpoint/sdk_file_end.go.tpl
     update_operation:
       custom_method_name: customUpdateVPCEndpoint
-
+

config/crd/bases/ec2.services.k8s.aws_flowlogs.yaml

@@ -0,0 +1,226 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: flowlogs.ec2.services.k8s.aws
+spec:
+  group: ec2.services.k8s.aws
+  names:
+    kind: FlowLog
+    listKind: FlowLogList
+    plural: flowlogs
+    singular: flowlog
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: FlowLog is the Schema for the FlowLogs API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: "FlowLogSpec defines the desired state of FlowLog. \n Describes
+              a flow log."
+            properties:
+              deliverLogsPermissionARN:
+                description: "The ARN for the IAM role that permits Amazon EC2 to
+                  publish flow logs to a CloudWatch Logs log group in your account.
+                  \n If you specify LogDestinationType as s3, do not specify DeliverLogsPermissionArn
+                  or LogGroupName."
+                type: string
+              destinationOptions:
+                description: The destination options.
+                properties:
+                  fileFormat:
+                    type: string
+                  hiveCompatiblePartitions:
+                    type: boolean
+                  perHourPartition:
+                    type: boolean
+                type: object
+              logDestination:
+                description: "The destination to which the flow log data is to be
+                  published. Flow log data can be published to a CloudWatch Logs log
+                  group or an Amazon S3 bucket. The value specified for this parameter
+                  depends on the value specified for LogDestinationType. \n If LogDestinationType
+                  is not specified or cloud-watch-logs, specify the Amazon Resource
+                  Name (ARN) of the CloudWatch Logs log group. For example, to publish
+                  to a log group called my-logs, specify arn:aws:logs:us-east-1:123456789012:log-group:my-logs.
+                  Alternatively, use LogGroupName instead. \n If LogDestinationType
+                  is s3, specify the ARN of the Amazon S3 bucket. You can also specify
+                  a subfolder in the bucket. To specify a subfolder in the bucket,
+                  use the following ARN format: bucket_ARN/subfolder_name/. For example,
+                  to specify a subfolder named my-logs in a bucket named my-bucket,
+                  use the following ARN: arn:aws:s3:::my-bucket/my-logs/. You cannot
+                  use AWSLogs as a subfolder name. This is a reserved term."
+                type: string
+              logDestinationType:
+                description: "The type of destination to which the flow log data is
+                  to be published. Flow log data can be published to CloudWatch Logs
+                  or Amazon S3. To publish flow log data to CloudWatch Logs, specify
+                  cloud-watch-logs. To publish flow log data to Amazon S3, specify
+                  s3. \n If you specify LogDestinationType as s3, do not specify DeliverLogsPermissionArn
+                  or LogGroupName. \n Default: cloud-watch-logs"
+                type: string
+              logFormat:
+                description: "The fields to include in the flow log record, in the
+                  order in which they should appear. For a list of available fields,
+                  see Flow log records (https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-log-records).
+                  If you omit this parameter, the flow log is created using the default
+                  format. If you specify this parameter, you must specify at least
+                  one field. \n Specify the fields using the ${field-id} format, separated
+                  by spaces. For the CLI, surround this parameter value with single
+                  quotes on Linux or double quotes on Windows."
+                type: string
+              logGroupName:
+                description: "The name of a new or existing CloudWatch Logs log group
+                  where Amazon EC2 publishes your flow logs. \n If you specify LogDestinationType
+                  as s3, do not specify DeliverLogsPermissionArn or LogGroupName."
+                type: string
+              maxAggregationInterval:
+                description: "The maximum interval of time during which a flow of
+                  packets is captured and aggregated into a flow log record. You can
+                  specify 60 seconds (1 minute) or 600 seconds (10 minutes). \n When
+                  a network interface is attached to a Nitro-based instance (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances),
+                  the aggregation interval is always 60 seconds or less, regardless
+                  of the value that you specify. \n Default: 600"
+                format: int64
+                type: integer
+              resourceID:
+                type: string
+              resourceType:
+                description: The type of resource for which to create the flow log.
+                  For example, if you specified a VPC ID for the ResourceId property,
+                  specify VPC for this property.
+                type: string
+              tags:
+                description: The tags. The value parameter is required, but if you
+                  don't want the tag to have a value, specify the parameter with no
+                  value, and we set the value to an empty string.
+                items:
+                  description: Describes a tag.
+                  properties:
+                    key:
+                      type: string
+                    value:
+                      type: string
+                  type: object
+                type: array
+              trafficType:
+                description: The type of traffic to log. You can log traffic that
+                  the resource accepts or rejects, or all traffic.
+                type: string
+            required:
+            - resourceID
+            - resourceType
+            type: object
+          status:
+            description: FlowLogStatus defines the observed state of FlowLog
+            properties:
+              ackResourceMetadata:
+                description: All CRs managed by ACK have a common `Status.ACKResourceMetadata`
+                  member that is used to contain resource sync state, account ownership,
+                  constructed ARN for the resource
+                properties:
+                  arn:
+                    description: 'ARN is the Amazon Resource Name for the resource.
+                      This is a globally-unique identifier and is set only by the
+                      ACK service controller once the controller has orchestrated
+                      the creation of the resource OR when it has verified that an
+                      "adopted" resource (a resource where the ARN annotation was
+                      set by the Kubernetes user on the CR) exists and matches the
+                      supplied CR''s Spec field values. TODO(vijat@): Find a better
+                      strategy for resources that do not have ARN in CreateOutputResponse
+                      https://github.com/aws/aws-controllers-k8s/issues/270'
+                    type: string
+                  ownerAccountID:
+                    description: OwnerAccountID is the AWS Account ID of the account
+                      that owns the backend AWS service API resource.
+                    type: string
+                  region:
+                    description: Region is the AWS region in which the resource exists
+                      or will exist.
+                    type: string
+                required:
+                - ownerAccountID
+                - region
+                type: object
+              clientToken:
+                description: Unique, case-sensitive identifier that you provide to
+                  ensure the idempotency of the request.
+                type: string
+              conditions:
+                description: All CRS managed by ACK have a common `Status.Conditions`
+                  member that contains a collection of `ackv1alpha1.Condition` objects
+                  that describe the various terminal states of the CR and its backend
+                  AWS service API resource
+                items:
+                  description: Condition is the common struct used by all CRDs managed
+                    by ACK service controllers to indicate terminal states  of the
+                    CR and its backend AWS service API resource
+                  properties:
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      format: date-time
+                      type: string
+                    message:
+                      description: A human readable message indicating details about
+                        the transition.
+                      type: string
+                    reason:
+                      description: The reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the Condition
+                      type: string
+                  required:
+                  - status
+                  - type
+                  type: object
+                type: array
+              flowLogID:
+                type: string
+              unsuccessful:
+                description: Information about the flow logs that could not be created
+                  successfully.
+                items:
+                  description: Information about items that were not successfully
+                    processed in a batch call.
+                  properties:
+                    error:
+                      description: Information about the error that occurred. For
+                        more information about errors, see Error codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html).
+                      properties:
+                        code:
+                          type: string
+                        message:
+                          type: string
+                      type: object
+                    resourceID:
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
   - common
   - bases/ec2.services.k8s.aws_dhcpoptions.yaml
   - bases/ec2.services.k8s.aws_elasticipaddresses.yaml
+  - bases/ec2.services.k8s.aws_flowlogs.yaml
   - bases/ec2.services.k8s.aws_instances.yaml
   - bases/ec2.services.k8s.aws_internetgateways.yaml
   - bases/ec2.services.k8s.aws_natgateways.yaml

config/rbac/cluster-role-controller.yaml

@@ -71,6 +71,26 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - ec2.services.k8s.aws
+  resources:
+  - flowlogs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ec2.services.k8s.aws
+  resources:
+  - flowlogs/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ec2.services.k8s.aws
   resources:

config/rbac/role-reader.yaml

@@ -11,6 +11,7 @@ rules:
   resources:
   - dhcpoptions
   - elasticipaddresses
+  - flowlogs
   - instances
   - internetgateways
   - natgateways

config/rbac/role-writer.yaml

@@ -11,6 +11,7 @@ rules:
   resources:
   - dhcpoptions
   - elasticipaddresses
+  - flowlogs
   - instances
   - internetgateways
   - natgateways
@@ -33,6 +34,7 @@ rules:
   resources:
   - dhcpoptions
   - elasticipaddresses
+  - flowlogs
   - instances
   - internetgateways
   - natgateways

generator.yaml

@@ -7,6 +7,11 @@ ignore:
     - AllocateAddressInput.TagSpecifications
     - CreateDhcpOptionsInput.DryRun
     - CreateDhcpOptionsInput.TagSpecifications
+    - CreateFlowLogsInput.DryRun
+    - CreateFlowLogsInput.ClientToken
+    - CreateFlowLogsInput.TagSpecifications
+    - CreateFlowLogsInput.ResourceIds
+    - CreateFlowLogsOutput.FlowLogIds
     - CreateInternetGatewayInput.DryRun
     - CreateInternetGatewayInput.TagSpecifications
     - CreateNatGatewayInput.ClientToken
@@ -141,6 +146,14 @@ operations:
     operation_type:
       - Delete
     resource_name: VpcEndpoint
+  CreateFlowLogs:
+    operation_type:
+    - Create
+    resource_name: FlowLog
+  DeleteFlowLogs:
+    operation_type:
+    - Delete 
+    resource_name: FlowLog
   RunInstances:
     #ouput shape: Reservation
     output_wrapper_field_path: Instances
@@ -268,6 +281,42 @@ resources:
         template_path: hooks/elastic_ip_address/sdk_file_end.go.tpl
     update_operation:
       custom_method_name: customUpdateElasticIP
+  FlowLog:
+    exceptions:
+      terminal_codes:
+      - InvalidParameter
+      - InvalidParameterValue
+    fields:
+      ResourceID:
+        type: string
+        is_required: true
+      FlowLogID:
+        type: string
+        is_read_only: true
+        is_primary_key: true
+      Tags:
+        from:
+          operation: CreateTags
+          path: Tags
+        compare:
+          is_ignored: True
+    hooks:
+      delta_pre_compare:
+        code: compareTags(delta, a, b)
+      sdk_create_post_build_request:
+        template_path: hooks/flow_log/sdk_create_post_build_request.go.tpl
+      sdk_create_post_set_output:
+        template_path: hooks/flow_log/sdk_create_post_set_output.go.tpl
+      sdk_delete_post_build_request:
+        template_path: hooks/flow_log/sdk_delete_post_build_request.go.tpl
+      sdk_read_many_pre_build_request:
+        template_path: hooks/flow_log/sdk_read_many_pre_build_request.go.tpl
+      sdk_read_many_post_build_request:
+        template_path: hooks/flow_log/sdk_read_many_post_build_request.go.tpl
+      sdk_file_end:
+        template_path: hooks/flow_log/sdk_file_end.go.tpl
+    update_operation:
+      custom_method_name: customUpdateFlowLog
   InternetGateway:
     fields:
       Tags:
@@ -649,4 +698,4 @@ resources:
         template_path: hooks/vpc_endpoint/sdk_file_end.go.tpl
     update_operation:
       custom_method_name: customUpdateVPCEndpoint
-
+

helm/crds/ec2.services.k8s.aws_flowlogs.yaml

@@ -0,0 +1,226 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: flowlogs.ec2.services.k8s.aws
+spec:
+  group: ec2.services.k8s.aws
+  names:
+    kind: FlowLog
+    listKind: FlowLogList
+    plural: flowlogs
+    singular: flowlog
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: FlowLog is the Schema for the FlowLogs API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: "FlowLogSpec defines the desired state of FlowLog. \n Describes
+              a flow log."
+            properties:
+              deliverLogsPermissionARN:
+                description: "The ARN for the IAM role that permits Amazon EC2 to
+                  publish flow logs to a CloudWatch Logs log group in your account.
+                  \n If you specify LogDestinationType as s3, do not specify DeliverLogsPermissionArn
+                  or LogGroupName."
+                type: string
+              destinationOptions:
+                description: The destination options.
+                properties:
+                  fileFormat:
+                    type: string
+                  hiveCompatiblePartitions:
+                    type: boolean
+                  perHourPartition:
+                    type: boolean
+                type: object
+              logDestination:
+                description: "The destination to which the flow log data is to be
+                  published. Flow log data can be published to a CloudWatch Logs log
+                  group or an Amazon S3 bucket. The value specified for this parameter
+                  depends on the value specified for LogDestinationType. \n If LogDestinationType
+                  is not specified or cloud-watch-logs, specify the Amazon Resource
+                  Name (ARN) of the CloudWatch Logs log group. For example, to publish
+                  to a log group called my-logs, specify arn:aws:logs:us-east-1:123456789012:log-group:my-logs.
+                  Alternatively, use LogGroupName instead. \n If LogDestinationType
+                  is s3, specify the ARN of the Amazon S3 bucket. You can also specify
+                  a subfolder in the bucket. To specify a subfolder in the bucket,
+                  use the following ARN format: bucket_ARN/subfolder_name/. For example,
+                  to specify a subfolder named my-logs in a bucket named my-bucket,
+                  use the following ARN: arn:aws:s3:::my-bucket/my-logs/. You cannot
+                  use AWSLogs as a subfolder name. This is a reserved term."
+                type: string
+              logDestinationType:
+                description: "The type of destination to which the flow log data is
+                  to be published. Flow log data can be published to CloudWatch Logs
+                  or Amazon S3. To publish flow log data to CloudWatch Logs, specify
+                  cloud-watch-logs. To publish flow log data to Amazon S3, specify
+                  s3. \n If you specify LogDestinationType as s3, do not specify DeliverLogsPermissionArn
+                  or LogGroupName. \n Default: cloud-watch-logs"
+                type: string
+              logFormat:
+                description: "The fields to include in the flow log record, in the
+                  order in which they should appear. For a list of available fields,
+                  see Flow log records (https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-log-records).
+                  If you omit this parameter, the flow log is created using the default
+                  format. If you specify this parameter, you must specify at least
+                  one field. \n Specify the fields using the ${field-id} format, separated
+                  by spaces. For the CLI, surround this parameter value with single
+                  quotes on Linux or double quotes on Windows."
+                type: string
+              logGroupName:
+                description: "The name of a new or existing CloudWatch Logs log group
+                  where Amazon EC2 publishes your flow logs. \n If you specify LogDestinationType
+                  as s3, do not specify DeliverLogsPermissionArn or LogGroupName."
+                type: string
+              maxAggregationInterval:
+                description: "The maximum interval of time during which a flow of
+                  packets is captured and aggregated into a flow log record. You can
+                  specify 60 seconds (1 minute) or 600 seconds (10 minutes). \n When
+                  a network interface is attached to a Nitro-based instance (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances),
+                  the aggregation interval is always 60 seconds or less, regardless
+                  of the value that you specify. \n Default: 600"
+                format: int64
+                type: integer
+              resourceID:
+                type: string
+              resourceType:
+                description: The type of resource for which to create the flow log.
+                  For example, if you specified a VPC ID for the ResourceId property,
+                  specify VPC for this property.
+                type: string
+              tags:
+                description: The tags. The value parameter is required, but if you
+                  don't want the tag to have a value, specify the parameter with no
+                  value, and we set the value to an empty string.
+                items:
+                  description: Describes a tag.
+                  properties:
+                    key:
+                      type: string
+                    value:
+                      type: string
+                  type: object
+                type: array
+              trafficType:
+                description: The type of traffic to log. You can log traffic that
+                  the resource accepts or rejects, or all traffic.
+                type: string
+            required:
+            - resourceID
+            - resourceType
+            type: object
+          status:
+            description: FlowLogStatus defines the observed state of FlowLog
+            properties:
+              ackResourceMetadata:
+                description: All CRs managed by ACK have a common `Status.ACKResourceMetadata`
+                  member that is used to contain resource sync state, account ownership,
+                  constructed ARN for the resource
+                properties:
+                  arn:
+                    description: 'ARN is the Amazon Resource Name for the resource.
+                      This is a globally-unique identifier and is set only by the
+                      ACK service controller once the controller has orchestrated
+                      the creation of the resource OR when it has verified that an
+                      "adopted" resource (a resource where the ARN annotation was
+                      set by the Kubernetes user on the CR) exists and matches the
+                      supplied CR''s Spec field values. TODO(vijat@): Find a better
+                      strategy for resources that do not have ARN in CreateOutputResponse
+                      https://github.com/aws/aws-controllers-k8s/issues/270'
+                    type: string
+                  ownerAccountID:
+                    description: OwnerAccountID is the AWS Account ID of the account
+                      that owns the backend AWS service API resource.
+                    type: string
+                  region:
+                    description: Region is the AWS region in which the resource exists
+                      or will exist.
+                    type: string
+                required:
+                - ownerAccountID
+                - region
+                type: object
+              clientToken:
+                description: Unique, case-sensitive identifier that you provide to
+                  ensure the idempotency of the request.
+                type: string
+              conditions:
+                description: All CRS managed by ACK have a common `Status.Conditions`
+                  member that contains a collection of `ackv1alpha1.Condition` objects
+                  that describe the various terminal states of the CR and its backend
+                  AWS service API resource
+                items:
+                  description: Condition is the common struct used by all CRDs managed
+                    by ACK service controllers to indicate terminal states  of the
+                    CR and its backend AWS service API resource
+                  properties:
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another.
+                      format: date-time
+                      type: string
+                    message:
+                      description: A human readable message indicating details about
+                        the transition.
+                      type: string
+                    reason:
+                      description: The reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the Condition
+                      type: string
+                  required:
+                  - status
+                  - type
+                  type: object
+                type: array
+              flowLogID:
+                type: string
+              unsuccessful:
+                description: Information about the flow logs that could not be created
+                  successfully.
+                items:
+                  description: Information about items that were not successfully
+                    processed in a batch call.
+                  properties:
+                    error:
+                      description: Information about the error that occurred. For
+                        more information about errors, see Error codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html).
+                      properties:
+                        code:
+                          type: string
+                        message:
+                          type: string
+                      type: object
+                    resourceID:
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

helm/templates/cluster-role-controller.yaml

@@ -86,6 +86,26 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - ec2.services.k8s.aws
+  resources:
+  - flowlogs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ec2.services.k8s.aws
+  resources:
+  - flowlogs/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ec2.services.k8s.aws
   resources:

helm/templates/deployment.yaml

@@ -18,10 +18,12 @@ spec:
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
+{{- if .Values.deployment.annotations }}
       annotations:
       {{- range $key, $value := .Values.deployment.annotations }}
         {{ $key }}: {{ $value | quote }}
       {{- end }}
+{{- end }}
       labels:
         app.kubernetes.io/name: {{ include "app.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
@@ -104,11 +106,19 @@ spec:
           value: {{ include "aws.credentials.path" . }}
         - name: AWS_PROFILE
           value: {{ .Values.aws.credentials.profile }}
+        {{- end }}
+        {{- if .Values.deployment.extraEnvVars -}}
+          {{ toYaml .Values.deployment.extraEnvVars | nindent 8 }}
+        {{- end }}
         volumeMounts:
+        {{- if .Values.aws.credentials.secretName }}
           - name: {{ .Values.aws.credentials.secretName }}
             mountPath: {{ include "aws.credentials.secret_mount_path" . }}
             readOnly: true
         {{- end }}
+        {{- if .Values.deployment.extraVolumeMounts -}}
+          {{ toYaml .Values.deployment.extraVolumeMounts | nindent 12 }}
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           privileged: false
@@ -133,9 +143,12 @@ spec:
       hostIPC: false
       hostNetwork: false
       hostPID: false
-      {{ if .Values.aws.credentials.secretName -}}
       volumes:
+      {{- if .Values.aws.credentials.secretName -}}
         - name: {{ .Values.aws.credentials.secretName }}
           secret:
             secretName: {{ .Values.aws.credentials.secretName }}
       {{ end -}}
+{{- if .Values.deployment.extraVolumes }}
+{{ toYaml .Values.deployment.extraVolumes | indent 8}}
+{{- end }}

helm/templates/role-reader.yaml

@@ -11,6 +11,7 @@ rules:
   resources:
   - dhcpoptions
   - elasticipaddresses
+  - flowlogs
   - instances
   - internetgateways
   - natgateways

helm/templates/role-writer.yaml

@@ -13,6 +13,8 @@ rules:
 
   - elasticipaddresses
 
+  - flowlogs
+
   - instances
 
   - internetgateways
@@ -44,6 +46,7 @@ rules:
   resources:
   - dhcpoptions
   - elasticipaddresses
+  - flowlogs
   - instances
   - internetgateways
   - natgateways

helm/values.schema.json

@@ -58,6 +58,15 @@
         },
         "priorityClassName": {
           "type": "string"
+        },
+        "extraVolumeMounts": {
+          "type": "array"
+        },
+        "extraVolumes": {
+          "type": "array"
+        },
+        "extraEnvVars": {
+          "type": "array"
         }
       },
       "required": [

helm/values.yaml

@@ -28,6 +28,26 @@ deployment:
   # Which priorityClassName to set?
   # See: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority
   priorityClassName: ""
+  extraVolumes: []
+  extraVolumeMounts: []
+
+  # Additional server container environment variables
+  #
+  # You specify this manually like you would a raw deployment manifest.
+  # This means you can bind in environment variables from secrets.
+  #
+  # e.g. static environment variable:
+  #  - name: DEMO_GREETING
+  #    value: "Hello from the environment"
+  #
+  # e.g. secret environment variable:
+  # - name: USERNAME
+  #   valueFrom:
+  #     secretKeyRef:
+  #       name: mysecret
+  #       key: username
+  extraEnvVars: []
+
 
 # If "installScope: cluster" then these labels will be applied to ClusterRole
 role:

pkg/resource/flow_log/hooks.go

@@ -0,0 +1,189 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package flow_log
+
+import (
+	"context"
+
+	svcapitypes "github.com/aws-controllers-k8s/ec2-controller/apis/v1alpha1"
+	ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare"
+	ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log"
+	svcsdk "github.com/aws/aws-sdk-go/service/ec2"
+)
+
+func (rm *resourceManager) customUpdateFlowLog(
+	ctx context.Context,
+	desired *resource,
+	latest *resource,
+	delta *ackcompare.Delta,
+) (updated *resource, err error) {
+	rlog := ackrtlog.FromContext(ctx)
+	exit := rlog.Trace("rm.customUpdateFlowLog")
+	defer exit(err)
+
+	// Default `updated` to `desired` because it is likely
+	// EC2 `modify` APIs do NOT return output, only errors.
+	// If the `modify` calls (i.e. `sync`) do NOT return
+	// an error, then the update was successful and desired.Spec
+	// (now updated.Spec) reflects the latest resource state.
+	updated = rm.concreteResource(desired.DeepCopy())
+
+	if delta.DifferentAt("Spec.Tags") {
+		if err := rm.syncTags(ctx, desired, latest); err != nil {
+			return nil, err
+		}
+	}
+
+	return updated, nil
+}
+
+// syncTags used to keep tags in sync by calling Create and Delete API's
+func (rm *resourceManager) syncTags(
+	ctx context.Context,
+	desired *resource,
+	latest *resource,
+) (err error) {
+	rlog := ackrtlog.FromContext(ctx)
+	exit := rlog.Trace("rm.syncTags")
+	defer func(err error) {
+		exit(err)
+	}(err)
+
+	resourceId := []*string{latest.ko.Status.FlowLogID}
+
+	toAdd, toDelete := computeTagsDelta(
+		desired.ko.Spec.Tags, latest.ko.Spec.Tags,
+	)
+
+	if len(toDelete) > 0 {
+		rlog.Debug("removing tags from FlowLog resource", "tags", toDelete)
+		_, err = rm.sdkapi.DeleteTagsWithContext(
+			ctx,
+			&svcsdk.DeleteTagsInput{
+				Resources: resourceId,
+				Tags:      rm.sdkTags(toDelete),
+			},
+		)
+		rm.metrics.RecordAPICall("UPDATE", "DeleteTags", err)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	if len(toAdd) > 0 {
+		rlog.Debug("adding tags to FlowLog resource", "tags", toAdd)
+		_, err = rm.sdkapi.CreateTagsWithContext(
+			ctx,
+			&svcsdk.CreateTagsInput{
+				Resources: resourceId,
+				Tags:      rm.sdkTags(toAdd),
+			},
+		)
+		rm.metrics.RecordAPICall("UPDATE", "CreateTags", err)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// sdkTags converts *svcapitypes.Tag array to a *svcsdk.Tag array
+func (rm *resourceManager) sdkTags(
+	tags []*svcapitypes.Tag,
+) (sdktags []*svcsdk.Tag) {
+
+	for _, i := range tags {
+		sdktag := rm.newTag(*i)
+		sdktags = append(sdktags, sdktag)
+	}
+
+	return sdktags
+}
+
+// computeTagsDelta returns tags to be added and removed from the resource
+func computeTagsDelta(
+	desired []*svcapitypes.Tag,
+	latest []*svcapitypes.Tag,
+) (toAdd []*svcapitypes.Tag, toDelete []*svcapitypes.Tag) {
+
+	desiredTags := map[string]string{}
+	for _, tag := range desired {
+		desiredTags[*tag.Key] = *tag.Value
+	}
+
+	latestTags := map[string]string{}
+	for _, tag := range latest {
+		latestTags[*tag.Key] = *tag.Value
+	}
+
+	for _, tag := range desired {
+		val, ok := latestTags[*tag.Key]
+		if !ok || val != *tag.Value {
+			toAdd = append(toAdd, tag)
+		}
+	}
+
+	for _, tag := range latest {
+		_, ok := desiredTags[*tag.Key]
+		if !ok {
+			toDelete = append(toDelete, tag)
+		}
+	}
+
+	return toAdd, toDelete
+
+}
+
+// compareTags is a custom comparison function for comparing lists of Tag
+// structs where the order of the structs in the list is not important.
+func compareTags(
+	delta *ackcompare.Delta,
+	a *resource,
+	b *resource,
+) {
+	if len(a.ko.Spec.Tags) != len(b.ko.Spec.Tags) {
+		delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
+	} else if len(a.ko.Spec.Tags) > 0 {
+		addedOrUpdated, removed := computeTagsDelta(a.ko.Spec.Tags, b.ko.Spec.Tags)
+		if len(addedOrUpdated) != 0 || len(removed) != 0 {
+			delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
+		}
+	}
+}
+
+// updateTagSpecificationsInCreateRequest adds
+// Tags defined in the Spec to CreateFlowLogsInput.TagSpecification
+// and ensures the ResourceType is always set to 'FlowLog'
+func updateTagSpecificationsInCreateRequest(r *resource,
+	input *svcsdk.CreateFlowLogsInput) {
+	input.TagSpecifications = nil
+	desiredTagSpecs := svcsdk.TagSpecification{}
+	if r.ko.Spec.Tags != nil {
+		requestedTags := []*svcsdk.Tag{}
+		for _, desiredTag := range r.ko.Spec.Tags {
+			// Add in tags defined in the Spec
+			tag := &svcsdk.Tag{}
+			if desiredTag.Key != nil && desiredTag.Value != nil {
+				tag.SetKey(*desiredTag.Key)
+				tag.SetValue(*desiredTag.Value)
+			}
+			requestedTags = append(requestedTags, tag)
+		}
+		desiredTagSpecs.SetResourceType("vpc-flow-log")
+		desiredTagSpecs.SetTags(requestedTags)
+		input.TagSpecifications = []*svcsdk.TagSpecification{&desiredTagSpecs}
+	}
+}

templates/hooks/flow_log/sdk_create_post_build_request.go.tpl

@@ -0,0 +1,2 @@
+    updateTagSpecificationsInCreateRequest(desired, input)
+    input.SetResourceIds([]*string{desired.ko.Spec.ResourceID})

templates/hooks/flow_log/sdk_create_post_set_output.go.tpl

@@ -0,0 +1,3 @@
+    if resp.FlowLogIds[0] != nil {
+		ko.Status.FlowLogID = resp.FlowLogIds[0]
+	}

templates/hooks/flow_log/sdk_delete_post_build_request.go.tpl

@@ -0,0 +1,4 @@
+ 	if r.ko.Status.FlowLogID == nil {
+		return nil, ackerr.NotFound
+	}
+    input.SetFlowLogIds([]*string{r.ko.Status.FlowLogID})

templates/hooks/flow_log/sdk_file_end.go.tpl

@@ -0,0 +1,23 @@
+{{ $CRD := .CRD }}
+{{ $SDKAPI := .SDKAPI }}
+
+{{/* Generate helper methods for Flow Log */}}
+{{- range $specFieldName, $specField := $CRD.Config.Resources.FlowLog.Fields }}
+{{- if $specField.From }}
+{{- $operationName := $specField.From.Operation }}
+{{- $operation := (index $SDKAPI.API.Operations $operationName) -}}
+{{- range $flowLogRefName, $flowLogMemberRefs := $operation.InputRef.Shape.MemberRefs -}}
+{{- if eq $flowLogRefName "Tags" }}
+{{- $flowLogRef := $flowLogMemberRefs.Shape.MemberRef }}
+{{- $flowLogRefName = "Tag" }}
+func (rm *resourceManager) new{{ $flowLogRefName }}(
+	    c svcapitypes.{{ $flowLogRefName }},
+) *svcsdk.{{ $flowLogRefName }} {
+	res := &svcsdk.{{ $flowLogRefName }}{}
+{{ GoCodeSetSDKForStruct $CRD "" "res" $flowLogRef "" "c" 1 }}
+	return res
+}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

templates/hooks/flow_log/sdk_read_many_post_build_request.go.tpl

@@ -0,0 +1 @@
+	input.SetFlowLogIds([]*string{r.ko.Status.FlowLogID})

templates/hooks/flow_log/sdk_read_many_pre_build_request.go.tpl

@@ -0,0 +1,3 @@
+    if r.ko.Status.FlowLogID == nil {
+		return nil, ackerr.NotFound
+	}

test/e2e/bootstrap_resources.py

@@ -15,11 +15,13 @@
 """
 from dataclasses import dataclass
 from acktest.bootstrapping import Resources
+from acktest.bootstrapping.s3 import Bucket
 from acktest.bootstrapping.vpc import VPC
 from e2e import bootstrap_directory
 
 @dataclass
 class BootstrapResources(Resources):
+    FlowLogsBucket: Bucket
     SharedTestVPC: VPC
 
 _bootstrap_resources = None
@@ -28,4 +30,4 @@ def get_bootstrap_resources(bootstrap_file_name: str = "bootstrap.pkl") -> Boots
     global _bootstrap_resources
     if _bootstrap_resources is None:
         _bootstrap_resources = BootstrapResources.deserialize(bootstrap_directory, bootstrap_file_name=bootstrap_file_name)
-    return _bootstrap_resources
+    return _bootstrap_resources

test/e2e/resources/flow_log.yaml

@@ -0,0 +1,13 @@
+apiVersion: ec2.services.k8s.aws/v1alpha1
+kind: FlowLog
+metadata:
+  name: $FLOWLOG_NAME 
+spec:
+  resourceID: $RESOURCE_ID
+  resourceType: $RESOURCE_TYPE 
+  logDestinationType: $LOG_DESTINATION_TYPE
+  logDestination: $LOG_DESTINATION 
+  trafficType: $TRAFFIC_TYPE 
+  tags:
+    - key: $TAG_KEY
+      value: $TAG_VALUE

test/e2e/resources/invalid/flow_log_invalid_parameter.yaml

@@ -0,0 +1,12 @@
+apiVersion: ec2.services.k8s.aws/v1alpha1
+kind: FlowLog
+metadata:
+  name: $FLOWLOG_NAME 
+spec:
+  resourceID: $RESOURCE_ID
+  resourceType: $RESOURCE_TYPE 
+  logDestinationType: $LOG_DESTINATION_TYPE
+  trafficType: $TRAFFIC_TYPE 
+  tags:
+    - key: $TAG_KEY
+      value: $TAG_VALUE

test/e2e/service_bootstrap.py

@@ -17,14 +17,18 @@
 
 from acktest.bootstrapping import Resources, BootstrapFailureException
 from acktest.bootstrapping.vpc import VPC
+from acktest.bootstrapping.s3 import Bucket
 from e2e import bootstrap_directory
 from e2e.bootstrap_resources import BootstrapResources
 
 def service_bootstrap() -> Resources:
     logging.getLogger().setLevel(logging.INFO)
 
     resources = BootstrapResources(
-        SharedTestVPC=VPC(name_prefix="e2e-test-vpc", num_public_subnet=1, num_private_subnet=0)
+        SharedTestVPC=VPC(name_prefix="e2e-test-vpc", num_public_subnet=1, num_private_subnet=0),
+        FlowLogsBucket=Bucket(
+            "ack-ec2-controller-flow-log-tests",
+        ),
     )
 
     try:
@@ -37,4 +41,4 @@ def service_bootstrap() -> Resources:
 if __name__ == "__main__":
     config = service_bootstrap()
     # Write config to current directory by default
-    config.serialize(bootstrap_directory)
+    config.serialize(bootstrap_directory)

test/e2e/tests/test_flow_logs.py

@@ -0,0 +1,150 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Integration tests for the Flow Log API.
+"""
+
+import pytest
+import time
+import logging
+
+from acktest import tags
+from acktest.resources import random_suffix_name
+from acktest.k8s import resource as k8s
+from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource
+from e2e.replacement_values import REPLACEMENT_VALUES
+from e2e.bootstrap_resources import get_bootstrap_resources
+
+RESOURCE_PLURAL = "flowlogs"
+
+CREATE_WAIT_AFTER_SECONDS = 10
+DELETE_WAIT_AFTER_SECONDS = 10
+MODIFY_WAIT_AFTER_SECONDS = 5
+
+
+def get_flow_log_ids(ec2_client, flow_log_id: str) -> list:
+    flow_log_ids = [flow_log_id]
+    try:
+        resp = ec2_client.describe_flow_logs(
+            FlowLogIds=flow_log_ids
+        )
+    except Exception as e:
+        logging.debug(e)
+        return None
+
+    flow_log_ids = []
+    for flow_log in resp['FlowLogs']:
+        flow_log_ids.append(flow_log['FlowLogId'])
+
+    if len(flow_log_ids) == 0:
+        return None
+    return flow_log_ids
+
+
+def flow_log_exists(ec2_client, flow_log_id: str) -> bool:
+    return get_flow_log_ids(ec2_client, flow_log_id) is not None
+
+@pytest.fixture
+def simple_flow_log(request):
+    resource_name = random_suffix_name("flow-log-ack-test", 24)
+    resource_file = "flow_log"
+    resources = get_bootstrap_resources()
+
+
+    replacements = REPLACEMENT_VALUES.copy()
+    replacements["FLOWLOG_NAME"] = resource_name
+    replacements["RESOURCE_ID"] = resources.SharedTestVPC.vpc_id
+    replacements["RESOURCE_TYPE"] = "VPC"
+    replacements["LOG_DESTINATION_TYPE"] = "s3"
+    replacements["LOG_DESTINATION"] = "arn:aws:s3:::" + resources.FlowLogsBucket.name 
+    replacements["TRAFFIC_TYPE"] = "ALL"
+    replacements["TAG_KEY"] = "Name"
+    replacements["TAG_VALUE"] = resource_name
+
+    marker = request.node.get_closest_marker("resource_data")
+    if marker is not None:
+        data = marker.args[0]
+        if 'resource_file' in data:
+            resource_file = data['resource_file']
+        if 'resource_type' in data:
+            replacements["RESOURCE_TYPE"] = data['resource_type']
+
+    # Load FlowLog CR
+    resource_data = load_ec2_resource(
+        resource_file,
+        additional_replacements=replacements,
+    )
+    logging.debug(resource_data)
+
+    # Create k8s resource
+    ref = k8s.CustomResourceReference(
+        CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
+        resource_name, namespace="default",
+    )
+    k8s.create_custom_resource(ref, resource_data)
+    time.sleep(CREATE_WAIT_AFTER_SECONDS)
+
+    cr = k8s.wait_resource_consumed_by_controller(ref)
+    assert cr is not None
+    assert k8s.get_resource_exists(ref)
+
+    yield (ref, cr)
+
+    # Try to delete, if doesn't already exist
+    try:
+        _, deleted = k8s.delete_custom_resource(ref, 3, 10)
+        assert deleted
+    except:
+        pass
+
+@service_marker
+@pytest.mark.canary
+class TestFlowLogs:
+    def test_create_delete(self, ec2_client, simple_flow_log):
+        (ref, cr) = simple_flow_log
+        resource_id = cr["status"]["flowLogID"]
+
+        # Check Flow Log exists
+        exists = flow_log_exists(ec2_client, resource_id)
+        assert exists
+
+        # Delete k8s resource
+        _, deleted = k8s.delete_custom_resource(ref, 2, 5)
+        assert deleted is True
+
+        time.sleep(DELETE_WAIT_AFTER_SECONDS)
+
+        # Check Flow Log doesn't exist
+        exists = flow_log_exists(ec2_client, resource_id)
+        assert not exists
+
+    @pytest.mark.resource_data({'resource_type': 'InvalidResource'})
+    def test_terminal_condition_invalid_parameter_value(self, simple_flow_log):
+        (ref, cr) = simple_flow_log
+
+        expected_msg = "InvalidParameterValue: "
+        terminal_condition = k8s.get_resource_condition(ref, "ACK.Terminal")
+        # Example condition message:
+        # InvalidParameterValue: 1 validation error detected: Value 'S3' at 'resourceType'
+        # has an invalid format
+        assert expected_msg in terminal_condition['message']
+
+    @pytest.mark.resource_data({'resource_file': 'invalid/flow_log_invalid_parameter'})
+    def test_terminal_condition_invalid_parameter(self, simple_flow_log):
+        (ref, cr) = simple_flow_log
+
+        expected_msg = "InvalidParameter: "
+        terminal_condition = k8s.get_resource_condition(ref, "ACK.Terminal")
+        # Example condition message:
+        # InvalidParameter: LogDestination can't be empty if LogGroupName is not provided.
+        assert expected_msg in terminal_condition['message']