Feature/alexaz private ip new standard support

.gitignore

@@ -100,6 +100,6 @@ cloudformation/deploy_execution_server_by_selected_version/*
 */.aws/*
 .aws/*
 /local/
-
 /aws_venv*
 /drivers/aws_shell/src/test_*
+/venv/*

.travis.yml

@@ -9,7 +9,7 @@ install:
   - pip install "cloudshell-core>=2.2.0,<2.3.0"
   - pip install "cloudshell-shell-core>=3.1.0,<3.2.0"
   - pip install "cloudshell-automation-api>=8.3.0.0,<8.3.1.0" --extra-index-url https://testpypi.python.org/simple
-  - pip install "cloudshell-cp-core>=1.0.0,<1.1.0" --extra-index-url https://test.pypi.org/simple
+  - pip install "cloudshell-cp-core>=1.1.0,<1.2.0" --extra-index-url https://test.pypi.org/simple
 script:
     - pushd package
     - python setup.py develop

drivers/aws_shell/src/drivermetadata.xml

@@ -1,4 +1,4 @@
-<Driver Description="This driver orchestrate all the command that will be executed on AWS" MainClass="driver.AWSShellDriver" Name="AWS Shell Driver" Version="2.5.1">
+<Driver Description="This driver orchestrate all the command that will be executed on AWS" MainClass="driver.AWSShellDriver" Name="AWS Shell Driver" Version="2.6.0">
     <Layout>
         <Category Name="Deployment">
             <Command Description="" DisplayName="Deploy From AMI" EnableCancellation="true" Name="deploy_ami" Tags="allow_unreserved" />

drivers/aws_shell/src/requirements.txt

@@ -1,5 +1,5 @@
 cloudshell-automation-api>=9.3
 cloudshell-shell-core>=3.1.0,<3.2.0
-cloudshell-cp-aws>=2.5.0,<2.6.0
+cloudshell-cp-aws>=2.6.0,<2.7.0
 jsonpickle==0.9.3
-cloudshell-cp-core>=1.0.4,<1.1.0
+cloudshell-cp-core>=1.1.0,<1.2.0

drivers/version.txt

@@ -1 +1 @@
-2.5.1
+2.6.0

package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py

@@ -2,6 +2,9 @@
 import uuid
 from multiprocessing import TimeoutError
 
+import ipaddress
+from typing import List, Dict
+
 from cloudshell.cp.aws.domain.common.cancellation_service import CommandCancellationService
 from cloudshell.cp.aws.domain.common.list_helper import first_or_default
 from cloudshell.cp.aws.domain.common.vm_details_provider import VmDetailsProvider
@@ -23,6 +26,7 @@
 from cloudshell.cp.aws.models.network_actions_models import DeployNetworkingResultModel
 from cloudshell.cp.core.models import ConnectToSubnetActionResult, ConnectToSubnetParams, ConnectSubnet, DeployAppResult
 from cloudshell.cp.core.utils import convert_dict_to_attributes_list
+from cloudshell.cp.core.requested_ips.mapper import RequestedIPsMapper
 
 
 class DeployAMIOperation(object):
@@ -177,11 +181,13 @@ def deploy(self, ec2_session, s3_session, name, reservation, aws_ec2_cp_resource
 
         deploy_app_result = DeployAppResult(vmName=self._get_name_from_tags(instance),
                                             vmUuid=instance.instance_id,
-                                            deployedAppAttributes=convert_dict_to_attributes_list(deployed_app_attributes),
+                                            deployedAppAttributes=convert_dict_to_attributes_list(
+                                                deployed_app_attributes),
                                             deployedAppAddress=instance.private_ip_address,
                                             vmDetailsData=vm_details_data,
-                                            deployedAppAdditionalData={'inbound_ports': ami_deployment_model.inbound_ports,
-                                                                       'public_ip': instance.public_ip_address})
+                                            deployedAppAdditionalData={
+                                                'inbound_ports': ami_deployment_model.inbound_ports,
+                                                'public_ip': instance.public_ip_address})
         deploy_app_result.actionId = ami_deploy_action.actionId
         network_actions_results_dtos.append(deploy_app_result)
         return network_actions_results_dtos
@@ -286,7 +292,7 @@ def _get_ami_credentials(self, s3_session, key_pair_location, reservation, wait_
                                                                           cancellation_context=cancellation_context)
             except TimeoutError:
                 logger.info(
-                        "Timeout when waiting for windows credentials. Traceback: {0}".format(traceback.format_exc()))
+                    "Timeout when waiting for windows credentials. Traceback: {0}".format(traceback.format_exc()))
                 return None
         else:
             return None if self._get_deployed_app_resource_user_attribute(ami_deploy_action) else \
@@ -305,7 +311,7 @@ def _get_deployed_app_resource_user_attribute(ami_deploy_action):
         :return:
         """
         attribute_names_in_deployed_resource = ami_deploy_action.actionParams.appResource.attributes.keys()
-        return next((attr for attr in attribute_names_in_deployed_resource if attr.split('.')[-1]=='User'))
+        return next((attr for attr in attribute_names_in_deployed_resource if attr.split('.')[-1] == 'User'))
 
     @staticmethod
     def _get_name_from_tags(result):
@@ -376,6 +382,7 @@ def _create_deployment_parameters(self,
         :param network_config_results: list of network configuration result objects
         :type network_config_results: list[DeployNetworkingResultModel]
         :param logging.Logger logger:
+        :rtype: AMIDeploymentModel
         """
         aws_model = AMIDeploymentModel()
         if not ami_deployment_model.aws_ami_id:
@@ -427,20 +434,16 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions
         """
         :param vpc: The reservation VPC
         :param DeployAWSEc2AMIInstanceResourceModel ami_deployment_model:
-        :param cloudshell.cp.core.models.ConnectSubnet network_actions:
+        :param List[cloudshell.cp.core.models.ConnectSubnet] network_actions:
         :param [str] security_group_ids:
-        :param list[DeployNetworkingResultModel] network_config_results: list of network configuration result objects
+        :param List[DeployNetworkingResultModel] network_config_results: list of network configuration result objects
         :param logging.Logger logger:
         :return:
         """
         if not network_actions:
             logger.info("Single subnet mode detected")
-            network_config_results[0].device_index = 0
-            return [self.network_interface_service.get_network_interface_for_single_subnet_mode(
-                add_public_ip=ami_deployment_model.add_public_ip,
-                security_group_ids=security_group_ids,
-                vpc=vpc,
-                private_ip=ami_deployment_model.private_ip_address)]
+            return self._prepare_nic_for_single_subnet(ami_deployment_model, network_config_results, security_group_ids,
+                                                       vpc)
 
         self._validate_network_interfaces_request(ami_deployment_model, network_actions, logger)
 
@@ -451,14 +454,17 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions
         logger.info("Applying device index strategy")
         self.device_index_strategy.apply(network_actions)
 
+        logger.info("Mapping network CIDRs to requested IPs")
+        cidr_to_requested_ips = RequestedIPsMapper(ami_deployment_model.private_ip_addresses_list)\
+            .map_network_to_requested_ips(network_actions)
+
         logger.info("Building network interface dtos")
         for net_config in network_actions:
             if not isinstance(net_config.actionParams, ConnectToSubnetParams):
                 continue
 
             device_index = net_config.actionParams.vnicName
-            private_ip = self._get_private_ip_for_subnet(ami_deployment_model,
-                                                         net_config.actionParams.cidr)
+            private_ips = self._get_private_ips_for_subnet(net_config.actionParams.cidr, cidr_to_requested_ips)
 
             net_interfaces.append(
                 # todo: maybe add fallback to find subnet by cidr if subnet id doesnt exist?
@@ -467,34 +473,49 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions
                     device_index=device_index,
                     groups=security_group_ids,  # todo: set groups by subnet id
                     public_ip=public_ip_prop_value,
-                    private_ip=private_ip))
+                    private_ips=private_ips))
 
             # set device index on action result object
             res = first_or_default(network_config_results, lambda x: x.action_id == net_config.actionId)
             res.device_index = device_index
 
         if len(net_interfaces) == 0:
             logger.info("No network interface dto was created, switching back to single subnet mode")
-            network_config_results[0].device_index = 0
-            return [self.network_interface_service.get_network_interface_for_single_subnet_mode(
-                add_public_ip=ami_deployment_model.add_public_ip,
-                security_group_ids=security_group_ids,
-                vpc=vpc,
-                private_ip=ami_deployment_model.private_ip_address)]
+            return self._prepare_nic_for_single_subnet(ami_deployment_model, network_config_results, security_group_ids,
+                                                       vpc)
 
         logger.info("Created dtos for {} network interfaces".format(len(net_interfaces)))
 
         return net_interfaces
 
-    def _get_private_ip_for_subnet(self, ami_deployment_model, subnet_cidr):
+    def _prepare_nic_for_single_subnet(self, ami_deployment_model, network_config_results, security_group_ids, vpc):
         """
+        :param vpc: The reservation VPC
         :param DeployAWSEc2AMIInstanceResourceModel ami_deployment_model:
+        :param [str] security_group_ids:
+        :param list[DeployNetworkingResultModel] network_config_results: list of network configuration result objects
+        :rtype: List[dict]
+        """
+        self._validate_private_ip_addresses_for_single_subnet_mode(ami_deployment_model)
+        network_config_results[0].device_index = 0
+        private_ips = ami_deployment_model.private_ip_addresses_list[0] if \
+            ami_deployment_model.private_ip_addresses_list else None
+        return [self.network_interface_service.get_network_interface_for_single_subnet_mode(
+            add_public_ip=ami_deployment_model.add_public_ip,
+            security_group_ids=security_group_ids,
+            vpc=vpc,
+            private_ips=private_ips)]
+
+    def _get_private_ips_for_subnet(self, subnet_cidr, cidr_to_requested_ips):
+        """
         :param str subnet_cidr:
-        :return:
+        :param Dict[str, List[str]] cidr_to_requested_ips:
+        :rtype: List[str]
         """
-        if subnet_cidr and ami_deployment_model.private_ip_addresses_dict and \
-                subnet_cidr in ami_deployment_model.private_ip_addresses_dict:
-            return ami_deployment_model.private_ip_addresses_dict.get(subnet_cidr)
+        if subnet_cidr and cidr_to_requested_ips:
+            return cidr_to_requested_ips.get(subnet_cidr, None)
+
+        return None
 
     def _validate_network_interfaces_request(self, ami_deployment_model, network_actions, logger):
         self._validate_public_ip_with_multiple_subnets(ami_deployment_model, network_actions, logger)
@@ -506,6 +527,11 @@ def _validate_public_ip_with_multiple_subnets(self, ami_deployment_model, networ
             logger.error("Requested public ip with multiple subnets")
             raise ValueError("Public IP option is not supported with multiple subnets")
 
+    def _validate_private_ip_addresses_for_single_subnet_mode(self, ami_deployment_model):
+        if ami_deployment_model.private_ip_addresses_list and len(ami_deployment_model.private_ip_addresses_list) != 1:
+            raise ValueError('Value in Private IP attribute is invalid. Expected IP request for a single subnet but '
+                             'instead detected request for multiple subnets.')
+
     def _get_instance_item(self, ami_deployment_model, aws_ec2_resource_model):
         return ami_deployment_model.instance_type \
             if ami_deployment_model.instance_type \

package/cloudshell/cp/aws/domain/context/client_error.py

@@ -12,5 +12,4 @@ def wrap(self):
             yield
         except ClientError as e:
             raise type(e), \
-                type(e)('AWS API Error. Please consider retrying the operation. ' + e.message), \
-                sys.exc_info()[2]
+                type(e)('AWS API Error. Please consider retrying the operation. ' + e.message, sys.exc_info()[2])

package/cloudshell/cp/aws/domain/operations/__init__.py

@@ -0,0 +1,3 @@
+__author__ = 'quali'
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

package/cloudshell/cp/aws/domain/services/cloudshell/__init__.py

@@ -0,0 +1,3 @@
+__author__ = 'quali'
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

package/cloudshell/cp/aws/domain/services/ec2/network_interface.py

@@ -6,22 +6,22 @@ def __init__(self, subnet_service):
         """
         self.subnet_service = subnet_service
 
-    def get_network_interface_for_single_subnet_mode(self, add_public_ip, security_group_ids, vpc, private_ip=None):
+    def get_network_interface_for_single_subnet_mode(self, add_public_ip, security_group_ids, vpc, private_ips=None):
         """
         :param bool add_public_ip:
         :param list[str] security_group_ids:
         :param vpc: VPC instance
-        :param str private_ip:
-        :return:
+        :param List[str] private_ips:
+        :rtype: dict
         """
         return self.build_network_interface_dto(
                 subnet_id=self.subnet_service.get_first_subnet_from_vpc(vpc).subnet_id,
                 device_index=0,
                 groups=security_group_ids,
                 public_ip=add_public_ip,
-                private_ip=private_ip)
+                private_ips=private_ips)
 
-    def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip=None, private_ip=None):
+    def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip=None, private_ips=None):
         net_if = {
             'SubnetId': subnet_id,
             'DeviceIndex': device_index,
@@ -31,7 +31,12 @@ def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip
         if public_ip:
             net_if['AssociatePublicIpAddress'] = public_ip
 
-        if private_ip:
-            net_if['PrivateIpAddress'] = private_ip
+        if private_ips:
+            if isinstance(private_ips, str):
+                net_if['PrivateIpAddress'] = private_ips
+            elif isinstance(private_ips, list):
+                net_if['PrivateIpAddresses'] = list(map(lambda x: {'PrivateIpAddress': x,
+                                                                   'Primary': True if x == private_ips[0] else False}
+                                                        , private_ips))
 
         return net_if

package/cloudshell/cp/aws/models/deploy_aws_ec2_ami_instance_resource_model.py

@@ -1,11 +1,9 @@
 import json
 
-from typing import Dict
-
-import cloudshell
-from cloudshell.cp.aws.domain.services.parsers.aws_model_parser import AWSModelsParser
-
+from typing import Dict, List
+from cloudshell.cp.core.requested_ips.parser import RequestedIPsParser
 from cloudshell.cp.aws.common.converters import convert_to_bool
+from cloudshell.cp.aws.domain.services.parsers.aws_model_parser import AWSModelsParser
 
 
 class DeployAWSEc2AMIInstanceResourceModel(object):
@@ -28,7 +26,7 @@ def __init__(self, attributes):  # todo handle the c=initialization of the objec
         self.iam_role = ''  # type: str
         self.security_group_ids = None  # type: str
         self.private_ip_address = ""  # type: str
-        self.private_ip_addresses_dict = None  # type: Dict
+        self.private_ip_addresses_list = None  # type: Dict
         self.root_volume_name = ''  # type: str
         self.delete_on_termination = True  # type: bool
         self.auto_power_off = False  # type: bool
@@ -63,20 +61,23 @@ def __init__(self, attributes):  # todo handle the c=initialization of the objec
         self.user_data_url = attributes['User Data URL']
         self.user_data_run_parameters = attributes['User Data Parameters']
 
-        private_ip_att_value = attributes['Private IP']
-        self.private_ip_address = self._get_primary_private_ip_address(private_ip_att_value)
-        self.private_ip_addresses_dict = self._get_private_ip_addresses_dict(private_ip_att_value)
+        self.private_ip_addresses_list = self._get_private_ip_addresses(attributes)
+        self.private_ip_address = self._get_primary_private_ip_address(self.private_ip_addresses_list)
 
-    def _get_private_ip_addresses_dict(self, private_ip_address):
-        try:
-            # if dict of private ip address then we take the first as the primary
-            return json.loads(private_ip_address)
-        except:
-            return None
+    def _get_private_ip_addresses(self, attributes):
+        """
+        :param dict attributes:
+        :rtype: List[List[str]]
+        """
+        if attributes:
+            return RequestedIPsParser.parse(attributes)
+        return None
 
-    def _get_primary_private_ip_address(self, private_ip_address):
-        try:
-            # if dict of private ip address then we take the first as the primary
-            return json.loads(private_ip_address).values()[0]
-        except:
-            return private_ip_address or None
+    def _get_primary_private_ip_address(self, private_ip_addresses_list):
+        """
+        :param List[List[str]] private_ip_addresses_list:
+        :rtype: str
+        """
+        if private_ip_addresses_list:
+            return private_ip_addresses_list[0][0]
+        return None

package/print_coverage.bat

@@ -0,0 +1,2 @@
+coverage run --source=cloudshell -m unittest
+coverage report -m

package/requirements.txt

@@ -1,7 +1,7 @@
 pyrsistent==0.16.1
 boto3>=1.9.0,<1.10.0
 typing==3.6.4
-ipaddress==1.0.22
+ipaddress==1.0.23
 netaddr==0.7.19
 cloudshell-automation-api>=9.3.0.0
 cloudshell-core>=2.2.0,<2.3.0
@@ -10,5 +10,5 @@ jsonpickle==0.9.3
 enum==0.4.6
 pycrypto==2.6.1
 retrying==1.3.3
-cloudshell-cp-core>=1.0.4,<1.1.0
+cloudshell-cp-core>=1.1.0,<1.2.0
 jsonschema~=3.2