From 3899ec578619d06a9e8239c283c65e0dd9cd84f3 Mon Sep 17 00:00:00 2001 From: alexazarh Date: Thu, 15 Oct 2020 15:13:17 -0500 Subject: [PATCH 1/6] partial commit, adding support for private ip attr new standard --- .gitignore | 1 + drivers/aws_shell/src/requirements.txt | 2 +- .../operations/deploy_operation.py | 92 ++++++++++++------- .../cp/aws/domain/operations/__init__.py | 3 + .../domain/services/cloudshell/__init__.py | 3 + .../domain/services/ec2/network_interface.py | 19 ++-- ...loy_aws_ec2_ami_instance_resource_model.py | 43 ++++----- .../test_network_interface_service.py | 2 +- package/version.txt | 2 +- 9 files changed, 102 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index 1276170e..56752368 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,4 @@ cloudformation/deploy_execution_server_by_selected_version/* /drivers/aws_shell/src/test_* /aws_venv +/venv/* diff --git a/drivers/aws_shell/src/requirements.txt b/drivers/aws_shell/src/requirements.txt index 85c8ec32..a5556038 100644 --- a/drivers/aws_shell/src/requirements.txt +++ b/drivers/aws_shell/src/requirements.txt @@ -1,5 +1,5 @@ cloudshell-automation-api>=9.3 cloudshell-shell-core>=3.1.0,<3.2.0 -cloudshell-cp-aws>=2.4.3,<2.5.0 +cloudshell-cp-aws>=2.5.0,<2.6.0 jsonpickle==0.9.3 cloudshell-cp-core>=1.0.4,<1.1.0 diff --git a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py index d95c0002..49db6400 100644 --- a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py +++ b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py @@ -2,6 +2,9 @@ import uuid from multiprocessing import TimeoutError +import ipaddress +from typing import List, Dict + from cloudshell.cp.aws.domain.common.cancellation_service import CommandCancellationService from cloudshell.cp.aws.domain.common.list_helper import first_or_default from cloudshell.cp.aws.domain.common.vm_details_provider import VmDetailsProvider @@ -23,6 +26,7 @@ from cloudshell.cp.aws.models.network_actions_models import DeployNetworkingResultModel from cloudshell.cp.core.models import ConnectToSubnetActionResult, ConnectToSubnetParams, ConnectSubnet, DeployAppResult from cloudshell.cp.core.utils import convert_dict_to_attributes_list +from cloudshell.cp.core.requested_ips.mapper import RequestedIPsMapper class DeployAMIOperation(object): @@ -170,11 +174,13 @@ def deploy(self, ec2_session, s3_session, name, reservation, aws_ec2_cp_resource deploy_app_result = DeployAppResult(vmName=self._get_name_from_tags(instance), vmUuid=instance.instance_id, - deployedAppAttributes=convert_dict_to_attributes_list(deployed_app_attributes), + deployedAppAttributes=convert_dict_to_attributes_list( + deployed_app_attributes), deployedAppAddress=instance.private_ip_address, vmDetailsData=vm_details_data, - deployedAppAdditionalData={'inbound_ports': ami_deployment_model.inbound_ports, - 'public_ip': instance.public_ip_address}) + deployedAppAdditionalData={ + 'inbound_ports': ami_deployment_model.inbound_ports, + 'public_ip': instance.public_ip_address}) deploy_app_result.actionId = ami_deploy_action.actionId network_actions_results_dtos.append(deploy_app_result) return network_actions_results_dtos @@ -279,7 +285,7 @@ def _get_ami_credentials(self, s3_session, key_pair_location, reservation, wait_ cancellation_context=cancellation_context) except TimeoutError: logger.info( - "Timeout when waiting for windows credentials. Traceback: {0}".format(traceback.format_exc())) + "Timeout when waiting for windows credentials. Traceback: {0}".format(traceback.format_exc())) return None else: return None if self._get_deployed_app_resource_user_attribute(ami_deploy_action) else \ @@ -298,7 +304,7 @@ def _get_deployed_app_resource_user_attribute(ami_deploy_action): :return: """ attribute_names_in_deployed_resource = ami_deploy_action.actionParams.appResource.attributes.keys() - return next((attr for attr in attribute_names_in_deployed_resource if attr.split('.')[-1]=='User')) + return next((attr for attr in attribute_names_in_deployed_resource if attr.split('.')[-1] == 'User')) @staticmethod def _get_name_from_tags(result): @@ -412,20 +418,16 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions """ :param vpc: The reservation VPC :param DeployAWSEc2AMIInstanceResourceModel ami_deployment_model: - :param cloudshell.cp.core.models.ConnectSubnet network_actions: + :param List[cloudshell.cp.core.models.ConnectSubnet] network_actions: :param [str] security_group_ids: - :param list[DeployNetworkingResultModel] network_config_results: list of network configuration result objects + :param List[DeployNetworkingResultModel] network_config_results: list of network configuration result objects :param logging.Logger logger: :return: """ if not network_actions: logger.info("Single subnet mode detected") - network_config_results[0].device_index = 0 - return [self.network_interface_service.get_network_interface_for_single_subnet_mode( - add_public_ip=ami_deployment_model.add_public_ip, - security_group_ids=security_group_ids, - vpc=vpc, - private_ip=ami_deployment_model.private_ip_address)] + return self._prepare_nic_for_single_subnet(ami_deployment_model, network_config_results, security_group_ids, + vpc) self._validate_network_interfaces_request(ami_deployment_model, network_actions, logger) @@ -436,23 +438,26 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions logger.info("Applying device index strategy") self.device_index_strategy.apply(network_actions) + logger.info("Mapping network CIDRs to requested IPs") + cidr_to_requested_ips = RequestedIPsMapper(ami_deployment_model.private_ip_addresses_list)\ + .map_network_to_requested_ips(network_actions) + logger.info("Building network interface dtos") for net_config in network_actions: if not isinstance(net_config.actionParams, ConnectToSubnetParams): continue device_index = net_config.actionParams.vnicName - private_ip = self._get_private_ip_for_subnet(ami_deployment_model, - net_config.actionParams.cidr) + private_ips = self._get_private_ips_for_subnet(net_config.actionParams.cidr, cidr_to_requested_ips) net_interfaces.append( - # todo: maybe add fallback to find subnet by cidr if subnet id doesnt exist? - self.network_interface_service.build_network_interface_dto( - subnet_id=net_config.actionParams.subnetId, - device_index=device_index, - groups=security_group_ids, # todo: set groups by subnet id - public_ip=public_ip_prop_value, - private_ip=private_ip)) + # todo: maybe add fallback to find subnet by cidr if subnet id doesnt exist? + self.network_interface_service.build_network_interface_dto( + subnet_id=net_config.actionParams.subnetId, + device_index=device_index, + groups=security_group_ids, # todo: set groups by subnet id + public_ip=public_ip_prop_value, + private_ips=private_ips)) # set device index on action result object res = first_or_default(network_config_results, lambda x: x.action_id == net_config.actionId) @@ -460,26 +465,39 @@ def _prepare_network_interfaces(self, vpc, ami_deployment_model, network_actions if len(net_interfaces) == 0: logger.info("No network interface dto was created, switching back to single subnet mode") - network_config_results[0].device_index = 0 - return [self.network_interface_service.get_network_interface_for_single_subnet_mode( - add_public_ip=ami_deployment_model.add_public_ip, - security_group_ids=security_group_ids, - vpc=vpc, - private_ip=ami_deployment_model.private_ip_address)] + return self._prepare_nic_for_single_subnet(ami_deployment_model, network_config_results, security_group_ids, + vpc) logger.info("Created dtos for {} network interfaces".format(len(net_interfaces))) return net_interfaces - def _get_private_ip_for_subnet(self, ami_deployment_model, subnet_cidr): + def _prepare_nic_for_single_subnet(self, ami_deployment_model, network_config_results, security_group_ids, vpc): """ + :param vpc: The reservation VPC :param DeployAWSEc2AMIInstanceResourceModel ami_deployment_model: + :param [str] security_group_ids: + :param list[DeployNetworkingResultModel] network_config_results: list of network configuration result objects + :rtype: List[dict] + """ + self._validate_private_ip_addresses_for_single_subnet_mode(ami_deployment_model) + network_config_results[0].device_index = 0 + return [self.network_interface_service.get_network_interface_for_single_subnet_mode( + add_public_ip=ami_deployment_model.add_public_ip, + security_group_ids=security_group_ids, + vpc=vpc, + private_ips=ami_deployment_model.private_ip_addresses_list[0])] + + def _get_private_ips_for_subnet(self, subnet_cidr, cidr_to_requested_ips): + """ :param str subnet_cidr: - :return: + :param Dict[str, List[str]] cidr_to_requested_ips: + :rtype: List[str] """ - if subnet_cidr and ami_deployment_model.private_ip_addresses_dict and \ - subnet_cidr in ami_deployment_model.private_ip_addresses_dict: - return ami_deployment_model.private_ip_addresses_dict.get(subnet_cidr) + if subnet_cidr and cidr_to_requested_ips: + return cidr_to_requested_ips.get(subnet_cidr, None) + + return None def _validate_network_interfaces_request(self, ami_deployment_model, network_actions, logger): self._validate_public_ip_with_multiple_subnets(ami_deployment_model, network_actions, logger) @@ -491,6 +509,11 @@ def _validate_public_ip_with_multiple_subnets(self, ami_deployment_model, networ logger.error("Requested public ip with multiple subnets") raise ValueError("Public IP option is not supported with multiple subnets") + def _validate_private_ip_addresses_for_single_subnet_mode(self, ami_deployment_model): + if ami_deployment_model.private_ip_addresses_list and len(ami_deployment_model.private_ip_addresses_list) != 1: + raise ValueError('Value in Private IP attribute is invalid. Expected IP request for a single subnet but ' + 'instead detected request for multiple subnets.') + def _get_instance_item(self, ami_deployment_model, aws_ec2_resource_model): return ami_deployment_model.instance_type if ami_deployment_model.instance_type else aws_ec2_resource_model.instance_type @@ -656,7 +679,8 @@ def _get_custom_tags(self, custom_tags): return res def _get_user_data(self, user_data_url, user_data_run_parameters): - data = "#!/bin/bash\n" + "curl --retry 10 --max-time 5 --retry-max-time 180 {0} > cs.sh \n".format(user_data_url) \ + data = "#!/bin/bash\n" + "curl --retry 10 --max-time 5 --retry-max-time 180 {0} > cs.sh \n".format( + user_data_url) \ + "chmod +x cs.sh \n" \ + "./cs.sh {0}".format(user_data_run_parameters) return data diff --git a/package/cloudshell/cp/aws/domain/operations/__init__.py b/package/cloudshell/cp/aws/domain/operations/__init__.py index e69de29b..b50dc4a9 100644 --- a/package/cloudshell/cp/aws/domain/operations/__init__.py +++ b/package/cloudshell/cp/aws/domain/operations/__init__.py @@ -0,0 +1,3 @@ +__author__ = 'quali' +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) \ No newline at end of file diff --git a/package/cloudshell/cp/aws/domain/services/cloudshell/__init__.py b/package/cloudshell/cp/aws/domain/services/cloudshell/__init__.py index e69de29b..b50dc4a9 100644 --- a/package/cloudshell/cp/aws/domain/services/cloudshell/__init__.py +++ b/package/cloudshell/cp/aws/domain/services/cloudshell/__init__.py @@ -0,0 +1,3 @@ +__author__ = 'quali' +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) \ No newline at end of file diff --git a/package/cloudshell/cp/aws/domain/services/ec2/network_interface.py b/package/cloudshell/cp/aws/domain/services/ec2/network_interface.py index 101cc120..61aa167e 100644 --- a/package/cloudshell/cp/aws/domain/services/ec2/network_interface.py +++ b/package/cloudshell/cp/aws/domain/services/ec2/network_interface.py @@ -6,22 +6,22 @@ def __init__(self, subnet_service): """ self.subnet_service = subnet_service - def get_network_interface_for_single_subnet_mode(self, add_public_ip, security_group_ids, vpc, private_ip=None): + def get_network_interface_for_single_subnet_mode(self, add_public_ip, security_group_ids, vpc, private_ips=None): """ :param bool add_public_ip: :param list[str] security_group_ids: :param vpc: VPC instance - :param str private_ip: - :return: + :param List[str] private_ips: + :rtype: dict """ return self.build_network_interface_dto( subnet_id=self.subnet_service.get_first_subnet_from_vpc(vpc).subnet_id, device_index=0, groups=security_group_ids, public_ip=add_public_ip, - private_ip=private_ip) + private_ips=private_ips) - def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip=None, private_ip=None): + def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip=None, private_ips=None): net_if = { 'SubnetId': subnet_id, 'DeviceIndex': device_index, @@ -31,7 +31,12 @@ def build_network_interface_dto(self, subnet_id, device_index, groups, public_ip if public_ip: net_if['AssociatePublicIpAddress'] = public_ip - if private_ip: - net_if['PrivateIpAddress'] = private_ip + if private_ips: + if isinstance(private_ips, str): + net_if['PrivateIpAddress'] = private_ips + elif isinstance(private_ips, list): + net_if['PrivateIpAddresses'] = list(map(lambda x: {'PrivateIpAddress': x, + 'Primary': True if x == private_ips[0] else False} + , private_ips)) return net_if diff --git a/package/cloudshell/cp/aws/models/deploy_aws_ec2_ami_instance_resource_model.py b/package/cloudshell/cp/aws/models/deploy_aws_ec2_ami_instance_resource_model.py index f6985abc..18c6c03d 100644 --- a/package/cloudshell/cp/aws/models/deploy_aws_ec2_ami_instance_resource_model.py +++ b/package/cloudshell/cp/aws/models/deploy_aws_ec2_ami_instance_resource_model.py @@ -1,11 +1,9 @@ import json -from typing import Dict - -import cloudshell -from cloudshell.cp.aws.domain.services.parsers.aws_model_parser import AWSModelsParser - +from typing import Dict, List +from cloudshell.cp.core.requested_ips.parser import RequestedIPsParser from cloudshell.cp.aws.common.converters import convert_to_bool +from cloudshell.cp.aws.domain.services.parsers.aws_model_parser import AWSModelsParser class DeployAWSEc2AMIInstanceResourceModel(object): @@ -28,7 +26,7 @@ def __init__(self, attributes): # todo handle the c=initialization of the objec self.iam_role = '' # type: str self.security_group_ids = None # type: str self.private_ip_address = "" # type: str - self.private_ip_addresses_dict = None # type: Dict + self.private_ip_addresses_list = None # type: Dict self.root_volume_name = '' # type: str self.delete_on_termination = True # type: bool self.auto_power_off = False # type: bool @@ -63,20 +61,23 @@ def __init__(self, attributes): # todo handle the c=initialization of the objec self.user_data_url = attributes['User Data URL'] self.user_data_run_parameters = attributes['User Data Parameters'] - private_ip_att_value = attributes['Private IP'] - self.private_ip_address = self._get_primary_private_ip_address(private_ip_att_value) - self.private_ip_addresses_dict = self._get_private_ip_addresses_dict(private_ip_att_value) + self.private_ip_addresses_list = self._get_private_ip_addresses(attributes) + self.private_ip_address = self._get_primary_private_ip_address(self.private_ip_addresses_list) - def _get_private_ip_addresses_dict(self, private_ip_address): - try: - # if dict of private ip address then we take the first as the primary - return json.loads(private_ip_address) - except: - return None + def _get_private_ip_addresses(self, attributes): + """ + :param dict attributes: + :rtype: List[List[str]] + """ + if attributes: + return RequestedIPsParser.parse(attributes) + return None - def _get_primary_private_ip_address(self, private_ip_address): - try: - # if dict of private ip address then we take the first as the primary - return json.loads(private_ip_address).values()[0] - except: - return private_ip_address or None + def _get_primary_private_ip_address(self, private_ip_addresses_list): + """ + :param List[List[str]] private_ip_addresses_list: + :rtype: str + """ + if private_ip_addresses_list: + return private_ip_addresses_list[0][0] + return None diff --git a/package/tests/test_domain_services/test_network_interface_service.py b/package/tests/test_domain_services/test_network_interface_service.py index a44c63d8..485f5258 100644 --- a/package/tests/test_domain_services/test_network_interface_service.py +++ b/package/tests/test_domain_services/test_network_interface_service.py @@ -67,4 +67,4 @@ def test_get_network_interface_for_single_subnet_mode(self): device_index=0, groups=security_group_ids, public_ip=add_public_ip, - private_ip=None) + private_ips=None) diff --git a/package/version.txt b/package/version.txt index 19244e85..fad066f8 100644 --- a/package/version.txt +++ b/package/version.txt @@ -1 +1 @@ -2.4.21 \ No newline at end of file +2.5.0 \ No newline at end of file From 64f824eddf9845e6a31a8cf66853e16e5661bacb Mon Sep 17 00:00:00 2001 From: alexazarh Date: Thu, 15 Oct 2020 22:27:34 -0500 Subject: [PATCH 2/6] update reference to new version of cloudshell-cp-core --- drivers/aws_shell/src/requirements.txt | 2 +- package/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/aws_shell/src/requirements.txt b/drivers/aws_shell/src/requirements.txt index a5556038..499defd2 100644 --- a/drivers/aws_shell/src/requirements.txt +++ b/drivers/aws_shell/src/requirements.txt @@ -2,4 +2,4 @@ cloudshell-automation-api>=9.3 cloudshell-shell-core>=3.1.0,<3.2.0 cloudshell-cp-aws>=2.5.0,<2.6.0 jsonpickle==0.9.3 -cloudshell-cp-core>=1.0.4,<1.1.0 +cloudshell-cp-core>=1.1.0,<1.2.0 diff --git a/package/requirements.txt b/package/requirements.txt index d6231f24..6f14892f 100644 --- a/package/requirements.txt +++ b/package/requirements.txt @@ -10,5 +10,5 @@ enum==0.4.6 pycrypto==2.6.1 retrying==1.3.3 pyrsistent==0.16.1 -cloudshell-cp-core>=1.0.4,<1.1.0 +cloudshell-cp-core>=1.1.0,<1.2.0 jsonschema~=3.2 \ No newline at end of file From 7dae918f865ee1a3c947c86c5702b56b68ed9d7d Mon Sep 17 00:00:00 2001 From: alexazarh Date: Fri, 16 Oct 2020 09:48:51 -0500 Subject: [PATCH 3/6] fix tests and bugs --- .../ami_management/operations/deploy_operation.py | 4 +++- .../test_operations/test_deploy_operation.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py index 49db6400..44bbd3c0 100644 --- a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py +++ b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py @@ -482,11 +482,13 @@ def _prepare_nic_for_single_subnet(self, ami_deployment_model, network_config_re """ self._validate_private_ip_addresses_for_single_subnet_mode(ami_deployment_model) network_config_results[0].device_index = 0 + private_ips = ami_deployment_model.private_ip_addresses_list[0] if \ + ami_deployment_model.private_ip_addresses_list else None return [self.network_interface_service.get_network_interface_for_single_subnet_mode( add_public_ip=ami_deployment_model.add_public_ip, security_group_ids=security_group_ids, vpc=vpc, - private_ips=ami_deployment_model.private_ip_addresses_list[0])] + private_ips=private_ips)] def _get_private_ips_for_subnet(self, subnet_cidr, cidr_to_requested_ips): """ diff --git a/package/tests/test_ami_management/test_operations/test_deploy_operation.py b/package/tests/test_ami_management/test_operations/test_deploy_operation.py index abd87aad..de13ea92 100644 --- a/package/tests/test_ami_management/test_operations/test_deploy_operation.py +++ b/package/tests/test_ami_management/test_operations/test_deploy_operation.py @@ -4,6 +4,7 @@ from cloudshell.cp.aws.domain.ami_management.operations.deploy_operation import DeployAMIOperation from cloudshell.cp.aws.domain.common.exceptions import CancellationException +from cloudshell.cp.aws.models.deploy_aws_ec2_ami_instance_resource_model import DeployAWSEc2AMIInstanceResourceModel from cloudshell.cp.aws.models.network_actions_models import DeployNetworkingResultModel from cloudshell.cp.core.models import ConnectToSubnetParams, PrepareCloudInfra, ConnectSubnet @@ -419,12 +420,12 @@ def test_create_deployment_parameters_no_ami_id(self): network_config_results=Mock(), logger=self.logger) - def test_create_deployment_parameters_single_subnet(self): + def test_create_deployment_parameters_single_subnet_no_requested_ips(self): image = Mock() image.state = 'available' ec2_session = Mock() ec2_session.Image = Mock(return_value=image) - ami_model = Mock() + ami_model = Mock(private_ip_addresses_list=None, private_ip_address=None) ami_model.aws_ami_id = 'asd' ami_model.storage_size = '0' ami_model.iam_role = "" @@ -456,7 +457,7 @@ def test_create_deployment_parameters_no_iam_role(self): image.state = 'available' ec2_session = Mock() ec2_session.Image = Mock(return_value=image) - ami_model = Mock() + ami_model = Mock(private_ip_addresses_list=None, private_ip_address=None) ami_model.iam_role = "" ami_model.custom_tags = "" network_actions = None @@ -482,7 +483,7 @@ def test_create_deployment_parameters_iam_role_not_arn(self): image.state = 'available' ec2_session = Mock() ec2_session.Image = Mock(return_value=image) - ami_model = Mock() + ami_model = Mock(private_ip_addresses_list=None, private_ip_address=None) ami_model.iam_role = "admin_role" ami_model.custom_tags = "" @@ -509,8 +510,8 @@ def test_create_deployment_parameters_iam_role_arn(self): image.state = 'available' ec2_session = Mock() ec2_session.Image = Mock(return_value=image) - ami_model = Mock() network_actions = None + ami_model = Mock(private_ip_addresses_list=None, private_ip_address=None) ami_model.iam_role = "arn:aws:iam::admin_role" ami_model.custom_tags = "" vpc = Mock() @@ -545,7 +546,7 @@ def test_prepare_network_interfaces_multi_subnets_with_public_ip(self): network_config_results=MagicMock(), logger=self.logger) - def test_prepare_network_interfaces_multi_subnets(self): + def test_prepare_network_interfaces_multi_subnets_no_requested_ips(self): def build_network_interface_handler(*args, **kwargs): return {'SubnetId': kwargs['subnet_id']} @@ -566,7 +567,7 @@ def build_network_interface_handler(*args, **kwargs): action2.actionParams.subnetId = 'sub2' action2.actionParams.vnicName = 1 - ami_model = Mock() + ami_model = Mock(private_ip_addresses_list=None, private_ip_address=None) network_actions = [action1, action2] ami_model.add_public_ip = False From 1d8e9af1d8d1105ec8e588d0164734ee5f60c9c8 Mon Sep 17 00:00:00 2001 From: alexazarh Date: Tue, 3 Nov 2020 18:35:10 -0600 Subject: [PATCH 4/6] more unit tests and fixing CI --- .travis.yml | 2 +- .../operations/deploy_operation.py | 1 + .../cp/aws/domain/context/client_error.py | 3 +- package/print_coverage.bat | 2 + package/requirements.txt | 2 +- .../test_operations/test_deploy_operation.py | 111 +++++++++++++++++- .../test_domain_services/test_model_parser.py | 30 ++++- 7 files changed, 140 insertions(+), 11 deletions(-) create mode 100644 package/print_coverage.bat diff --git a/.travis.yml b/.travis.yml index ee5c5f64..337144f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - pip install "cloudshell-core>=2.2.0,<2.3.0" - pip install "cloudshell-shell-core>=3.1.0,<3.2.0" - pip install "cloudshell-automation-api>=8.3.0.0,<8.3.1.0" --extra-index-url https://testpypi.python.org/simple - - pip install "cloudshell-cp-core>=1.0.0,<1.1.0" --extra-index-url https://test.pypi.org/simple + - pip install "cloudshell-cp-core>=1.1.0,<1.2.0" --extra-index-url https://test.pypi.org/simple script: - pushd package - python setup.py develop diff --git a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py index 44bbd3c0..9230cc46 100644 --- a/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py +++ b/package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py @@ -367,6 +367,7 @@ def _create_deployment_parameters(self, ec2_session, aws_ec2_resource_model, ami :param network_config_results: list of network configuration result objects :type network_config_results: list[DeployNetworkingResultModel] :param logging.Logger logger: + :rtype: AMIDeploymentModel """ aws_model = AMIDeploymentModel() if not ami_deployment_model.aws_ami_id: diff --git a/package/cloudshell/cp/aws/domain/context/client_error.py b/package/cloudshell/cp/aws/domain/context/client_error.py index c1cbb3bf..0b082981 100644 --- a/package/cloudshell/cp/aws/domain/context/client_error.py +++ b/package/cloudshell/cp/aws/domain/context/client_error.py @@ -12,5 +12,4 @@ def wrap(self): yield except ClientError as e: raise type(e), \ - type(e)('AWS API Error. Please consider retrying the operation. ' + e.message), \ - sys.exc_info()[2] + type(e)('AWS API Error. Please consider retrying the operation. ' + e.message, sys.exc_info()[2]) diff --git a/package/print_coverage.bat b/package/print_coverage.bat new file mode 100644 index 00000000..45319280 --- /dev/null +++ b/package/print_coverage.bat @@ -0,0 +1,2 @@ +coverage run --source=cloudshell -m unittest +coverage report -m \ No newline at end of file diff --git a/package/requirements.txt b/package/requirements.txt index 6f14892f..98b08d17 100644 --- a/package/requirements.txt +++ b/package/requirements.txt @@ -1,6 +1,6 @@ boto3>=1.9.0,<1.10.0 typing==3.6.4 -ipaddress==1.0.22 +ipaddress==1.0.23 netaddr==0.7.19 cloudshell-automation-api>=9.3.0.0 cloudshell-core>=2.2.0,<2.3.0 diff --git a/package/tests/test_ami_management/test_operations/test_deploy_operation.py b/package/tests/test_ami_management/test_operations/test_deploy_operation.py index de13ea92..fdd8b514 100644 --- a/package/tests/test_ami_management/test_operations/test_deploy_operation.py +++ b/package/tests/test_ami_management/test_operations/test_deploy_operation.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import Mock, call, MagicMock +from mock import Mock, call, MagicMock, ANY, patch from cloudshell.cp.aws.domain.ami_management.operations.deploy_operation import DeployAMIOperation from cloudshell.cp.aws.domain.common.exceptions import CancellationException @@ -452,6 +452,43 @@ def test_create_deployment_parameters_single_subnet_no_requested_ips(self): self.assertTrue(len(aws_model.security_group_ids) == 1) self.assertTrue(len(aws_model.network_interfaces) == 1) + def test_create_deployment_parameters_single_subnet_single_requested_ip(self): + image = Mock() + image.state = 'available' + ec2_session = Mock() + ec2_session.Image = Mock(return_value=image) + ami_model = Mock() + private_ip_request = Mock() + ami_model.private_ip_address = private_ip_request + ami_model.private_ip_addresses_list = [[private_ip_request]] + ami_model.aws_ami_id = 'asd' + ami_model.storage_size = '0' + ami_model.iam_role = "" + ami_model.custom_tags = "" + + network_actions = None + vpc = Mock() + self.deploy_operation._get_block_device_mappings = Mock() + + aws_model = self.deploy_operation._create_deployment_parameters(ec2_session=ec2_session, + aws_ec2_resource_model=self.ec2_datamodel, + ami_deployment_model=ami_model, + network_actions=network_actions, + vpc=vpc, + security_group=None, + key_pair='keypair', + reservation=Mock(), + network_config_results=MagicMock(), + logger=self.logger) + + self.assertEquals(aws_model.min_count, 1) + self.assertEquals(aws_model.max_count, 1) + self.assertEquals(aws_model.aws_key, 'keypair') + self.assertTrue(len(aws_model.security_group_ids) == 1) + self.assertTrue(len(aws_model.network_interfaces) == 1) + self.network_interface_service.get_network_interface_for_single_subnet_mode.assert_called_once_with( + add_public_ip=ANY, security_group_ids=ANY, vpc=ANY, private_ips=[private_ip_request]) + def test_create_deployment_parameters_no_iam_role(self): image = Mock() image.state = 'available' @@ -531,7 +568,7 @@ def test_create_deployment_parameters_iam_role_arn(self): # if instance has iam role, but not in the form of arn, will return dict with iam role name self.assertTrue(aws_model.iam_role['Arn'] == ami_model.iam_role) - def test_prepare_network_interfaces_multi_subnets_with_public_ip(self): + def test_prepare_network_interfaces_multi_subnets_with_public_ip_throws(self): ami_model = Mock() ami_model.add_public_ip = True ami_model.network_configurations = [Mock(), Mock()] @@ -594,6 +631,72 @@ def build_network_interface_handler(*args, **kwargs): self.assertEquals(net_interfaces[0]['SubnetId'], 'sub1') self.assertEquals(net_interfaces[1]['SubnetId'], 'sub2') + @patch('cloudshell.cp.aws.domain.ami_management.operations.deploy_operation.RequestedIPsMapper') + def test_prepare_network_interfaces_multi_subnets_with_requested_ips(self, requested_ips_mapper_class_mock): + def build_network_interface_handler(*args, **kwargs): + return {'SubnetId': kwargs['subnet_id'], + 'PrivateIpAddresses': kwargs['private_ips']} + + # arrange + vpc = Mock() + security_group_ids = MagicMock() + + action1 = ConnectSubnet() + action1.actionId = 'action1' + action1.actionParams = ConnectToSubnetParams() + action1.actionParams.subnetId = 'sub1' + action1.actionParams.vnicName = 0 + action1.actionParams.isPublic = True + action1.actionParams.cidr = 'cidr1' + + action2 = ConnectSubnet() + action2.actionId = 'action2' + action2.actionParams = ConnectToSubnetParams() + action2.actionParams.subnetId = 'sub2' + action2.actionParams.vnicName = 1 + action2.actionParams.cidr = 'cidr2' + + ami_model = Mock() + ami_model.private_ip_addresses_list = MagicMock() + # ami_model.private_ip_address=Mock() + network_actions = [action1, action2] + ami_model.add_public_ip = False + + res_model_1 = DeployNetworkingResultModel('action1') + res_model_2 = DeployNetworkingResultModel('action2') + network_config_results = [res_model_1, res_model_2] + + self.deploy_operation.network_interface_service.build_network_interface_dto = \ + Mock(side_effect=build_network_interface_handler) + + cidr1_requested_ip = MagicMock() + cidr2_requested_ip = MagicMock() + + requested_ips_mapper_instance_mock = Mock() + requested_ips_mapper_instance_mock.map_network_to_requested_ips.return_value = { + 'cidr1': cidr1_requested_ip, + 'cidr2': cidr2_requested_ip + } + requested_ips_mapper_class_mock.return_value = requested_ips_mapper_instance_mock + + # act + net_interfaces = self.deploy_operation._prepare_network_interfaces(ami_deployment_model=ami_model, + network_actions=network_actions, + vpc=vpc, + security_group_ids=security_group_ids, + network_config_results=network_config_results, + logger=self.logger) + + # assert + # print res_model_1.device_index + self.assertEquals(res_model_1.device_index, 0) + self.assertEquals(res_model_2.device_index, 1) + self.assertEquals(len(net_interfaces), 2) + self.assertEquals(net_interfaces[0]['SubnetId'], 'sub1') + self.assertEquals(net_interfaces[0]['PrivateIpAddresses'], cidr1_requested_ip) + self.assertEquals(net_interfaces[1]['SubnetId'], 'sub2') + self.assertEquals(net_interfaces[1]['PrivateIpAddresses'], cidr2_requested_ip) + def test_prepare_network_config_results_dto_returns_empty_array_when_no_network_config(self): # arrange network_actions = None @@ -673,7 +776,7 @@ def test_deploy_raised_no_vpc(self): cancellation_context=Mock(), logger=self.logger) - def test__prepare_network_result_models_returns_empty_model_when_no_network_config(self): + def test_prepare_network_result_models_returns_empty_model_when_no_network_config(self): # arrange network_actions = None @@ -685,7 +788,7 @@ def test__prepare_network_result_models_returns_empty_model_when_no_network_conf self.assertEquals(models[0].action_id, '') self.assertTrue(isinstance(models[0], DeployNetworkingResultModel)) - def test__prepare_network_result_models_returns_result_model_per_action(self): + def test_prepare_network_result_models_returns_result_model_per_action(self): # arrange action1 = Mock(spec=PrepareCloudInfra, actionId=Mock(spec=str)) action1.actionParams = Mock(spec=ConnectToSubnetParams) diff --git a/package/tests/test_domain_services/test_model_parser.py b/package/tests/test_domain_services/test_model_parser.py index de8d1c1e..d187e8f7 100644 --- a/package/tests/test_domain_services/test_model_parser.py +++ b/package/tests/test_domain_services/test_model_parser.py @@ -199,6 +199,8 @@ def test_convert_to_deployment_resource_model(self): self.assertFalse(model.actionParams.deployment.customModel.wait_for_credentials) self.assertFalse(model.actionParams.deployment.customModel.add_public_ip) self.assertTrue(model.actionParams.deployment.customModel.allocate_elastic_ip) + self.assertIsNone(model.actionParams.deployment.customModel.private_ip_addresses_list) + self.assertIsNone(model.actionParams.deployment.customModel.private_ip_address) def test_convert_to_deployment_resource_model_with_network(self): json = '{'\ @@ -261,7 +263,22 @@ def test_convert_to_deployment_resource_model_with_network(self): '"attributeName": "Allow all Sandbox Traffic",'\ '"attributeValue": "True",'\ '"type": "attribute"'\ - '},'\ + '},' \ + '{' \ + '"attributeName": "Custom Tags",' \ + '"attributeValue": "custom_tags",' \ + '"type": "attribute"' \ + '},' \ + '{' \ + '"attributeName": "User Data URL",' \ + '"attributeValue": "user_data_url",' \ + '"type": "attribute"' \ + '},' \ + '{' \ + '"attributeName": "User Data Parameters",' \ + '"attributeValue": "user_data_parameters",' \ + '"type": "attribute"' \ + '},' \ '{'\ '"attributeName": "Wait for IP",'\ '"attributeValue": "False",'\ @@ -321,8 +338,13 @@ def test_convert_to_deployment_resource_model_with_network(self): '"attributeName": "Inbound Ports",'\ '"attributeValue": "",'\ '"type": "attribute"'\ - '}'\ - '],'\ + '},' \ + '{' \ + '"attributeName": "Private IP",' \ + '"attributeValue": "10.0.1.6, 10.0.1.8",' \ + '"type": "attribute"' \ + '}' \ + '],'\ '"type": "deployAppDeploymentInfo"'\ '},'\ '"appResource": {'\ @@ -366,6 +388,8 @@ def test_convert_to_deployment_resource_model_with_network(self): self.assertEquals(len(model.actionParams.subnetServiceAttributes), 6) self.assertEquals(model.actionParams.subnetServiceAttributes["Public"], 'True') self.assertTrue(isinstance(model.actionParams, ConnectToSubnetParams)) + self.assertEqual(model.actionParams.deployment.customModel.private_ip_addresses_list, ['10.0.1.6', '10.0.1.8']) + self.assertIsNone(model.actionParams.deployment.customModel.private_ip_address, '10.0.1.6') def test_subnet_connection_params_check_is_public_subnet_true(self): # arrange From 3427ad9480960f3cf7d7745d585b87db392579fc Mon Sep 17 00:00:00 2001 From: alexazarh Date: Tue, 3 Nov 2020 21:43:34 -0600 Subject: [PATCH 5/6] fixing CI --- .../test_domain_services/test_model_parser.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/package/tests/test_domain_services/test_model_parser.py b/package/tests/test_domain_services/test_model_parser.py index d187e8f7..cc65eda0 100644 --- a/package/tests/test_domain_services/test_model_parser.py +++ b/package/tests/test_domain_services/test_model_parser.py @@ -1,7 +1,8 @@ from unittest import TestCase from cloudshell.cp.core import DriverRequestParser -from cloudshell.cp.core.models import PrepareSubnetParams, PrepareCloudInfra, ConnectToSubnetParams +from cloudshell.cp.core.models import PrepareSubnetParams, PrepareCloudInfra, ConnectToSubnetParams, \ + ConnectToSubnetActionResult, ConnectSubnet, DeployApp from mock import Mock from cloudshell.cp.aws.common.converters import convert_to_bool @@ -381,15 +382,19 @@ def test_convert_to_deployment_resource_model_with_network(self): # Act self.request_parser = DriverRequestParser() self.request_parser.add_deployment_model(deployment_model_cls=DeployAWSEc2AMIInstanceResourceModel) - model = self.request_parser.convert_driver_request_to_actions(json)[0] + results = self.request_parser.convert_driver_request_to_actions(json) # Assert - self.assertEquals(model.actionId, "some_id") - self.assertEquals(len(model.actionParams.subnetServiceAttributes), 6) - self.assertEquals(model.actionParams.subnetServiceAttributes["Public"], 'True') - self.assertTrue(isinstance(model.actionParams, ConnectToSubnetParams)) - self.assertEqual(model.actionParams.deployment.customModel.private_ip_addresses_list, ['10.0.1.6', '10.0.1.8']) - self.assertIsNone(model.actionParams.deployment.customModel.private_ip_address, '10.0.1.6') + connect_subnet_action = next(iter(filter(lambda x: isinstance(x, ConnectSubnet), results)), None) + self.assertEquals(connect_subnet_action.actionId, "some_id") + self.assertEquals(len(connect_subnet_action.actionParams.subnetServiceAttributes), 6) + self.assertEquals(connect_subnet_action.actionParams.subnetServiceAttributes["Public"], 'True') + self.assertTrue(isinstance(connect_subnet_action.actionParams, ConnectToSubnetParams)) + + deployment_action = next(iter(filter(lambda x: isinstance(x, DeployApp), results)), None) + self.assertEqual([['10.0.1.6', '10.0.1.8']], + deployment_action.actionParams.deployment.customModel.private_ip_addresses_list) + self.assertEquals(deployment_action.actionParams.deployment.customModel.private_ip_address, '10.0.1.6') def test_subnet_connection_params_check_is_public_subnet_true(self): # arrange From f99618001eee69506f3bf709cd0e47381df25f5b Mon Sep 17 00:00:00 2001 From: alexazarh Date: Tue, 3 Nov 2020 22:20:10 -0600 Subject: [PATCH 6/6] increasing minor version --- drivers/aws_shell/src/drivermetadata.xml | 2 +- drivers/aws_shell/src/requirements.txt | 2 +- drivers/version.txt | 2 +- version.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/aws_shell/src/drivermetadata.xml b/drivers/aws_shell/src/drivermetadata.xml index 29ef0567..c37587a4 100644 --- a/drivers/aws_shell/src/drivermetadata.xml +++ b/drivers/aws_shell/src/drivermetadata.xml @@ -1,4 +1,4 @@ - + diff --git a/drivers/aws_shell/src/requirements.txt b/drivers/aws_shell/src/requirements.txt index 499defd2..c2f4019e 100644 --- a/drivers/aws_shell/src/requirements.txt +++ b/drivers/aws_shell/src/requirements.txt @@ -1,5 +1,5 @@ cloudshell-automation-api>=9.3 cloudshell-shell-core>=3.1.0,<3.2.0 -cloudshell-cp-aws>=2.5.0,<2.6.0 +cloudshell-cp-aws>=2.6.0,<2.7.0 jsonpickle==0.9.3 cloudshell-cp-core>=1.1.0,<1.2.0 diff --git a/drivers/version.txt b/drivers/version.txt index 26f8b8bc..914ec967 100644 --- a/drivers/version.txt +++ b/drivers/version.txt @@ -1 +1 @@ -2.4.5 \ No newline at end of file +2.6.0 \ No newline at end of file diff --git a/version.txt b/version.txt index 26f8b8bc..914ec967 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -2.4.5 \ No newline at end of file +2.6.0 \ No newline at end of file