Merge pull request #62 from softwareone-platform/feature/MPT-6729-sync-with-integration-20250205

Feature/mpt 6729 sync with integration 20250205

Author: michal-zielonka

auth/auth_server/controllers/user.py

@@ -38,6 +38,8 @@ def _get_input(self, **input_):
             'is_password_autogenerated', False)
         self_registration = input_.get('self_registration', False)
         verified = input_.get('verified', False)
+        if self._config.disable_email_verification():
+            verified = True
         if self_registration:
             root_type = self.session.query(Type).filter(
                 and_(

auth/auth_server/tests/unittests/test_api_base.py

@@ -44,6 +44,9 @@ def setUp(self, version='v1'):
               return_value=secret).start()
         patch('optscale_client.config_client.client.Client.zoho_params',
               return_value={}).start()
+        patch('optscale_client.config_client.client.'
+              'Client.disable_email_verification',
+              return_value=None).start()
         patch('auth.auth_server.utils.get_encryption_salt',
               return_value=gen_salt()).start()
         patch('auth.auth_server.controllers.token.TokenController.expiration',

auth/auth_server/tests/unittests/test_api_user.py

@@ -872,3 +872,12 @@ def test_create_verified_user(self):
                                              display_name='test',
                                              verified=True)
         self.assertTrue(user['verified'])
+
+    def test_disabled_email_verification(self):
+        patch('optscale_client.config_client.client.'
+              'Client.disable_email_verification',
+              return_value=True).start()
+        self.user_verified_mock.stop()
+        code, user = self.client.user_create('[email protected]', 'pass1',
+                                             display_name='test')
+        self.assertTrue(user['verified'])

bulldozer/bulldozer_api/server.py

@@ -1,4 +1,3 @@
-import datetime
 import asyncio
 from typing import Tuple
 from urllib.parse import urlparse

bulldozer/bulldozer_worker/generator/aws.py

@@ -48,6 +48,7 @@ def __init__(self, seed,
                  key=None,
                  tags=None,
                  open_ingress=False,
+                 spot_price=None,
                  ):
         if tags is None:
             tags = dict()
@@ -63,6 +64,7 @@ def __init__(self, seed,
         self.user_data = user_data
         self.tags = tags
         self.open_ingress = open_ingress
+        self.spot_price = spot_price
 
     @property
     def image(self):
@@ -102,6 +104,8 @@ def generate_payload(self, spot=False):
         }
         if self.user_data:
             d.update({"user_data": self.user_data})
+        if self.spot_price:
+            d.update({"spot_price": self.spot_price})
         return d
 
 

bulldozer/bulldozer_worker/generator/templates/aws.tft

@@ -51,6 +51,9 @@ resource "aws_spot_instance_request" "optscale_instance_{{seed}}" {
   {{#key}}
   key_name = "{{key_name}}"
   {{/key}}
+  {{#spot_price}}
+  spot_price = "{{spot_price}}"
+  {{/spot_price}}
   instance_type = "{{instance_type}}"
   vpc_security_group_ids = ["${aws_security_group.optscale_sg_{{seed}}.id}"]
   root_block_device {

bulldozer/bulldozer_worker/infra.py

@@ -211,6 +211,7 @@ def start(
             key=None,
             tags=None,
             open_ingress=False,
+            spot_price=None,
     ):
         gen_cls = self.generator
         gen = gen_cls(
@@ -223,7 +224,8 @@ def start(
             user_data,
             key,
             tags,
-            open_ingress
+            open_ingress,
+            spot_price
         )
         template = gen.render()
         path = os.path.join(os.path.dirname(self.path), self.seed)

bulldozer/bulldozer_worker/tasks.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 
 from bulldozer.bulldozer_worker.infra import Infra, InfraException
@@ -74,7 +73,7 @@ class RunsetState:
 
 
 class TaskReason:
-    COMPLETED = "task completed successfully"
+    COMPLETED = "Task completed successfully"
 
 
 class Base:
@@ -237,7 +236,7 @@ def check_destroy_conditions(self):
         _, runner = self.bulldozer_cl.get_runner(runner_id)
         # check for destroy flag set
         if runner.get("destroy"):
-            raise DestroyFlagSet("Destroy flag is set")
+            raise DestroyFlagSet("Aborted - Destroy flag is set")
 
         destroy_conditions = runner.get("destroy_conditions", {})
         # check budget condition
@@ -250,7 +249,8 @@ def check_destroy_conditions(self):
                      "current (estimated): %f", runner_id, max_budget, cost)
             if max_budget < cost:
                 raise BudgetExceeded(
-                    f"Budget exceeded max: {max_budget}, current: {cost}")
+                    f"Aborted - Budget exceeded max: {max_budget}, "
+                    f"current: {cost}")
         max_duration = destroy_conditions.get("max_duration")
         if max_duration:
             LOG.info("checking for max duration %d for runner %s",
@@ -263,7 +263,7 @@ def check_destroy_conditions(self):
                          runner_id, now, threshold)
                 if now > threshold:
                     raise TimeoutConditionExceeded(
-                        f"Duration exceeded: current time: {now} "
+                        f"Aborted - Duration exceeded: current time: {now} "
                         f"threshold: {threshold}"
                     )
 
@@ -366,15 +366,18 @@ def _exec(self):
                 }
             )
             infra.destroy()
-
+            self.bulldozer_cl.update_runner(
+                runner_id,
+                state=TaskState.DESTROYED,
+                destroyed_at=utcnow_timestamp()
+            )
         except Exception as exc:
             # basically exception
             LOG.exception("Cleanup problem: %s", str(exc))
-        finally:
             self.bulldozer_cl.update_runner(
                 runner_id,
-                state=TaskState.ERROR,
-                destroyed_at=utcnow_timestamp())
+                state=TaskState.ERROR)
+        finally:
             self.update_reason()
         self.message.ack()
 
@@ -402,6 +405,11 @@ def _exec(self):
         tags = runner.get("tags", dict())
         # opens ingress ports for runner instance
         open_ingress = runner.get("open_ingress", False)
+        _, runset = self.bulldozer_cl.runset_get(runner["runset_id"])
+        spot_settings = runset.get("spot_settings")
+        spot_price = None
+        if spot_settings:
+            spot_price = spot_settings.get("spot_price")
 
         if hp is not None and isinstance(hp, dict):
             for k, v in hp.items():
@@ -413,7 +421,7 @@ def _exec(self):
             state=TaskState.STARTING)
         _, cloud_account = self.rest_cl.cloud_account_get(
             cloud_account_id, True)
-        # TODO: get cloud type form cloud account to support multi-cloud
+        # TODO: get cloud type from cloud account to support multi-cloud
         # Now only AWS is supported
         c_type = "AWS"
         if not self.body.get("type"):
@@ -448,6 +456,7 @@ def _exec(self):
             key=None,
             tags=tags,
             open_ingress=open_ingress,
+            spot_price=spot_price
         )
 
         LOG.info("Created runner id=%s, instance=%s, ip=%s",
@@ -494,7 +503,7 @@ def _exec(self):
                      current_time, wait_time)
             if current_time > wait_time:
                 # TODO: Do we need automatically destroy env?
-                raise ArceeWaitException("Arcee wait exceeded")
+                raise ArceeWaitException("Aborted - Arcee wait exceeded")
         else:
             self.update_run_info(run_id, runner)
             self.bulldozer_cl.update_runner(

bumiworker/bumiworker/modules/rightsizing_base.py

@@ -93,7 +93,7 @@ def get_recommended_cpu(self, flavor_cpu, instance_metrics,
             flavor_cpu, instance_metrics, optimization_metric)
         if min_recommended:
             recommended_cpu_list = [
-                x for x in range(int(min_recommended), flavor_cpu)
+                x for x in range(int(min_recommended), int(flavor_cpu))
                 if x >= recommended_flavor_cpu_min and bool(
                     x == 1 or x > 1 and x % 2 == 0)]
         return recommended_cpu_list

diworker/diworker/importers/aws.py

@@ -756,7 +756,8 @@ def extract_data_transfer():
             volume_type: usage_type and volume_type in usage_type,
             'Bucket': product and 'AmazonS3' in product and (
                     bool(resource_id) and bucket_type in operation),
-            ip_address_type: extract_type_by_product_type(ip_address_type),
+            ip_address_type: (extract_type_by_product_type(ip_address_type) or
+                              'PublicIP' in usage_type),
             sp_type: bool(sp_id) and 'SavingsPlan' in item_type,
             ri_type: bool(ri_id),
             'Other': (tax_type or resource_type or product_family or
@@ -915,6 +916,16 @@ def _get_cloud_extras(self, info):
                 res['meta'][k] = val
         return res
 
+    def update_cloud_account_config(self):
+        config = self.cloud_acc.get('config')
+        if not config.get('cur_version'):
+            cur_version = self.cloud_adapter.config.get('cur_version')
+            if cur_version:
+                config.pop('region_name', None)
+                config.update({'cur_version': cur_version})
+                self.rest_cl.cloud_account_update(
+                    self.cloud_acc_id, {'config': config})
+
     def create_traffic_processing_tasks(self):
         self._create_traffic_processing_tasks()