feat(throttling): throttle jobs resources based on parameters size

  step 1: define a priority scaling for jobs resources based on a configuration
    assign the configuration to parameter prio_throttling_parameters in openqa.ini
  step 2: apply changes requested in PR#6952
    Apply the priority throttling as by ticket's Suggestions in the description
    use configuration format "<parameter_name>:<scale_decimal>:<reference_valuee>,..."
    Set throttling code in create_from_settings,
    Add 2 subtests in priority tests of t/api/04-jobs.t
  step 3: move in a routine load_prio_throttling the format checking for throttling
    load the resulting hash after validated the throtting configuration
    and fix _load_prio_throttling,in Setup.pm
    calculate the resulting priority and update debug_message,in Jobs.pm
    update tests t/api/04-jobs.t and t/config.t to manage the throttling
    update the throttling description in openqa.ini
    fix comments in sub load_throttling_config

Ref. ticket poo#192952

Author: m-dati

etc/openqa/openqa.ini

@@ -369,6 +369,11 @@ concurrent = 0
 ## scheduled products without any jobs (e.g. due to errors) are not immediately
 ## cleaned up.
 #scheduled_product_min_storage_duration = 34
+## Throttling: set a comma-separated list of test parameters that trigger throttling
+## of scheduled openQA jobs by priority. Example: PARAM1:SCALE1[:THRESH1][,...]
+## Set a scale that a parameter value, minus an optional threshold, is multiplied by 
+## and added to each job prio, as: '$base_prio + $scale# * ($param# - $thresh#)'
+#prio_throttling_parameters =
 
 [archiving]
 ## Moves logs of jobs which are preserved during the cleanup because they are

lib/OpenQA/Schema/ResultSet/Jobs.pm

@@ -158,6 +158,21 @@ sub create_from_settings ($self, $settings, $scheduled_product_id = undef) {
             $new_job_args{priority} += $malus;
         }
     }
+    # apply resources throttling control
+    if (my $throttling = OpenQA::App->singleton->config->{misc_limits}->{prio_throttling_data}) {
+        my $throttling_info;
+        for my $resource (keys %$throttling) {
+            next if !defined $settings{$resource};
+            my $scale = $throttling->{$resource}->{scale};
+            my $reference = $throttling->{$resource}->{reference};
+            my $prio = int(($settings{$resource} - $reference) * $scale);
+            $throttling_info .= $resource . ", scale: $scale" . ($reference ? ", reference: $reference;" : ';');
+            $new_job_args{priority} += $prio;
+        }
+        $debug_msg .= sprintf("\nAdjusting job priority by %s based on resource requirement(s): %s",
+            $new_job_args{priority}, $throttling_info)
+          if $throttling_info;
+    }
 
     my $job = $self->create(\%new_job_args);
     log_debug(sprintf "(Job %d) $debug_msg", $job->id) if $debug_msg;

lib/OpenQA/Setup.pm

@@ -54,13 +54,29 @@ sub _load_config ($app, $defaults, $mode_specific_defaults) {
         _read_config_file($config, $config_file, $defaults, $mode_defaults);
         $config->{ini_config} = $config_file;
     }
-
     # ensure default values are assigned; warn if config files were supplied at all
     _read_config_file($config, undef, $defaults, $mode_defaults);
     $app->log->warn('No configuration files supplied, will fallback to default configuration') unless $config_file;
     return $config;
 }
 
+sub _load_prio_throttling ($app, $config) {
+    return
+      unless my $throttling = $config->{misc_limits}->{prio_throttling_parameters};
+    # floating point number
+    my $n = qr/[+-]?(?:\d+\.?\d*|\.\d+)/;
+    # Unit format = Param.name: scale: reference(optional)
+    my $u = qr/([A-Z_][A-Z0-9_]*):($n)(?::($n))?/i;
+    $throttling =~ s/\s+//g;
+    unless ($throttling =~ qr/^$u(?:,$u)*$/i) {
+        $app->log->warn("Wrong format in openqa.ini 'prio_throttling_parameters': $throttling");
+        return;
+    }
+    my %hash = map { my ($k, $s, $r) = $_ =~ /$u/g; $k => {scale => $s, reference => $r // 0} }
+      split(',', $throttling);
+    return \%hash;
+}
+
 sub read_config ($app) {
     my %defaults = (
         global => {
@@ -243,6 +259,8 @@ sub read_config ($app) {
             mcp_max_result_size => 500000,
             max_job_time_prio_scale => 100,
             scheduled_product_min_storage_duration => 34,
+            prio_throttling_parameters => '',
+            prio_throttling_data => undef,
         },
         archiving => {
             archive_preserved_important_jobs => 0,
@@ -293,6 +311,7 @@ sub read_config ($app) {
     if (my $minion_fail_job_blocklist = $config->{influxdb}->{ignored_failed_minion_jobs}) {
         $config->{influxdb}->{ignored_failed_minion_jobs} = [split(/\s+/, $minion_fail_job_blocklist)];
     }
+    $config->{misc_limits}->{prio_throttling_data} = _load_prio_throttling($app, $config);
     my $results = delete $global_config->{parallel_children_collapsable_results};
     $global_config->{parallel_children_collapsable_results_sel}
       = ' .status' . join('', map { ":not(.result_$_)" } split(/\s+/, $results));

t/api/04-jobs.t

@@ -1052,6 +1052,42 @@ subtest 'priority correctly assigned when posting job' => sub {
         $t->json_is('/job/priority', 50, 'feature disabled: prio value unchanged');
     };
 
+    subtest 'priority scaled up due to QEMURAM demand' => sub {
+        my $add = 20;
+        local $jobs_post_params{QEMURAM} = 4096;
+
+        my $config = OpenQA::Setup::read_config($t->app);
+        $config->{misc_limits}->{prio_throttling_parameters} = 'XXX :0.2, QEMURAM:0.01:2048';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($t->app, $config);
+        $t->post_ok('/api/v1/jobs', form => \%jobs_post_params)->status_is(200);
+        $t->get_ok('/api/v1/jobs/' . $t->tx->res->json->{id})->status_is(200);
+        $t->json_is('/job/priority', 50 + $add, 'increased prio value');
+    };
+
+    subtest 'priority improved due to QEMURAM low demand' => sub {
+        my $add = -10;
+        local $jobs_post_params{QEMURAM} = 1024;
+
+        my $config = OpenQA::Setup::read_config($t->app);    # OpenQA::App->singleton->config;
+        $config->{misc_limits}->{prio_throttling_parameters} = 'XXX :0.2, QEMURAM:0.01:2048';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($t->app, $config);
+        $t->post_ok('/api/v1/jobs', form => \%jobs_post_params)->status_is(200);
+        $t->get_ok('/api/v1/jobs/' . $t->tx->res->json->{id})->status_is(200);
+        $t->json_is('/job/priority', 50 + $add, 'decreased prio value');
+    };
+
+    subtest 'priority scaled up due to HDDSIZEGB demand' => sub {
+        my $add = 2;
+        local $jobs_post_params{HDDSIZEGB} = 40;
+        my $config = OpenQA::Setup::read_config($t->app);    # OpenQA::App->singleton->config;
+        $config->{misc_limits}->{prio_throttling_parameters}
+          = 'XXX :0.2,  FAKE_HDDSIZEGB:0.01, HDDSIZEGB:0.05,  YYY: 0.1';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($t->app, $config);
+        $t->post_ok('/api/v1/jobs', form => \%jobs_post_params)->status_is(200);
+        $t->get_ok('/api/v1/jobs/' . $t->tx->res->json->{id})->status_is(200);
+        $t->json_is('/job/priority', 50 + $add, 'increased prio value');
+    };
+
     # post new job in job group with customized default priority
     $schema->resultset('JobGroups')->find({name => 'opensuse test'})->update({default_priority => 42});
     $jobs_post_params{_GROUP} = 'opensuse test';

t/config.t

@@ -9,6 +9,7 @@ use lib "$FindBin::Bin/lib", "$FindBin::Bin/../external/os-autoinst-common/lib";
 use Test::Warnings ':report_warnings';
 use Test::Output 'combined_like';
 use Test::MockModule;
+use Test::MockObject;
 use Mojolicious;
 use Mojo::Base -signatures;
 use Mojo::Log;
@@ -193,6 +194,7 @@ subtest 'Test configuration default modes' => sub {
             mcp_max_result_size => 500000,
             max_job_time_prio_scale => 100,
             scheduled_product_min_storage_duration => 34,
+            prio_throttling_parameters => '',
         },
         archiving => {
             archive_preserved_important_jobs => 0,
@@ -215,6 +217,7 @@ subtest 'Test configuration default modes' => sub {
     $test_config->{_openid_secret} = $config->{_openid_secret};
     $test_config->{logging}->{level} = 'debug';
     $test_config->{global}->{service_port_delta} = 2;
+    $test_config->{misc_limits}->{prio_throttling_data} = undef;
     is ref delete $config->{global}->{auto_clone_regex}, 'Regexp', 'auto_clone_regex parsed as regex';
     ok delete $config->{'test_preset example'}, 'default values for example tests assigned';
     is_deeply $config, $test_config, '"test" configuration';
@@ -392,4 +395,56 @@ subtest 'Lookup precedence/hiding' => sub {
     is_deeply lookup_config_files(@args), \@expected, 'drop-in in overriden dir hides all other config';
 };
 
+subtest 'check throttling configuration validation and application' => sub {
+    my $warn_called = 0;
+    my $log_mock = Test::MockObject->new();
+    my $app_mock = Test::MockObject->new();
+    $log_mock->mock('warn', sub { $warn_called = 1; });
+    $app_mock->mock('log', sub { $log_mock });
+    my $config = OpenQA::App->set_singleton(my $app = Mojolicious->new(log => $quiet_log));
+    # my $config = OpenQA::Setup::read_config($app);
+
+    subtest 'invalid prio_throttling_parameters' => sub {
+        $config->{misc_limits}->{prio_throttling_parameters} = 'invalid';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($app_mock, $config);
+        is_deeply $config->{misc_limits}->{prio_throttling_data}, undef,
+          'prio_throttling_data is empty hash for invalid';
+        is $warn_called, 1, 'warning called for invalid format';
+    };
+    subtest 'no prio_throttling_parameters' => sub {
+        $warn_called = 0;
+        $config->{misc_limits}->{prio_throttling_parameters} = undef;
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($app_mock, $config);
+        is $config->{misc_limits}->{prio_throttling_data}, undef, 'prio_throttling_data is undef when no parameters';
+        is $warn_called, 0, 'no warning for undef';
+    };
+    subtest 'empty string' => sub {
+        $warn_called = 0;
+        $config->{misc_limits}->{prio_throttling_parameters} = '';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($app_mock, $config);
+        is $config->{misc_limits}->{prio_throttling_data}, undef, 'prio_throttling_data is undef for empty string';
+        is $warn_called, 0, 'no warning for empty';
+    };
+    subtest 'valid prio_throttling_parameter with space separators' => sub {
+        $warn_called = 0;
+        $config->{misc_limits}->{prio_throttling_parameters} = 'KEY1ONE:  1.5';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($app_mock, $config);
+        is_deeply $config->{misc_limits}->{prio_throttling_data},
+          {KEY1ONE => {scale => 1.5, reference => 0}},
+          'prio_throttling_data parsed correctly';
+    };
+    subtest 'valid multiple prio_throttling_parameters keys' => sub {
+        $warn_called = 0;
+        $config->{misc_limits}->{prio_throttling_parameters} = 'A:1.04,B:2:3,C:0.04:5';
+        $config->{misc_limits}->{prio_throttling_data} = OpenQA::Setup::_load_prio_throttling($app_mock, $config);
+        is_deeply $config->{misc_limits}->{prio_throttling_data},
+          {
+            A => {scale => 1.04, reference => 0},
+            B => {scale => 2, reference => 3},
+            C => {scale => 0.04, reference => 5}
+          },
+          'prio_throttling_data parses multiple correctly';
+    };
+};
+
 done_testing();