Throttle openQA jobs resources based on test parameters size

  Define a priority malus for jobs resources exceeding given limit.
  Store the configuration in openqa.ini file 'prio_throttling_parameters',
  in the format therein described "PAR1=MAX1:MALUS1,PAR2=..:..,",
  to load the paramters details at server start.

  - Ticket poo#192952

Author: m-dati

etc/openqa/openqa.ini

@@ -369,6 +369,13 @@ concurrent = 0
 ## scheduled products without any jobs (e.g. due to errors) are not immediately
 ## cleaned up.
 #scheduled_product_min_storage_duration = 34
+## Job resources throttling configuration:
+## a string of comma-separated jobs resources, 
+## formatted like: "RES1=THR1:MAL1,RES2=THR2:MAL2,..."
+## where THR# is the maximum threshold of a RESOURCE# value,
+## and MAL# is a malus priority to multiply as many times as THR# increases:
+## PRIO1 = base + int(RESOURCE1/THR1)*MAL1
+#prio_throttling_parameters
 
 [archiving]
 ## Moves logs of jobs which are preserved during the cleanup because they are

lib/OpenQA/Schema/ResultSet/Jobs.pm

@@ -101,6 +101,14 @@ sub latest_jobs ($self, $until = undef) {
     return @latest;
 }
 
+sub load_throttling_config ($config_string) {
+    # parse to hash "{ PAR1 => [THR1, MAL1], PAR2 => [THR2, MAL2], ...}"
+    # the prio_throttling_parameters configuration string; see openqa.ini
+    return unless ($config_string && $config_string =~ /=\d+:/);
+    my %hash = map { my ($k, $v) = split /=/, $_, 2; uc($k) => [split(/:/, $v)] } split /,/, $config_string;
+    return \%hash;
+}
+
 sub create_from_settings ($self, $settings, $scheduled_product_id = undef) {
     my %settings = %$settings;
     my %new_job_args;
@@ -158,6 +166,21 @@ sub create_from_settings ($self, $settings, $scheduled_product_id = undef) {
             $new_job_args{priority} += $malus;
         }
     }
+    my $throttling = OpenQA::App->singleton->config->{misc_limits}->{prio_throttling_parameters};
+    $throttling = load_throttling_config($throttling);
+    # apply resources throttling control
+    if ($throttling && ref $throttling eq 'HASH') {
+        for my $resource (keys %$throttling) {
+            next if !defined $settings{$resource};
+            my @limits = $throttling->{$resource};
+            if ($settings{$resource} > $limits[0]) {
+                my $malus = int($settings{$resource} / $limits[0]) * $limits[1];
+                $debug_msg = sprintf 'Adding priority malus to newly created job (old: %d, malus: %s)',
+                  $new_job_args{priority}, $malus;
+                $new_job_args{priority} += $malus;
+            }
+        }
+    }
 
     my $job = $self->create(\%new_job_args);
     log_debug(sprintf "(Job %d) $debug_msg", $job->id) if $debug_msg;

lib/OpenQA/Setup.pm

@@ -243,6 +243,7 @@ sub read_config ($app) {
             mcp_max_result_size => 500000,
             max_job_time_prio_scale => 100,
             scheduled_product_min_storage_duration => 34,
+            prio_throttling_parameters => '',
         },
         archiving => {
             archive_preserved_important_jobs => 0,
@@ -293,6 +294,12 @@ sub read_config ($app) {
     if (my $minion_fail_job_blocklist = $config->{influxdb}->{ignored_failed_minion_jobs}) {
         $config->{influxdb}->{ignored_failed_minion_jobs} = [split(/\s+/, $minion_fail_job_blocklist)];
     }
+    my $throttling = $config->{misc_limits}->{prio_throttling_parameters};
+    if ($throttling && length $throttling) {
+        $config->{misc_limits}->{prio_throttling_parameters} =~ s/\s+//g;
+        die("Wrong formatting for 'prio_throttling_parameters' in openqa.ini")
+          unless ($throttling =~ /^[A-Z_]+=\d+:\d+(?:,[A-Z_]+=\d+:\d+)*$/i);
+    }
     my $results = delete $global_config->{parallel_children_collapsable_results};
     $global_config->{parallel_children_collapsable_results_sel}
       = ' .status' . join('', map { ":not(.result_$_)" } split(/\s+/, $results));

t/21-needles.t

@@ -43,8 +43,9 @@ my %settings = (
     ISO => 'whatever.iso',
     MACHINE => 'alpha',
     ARCH => 'x86_64',
+    MAX_JOB_TIME => '7600',
+    _PRIORITY => 50,
 );
-
 my $mock_utils = Test::MockModule->new('OpenQA::Utils');
 $mock_utils->redefine('run_cmd_with_log', 1);
 $mock_utils->redefine('run_cmd_with_log_return_error', 1);
@@ -54,6 +55,23 @@ my $schema = OpenQA::Test::Database->new->create;
 my $needledir_archlinux = 't/data/openqa/share/tests/archlinux/needles';
 my $needledir_fedora = 't/data/openqa/share/tests/fedora/needles';
 # create dummy job
+
+my $mock_config = {
+    misc_limits => {prio_throttling_parameters => '', max_job_time_prio_scale => 100},
+    global => {worker_timeout => 900},
+    'scm git' => {checkout_needles_sha => 'yes', allow_arbitrary_url_fetch => 'yes'}};
+my $mock_app = Test::MockModule->new('OpenQA::App');
+my $mock_log = Test::MockModule->new('Mojo::Log');
+$mock_log->redefine('error', sub { print STDERR $_[1] . "\n" });
+$mock_log->redefine('info', sub { print STDOUT $_[1] . "\n" });
+$mock_log->redefine('debug', sub { print STDOUT $_[1] . "\n" });
+my $mock_singleton = bless {config => $mock_config}, 'OpenQA::App';
+# Add config and log as methods
+no strict 'refs';
+*{'OpenQA::App::config'} = sub { return $_[0]->{config} };
+*{'OpenQA::App::log'} = sub { return bless {}, 'Mojo::Log' };
+use strict 'refs';
+$mock_app->redefine('singleton', sub { return $mock_singleton });
 my $job = $schema->resultset('Jobs')->create_from_settings(\%settings);
 # create dummy module
 $job->insert_module({name => 'a', category => 'a', script => 'a', flags => {}});

t/config.t

@@ -193,6 +193,7 @@ subtest 'Test configuration default modes' => sub {
             mcp_max_result_size => 500000,
             max_job_time_prio_scale => 100,
             scheduled_product_min_storage_duration => 34,
+            prio_throttling_parameters => '',
         },
         archiving => {
             archive_preserved_important_jobs => 0,