Merge pull request #625 from loganharbour/ready_jobs

Add ready jobs page

Author: loganharbour

ci/client/ReadyJobs.py

@@ -29,17 +29,16 @@ def get_ready_jobs():
                           'recipe__build_user')
           .order_by(F('prioritized').desc(nulls_last=True), '-recipe__priority', 'created'))
 
-    ready_jobs = []
-    current_push_event_branches = set()
+    first_jobs = []
+    later_jobs = []
     for job in jobs.all():
-      if job.event.cause == models.Event.PUSH and job.event.auto_cancel_event_except_current():
-          current_push_event_branches.add(job.event.base.branch)
-      else:
-          ready_jobs.append(job)
+        # Delay push jobs that are not prioritized and have no set priority
+        delay = job.event.cause == models.Event.PUSH and \
+            job.prioritized is None and job.recipe.priority == 0
 
-    if current_push_event_branches:
-      jobs = jobs.filter(event__base__branch__in=current_push_event_branches).order_by('created', '-recipe__priority')
-      for job in jobs.all():
-          ready_jobs.append(job)
+        if delay:
+            later_jobs.append(job)
+        else:
+            first_jobs.append(job)
 
-    return ready_jobs
+    return first_jobs + later_jobs

ci/client/tests/test_views.py

@@ -117,59 +117,6 @@ def test_get_job_order(self):
             data = response.json()
             self.assertEqual(data['job_id'], expected_jobs[i].pk)
 
-    def test_ready_jobs_with_current_event(self):
-        """
-        If a branch is specified with "auto_cancel_push_events_except_current" then
-        jobs on the "current" event get priority over subsequent events. Basically
-        the normal sort of (-priority, created) gets changed to (created, -priority)
-        for those jobs.
-        """
-        url = reverse('ci:client:get_job')
-        user = utils.get_test_user()
-
-        recipes = []
-        for i in range(3):
-            recipe = utils.create_recipe(name=f'recipe{i}', user=user)
-            recipe.priority = int(30 - 10 * i)
-            recipe.save()
-            recipes.append(recipe)
-
-        e0 = utils.create_event(user=user, cause=models.Event.PUSH)
-        e1 = utils.create_event(user=user, cause=models.Event.PUSH, commit1=12345)
-
-        jobs = []
-        for i in range(3):
-            jobs.append(utils.create_job(recipe=recipes[i], event=e0, user=user))
-        for i in range(3):
-            jobs.append(utils.create_job(recipe=recipes[i], event=e1, user=user))
-
-        for job in jobs:
-            job.active = True
-            job.ready = True
-            job.save()
-
-        clients = []
-        for i in range(len(jobs)):
-            client = utils.create_client(name=f'client{i}')
-            clients.append(client)
-
-        url = reverse('ci:client:get_job')
-        post_data = {'build_keys': [user.build_key],
-                     'build_configs': [jobs[0].config.name]}
-
-        repo_name = "%s/%s" % (e0.base.branch.repository.user.name, e0.base.branch.repository.name)
-        branch_name = e0.base.branch.name
-        repo_settings={repo_name: {"branch_settings": {branch_name: {"auto_cancel_push_events_except_current": True}}}}
-        with self.settings(INSTALLED_GITSERVERS=[utils.github_config(repo_settings=repo_settings)]):
-            self.set_counts()
-            for i in range(len(jobs)):
-                post_data['client_name'] = clients[i].name
-                response = self.client_post_json(url, post_data)
-                self.compare_counts(active_branches=1)
-                self.assertEqual(response.status_code, 200)
-                data = response.json()
-                self.assertEqual(data['job_id'], jobs[i].pk)
-
     def json_post_request(self, data):
         jdata = json.dumps(data)
         return self.factory.post('/', jdata, content_type='application/json')

ci/templates/ci/base.html

@@ -92,6 +92,7 @@
               <li><a href="{% url 'ci:event_list' %}">Events</a></li>
               <li><a href="{% url 'ci:pullrequest_list' %}">Pull Requests</a></li>
               <li><a href="{% url 'ci:client_list' %}">Job Clients</a></li>
+              <li><a href="{% url 'ci:ready_jobs' %}">Ready Jobs</a></li>
               <li><a href="{% url 'ci:scheduled' %}">Scheduled Events</a></li>
               <li><a href="{% url 'ci:cronjobs' %}">Cron Recipes</a></li>
               <li><a href="{% url 'ci:job_info_search' %}">Filter Job by modules</a></li>

ci/templates/ci/ready_jobs.html

@@ -0,0 +1,76 @@
+{% extends "ci/base.html" %}
+{% load tz %}
+{% load humanize %}
+{% comment %}
+  Copyright 2016-2025 Battelle Energy Alliance, LLC
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+{% endcomment %}
+{% load humanize %}
+{% block title %}Civet: Ready jobs{% endblock %}
+{% block content %}
+<div class="center">
+{% if num_jobs %}
+    <h2>{{ num_jobs }} ready jobs</h2>
+{% else %}
+    <h2>Ready jobs</h2>
+{% endif %}
+</div>
+
+{% if not allowed %}
+  You are not allowed to view the ready jobs.
+  <br/>Please sign in and try again.
+{% elif num_jobs %}
+{% timezone "US/Mountain" %}
+    {% for config, jobs in jobs_by_config.items %}
+    <table class="table table-hover table-bordered table-condensed">
+        <thead>
+        <tr>
+        <th colspan="6" style="text-align:center;">Build config {{ config }}</tr>
+        </tr>
+        <tr>
+        <th>Created</th>
+        <th>Repo</th>
+        <th>Event</th>
+        <th>Job</th>
+        <th>Priority</th>
+        <th>Prioritized</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for job in jobs %}
+        <tr>
+            <td>{{ job.created|date:"m/d/y h:i:s" }}, {{ job.created|naturaltime }}</td>
+            <td>{{ job.recipe.repository.name }}</td>
+            {% if job.event.pull_request %}
+            <td><a href="{% url "ci:view_event" job.event.pk %}">{{ job.event.pull_request }}</a></td>
+            {% else %}
+            <td><a href="{% url "ci:view_event" job.event.pk %}">{{ job.event.description }}</a></td>
+            {% endif %}
+            <td><a href="{% url "ci:view_job" job.pk %}">{{ job.recipe.display_name }}</a></td>
+            <td>{{ job.recipe.priority }}</td>
+            {% if job.prioritized %}
+            <td>{{ job.prioritized|date:"m/d/y h:i:s" }}, {{ job.prioritized|naturaltime }}</td>
+            {% else %}
+            <td>No</td>
+            {% endif %}
+        </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+  {% endfor %}
+{% endtimezone %}
+{% else %}
+No ready jobs
+{% endif %}
+{% endblock %}

ci/tests/test_views.py

@@ -729,6 +729,43 @@ def test_prioritize_event(self, mock_perms):
             j.refresh_from_db()
             self.assertIsNotNone(j.prioritized)
 
+    @patch.object(Permissions, 'is_allowed_to_see_clients')
+    @override_settings(PERMISSION_CACHE_TIMEOUT=0)
+    def test_ready_jobs(self, mock_perms):
+        # not allowed
+        mock_perms.return_value = False
+        response = self.client.get(reverse('ci:ready_jobs'))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('You are not allowed to view the ready jobs', response.content.decode("utf-8"))
+
+        # no ready jobs
+        mock_perms.return_value = True
+        response = self.client.get(reverse('ci:ready_jobs'))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('No ready jobs', response.content.decode("utf-8"))
+
+        j0, j1, j2, j3 = utils.create_test_jobs()
+        client = utils.create_client()
+        for j in [j0, j1, j2, j3]:
+            j.client = client
+            j.ready = True
+            j.event.complete = False
+            j.save()
+
+        repo = j0.recipe.repository
+        repo.active = True
+        repo.save()
+
+        mock_perms.return_value = True
+        response = self.client.get(reverse('ci:ready_jobs'))
+        self.assertEqual(response.status_code, 200)
+        content = response.content.decode("utf-8")
+        self.assertIn('4 ready jobs', content)
+        for j in [j0, j1, j2, j3]:
+            self.assertIn(j.recipe.repository.name, content)
+            self.assertIn(j.event.description, content)
+            self.assertIn(j.recipe.display_name, content)
+
     @patch.object(Permissions, 'is_collaborator')
     @override_settings(PERMISSION_CACHE_TIMEOUT=0)
     def test_cancel_event(self, mock_collab):

ci/urls.py

@@ -38,6 +38,7 @@
     re_path(r'^recipe_crons/(?P<recipe_id>[0-9]+)/$', views.recipe_crons, name='recipe_crons'),
     re_path(r'^manual_cron/(?P<recipe_id>[0-9]+)/$', views.manual_cron, name='manual_cron'),
     re_path(r'^cronjobs/$', views.cronjobs, name='cronjobs'),
+    re_path(r'^ready_jobs/$', views.ready_jobs, name='ready_jobs'),
     re_path(r'^manual_branch/(?P<build_key>[0-9]+)/(?P<branch_id>[0-9]+)/$',
         views.manual_branch, name='manual_branch'),
     re_path(r'^manual_branch/(?P<build_key>[0-9]+)/(?P<branch_id>[0-9]+)/(?P<label>[A-Za-z0-9_.-]+)/$',

ci/views.py

@@ -29,6 +29,7 @@
 import tarfile
 from io import BytesIO
 from ci import RepositoryStatus, EventsStatus, Permissions, PullRequestEvent, ManualEvent, TimeUtils
+from ci.client.ReadyJobs import get_ready_jobs
 from django.utils.html import escape
 from django.utils.text import get_valid_filename
 from django.views.decorators.cache import never_cache
@@ -37,6 +38,7 @@
 from datetime import datetime
 from croniter import croniter
 import pytz
+from collections import defaultdict
 
 import logging, traceback
 logger = logging.getLogger('ci')
@@ -599,6 +601,22 @@ def cronjobs(request):
     data = {'recipes': recipe_list, 'allowed': True, 'update_interval': settings.HOME_PAGE_UPDATE_INTERVAL, }
     return render(request, 'ci/cronjobs.html', data)
 
+def ready_jobs(request):
+    allowed = Permissions.is_allowed_to_see_clients(request.session)
+    if not allowed:
+        return render(request, 'ci/ready_jobs.html', {'allowed': False, 'jobs_by_config': None, 'num_jobs': None})
+
+    num_jobs = 0
+    jobs_by_config = defaultdict(list)
+    for job in get_ready_jobs():
+        jobs_by_config[str(job.config)].append(job)
+        num_jobs += 1
+
+    data = {'allowed': True,
+            'jobs_by_config': dict(jobs_by_config),
+            'num_jobs': num_jobs}
+    return render(request, 'ci/ready_jobs.html', data)
+
 def clients_info():
     """
     Gets the information on all the currently active clients.