ci: merge main to release (#8407)

Author: rjsparks

dev/INSTALL

@@ -64,6 +64,7 @@
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/test/staging/'

dev/deploy-to-container/settings_local.py

@@ -60,6 +60,7 @@
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'

dev/diff/settings_local.py

@@ -59,6 +59,7 @@
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'

dev/tests/settings_local.py

@@ -50,6 +50,7 @@
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/assets/www6s/staging/'

docker/configs/settings_local.py

@@ -29,6 +29,7 @@ for sub in \
     /assets/www6/iesg \
     /assets/www6/iesg/evaluation \
     /assets/media/photo \
+    /assets/tmp \
     /assets/ftp \
     /assets/ftp/charter \
     /assets/ftp/internet-drafts \

docker/scripts/app-create-dirs.sh

@@ -970,6 +970,14 @@ def test_api_appauth(self):
             self.assertEqual(jsondata['success'], True)
             self.client.logout()
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.nfs_metrics": ["valid-token"]})
+    def test_api_nfs_metrics(self):
+        url = urlreverse("ietf.api.views.nfs_metrics")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, 'nfs_latency_seconds{operation="write"}')
+
     def test_api_get_session_matherials_no_agenda_meeting_url(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(meeting=meeting)

ietf/api/tests.py

@@ -82,6 +82,8 @@
     url(r'^version/?$', api_views.version),
     # Application authentication API key
     url(r'^appauth/(?P<app>authortools|bibxml)$', api_views.app_auth),
+    # NFS metrics endpoint
+    url(r'^metrics/nfs/?$', api_views.nfs_metrics),
     # latest versions
     url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json),
     url(r'^rfcdiff-latest-json/(?P<name>[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json),

ietf/api/urls.py

@@ -3,7 +3,10 @@
 
 import base64
 import binascii
+import datetime
 import json
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 import jsonschema
 import pytz
 import re
@@ -264,7 +267,22 @@ def app_auth(request, app: Literal["authortools", "bibxml"]):
             json.dumps({'success': True}),
             content_type='application/json')
 
-
+@requires_api_token
+@csrf_exempt
+def nfs_metrics(request):
+    with NamedTemporaryFile(dir=settings.NFS_METRICS_TMP_DIR,delete=False) as fp:
+        fp.close()
+        mark = datetime.datetime.now()
+        with open(fp.name, mode="w") as f:
+            f.write("whyioughta"*1024)
+        write_latency = (datetime.datetime.now() - mark).total_seconds()
+        mark = datetime.datetime.now()
+        with open(fp.name, "r") as f:
+            _=f.read()
+        read_latency = (datetime.datetime.now() - mark).total_seconds()
+        Path(f.name).unlink()
+    response=f'nfs_latency_seconds{{operation="write"}} {write_latency}\nnfs_latency_seconds{{operation="read"}} {read_latency}\n'
+    return HttpResponse(response)
 
 def find_doc_for_rfcdiff(name, rev):
     """rfcdiff lookup heuristics

ietf/api/views.py

@@ -13,10 +13,10 @@
 
 from typing import List, Optional      # pyflakes:ignore
 
-from ietf.doc.utils import new_state_change_event, update_action_holders
+from ietf.doc.utils import update_action_holders
 from ietf.utils import log
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, StateDocEvent
+from ietf.doc.models import Document, DocEvent, State
 from ietf.person.models import Person 
 from ietf.meeting.models import Meeting
 from ietf.mailtrigger.utils import gather_address_lists
@@ -213,11 +213,11 @@ def splitext(fn):
 
         def move_file_to(subdir):
             # Similar to move_draft_files_to_archive
-            # ghostlinkd would keep this in the combined all archive since it would
-            # be sourced from a different place. But when ghostlinkd is removed, nothing
-            # new is needed here - the file will already exist in the combined archive
             shutil.move(path,
                         os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, subdir, basename))
+            mark = Path(settings.FTP_DIR) / "internet-drafts" / basename
+            if mark.exists():
+                mark.unlink()
 
         try:
             doc = Document.objects.get(name=filename, rev=revision)
@@ -235,41 +235,3 @@ def move_file_to(subdir):
             # All uses of this past 2014 seem related to major system failures.
             move_file_to("unknown_ids")
 
-
-def repair_dead_on_expire():
-    by = Person.objects.get(name="(System)")
-    id_exists = State.objects.get(type="draft-iesg", slug="idexists")
-    dead = State.objects.get(type="draft-iesg", slug="dead")
-    dead_drafts = Document.objects.filter(
-        states__type="draft-iesg", states__slug="dead", type_id="draft"
-    )
-    for d in dead_drafts:
-        dead_event = d.latest_event(
-            StateDocEvent, state_type="draft-iesg", state__slug="dead"
-        )
-        if dead_event is not None:
-            if d.docevent_set.filter(type="expired_document").exists():
-                closest_expiry = min(
-                    [
-                        abs(e.time - dead_event.time)
-                        for e in d.docevent_set.filter(type="expired_document")
-                    ]
-                )
-                if closest_expiry.total_seconds() < 60:
-                    d.set_state(id_exists)
-                    events = []
-                    e = DocEvent(
-                        doc=d,
-                        rev=d.rev,
-                        type="added_comment",
-                        by=by,
-                        desc="IESG Dead state was set due only to document expiry - changing IESG state to ID-Exists",
-                    )
-                    e.skip_community_list_notification = True
-                    e.save()
-                    events.append(e)
-                    e = new_state_change_event(d, by, dead, id_exists)
-                    e.skip_community_list_notification = True
-                    e.save()
-                    events.append(e)
-                    d.save_with_history(events)

ietf/doc/expire.py

@@ -18,7 +18,6 @@
     in_draft_expire_freeze,
     get_expired_drafts,
     expirable_drafts,
-    repair_dead_on_expire,
     send_expire_notice_for_draft,
     expire_draft,
     clean_up_draft_files,
@@ -62,11 +61,6 @@ def expire_ids_task():
         raise
 
 
-@shared_task
-def repair_dead_on_expire_task():
-    repair_dead_on_expire()
-
-
 @shared_task
 def notify_expirations_task(notify_days=14):
     for doc in get_soon_to_expire_drafts(notify_days):

ietf/doc/tasks.py

@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 
 from pyquery import PyQuery
 from textwrap import wrap
@@ -387,7 +388,7 @@ def setUp(self):
 
 
 class ConflictReviewSubmitTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH','FTP_PATH']
     def test_initial_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         url = urlreverse('ietf.doc.views_conflict_review.submit',kwargs=dict(name=doc.name))
@@ -403,16 +404,23 @@ def test_initial_submission(self):
         # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
 
         # sane post using textbox
-        path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.name, doc.rev))
+        basename = f"{doc.name}-{doc.rev}.txt"
+        path = Path(settings.CONFLICT_REVIEW_PATH) / basename
+        ftp_dir = Path(settings.FTP_DIR) / "conflict-reviews"
+        if not ftp_dir.exists():
+            ftp_dir.mkdir()
+        ftp_path = ftp_dir / basename
         self.assertEqual(doc.rev,'00')
-        self.assertFalse(os.path.exists(path))
+        self.assertFalse(path.exists())
+        self.assertFalse(ftp_path.exists())
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         self.assertEqual(doc.rev,'00')
         with io.open(path) as f:
             self.assertEqual(f.read(),"Some initial review text\n")
             f.close()
+        self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
     def test_subsequent_submission(self):