identifiers: drop proquest

subjects: differentiate euproject subjects, freetext and controlled vocab
identifiers: add ISSN
identifiers: handle different variations of HDL
internal_notes: process internal notes
affiliations: validate affiliations in 901 field
collection: validate collections, allow EU project thesis

Author: kpsherva

cds_migrator_kit/rdm/data/thesis/README.md

@@ -15,6 +15,21 @@ invenio rdm-records custom-fields init
 1. Run affiliations
 2. Run users
 3. Run duplicates (and 981__b) mergers
+   3.1. add duplicated_pids.json file
 4. Identify UDC records
 5. Identify records with relations (2)
 6. irecords with comments, migrate comments
+
+
+next deployment
+
+change branch installed in migrator-kit from feature to master
+
+1. on worker pod
+2.
+invenio rdm-records add-to-fixture programmes
+invenio rdm-records add-to-fixture awards
+
+2. both on migration and worker pod
+invenio rdm-records custom-fields init
+invenio communities custom-fields init

cds_migrator_kit/rdm/records/load/load.py

@@ -228,10 +228,9 @@ def _pre_publish(self, identity, entry, version, draft):
             )
             if draft.errors:
                 raise ManualImportRequired(
-                    message=str(draft.errors),
+                    message=f"{str(draft.errors)}: {str(entry['record']['json'])}",
                     field="validation",
                     stage="load",
-                    description="Draft has errors",
                     recid=entry["record"]["recid"],
                     priority="warning",
                     value=draft._record.pid.pid_value,

cds_migrator_kit/rdm/records/transform/config.py

@@ -13,17 +13,17 @@
 PIDS_SCHEMES_ALLOWED = ["DOI"]
 
 # stores the identifiers found in PIDs field in the alternative identifiers instead
-PID_SCHEMES_TO_STORE_IN_IDENTIFIERS = ["ARXIV", "HDL", "HANDLE", "URN", "INIS"]
+PID_SCHEMES_TO_STORE_IN_IDENTIFIERS = ["ARXIV", "HDL", "HANDLE", "URN", "INIS", "CERCER"]
 
-IDENTIFIERS_SCHEMES_TO_DROP = ["SPIRES", "HAL", "OSTI", "SLAC", ]
+IDENTIFIERS_SCHEMES_TO_DROP = ["SPIRES", "HAL", "OSTI", "SLAC", "PROQUEST"]
 IDENTIFIERS_VALUES_TO_DROP = "oai:arXiv.org"
 
 CONTROLLED_SUBJECTS_SCHEMES = ["szgecern", "cern", "cds"]
 
-RECOGNISED_KEYWORD_SCHEMES = ["author", "cms", "arxiv", "inspire", "spr", "inis", "lanl eds", "in2p3"]
-KEYWORD_SCHEMES_TO_DROP = ["proquest"]
+RECOGNISED_KEYWORD_SCHEMES = ["author", "cms", "arxiv", "inspire", "spr", "inis", "lanl eds", "in2p3", "eucard", "inspec"]
+KEYWORD_SCHEMES_TO_DROP = ["proquest", "disxa"]
 
-ALLOWED_THESIS_COLLECTIONS = ["thesis", "publcms", "book"]
+ALLOWED_THESIS_COLLECTIONS = ["thesis", "publcms", "book", "aida", "eucard", "eucard2", "cern"]
 IGNORED_THESIS_COLLECTIONS = ["cern"]
 
 

cds_migrator_kit/rdm/records/transform/models/thesis.py

@@ -46,6 +46,7 @@ class ThesisModel(CdsOverdo):
         "035__m",  # oai harvest tag
         "035__t",  # oai harvest tag
         "035__u",  # oai harvest tag
+        "035__z",  # oai harvest tag
         "037__c",  # arxiv subject
         # "084__2",  # SORT ID, spreadsheet, also PACS and missing report numbers, requested curation
         # "084__a",  # SORT ID, spreadsheet
@@ -69,6 +70,7 @@ class ThesisModel(CdsOverdo):
         "701__m",  # supervisors's email <-- decided not to keep in RDM,
         "720__a",  # author's duplicate
         "773__o",  # spreadsheet
+        "773__x",  # spreadsheet
         "8564_8",  # Files system field
         "8564_s",  # Files system field
         "8564_x",  # Files system field
@@ -82,7 +84,7 @@ class ThesisModel(CdsOverdo):
         "961__x",  # CDS modification tag # TODO
         "964__a",  # spreadsheet
         "970__b",  # spreadsheet
-        "980__a",  # collection tag
+        # "980__a",  # collection tag
         "981__a",  # duplicated record marker # TODO -> decide how to handle these
         "999C50",  # https://cds.cern.ch/record/2284609/export/hm?ln=en CMS contributions
         "999C52",  # https://cds.cern.ch/record/2640188/export/hm?ln=en

cds_migrator_kit/rdm/records/transform/transform.py

@@ -15,6 +15,7 @@
 import arrow
 from cds_rdm.legacy.models import CDSMigrationAffiliationMapping
 from idutils import normalize_ror
+from idutils.validators import is_doi
 from invenio_access.permissions import system_identity
 from invenio_accounts.models import User
 from invenio_db import db
@@ -51,6 +52,9 @@
 def search_vocabulary(term, vocab_type):
     """Search vocabulary utility function."""
     service = current_service_registry.get("vocabularies")
+    if "/" in term:
+        # escape the slashes
+        term = term.replace("/", "\\/")
     try:
         vocabulary_result = service.search(
             system_identity, type=vocab_type, q=f"{term}"
@@ -141,7 +145,7 @@ def _pids(self, json_entry):
             if key.upper() in PIDS_SCHEMES_TO_DROP:
                 del output_pids[key]
 
-            elif key.upper() not in PIDS_SCHEMES_ALLOWED:
+            elif key and key.upper() not in PIDS_SCHEMES_ALLOWED:
                 raise UnexpectedValue(
                     field=key,
                     subfield="2",
@@ -150,9 +154,14 @@ def _pids(self, json_entry):
                     stage="transform",
                     value=identifier
                 )
+            elif not key and is_doi(identifier):
+                # assume it is DOI
+                key = "DOI"
             if key.upper() == "DOI":
                 doi_identifier = deepcopy(identifier)
                 if identifier["identifier"].startswith(DATACITE_PREFIX):
+                    if not json_entry.get("publisher"):
+                        json_entry["publisher"] = "CERN"
                     doi_identifier["provider"] = "datacite"
                 else:
                     doi_identifier["provider"] = "external"