Merge branch 'release/5.18.0'

Author: lukavdplas

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 5.17.0
-date-released: '2025-02-13'
+version: 5.18.0
+date-released: '2025-04-10'

LICENSE

@@ -1,5 +1,5 @@
 Copyright (c) 2017-2022 Digital Humanities Lab, Faculty of Humanities, Utrecht University
-Copyright (c) 2023-2024 Research Software Lab, Centre for Digital Humanities, Utrecht University
+Copyright (c) 2023-2025 Research Software Lab, Centre for Digital Humanities, Utrecht University
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -31,7 +31,15 @@ The [documentation directory](./documentation/) contains documentation for devel
 
 ## Licence
 
-I-analyzer is shared under an MIT licence. See [LICENSE](./LICENSE) for more information.
+The source code of I-analyzer is shared under an MIT licence. See [LICENSE](./LICENSE) for the full licence statement.
+
+### Images
+
+This licence does *not* cover the images used for corpora, which are licensed individually. These images are located in the [corpora directory](/backend/corpora/), in the "images" folder for each corpus.
+
+Each image is accompanied by a `*.license` file that provides information on licensing for that image. If you wish to reuse or distribute this repository including these images, you will have to ensure that you comply with the license terms of the image as well.
+
+Some images currently lack a licence file. We are working on providing clear copyright information for all images; until then, assume that these images are protected under copyright.
 
 ## Citation
 

backend/addcorpus/models.py

@@ -32,7 +32,7 @@
     validate_default_sort,
     validate_ngram_has_date_field,
 )
-from ianalyzer.elasticsearch import elasticsearch
+from es.client import elasticsearch
 
 MAX_LENGTH_NAME = 126
 MAX_LENGTH_DESCRIPTION = 254

backend/addcorpus/python_corpora/tests/test_times_source.py

@@ -12,7 +12,7 @@ def times_test_settings(settings):
         'times': join(settings.BASE_DIR, 'corpora/times/times.py')
     }
     settings.TIMES_DATA = join(settings.BASE_DIR, 'addcorpus/python_corpora/tests')
-    settings.TIMES_ES_INDEX = 'times-test'
+    settings.TIMES_ES_INDEX = 'test-times'
 
 
 

backend/addcorpus/tests/test_corpus_views.py

@@ -6,6 +6,7 @@
 from users.models import CustomUser
 from addcorpus.models import Corpus, CorpusDocumentationPage
 from addcorpus.python_corpora.save_corpus import load_and_save_all_corpora
+from addcorpus.json_corpora.validate import corpus_schema
 
 def test_no_corpora(db, settings, admin_client):
     Corpus.objects.all().delete()
@@ -160,3 +161,7 @@ def test_corpus_edit_views(admin_client: Client, json_corpus_definition: Dict, j
     assert status.is_success(response.status_code)
     assert len(response.data) == 1
 
+
+def test_corpus_schema_view(client):
+    response = client.get('/api/corpus/definition-schema')
+    assert response.data == corpus_schema()

backend/addcorpus/urls.py

@@ -1,8 +1,11 @@
 from django.urls import path
-from addcorpus.views import CorpusImageView, CorpusView, CorpusDocumentationPageViewset, CorpusDocumentView
+from addcorpus.views import (
+    CorpusImageView, CorpusView, CorpusDocumentView, CorpusDefinitionSchemaView
+)
 
 urlpatterns = [
     path('', CorpusView.as_view({'get': 'list'})),
     path('image/<str:corpus>', CorpusImageView.as_view()),
     path('document/<str:corpus>/<str:filename>', CorpusDocumentView.as_view()),
+    path('definition-schema', CorpusDefinitionSchemaView.as_view()),
 ]

backend/addcorpus/views.py

@@ -19,6 +19,7 @@
 from rest_framework.response import Response
 from rest_framework.status import HTTP_200_OK
 from rest_framework.views import APIView
+from addcorpus.json_corpora.validate import corpus_schema
 
 
 class CorpusView(viewsets.ReadOnlyModelViewSet):
@@ -124,3 +125,13 @@ def info(self, request, pk):
         info = get_csv_info(obj.file.path, sep=delimiter if delimiter else ',')
 
         return Response(info, HTTP_200_OK)
+
+
+class CorpusDefinitionSchemaView(APIView):
+    '''
+    View the JSON schema for corpus definitions
+    '''
+
+    def get(self, request, *args, **kwargs):
+        schema = corpus_schema()
+        return Response(schema, HTTP_200_OK)

backend/conftest.py

@@ -7,14 +7,18 @@
 from elasticsearch import Elasticsearch
 import warnings
 
-from ianalyzer.elasticsearch import client_from_config
+from es.client import client_from_config
 from addcorpus.python_corpora.save_corpus import load_and_save_all_corpora
-from es import es_index as index, sync
+from es import sync
+from indexing.create_job import create_indexing_job
+from indexing.run_job import perform_indexing
 from django.conf import settings
 from django.contrib.auth.models import Group
 from addcorpus.models import Corpus
 from addcorpus.serializers import CorpusJSONDefinitionSerializer
 from es.models import Server
+from rest_framework.test import APIClient
+
 
 @pytest.fixture(autouse=True)
 def media_dir(tmpdir, settings):
@@ -165,8 +169,8 @@ def _index_test_corpus(es_client: Elasticsearch, corpus_name: str):
 
     if not es_client.indices.exists(index=corpus.configuration.es_index):
         with warnings.catch_warnings():
-            job = index.create_indexing_job(corpus)
-            index.perform_indexing(job)
+            job = create_indexing_job(corpus)
+            perform_indexing(job)
 
         # ES is "near real time", so give it a second before we start searching the index
         sleep(2)
@@ -236,3 +240,11 @@ def json_mock_corpus(db, json_corpus_definition) -> Corpus:
     corpus.configuration.save()
 
     return corpus
+
+@pytest.fixture(scope='session')
+def celery_config():
+    return {
+        'task_serializer': 'pickle',
+        'result_serializer': 'pickle',
+        'accept_content': ['json', 'pickle'],
+    }

backend/corpora/dbnl/dbnl.py

@@ -360,9 +360,31 @@ def _xml_files(self):
         extractor=XML(
             Tag(utils.LINE_TAG),
             TransformTag(utils.pad_content),
+            TransformTag(utils.replace_notes_with_ref),
             multiple=True,
             flatten=True,
-            transform=lambda lines: '\n'.join(lines).strip() if lines else None,
+            transform=utils.join_paragraphs,
+        ),
+        es_mapping=main_content_mapping(token_counts=True),
+        visualizations=['wordcloud'],
+        language='dynamic',
+    )
+
+    notes = FieldDefinition(
+        name='notes',
+        display_name='Notes',
+        description='Notes (e.g. footnotes) added to the content',
+        display_type='text_content',
+        results_overview=False,
+        search_field_core=True,
+        csv_core=True,
+        extractor=XML(
+            Tag('note'),
+            TransformTag(utils.pad_content),
+            TransformTag(utils.insert_ref),
+            multiple=True,
+            flatten=True,
+            transform=utils.join_paragraphs,
         ),
         es_mapping=main_content_mapping(token_counts=True),
         visualizations=['wordcloud'],
@@ -418,6 +440,7 @@ def _xml_files(self):
         chapter_title,
         chapter_index,
         content,
+        notes,
         has_content,
         is_primary,
     ]

backend/corpora/dbnl/images/dbnl.png.license

@@ -0,0 +1,3 @@
+No copyright restrictions.
+
+Image source: https://www.dbnl.org/tekst/alph002vana01_01/alph002vana01_01_0001.php

backend/corpora/dbnl/tests/test_dbnl_extraction.py

@@ -107,6 +107,7 @@ def test_append_to_tag(xml, tag, padding, original_output, new_output):
             'Cornelis Maertsz. tot Wervers hoof.',
             '\'t Amsterdam Voor Michiel de Groot, Boek-Verkooper op den Nieuwen Dijck, 1671.',
         ]),
+        'notes': None,
         'chapter_title': None,
         'chapter_index': 1,
         'has_content': True,
@@ -199,4 +200,25 @@ def test_dbnl_extraction(dbnl_corpus):
             assert expected[key] == actual[key]
         assert expected.items() <= actual.items()
 
+section_with_footnote = '''
+<div>
+<p>
+<pb n="128"/>geen zekerlijk in de twee bedoelde taalen zeer in elkaâr loopt. Althans in de Constructie geloof ik niet dat de reden kan gezocht worden<note n="a" place="foot">Eenigsints belagchelijk wordt het, wanneer men zich te Stockholm geduurig, tot bewijs der overëenkomst tusschen Zweedsch en Engelsch, de zelfde drie of vier woorden hoort voorzeggen, zonder dat men, om meerdere voorbeelden vraagende, zoo ligtelijk antwoord ontvangt.</note>. Voor 't overige kan ieder Hollander, die er nog een paar der gewoonste Europaesche taalen bij bezit, het Zweedsch, even als het Deensch, zich zelve leeren. In de Poësie evenwel ontbreekt het niet aan eene meenigte woorden, die men vruchteloos uit de Analogie zou willen verklaaren, en die het leezen der Dichters zeer vermoeiëlijken.<note n="b" place="foot">Van een Hoogduitsch - Zweedsch, en Zweedsch - Hoogduitsch Woordenboek van <hi rend="sc">möller</hi> bezit ik nog maar de twee eerste deelen in 40., welke het Hoogduitsch gedeelte bevatten; ik weet niet of het overige reeds gevolgd is, of nog volgen zal. Het is reeds van 1785. Eene kleine <hi rend="i">Grammatica</hi> van <hi rend="sc">abr. Sahlstedt</hi> is in 1796 in 't Hoogduitsch overgezet. Over het Lapsch en Finsch, twee van het Zweedsch geheel onderscheidene taalen, welke ook in dit Koninkrijk gesproken worden, zal het voegsaamer zijn op eene andere plaats te handelen.</note>
+</p>
+</div>
+'''
 
+expected_content = '''geen zekerlijk in de twee bedoelde taalen zeer in elkaâr loopt. Althans in de Constructie geloof ik niet dat de reden kan gezocht worden[1]. Voor 't overige kan ieder Hollander, die er nog een paar der gewoonste Europaesche taalen bij bezit, het Zweedsch, even als het Deensch, zich zelve leeren. In de Poësie evenwel ontbreekt het niet aan eene meenigte woorden, die men vruchteloos uit de Analogie zou willen verklaaren, en die het leezen der Dichters zeer vermoeiëlijken.[2]'''
+
+expected_notes = '''[1] Eenigsints belagchelijk wordt het, wanneer men zich te Stockholm geduurig, tot bewijs der overëenkomst tusschen Zweedsch en Engelsch, de zelfde drie of vier woorden hoort voorzeggen, zonder dat men, om meerdere voorbeelden vraagende, zoo ligtelijk antwoord ontvangt.
+[2] Van een Hoogduitsch - Zweedsch, en Zweedsch - Hoogduitsch Woordenboek van möller bezit ik nog maar de twee eerste deelen in 40., welke het Hoogduitsch gedeelte bevatten; ik weet niet of het overige reeds gevolgd is, of nog volgen zal. Het is reeds van 1785. Eene kleine Grammatica van abr. Sahlstedt is in 1796 in 't Hoogduitsch overgezet. Over het Lapsch en Finsch, twee van het Zweedsch geheel onderscheidene taalen, welke ook in dit Koninkrijk gesproken worden, zal het voegsaamer zijn op eene andere plaats te handelen.'''
+
+def test_footnotes_extraction(dbnl_corpus):
+    corpus = load_corpus_definition(dbnl_corpus)
+    soup = BeautifulSoup(section_with_footnote, 'lxml-xml')
+
+    content = corpus.content.extractor.apply(None, soup)
+    assert content == expected_content
+
+    notes = corpus.notes.extractor.apply(None, soup)
+    assert notes == expected_notes

backend/corpora/dbnl/utils.py

@@ -2,7 +2,9 @@
 from bs4 import BeautifulSoup
 import os
 from langcodes import standardize_tag, Language
+from typing import List
 
+from copy import copy
 from addcorpus.python_corpora.extract import Pass, Combined, CSV
 
 # === METADATA EXTRACTION ===
@@ -179,13 +181,6 @@ def append_to_tag(soup, tag, padding):
         tag.append(padding)
 
     return soup
-
-def pad_content(node):
-    pad_cells = lambda n: append_to_tag(n, 'cell', ' ')
-    pad_linebreaks = lambda n: append_to_tag(n, 'lb', '\n')
-    pad_cells(pad_linebreaks(node))
-    return [node]
-
 def standardize_language_code(code):
     if code:
         return standardize_tag(code)
@@ -205,3 +200,41 @@ def language_name(code):
         codes
     ))
     return ', '.join(names)
+
+## ======== TEXT FORMATTING =============
+
+def pad_content(node):
+    pad_cells = lambda n: append_to_tag(n, 'cell', ' ')
+    pad_linebreaks = lambda n: append_to_tag(n, 'lb', '\n')
+    pad_cells(pad_linebreaks(node))
+    return [node]
+
+
+def get_ref(node):
+    prev = node.find_all_previous('note')
+    return len(prev) + 1
+
+def insert_ref(node):
+    '''
+    Adds a reference, e.g. `[1]` at the start of a node's contents
+    '''
+    ref = get_ref(node)
+    node.insert(0, f'[{ref}] ')
+    yield node
+
+def replace_notes_with_ref(node):
+    '''
+    Replaces all `<note>` tags in the a beautiful soup node with a reference, e.g. `[1]`
+    '''
+    new_node = copy(node) # make a copy to avoid altering the original document
+    tags = zip(node.find_all('note'), new_node.find_all('note'))
+
+    for old_tag, to_replace in tags:
+        ref = get_ref(old_tag)
+        to_replace.replace_with(f'[{ref}]')
+
+    yield new_node
+
+def join_paragraphs(paragraphs: List[str]):
+    text = '\n'.join(paragraphs).strip()
+    return text if text else None

backend/corpora/dutchannualreports/images/dutchannualreports.jpg

@@ -0,0 +1,7 @@
+(c) Massimo Catarinella
+
+Source: https://commons.wikimedia.org/wiki/File:RotterdamMaasNederland.jpg
+
+Licensed under a Creative Commons Attribution 3.0 Unported licence (https://creativecommons.org/licenses/by/3.0/deed.en)
+
+Modifications made from original: cropped image

backend/corpora/dutchannualreports/images/dutchannualreports.jpg.license

@@ -0,0 +1,16 @@
+This corpus contains the public portion of the [Delpher newspapers dataset](https://www.kb.nl/en/research-find/datasets/delpher-newspapers). This is a collection of Dutch newspapers published between 1618 and 1876, maintained by the KB, the National Library of the Netherlands.
+
+### What is in Delpher Newspapers?
+
+Are you interested to know which countries featured in the news in the 18th century, and which did not? Or how newspapers reported on reforms in the Batavian Republic? Delpher newspapers is a great resource for research into historical newspapers. But this dataset is also useful for broader-based history projects. It gives a peek behind the scenes of what was going on in society.
+
+The full *Delpher newspapers* collection comprises almost 2 million newspapers dating from 1618 until 1995. This collection contains publications published at least 140 years ago, which are no longer subject to copyright.
+
+It includes major national newspapers, such as the *Algemeen Handelsblad*, as well as regional newspapers, such as the *Leydse courant*, and colonial papers, such as the *Java-bode*.
+
+Please note that our collection currently includes only the newspaper set from Delpher and does not (yet) contain the [historical magazine collection](https://www.kb.nl/en/research-find/datasets/delpher-magazines) from the period 1800-2000.
+
+### Recent newspapers (1876-1995)
+
+All users have direct access to the Delpher newspaper dataset for the period 1600–1876. A more recent dataset, which includes newspapers up to 1995, is also available for exploration in I-Analyzer. However, access requires prior permission from the rightsholder, The National Library of the Netherlands (KB). To use the full newspaper dataset, you must first complete and sign an individual agreement to comply with the KB's usage conditions. This form can be requested via
+[[email protected]](mailto:[email protected]), after which we can assist you with finalizing the application process.

backend/corpora/dutchnewspapers/description/description_public.md

@@ -38,6 +38,7 @@ class DutchNewspapersPublic(XMLCorpusDefinition):
     image = 'dutchnewspapers.jpg'
     languages = ['nl']
     category = 'periodical'
+    description_page = 'description_public.md'
     citation_page = 'citation_public.md'
 
     @property

backend/corpora/dutchnewspapers/dutchnewspapers_public.py

@@ -0,0 +1,5 @@
+(c) Roman Kraft
+
+Source: https://unsplash.com/photos/man-sitting-on-bench-reading-newspaper-_Zua2hyvTBk
+
+Licensed under the Unsplash licence (https://unsplash.com/license)

backend/corpora/dutchnewspapers/images/dutchnewspapers.jpg.license

@@ -0,0 +1,3 @@
+No copyright restrictions
+
+Image source: https://commons.wikimedia.org/wiki/File:English_humorists_of_the_eighteenth_century_-_Sir_Richard_Steele,_Joseph_Addison,_Laurence_Sterne,_Oliver_Goldsmith_(1906)_(14802513503).jpg

backend/corpora/ecco/images/ecco.jpg

@@ -1 +1,5 @@
-DIOPTRA-L contains Goodreads reviews of contemporary literary texts that have been translated from and into the following languages: Afrikaans, Dutch, English, French, German, Italian, Portuguese and Spanish. A variety of genres are represented, and the corpus is searchable by, for example, language pair, author, title and genre. For more information, or suggestions, please contact Haidee Kotze ([email protected]) and Gys-Walt van Egdom ([email protected])
+DIOPTRA-L contains Goodreads reviews of contemporary literary texts that have been translated from and into the following languages: Afrikaans, Dutch, English, French, German, Italian, Portuguese and Spanish. A variety of genres are represented, and the corpus is searchable by, for example, language pair, author, title and genre. For more information, or suggestions, please contact Haidee Kotze ([email protected]) and Gys-Walt van Egdom ([email protected])
+
+### Image attribution
+
+The DIOPTRA-L logo used as an image for this corpus was created by Gys-Walt van Egdom and Haidee Kotze and is licensed under a [Creative Commons Attribution-Share Alike 4.0 licence](https://creativecommons.org/licenses/by-sa/4.0/deed.en).

backend/corpora/ecco/images/ecco.jpg.license

@@ -0,0 +1,3 @@
+(c) Gys-Walt van Egdom (https://www.uu.nl/staff/GMWvanEgdom) and Haidee Kotze (https://www.uu.nl/staff/HKotze)
+
+Licensed under a Creative Commons Attribution-Share Alike 4.0 International licence (https://creativecommons.org/licenses/by-sa/4.0/deed.en).

backend/corpora/goodreads/description/goodreads.md

@@ -28,3 +28,7 @@ Political alignment: Centre-left; British republicanism
 ### Availability
 
 The Guardian/Observer corpus is published by [ProQuest](https://en.wikipedia.org/wiki/ProQuest) and is only available to members of Utrecht University.
+
+### Image attribution
+
+The image used for this corpus was created by Michael Brunton-Spall ([image source](https://www.flickr.com/photos/27778872@N08/5084043824)) and is shared under a [Creative Commons Attribution 2.0 licence](https://creativecommons.org/licenses/by/2.0/).

backend/corpora/goodreads/images/DioptraL.png.license

@@ -15,7 +15,7 @@
 
 from django.conf import settings
 
-from es.es_update import update_document
+from indexing.run_update_task import update_document
 from addcorpus.python_corpora import extract
 from addcorpus.python_corpora import filters
 from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition, until, after, string_contains, consolidate_start_end_years

backend/corpora/guardianobserver/description/guardianobserver.md

@@ -0,0 +1,5 @@
+(c) Michael Brunton-Spall
+
+Image source: https://www.flickr.com/photos/27778872@N08/5084043824
+
+Licensed under a Creative Commons Attribution 2.0 licence (https://creativecommons.org/licenses/by/2.0/).

backend/corpora/guardianobserver/guardianobserver.py

@@ -0,0 +1,5 @@
+(c) Tadeáš Bednarz (https://www.wikidata.org/wiki/Q99692082)
+
+Image source: https://commons.wikimedia.org/wiki/File:Jewish_cemetery_in_Cieszyn_2017.jpg
+
+Licensed under a Creative Commons Attribution-Share Alike 4.0 International licence (https://creativecommons.org/licenses/by-sa/4.0/deed.en).

backend/corpora/guardianobserver/images/guardianobserver.jpg.license

@@ -22,12 +22,11 @@ class JewishInscriptions(XMLCorpusDefinition):
     max_date = datetime(year=849, month=12, day=31)
     data_directory = settings.JEWISH_INSCRIPTIONS_DATA
     es_index = getattr(settings, 'JEWISH_INSCRIPTIONS_ES_INDEX', 'jewishinscriptions')
-    image = 'jewish_inscriptions.jpg'
+    image = 'jewishinscriptions.jpg'
     visualize = []
     languages = ['heb', 'lat']
     category = 'inscription'
 
-
     tag_toplevel = CurrentTag()
     tag_entry = Tag('TEI')
 

backend/corpora/jewishinscriptions/images/jewish_inscriptions.jpg

@@ -0,0 +1,5 @@
+(c) KannanVM (https://commons.wikimedia.org/wiki/User:KannanVM)
+
+Image source: https://commons.wikimedia.org/wiki/File:Inscriptions_at_Jewish_Synagogue_at_Kottayil_Kovilakom1.jpg
+
+Licensed under a Creative Commons Attribution-Share Alike 4.0 International licence (https://creativecommons.org/licenses/by-sa/4.0/deed.en).