Merge branch 'release/5.16.0'

Author: BeritJanssen

.github/workflows/scheduled-build-and-push.yml

@@ -39,12 +39,3 @@ jobs:
             tags: ghcr.io/centrefordigitalhumanities/ianalyzer-backend:latest
             cache-from: type=registry,ref=ghcr.io/centrefordigitalhumanities/ianalyzer-backend:latest
             cache-to: type=inline
-        - name: Build Elasticsearch image, using cache from Github registry
-          uses: docker/build-push-action@v6
-          with:
-            context: .
-            file: DockerfileElastic
-            push: true
-            tags: ghcr.io/centrefordigitalhumanities/ianalyzer-elasticsearch:latest
-            cache-from: type=registry,ref=ghcr.io/centrefordigitalhumanities/ianalyzer-elasticsearch:latest
-            cache-to: type=inline

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 5.15.0
-date-released: '2024-12-09'
+version: 5.16.0
+date-released: '2025-01-22'

DockerfileElastic

@@ -1,5 +1,5 @@
 from django.contrib import admin, messages
-from .models import Corpus, CorpusConfiguration, Field, CorpusDocumentationPage
+from .models import Corpus, CorpusConfiguration, CorpusDataFile, Field, CorpusDocumentationPage
 
 def show_warning_message(request):
     '''
@@ -14,6 +14,12 @@ def show_warning_message(request):
     )
 
 
+class InlineDatafileAdmin(admin.StackedInline):
+    model = CorpusDataFile
+    fields = ['file', 'is_sample']
+    show_change_link = True,
+    extra = 0
+
 class CorpusAdmin(admin.ModelAdmin):
     readonly_fields = [
         'configuration', 'ready_to_index', 'ready_to_publish', 'date_created',
@@ -24,13 +30,15 @@ class CorpusAdmin(admin.ModelAdmin):
     ]
     list_display = ['name', 'active']
     list_filter = ['groups', 'active']
+    inlines = [InlineDatafileAdmin]
 
 class InlineFieldAdmin(admin.StackedInline):
     model = Field
     fields = ['display_name', 'description']
     show_change_link = True
     extra = 0
 
+
 class CorpusConfigurationAdmin(admin.ModelAdmin):
     readonly_fields = ['corpus']
 

backend/addcorpus/admin.py

@@ -26,7 +26,6 @@ class MappingType(Enum):
     FLOAT = 'float'
     BOOLEAN = 'boolean'
     GEO_POINT = 'geo_point'
-    ANNOTATED_TEXT = 'annotated_text'
 
 
 class VisualizationType(Enum):

backend/addcorpus/constants.py

@@ -92,13 +92,12 @@ def float_mapping():
         'type': 'float'
     }
 
-
 def bool_mapping():
     return {'type': 'boolean'}
 
 def geo_mapping():
     return {'type': 'geo_point'}
 
 
-def annotated_text_mapping():
-    return {'type': 'annotated_text'}
+def non_indexed_text_mapping():
+    return {'type': 'text', 'index': False}

backend/addcorpus/es_mappings.py

@@ -1,2 +1,8 @@
+from datetime import date
+
+
 DEFAULT_CSV_DELIMITER = ','
 DATE_FORMAT = '%Y-%m-%d'
+
+DEFAULT_MIN_DATE = date(1800, 1, 1)
+DEFAULT_MAX_DATE = date.today()

backend/addcorpus/json_corpora/constants.py

@@ -1,14 +1,13 @@
 from typing import List, Dict, Iterable, Optional
-from datetime import datetime
+from datetime import date, datetime
 
 
-from addcorpus.models import Corpus, CorpusConfiguration, Field
+from addcorpus.models import Field
 from addcorpus.json_corpora.utils import get_path
 from addcorpus import es_mappings
 from addcorpus.constants import VisualizationType
-from addcorpus.validation.publishing import _any_date_fields
 from django.conf import settings
-from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER, DATE_FORMAT
+from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER, DATE_FORMAT, DEFAULT_MAX_DATE, DEFAULT_MIN_DATE
 
 def import_json_corpus(data: Dict) -> Dict:
     name = get_path(data, 'name')
@@ -33,9 +32,9 @@ def _parse_configuration(data: Dict) -> Dict:
         'es_index': create_index_name(get_path(data, 'name')),
         'languages': get_path(data, 'meta', 'languages'),
         'min_date': _parse_date(
-            get_path(data, 'meta', 'date_range', 'min')),
+            get_path(data, 'meta', 'date_range', 'min'), DEFAULT_MIN_DATE),
         'max_date': _parse_date(
-            get_path(data, 'meta', 'date_range', 'max')),
+            get_path(data, 'meta', 'date_range', 'max'), DEFAULT_MAX_DATE),
         'default_sort': get_path(
             data, 'options', 'default_sort') or {},
         'language_field': get_path(
@@ -48,7 +47,9 @@ def _parse_configuration(data: Dict) -> Dict:
     }
 
 
-def _parse_date(date: str):
+def _parse_date(date: Optional[str], fallback: Optional[date]):
+    if not date:
+        return fallback
     return datetime.strptime(date, DATE_FORMAT).date()
 
 

backend/addcorpus/json_corpora/import_json.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.2.17 on 2024-12-10 13:25
+
+import addcorpus.models
+import addcorpus.validation.creation
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('addcorpus', '0026_corpus_date_created'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='category',
+            field=models.CharField(blank=True, choices=[('parliament', 'Parliamentary debates'), ('periodical', 'Newspapers and other periodicals'), ('finance', 'Financial reports'), ('ruling', 'Court rulings'), ('review', 'Online reviews'), ('inscription', 'Funerary inscriptions'), ('oration', 'Orations'), ('book', 'Books'), ('informative', 'Informative')], help_text='category/medium of documents in this dataset', max_length=64, null=True),
+        ),
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='description',
+            field=models.CharField(blank=True, help_text='short description of the corpus', max_length=254, null=True),
+        ),
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='languages',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=8, validators=[addcorpus.validation.creation.validate_language_code]), blank=True, help_text='languages used in the content of the corpus (from most to least frequent)', size=None),
+        ),
+        migrations.CreateModel(
+            name='CorpusDataFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file', models.FileField(help_text='file containing corpus data', upload_to=addcorpus.models.CorpusDataFile.upload_path)),
+                ('is_sample', models.BooleanField(default=False, help_text='This file is used in creating the corpus definition, it may additonaly reflect (part of) the actual data.')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('corpus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='addcorpus.corpus')),
+            ],
+        ),
+    ]

backend/addcorpus/migrations/0027_alter_corpusconfiguration_category_and_more.py

@@ -1,3 +1,4 @@
+import os
 import warnings
 
 from django.contrib import admin
@@ -178,10 +179,13 @@ class CorpusConfiguration(models.Model):
         max_length=64,
         choices=CATEGORIES,
         help_text='category/medium of documents in this dataset',
+        blank=True,
+        null=True
     )
     description = models.CharField(
         max_length=MAX_LENGTH_DESCRIPTION,
         blank=True,
+        null=True,
         help_text='short description of the corpus',
     )
     document_context = models.JSONField(
@@ -211,6 +215,7 @@ class CorpusConfiguration(models.Model):
             blank=True,
         ),
         help_text='languages used in the content of the corpus (from most to least frequent)',
+        blank=True,
     )
     min_date = models.DateField(
         help_text='earliest date for the data in the corpus',
@@ -514,3 +519,18 @@ class Meta:
                 name='unique_documentation_type_for_corpus'
             )
         ]
+
+
+class CorpusDataFile(models.Model):
+    def upload_path(self, filename):
+        return os.path.join('corpus_datafiles', f'{self.corpus.pk}', filename)
+
+    corpus = models.ForeignKey(to=Corpus, on_delete=models.CASCADE)
+    file = models.FileField(upload_to=upload_path,
+                            help_text='file containing corpus data')
+    is_sample = models.BooleanField(
+        default=False, help_text='This file is used in creating the corpus definition, it may additonaly reflect (part of) the actual data.')
+    created = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f'{self.file.name}'

backend/addcorpus/models.py

@@ -1,9 +1,10 @@
 from rest_framework import permissions
 from rest_framework.exceptions import NotFound
 from rest_framework.request import Request
-from addcorpus.models import Corpus
+from addcorpus.models import Corpus, CorpusConfiguration
 
-def corpus_name_from_request(request):
+
+def corpus_name_from_request(request: Request):
     '''
     Extract the corpus name from a request
     '''
@@ -24,6 +25,11 @@ def corpus_name_from_request(request):
     return corpus
 
 
+def corpus_config_from_request(request: Request) -> CorpusConfiguration:
+    corpus_name = corpus_name_from_request(request)
+    return CorpusConfiguration.objects.get(corpus__name=corpus_name)
+
+
 class CanSearchCorpus(permissions.BasePermission):
     message = 'You do not have permission to access this corpus'
 

backend/addcorpus/permissions.py

@@ -1,12 +1,14 @@
-from rest_framework import serializers
 from typing import Dict
 
-from addcorpus.models import Corpus, CorpusConfiguration, Field, CorpusDocumentationPage
 from addcorpus.constants import CATEGORIES
-from langcodes import Language, standardize_tag
 from addcorpus.documentation import render_documentation_context
 from addcorpus.json_corpora.export_json import export_json_corpus
 from addcorpus.json_corpora.import_json import import_json_corpus
+from addcorpus.models import (Corpus, CorpusConfiguration, CorpusDataFile,
+                              CorpusDocumentationPage, Field)
+from django.core.files import File
+from langcodes import Language, standardize_tag
+from rest_framework import serializers
 
 
 class NonEmptyJSONField(serializers.JSONField):
@@ -197,9 +199,12 @@ def update(self, instance: Corpus, validated_data: Dict):
         configuration.save()
 
         for field_data in fields_data:
-            field, _ = Field.objects.get_or_create(
-                corpus_configuration=configuration, name=field_data['name']
-            )
+            try:
+                field = Field.objects.get(
+                    corpus_configuration=configuration, name=field_data['name'])
+            except Field.DoesNotExist:
+                field = Field(corpus_configuration=configuration,
+                              name=field_data['name'])
             for attr in field_data:
                 setattr(field, attr, field_data[attr])
             field.save()
@@ -211,3 +216,19 @@ def update(self, instance: Corpus, validated_data: Dict):
             corpus.save()
 
         return corpus
+
+
+class DataFileField(serializers.FileField):
+    def to_representation(self, value: File) -> Dict:
+        return value.name
+
+    def to_internal_value(self, data):
+        return super().to_internal_value(data)
+
+
+class CorpusDataFileSerializer(serializers.ModelSerializer):
+    file = DataFileField()
+
+    class Meta:
+        model = CorpusDataFile
+        fields = ('id', 'corpus', 'file', 'created', 'is_sample')

backend/addcorpus/serializers.py

@@ -0,0 +1,11 @@
+character,line,date-column,FLOAT COLUMN,int_column,bool column
+"HAMLET","Whither wilt thou lead me? Speak, I'll go no further.","1256-10-08","7.03","13","True"
+"GHOST","Mark me.","1435-10-07","4.74","89","True"
+"HAMLET","I will.","1463-07-16","5.55","29","True"
+"GHOST","My hour is almost come,","1634-08-09","3.04","100","True"
+"GHOST","When I to sulph'rous and tormenting flames","1982-09-01","6.73","34","True"
+"GHOST","Must render up myself.","1756-11-22","-0.58","-12","False"
+"HAMLET","Alas, poor ghost!","1200-09-05","9.38","6","False"
+"GHOST","Pity me not, but lend thy serious hearing","1633-11-18","8.84","83","False"
+"GHOST","To what I shall unfold.","1445-11-09","3.6","97","False"
+"HAMLET","Speak, I am bound to hear.","1984-08-12","-1.89","-4","False"

backend/addcorpus/tests/files/example.csv

@@ -0,0 +1,28 @@
+import os
+
+from rest_framework.status import HTTP_200_OK, HTTP_201_CREATED
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+def test_csv_upload(admin_client, json_mock_corpus):
+    fp = os.path.join(here, 'files', 'example.csv')
+
+    # Test file upload
+    with open(fp, 'rb') as f:
+        data = {'corpus': json_mock_corpus.pk, 'is_sample': True, 'file': f}
+        res = admin_client.post('/api/corpus/datafiles/', data)
+    assert res.status_code == HTTP_201_CREATED
+    file_pk = res.data.get('id')
+
+    # Test file info
+    info_res = admin_client.get(f'/api/corpus/datafiles/{file_pk}/info/')
+    assert info_res.status_code == HTTP_200_OK
+    assert info_res.data == {
+        'character': 'text',
+        'line': 'text',
+        'date-column': 'date',
+        'FLOAT COLUMN': 'float',
+        'int_column': 'integer',
+        'bool column': 'boolean'
+    }

backend/addcorpus/tests/test_datafiles.py

@@ -0,0 +1,19 @@
+from addcorpus.utils import is_date, is_date_col
+import pandas as pd
+
+
+def test_is_date():
+    assert is_date('2024-01-01')
+    assert not is_date(None)
+    assert not is_date(5)
+    assert not is_date('01-01-2024')
+
+
+def test_is_date_col():
+    clean_date_series = pd.Series(['1800-01-01', '2024-01-01'])
+    dirty_date_series = pd.concat([clean_date_series, pd.Series([None, ''])])
+    empty_series = pd.Series([None, None])
+
+    assert is_date_col(clean_date_series)
+    assert is_date_col(dirty_date_series)
+    assert not is_date_col(empty_series)

backend/addcorpus/tests/test_utils.py

@@ -1,7 +1,7 @@
 import pytest
 from addcorpus.models import Field
 from addcorpus.es_mappings import (
-    annotated_text_mapping,
+    non_indexed_text_mapping,
     date_mapping,
     int_mapping,
     text_mapping,
@@ -29,7 +29,7 @@ def test_validate_ner_slug():
         validate_ner_slug({}, "some:ner_inslug")
     with pytest.raises(ValidationError):
         validate_ner_slug(keyword_mapping(), "slug:ner")
-    validate_ner_slug(annotated_text_mapping(), "slug:ner")
+    validate_ner_slug(non_indexed_text_mapping(), "slug:ner")
     with pytest.raises(ValidationError):
         validate_ner_slug(date_mapping(), "slug:ner-kw")
     validate_ner_slug(keyword_mapping(), "slug:ner-kw")