Merge branch 'release/5.17.0'

Author: lukavdplas

.github/workflows/scheduled-build-and-push.yml

@@ -21,21 +21,20 @@ jobs:
             registry: ghcr.io
             username: ${{ github.actor }}
             password: ${{ secrets.GITHUB_TOKEN }}
-        - name: Build frontend image, using cache from Github registry
+        - name: Build frontend image
           uses: docker/build-push-action@v6
           with:
             context: frontend/.
             file: frontend/Dockerfile.base
             push: true
             tags: ghcr.io/centrefordigitalhumanities/ianalyzer-frontend:latest
-            cache-from: type=registry,ref=ghcr.io/centrefordigitalhumanities/ianalyzer-frontend:latest
-            cache-to: type=inline
-        - name: Build backend image, using cache from Github registry
+            platforms: linux/amd64,linux/arm64
+        - name: Build backend image
           uses: docker/build-push-action@v6
           with:
             context: backend/.
             file: backend/Dockerfile.base
             push: true
             tags: ghcr.io/centrefordigitalhumanities/ianalyzer-backend:latest
-            cache-from: type=registry,ref=ghcr.io/centrefordigitalhumanities/ianalyzer-backend:latest
-            cache-to: type=inline
+            platforms: linux/amd64,linux/arm64
+

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 5.16.0
-date-released: '2025-01-22'
+version: 5.17.0
+date-released: '2025-02-13'

backend/addcorpus/exceptions.py

@@ -0,0 +1,33 @@
+from addcorpus.models import Corpus
+
+
+class PythonDefinitionRequired(Exception):
+    '''
+    Exception that can be raised when attempting to use functionality only applicable for
+    Python corpora, on a corpus that does not have a Python definition.
+    '''
+
+    def __init__(self, corpus: Corpus, message: str, *args):
+        self.corpus = corpus
+        self.message = message
+        super().__init__(*args)
+
+
+    def __str__(self):
+        return f'{self.message} (corpus: {self.corpus})'
+
+class NoPythonDefinitionAllowed(Exception):
+    '''
+    Exception that can be raised when attempting to use functionality only applicable for
+    database-only corpora, on a corpus with a Python definition.
+    '''
+
+    def __init__(self, corpus: Corpus, message: str, *args):
+        self.corpus = corpus
+        self.message = message
+        super().__init__(*args)
+
+
+    def __str__(self):
+        return f'{self.message} (corpus: {self.corpus})'
+

backend/addcorpus/json_corpora/constants.py

@@ -1,8 +1 @@
-from datetime import date
-
-
 DEFAULT_CSV_DELIMITER = ','
-DATE_FORMAT = '%Y-%m-%d'
-
-DEFAULT_MIN_DATE = date(1800, 1, 1)
-DEFAULT_MAX_DATE = date.today()

backend/addcorpus/json_corpora/export_json.py

@@ -1,7 +1,6 @@
 from typing import Dict
-from datetime import date
 from addcorpus.models import Corpus, CorpusConfiguration, Field
-from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER, DATE_FORMAT
+from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER
 from addcorpus.es_mappings import primary_mapping_type
 
 def export_json_corpus(corpus: Corpus) -> Dict:
@@ -24,13 +23,11 @@ def export_corpus_meta(configuration: CorpusConfiguration) -> Dict:
         'description': configuration.description,
         'languages': configuration.languages,
         'date_range': {
-            'min': export_date(configuration.min_date),
-            'max': export_date(configuration.max_date),
+            'min': configuration.min_year,
+            'max': configuration.max_year,
         }
     }
 
-def export_date(date: date):
-    return date.strftime(DATE_FORMAT)
 
 def export_corpus_source_data(configuration: CorpusConfiguration) -> Dict:
     data = {

backend/addcorpus/json_corpora/import_json.py

@@ -1,13 +1,11 @@
-from typing import List, Dict, Iterable, Optional
-from datetime import date, datetime
-
+from typing import List, Dict, Iterable
 
 from addcorpus.models import Field
 from addcorpus.json_corpora.utils import get_path
 from addcorpus import es_mappings
 from addcorpus.constants import VisualizationType
 from django.conf import settings
-from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER, DATE_FORMAT, DEFAULT_MAX_DATE, DEFAULT_MIN_DATE
+from addcorpus.json_corpora.constants import DEFAULT_CSV_DELIMITER
 
 def import_json_corpus(data: Dict) -> Dict:
     name = get_path(data, 'name')
@@ -31,10 +29,8 @@ def _parse_configuration(data: Dict) -> Dict:
         'category': get_path(data, 'meta', 'category'),
         'es_index': create_index_name(get_path(data, 'name')),
         'languages': get_path(data, 'meta', 'languages'),
-        'min_date': _parse_date(
-            get_path(data, 'meta', 'date_range', 'min'), DEFAULT_MIN_DATE),
-        'max_date': _parse_date(
-            get_path(data, 'meta', 'date_range', 'max'), DEFAULT_MAX_DATE),
+        'min_year': get_path(data, 'meta', 'date_range', 'min'),
+        'max_year': get_path(data, 'meta', 'date_range', 'max'),
         'default_sort': get_path(
             data, 'options', 'default_sort') or {},
         'language_field': get_path(
@@ -47,12 +43,6 @@ def _parse_configuration(data: Dict) -> Dict:
     }
 
 
-def _parse_date(date: Optional[str], fallback: Optional[date]):
-    if not date:
-        return fallback
-    return datetime.strptime(date, DATE_FORMAT).date()
-
-
 def _import_fields(data: Dict) -> List[Dict]:
     fields_data = get_path(data, 'fields')
 

backend/addcorpus/json_corpora/tests/test_import.py

@@ -1,8 +1,8 @@
-from datetime import date
 from addcorpus.json_corpora.import_json import _parse_field
 from addcorpus.models import Field, Corpus
 from addcorpus.serializers import CorpusJSONDefinitionSerializer
 from addcorpus.models import Corpus, CorpusConfiguration
+from addcorpus.json_corpora.export_json import export_json_corpus
 
 def test_json_corpus_import(db, json_mock_corpus, json_corpus_definition):
     json_mock_corpus.delete()
@@ -24,8 +24,8 @@ def test_json_corpus_import(db, json_mock_corpus, json_corpus_definition):
     assert config.description == 'Example corpus'
     assert config.languages == ['en']
     assert config.category == 'book'
-    assert config.min_date == date(1500, 1, 1)
-    assert config.max_date == date(1700, 12, 31)
+    assert config.min_year == 1500
+    assert config.max_year == 1700
     assert config.source_data_delimiter == ','
     assert config.es_index == 'test-example'
 
@@ -76,6 +76,20 @@ def test_serializer_update(db, json_corpus_definition, json_mock_corpus: Corpus)
     serializer.update(json_mock_corpus, serializer.validated_data)
     assert Field.objects.filter(corpus_configuration__corpus=json_mock_corpus).count() == 1
 
+def test_serializer_update_field_order(db, json_corpus_definition, json_mock_corpus: Corpus):
+    # send corpus with reverse field order
+    data = {
+        'definition': json_corpus_definition,
+        'active': True,
+    }
+    data['definition']['fields'] = list(reversed(data['definition']['fields']))
+    serializer = CorpusJSONDefinitionSerializer(data=data)
+    assert serializer.is_valid()
+    serializer.update(json_mock_corpus, serializer.validated_data)
+
+    json_mock_corpus.refresh_from_db()
+    assert export_json_corpus(json_mock_corpus) == data['definition']
+
 
 def test_parse_content_field(content_field_json):
     data = _parse_field(content_field_json)
@@ -195,4 +209,3 @@ def test_parse_geo_field(geo_field_json):
     assert field.hidden == False
     assert field.sortable == False
     assert field.searchable == False
-

backend/addcorpus/migrations/0028_corpusconfiguration_minyear_maxyear.py

@@ -0,0 +1,65 @@
+import addcorpus.models
+from django.db import migrations, models
+from django.apps import AppConfig
+from datetime import date
+
+
+def fill_min_max_years_from_date(apps: AppConfig, schema_editor):
+    CorpusConfiguration = apps.get_model('addcorpus', 'CorpusConfiguration')
+    for config in CorpusConfiguration.objects.all():
+        config.min_year = config.min_date.year
+        config.max_year = config.max_date.year
+        config.save()
+
+
+def fill_min_max_date_from_year(apps: AppConfig, schema_editor):
+    CorpusConfiguration = apps.get_model('addcorpus', 'CorpusConfiguration')
+    for config in CorpusConfiguration.objects.all():
+        config.min_date = date(year=config.min_year, month=1, day=1)
+        config.max_date = date(year=config.max_year, month=12, day=31)
+        config.save()
+
+
+class Migration(migrations.Migration):
+
+    replaces = [('addcorpus', '0028_corpusconfiguration_max_year_and_more'), ('addcorpus', '0029_fill_minyear_maxyear'), ('addcorpus', '0030_remove_corpusconfiguration_max_date_and_more'), ('addcorpus', '0031_alter_corpusconfiguration_max_year_and_more')]
+
+    dependencies = [
+        ('addcorpus', '0027_alter_corpusconfiguration_category_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='corpusconfiguration',
+            name='max_year',
+            field=models.IntegerField(default=addcorpus.models.default_max_year, help_text='latest year for the data in the corpus'),
+        ),
+        migrations.AddField(
+            model_name='corpusconfiguration',
+            name='min_year',
+            field=models.IntegerField(default=1800, help_text='earliest year for the data in the corpus'),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='min_date',
+            field=models.DateField(null=True, help_text='earliest date for the data in the corpus'),
+        ),
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='max_date',
+            field=models.DateField(null=True, help_text='latest date for the data in the corpus'),
+        ),
+        migrations.RunPython(
+            code=fill_min_max_years_from_date,
+            reverse_code=fill_min_max_date_from_year,
+        ),
+        migrations.RemoveField(
+            model_name='corpusconfiguration',
+            name='max_date',
+        ),
+        migrations.RemoveField(
+            model_name='corpusconfiguration',
+            name='min_date',
+        ),
+    ]

backend/addcorpus/migrations/0029_alter_corpusconfiguration_min_year.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.17 on 2025-02-13 16:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('addcorpus', '0028_corpusconfiguration_minyear_maxyear'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='corpusconfiguration',
+            name='min_year',
+            field=models.IntegerField(default=1800, help_text='earliest year for the data in the corpus'),
+        ),
+    ]

backend/addcorpus/models.py

@@ -1,5 +1,6 @@
 import os
 import warnings
+from datetime import datetime
 
 from django.contrib import admin
 from django.contrib.auth.models import Group
@@ -36,6 +37,11 @@
 MAX_LENGTH_NAME = 126
 MAX_LENGTH_DESCRIPTION = 254
 MAX_LENGTH_TITLE = 256
+DEFAULT_MIN_YEAR = 1800
+
+def default_max_year() -> int:
+    return datetime.now().year
+
 
 class Corpus(models.Model):
     name = models.SlugField(
@@ -217,11 +223,13 @@ class CorpusConfiguration(models.Model):
         help_text='languages used in the content of the corpus (from most to least frequent)',
         blank=True,
     )
-    min_date = models.DateField(
-        help_text='earliest date for the data in the corpus',
+    min_year = models.IntegerField(
+        help_text='earliest year for the data in the corpus',
+        default=DEFAULT_MIN_YEAR,
     )
-    max_date = models.DateField(
-        help_text='latest date for the data in the corpus',
+    max_year = models.IntegerField(
+        help_text='latest year for the data in the corpus',
+        default=default_max_year,
     )
     scan_image_type = models.CharField(
         max_length=64,

backend/addcorpus/python_corpora/corpus.py

@@ -2,9 +2,9 @@
 Module contains the base classes from which corpora can derive;
 '''
 
-from typing import Optional, List, Dict
+from typing import Optional, List, Dict, Union
 from ianalyzer_readers import extract
-from datetime import datetime
+from datetime import datetime, date
 from os.path import isdir
 import os
 
@@ -49,16 +49,20 @@ def description(self):
         raise NotImplementedError('CorpusDefinition missing description')
 
     @property
-    def min_date(self):
+    def min_date(self) -> Union[datetime, date, int]:
         '''
         Minimum timestamp for data files.
+
+        Can be a datetime, date, or integer (representing the year).
         '''
         raise NotImplementedError('CorpusDefinition missing min_date')
 
     @property
-    def max_date(self):
+    def max_date(self) -> Union[datetime, date, int]:
         '''
         Maximum timestamp for data files.
+
+        Can be a datetime, date, or integer (representing the year).
         '''
         raise NotImplementedError('CorpusDefinition missing max_date')
 
@@ -512,16 +516,26 @@ def f(metadata):
     return f
 
 
-def consolidate_start_end_years(start, end, min_date, max_date):
+def consolidate_start_end_years(
+        start: Union[datetime, date, int],
+        end: Union[datetime, date, int],
+        min_date: datetime,
+        max_date: datetime
+):
     ''' given a start and end date provided by the user, make sure
     - that start is not before end
     - that start is not before min_date (corpus variable)
     - that end is not after max_date (corpus variable)
     '''
     if isinstance(start, int):
         start = datetime(year=start, month=1, day=1)
+    elif isinstance(start, date):
+        start = datetime(year=start.year, month=start.month, day=start.day)
     if isinstance(end, int):
         end = datetime(year=end, month=12, day=31)
+    elif isinstance(end, date):
+        end = datetime(year=end.year, month=end.month, day=end.day)
+
     if start > end:
         tmp = start
         start = end

backend/addcorpus/python_corpora/filters.py

@@ -3,7 +3,7 @@
 passed through to ElasticSearch.
 '''
 
-from datetime import datetime
+from datetime import datetime, date
 from addcorpus.constants import MappingType
 
 class Filter(object):
@@ -24,7 +24,7 @@ def serialize(self):
         search_dict = {'name': name}
         for key, value in self.__dict__.items():
             if key == 'search_filter' or key != 'field':
-                if type(value) == datetime:
+                if isinstance(value, datetime) or isinstance(value, date):
                     search_dict[key] = value.isoformat()
                 else:
                     search_dict[key] = value