Implement ChRIS folders and links (#545)

* Add a new ChrisFolder model and update the file models accordingly

* Reimplement the file browser using the new underlying ChrisFolder model

* Refactor automated test to work with the new underlying ChrisFolder model

* Update chrisomatic to support userfiles change

* Add ChRIS links models, serializers and views

* Create ChRIS links when handling unextracted path parameters

* Find all file paths under a ChRIS link's pointed path

* Fix bug when multiple input directories are passed to create_zip_file

* Make output files consistent across different storage environments

* Add output_dir to the pfcon request parameters for any pfcon in network

* Fix bug with output dir creation for fslink

* Add fslink storage environment

* Fix bug in the filebrowser associated with the SERVICES top level folder and add unit tests

* Add more automated tests to the filebrowser app

---------

Co-authored-by: Jennings Zhang <[email protected]>

Author: jbernal0019

chris_backend/config/settings/local.py

@@ -84,7 +84,7 @@
 
 # Storage Settings
 STORAGE_ENV = os.getenv('STORAGE_ENV', 'swift')
-if STORAGE_ENV not in ('swift', 'filesystem'):
+if STORAGE_ENV not in ('swift', 'fslink', 'filesystem'):
     raise ImproperlyConfigured(f"Unsupported value '{STORAGE_ENV}' for STORAGE_ENV")
 
 STORAGES['default'] = {'BACKEND': 'swift.storage.SwiftStorage'}
@@ -96,7 +96,7 @@
                            'key': SWIFT_KEY,
                            'authurl': SWIFT_AUTH_URL}
 MEDIA_ROOT = None
-if STORAGE_ENV == 'filesystem':
+if STORAGE_ENV in ('fslink', 'filesystem'):
     STORAGES['default'] = {'BACKEND': 'django.core.files.storage.FileSystemStorage'}
     MEDIA_ROOT = '/var/chris'  # local filesystem storage settings
 

chris_backend/config/settings/production.py

@@ -55,7 +55,7 @@ def get_secret(setting, secret_type=env):
 # STORAGE CONFIGURATION
 # ------------------------------------------------------------------------------
 STORAGE_ENV = get_secret('STORAGE_ENV')
-if STORAGE_ENV not in ('swift', 'filesystem'):
+if STORAGE_ENV not in ('swift', 'fslink', 'filesystem'):
     raise ImproperlyConfigured(f"Unsupported value '{STORAGE_ENV}' for STORAGE_ENV")
 
 if STORAGE_ENV == 'swift':
@@ -72,7 +72,7 @@ def get_secret(setting, secret_type=env):
         SWIFT_CONTAINER_NAME=SWIFT_CONTAINER_NAME,
         SWIFT_CONNECTION_PARAMS=SWIFT_CONNECTION_PARAMS
     )
-elif STORAGE_ENV == 'filesystem':
+elif STORAGE_ENV in ('fslink', 'filesystem'):
     STORAGES['default'] = {'BACKEND': 'django.core.files.storage.FileSystemStorage'}
     MEDIA_ROOT = get_secret('MEDIA_ROOT')
     verify_storage = lambda: verify_storage_connection(DEFAULT_FILE_STORAGE=STORAGES['default']['BACKEND'],

chris_backend/core/api.py

@@ -56,9 +56,6 @@
     path('v1/comments/<int:pk>/',
         feed_views.CommentDetail.as_view(), name='comment-detail'),
 
-    path('v1/<int:pk>/files/',
-        feed_views.FeedFileList.as_view(), name='feedfile-list'),
-
     path('v1/<int:pk>/plugininstances/',
         feed_views.FeedPluginInstanceList.as_view(), name='feed-plugininstance-list'),
 
@@ -235,26 +232,6 @@
         plugininstance_views.PluginInstanceDescendantList.as_view(),
         name='plugininstance-descendant-list'),
 
-    path('v1/plugins/instances/<int:pk>/files/',
-        plugininstance_views.PluginInstanceFileList.as_view(),
-        name='plugininstancefile-list'),
-
-    path('v1/files/',
-        plugininstance_views.AllPluginInstanceFileList.as_view(),
-        name='allplugininstancefile-list'),
-
-    path('v1/files/search/',
-        plugininstance_views.AllPluginInstanceFileListQuerySearch.as_view(),
-        name='allplugininstancefile-list-query-search'),
-
-    path('v1/files/<int:pk>/',
-        plugininstance_views.PluginInstanceFileDetail.as_view(),
-        name='plugininstancefile-detail'),
-
-    re_path(r'^v1/files/(?P<pk>[0-9]+)/.*$',
-        plugininstance_views.FileResource.as_view(),
-        name='plugininstancefile-resource'),
-
     path('v1/plugins/instances/<int:pk>/parameters/',
         plugininstance_views.PluginInstanceParameterList.as_view(),
         name='plugininstance-parameter-list'),
@@ -378,20 +355,36 @@
 
 
     path('v1/filebrowser/',
-         filebrowser_views.FileBrowserPathList.as_view(),
-         name='filebrowserpath-list'),
+         filebrowser_views.FileBrowserFolderList.as_view(),
+         name='chrisfolder-list'),
 
     path('v1/filebrowser/search/',
-         filebrowser_views.FileBrowserPathListQuerySearch.as_view(),
-         name='filebrowserpath-list-query-search'),
+         filebrowser_views.FileBrowserFolderListQuerySearch.as_view(),
+         name='chrisfolder-list-query-search'),
+
+    path('v1/filebrowser/<int:pk>/',
+         filebrowser_views.FileBrowserFolderDetail.as_view(),
+         name='chrisfolder-detail'),
+
+    path('v1/filebrowser/<int:pk>/children/',
+         filebrowser_views.FileBrowserFolderChildList.as_view(),
+         name='chrisfolder-child-list'),
+
+    path('v1/filebrowser/<int:pk>/files/',
+         filebrowser_views.FileBrowserFolderFileList.as_view(),
+         name='chrisfolder-file-list'),
+
+    path('v1/filebrowser/<int:pk>/linkfiles/',
+         filebrowser_views.FileBrowserFolderLinkFileList.as_view(),
+         name='chrisfolder-linkfile-list'),
 
-    path('v1/filebrowser/<path:path>/',
-         filebrowser_views.FileBrowserPath.as_view(),
-         name='filebrowserpath'),
+    path('v1/filebrowser/linkfiles/<int:pk>/',
+         filebrowser_views.FileBrowserLinkFileDetail.as_view(),
+         name='chrislinkfile-detail'),
 
-    path('v1/filebrowser-files/<path:path>/',
-         filebrowser_views.FileBrowserPathFileList.as_view(),
-         name='filebrowserpathfile-list'),
+    re_path(r'^v1/filebrowser/linkfiles/(?P<pk>[0-9]+)/.*$',
+            filebrowser_views.FileBrowserLinkFileResource.as_view(),
+            name='chrislinkfile-resource')
 
 ])
 

chris_backend/core/migrations/0004_chrisfolder_chrislinkfile.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.2.5 on 2023-12-19 04:41
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('core', '0003_alter_chrisinstance_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ChrisFolder',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('creation_date', models.DateTimeField(auto_now_add=True)),
+                ('path', models.CharField(max_length=1024, unique=True)),
+                ('size', models.BigIntegerField(default=0)),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('parent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='core.chrisfolder')),
+            ],
+            options={
+                'ordering': ('-path',),
+            },
+        ),
+        migrations.CreateModel(
+            name='ChrisLinkFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('creation_date', models.DateTimeField(auto_now_add=True)),
+                ('path', models.CharField(max_length=1024)),
+                ('is_folder', models.BooleanField()),
+                ('fname', models.FileField(max_length=1024, unique=True, upload_to='')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('parent_folder', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chris_link_files', to='core.chrisfolder')),
+            ],
+        ),
+    ]

chris_backend/core/migrations/0005_remove_chrisfolder_size.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.5 on 2024-01-12 23:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_chrisfolder_chrislinkfile'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='chrisfolder',
+            name='size',
+        ),
+    ]

chris_backend/core/migrations/0006_remove_chrislinkfile_is_folder_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2024-01-31 04:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_remove_chrisfolder_size'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='chrislinkfile',
+            name='is_folder',
+        ),
+        migrations.AlterField(
+            model_name='chrislinkfile',
+            name='path',
+            field=models.CharField(db_index=True, max_length=1024),
+        ),
+    ]

chris_backend/core/models.py

@@ -1,7 +1,16 @@
 
 import uuid
+import io
+import os
 
 from django.db import models
+from django.conf import settings
+from django.contrib.auth.models import User
+import django_filters
+from django_filters.rest_framework import FilterSet
+
+from .storage import connect_storage
+#from django.core.files.base import ContentFile
 
 
 class ChrisInstance(models.Model):
@@ -38,3 +47,88 @@ def load(cls):
             obj = cls()
             obj.save()
         return obj
+
+
+class ChrisFolder(models.Model):
+    creation_date = models.DateTimeField(auto_now_add=True)
+    path = models.CharField(max_length=1024, unique=True)  # folder's path
+    parent = models.ForeignKey('self', on_delete=models.CASCADE, null=True,
+                               related_name='children')
+    owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
+
+    class Meta:
+        ordering = ('-path',)
+
+    def __str__(self):
+        return self.path
+
+    def save(self, *args, **kwargs):
+        """
+        Overriden to recursively create parent folders when first saving the folder
+        to the DB.
+        """
+        if self.path:
+            parent_path = os.path.dirname(self.path)
+            try:
+                parent = ChrisFolder.objects.get(path=parent_path)
+            except ChrisFolder.DoesNotExist:
+                parent = ChrisFolder(path=parent_path, owner=self.owner)
+                parent.save()  # recursive call
+            self.parent = parent
+
+        if self.path in ('', 'home') or self.path.startswith(('PIPELINES', 'SERVICES')):
+            self.owner = User.objects.get(username='chris')
+        super(ChrisFolder, self).save(*args, **kwargs)
+
+    def get_descendants(self):
+        """
+        Custom method to return all the folders that are a descendant of this
+        folder.
+        """
+        descendants = []
+        queue = [self]
+        while len(queue) > 0:
+            visited = queue.pop()
+            queue.extend(list(visited.children.all()))
+            descendants.append(visited)
+        return descendants
+
+
+class ChrisFolderFilter(FilterSet):
+    path = django_filters.CharFilter(field_name='path')
+
+    class Meta:
+        model = ChrisFolder
+        fields = ['id', 'path']
+
+
+class ChrisLinkFile(models.Model):
+    creation_date = models.DateTimeField(auto_now_add=True)
+    path = models.CharField(max_length=1024, db_index=True)  # pointed path
+    fname = models.FileField(max_length=1024, unique=True)
+    parent_folder = models.ForeignKey(ChrisFolder, on_delete=models.CASCADE,
+                                      related_name='chris_link_files')
+    owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
+
+    def __str__(self):
+        return self.fname.name
+
+    def save(self, *args, **kwargs):
+        """
+        Overriden to create and save the associated link file when the link is
+        saved.
+        """
+        path = self.path  # pointed path
+        name = kwargs.pop('name')  # must provide a name for the link
+        link_file_path = os.path.join(self.parent_folder.path, f'{name}.chrislink')
+        link_file_contents = f'{path}'
+
+        storage_manager = connect_storage(settings)
+
+        with io.StringIO(link_file_contents) as f:
+            if storage_manager.obj_exists(link_file_path):
+                storage_manager.delete_obj(link_file_path)
+            storage_manager.upload_obj(link_file_path, f.read(),
+                                       content_type='text/plain')
+        self.fname.name = link_file_path
+        super(ChrisLinkFile, self).save(*args, **kwargs)

chris_backend/core/serializers.py

@@ -1,7 +1,7 @@
 
 from rest_framework import serializers
 
-from .models import ChrisInstance
+from .models import ChrisInstance, ChrisFolder
 
 
 class ChrisInstanceSerializer(serializers.HyperlinkedModelSerializer):
@@ -10,3 +10,44 @@ class Meta:
         model = ChrisInstance
         fields = ('url', 'id', 'creation_date', 'name', 'uuid', 'job_id_prefix',
                   'description')
+
+
+class ChrisFolderSerializer(serializers.HyperlinkedModelSerializer):
+    parent = serializers.HyperlinkedRelatedField(view_name='chrisfolder-detail',
+                                                 read_only=True)
+    children = serializers.HyperlinkedIdentityField(view_name='chrisfolder-child-list')
+    user_files = serializers.HyperlinkedIdentityField(
+        view_name='chrisfolder-userfile-list')
+    pacs_files = serializers.HyperlinkedIdentityField(
+        view_name='chrisfolder-pacsfile-list')
+    service_files = serializers.HyperlinkedIdentityField(
+        view_name='chrisfolder-servicefile-list')
+    pipeline_source_files = serializers.HyperlinkedIdentityField(
+        view_name='chrisfolder-pipelinesourcefile-list')
+    link_files = serializers.HyperlinkedIdentityField(
+        view_name='chrisfolder-linkfile-list')
+    owner = serializers.HyperlinkedRelatedField(view_name='user-detail', read_only=True)
+
+    class Meta:
+        model = ChrisFolder
+        fields = ('url', 'id', 'creation_date', 'path', 'parent', 'children',
+                  'user_files', 'pacs_files', 'service_files', 'link_files',
+                  'pipeline_source_files', 'owner')
+
+    def validate_path(self, path):
+        """
+        Overriden to check whether the provided path is under home/<username>/ but not
+        under home/<username>/feeds/.
+        """
+        # remove leading and trailing slashes
+        path = path.strip(' ').strip('/')
+        user = self.context['request'].user
+        prefix = f'home/{user.username}/'
+        if path.startswith(prefix + 'feeds/'):
+            error_msg = f"Invalid file path. Creating folders with a path under the " \
+                        f"feed's directory '{prefix + 'feeds/'}' is not allowed."
+            raise serializers.ValidationError([error_msg])
+        if not path.startswith(prefix):
+            error_msg = f"Invalid file path. Path must start with '{prefix}'."
+            raise serializers.ValidationError([error_msg])
+        return path

chris_backend/core/storage/plain_fs.py

@@ -22,6 +22,8 @@ def create_container(self) -> None:
         self.__base.mkdir(exist_ok=True, parents=True)
 
     def ls(self, path_prefix: str) -> List[str]:
+        if self.obj_exists(path_prefix):
+            return [path_prefix]
         all_paths = (self.__base / path_prefix).rglob('*')
         return [str(p.relative_to(self.__base)) for p in all_paths if p.is_file()]
 

chris_backend/core/views.py

@@ -1,10 +1,14 @@
 
-from rest_framework import generics, permissions
+import logging
 
+from rest_framework import generics, permissions
 from .models import ChrisInstance
 from .serializers import ChrisInstanceSerializer
 
 
+logger = logging.getLogger(__name__)
+
+
 class ChrisInstanceDetail(generics.RetrieveAPIView):
     """
     A compute resource view.