Web API: file upload & proposal_pipeline support, tests, and some cleanup

regparser/web/jobs/migrations/0003_auto_20160810_0027.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-10 00:27
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0002_auto_20160728_2331'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PipelineJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('clear_cache', models.BooleanField(default=False)),
+                ('destination', models.URLField(default=b'http://localhost:8888/api', max_length=2000)),
+                ('notification_email', models.EmailField(blank=b'True', max_length=254)),
+                ('job_id', models.UUIDField(default=None, null=True)),
+                ('use_uploaded_metadata', models.UUIDField(default=None, null=True)),
+                ('use_uploaded_regulation', models.UUIDField(default=None, null=True)),
+                ('parser_errors', models.TextField(blank=True)),
+                ('regulation_url', models.URLField(blank=True, max_length=2000)),
+                ('status', models.CharField(choices=[(b'received', b'received'), (b'in_progress', b'in_progress'), (b'failed', b'failed'), (b'complete', b'complete'), (b'complete_with_errors', b'complete_with_errors')], default=b'received', max_length=32)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('cfr_title', models.IntegerField()),
+                ('cfr_part', models.IntegerField()),
+                ('only_latest', models.BooleanField(default=False)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.DeleteModel(
+            name='ParsingJob',
+        ),
+    ]

regparser/web/jobs/migrations/0004_uploadedfile.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-11 00:19
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0003_auto_20160810_0027'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UploadedFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file', models.FileField(upload_to=b'')),
+            ],
+        ),
+    ]

regparser/web/jobs/migrations/0005_auto_20160811_2132.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-11 21:32
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0004_uploadedfile'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RegulationFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('hexhash', models.CharField(default=None, max_length=32, null=True)),
+                ('filename', models.CharField(default=None, max_length=512, null=True)),
+                ('contents', models.BinaryField()),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='UploadedFile',
+        ),
+    ]

regparser/web/jobs/migrations/0006_regulationfile_file.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-11 22:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0005_auto_20160811_2132'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='regulationfile',
+            name='file',
+            field=models.FileField(null=True, upload_to=b''),
+        ),
+    ]

regparser/web/jobs/migrations/0007_regulationfile_url.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-19 04:28
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0006_regulationfile_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='regulationfile',
+            name='url',
+            field=models.URLField(blank=True, max_length=2000),
+        ),
+    ]

regparser/web/jobs/migrations/0008_proposalpipelinejob.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-19 04:42
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0007_regulationfile_url'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ProposalPipelineJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('clear_cache', models.BooleanField(default=False)),
+                ('destination', models.URLField(default=b'http://localhost:8888/api', max_length=2000)),
+                ('notification_email', models.EmailField(blank=b'True', max_length=254)),
+                ('job_id', models.UUIDField(default=None, null=True)),
+                ('use_uploaded_metadata', models.UUIDField(default=None, null=True)),
+                ('use_uploaded_regulation', models.UUIDField(default=None, null=True)),
+                ('parser_errors', models.TextField(blank=True)),
+                ('regulation_url', models.URLField(blank=True, max_length=2000)),
+                ('status', models.CharField(choices=[(b'received', b'received'), (b'in_progress', b'in_progress'), (b'failed', b'failed'), (b'complete', b'complete'), (b'complete_with_errors', b'complete_with_errors')], default=b'received', max_length=32)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('file_hexhash', models.CharField(max_length=32)),
+                ('only_latest', models.BooleanField(default=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

regparser/web/jobs/migrations/0009_auto_20160824_2347.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2016-08-24 23:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0008_proposalpipelinejob'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='pipelinejob',
+            name='destination',
+            field=models.URLField(max_length=2000),
+        ),
+        migrations.AlterField(
+            model_name='proposalpipelinejob',
+            name='destination',
+            field=models.URLField(max_length=2000),
+        ),
+    ]

regparser/web/jobs/models.py

@@ -1,16 +1,24 @@
 from django.db import models
 
+job_status_pairs = (
+    ("complete", "complete"),
+    ("complete_with_errors", "complete_with_errors"),
+    ("failed", "failed"),
+    ("in_progress", "in_progress"),
+    ("received", "received")
+)
+job_status_values = [j[0] for j in job_status_pairs]
+
 
 class ParsingJob(models.Model):
 
+    class Meta:
+        abstract = True
+
     created = models.DateTimeField(auto_now_add=True)
-    cfr_title = models.IntegerField()
-    cfr_part = models.IntegerField()
     clear_cache = models.BooleanField(default=False)
-    destination = models.URLField(default="http://fake-reg-site.gov/api",
-                                  max_length=2000)
+    destination = models.URLField(max_length=2000)
     notification_email = models.EmailField(blank="True", max_length=254)
-    only_latest = models.BooleanField(default=False)
     job_id = models.UUIDField(default=None, null=True)
     use_uploaded_metadata = models.UUIDField(default=None, null=True)
     use_uploaded_regulation = models.UUIDField(default=None, null=True)
@@ -25,3 +33,28 @@ class ParsingJob(models.Model):
         ("complete_with_errors", "complete_with_errors")
     ), default="received")
     url = models.URLField(blank=True, max_length=2000)
+
+    def save(self, *args, **kwargs):
+        super(ParsingJob, self).save(*args, **kwargs)
+
+
+class PipelineJob(ParsingJob):
+
+    cfr_title = models.IntegerField()
+    cfr_part = models.IntegerField()
+    only_latest = models.BooleanField(default=False)
+
+
+class ProposalPipelineJob(ParsingJob):
+
+    file_hexhash = models.CharField(max_length=32)
+    only_latest = models.BooleanField(default=True)
+
+
+class RegulationFile(models.Model):
+
+    contents = models.BinaryField()
+    file = models.FileField(null=True)
+    filename = models.CharField(default=None, max_length=512, null=True)
+    hexhash = models.CharField(default=None, max_length=32, null=True)
+    url = models.URLField(blank=True, max_length=2000)

regparser/web/jobs/serializers.py

@@ -1,13 +1,17 @@
-from regparser.web.jobs.models import ParsingJob
+from regparser.web.jobs.models import (
+    ParsingJob,
+    PipelineJob,
+    ProposalPipelineJob,
+    RegulationFile
+)
 from rest_framework import serializers
 
 
 class ParsingJobSerializer(serializers.ModelSerializer):
+
     class Meta:
         model = ParsingJob
         fields = (
-            "cfr_title",
-            "cfr_part",
             "clear_cache",
             "destination",  # Unsure about whether this should accept user
                             # input or be set by the system.
@@ -33,3 +37,47 @@ class Meta:
 
     def save(self, **kwargs):
         super(ParsingJobSerializer, self).save(**kwargs)
+
+
+class PipelineJobSerializer(ParsingJobSerializer):
+
+    class Meta(ParsingJobSerializer.Meta):
+        model = PipelineJob
+        fields = ParsingJobSerializer.Meta.fields + (
+                "cfr_title",
+                "cfr_part"
+            )
+
+
+class ProposalPipelineJobSerializer(ParsingJobSerializer):
+
+    class Meta(ParsingJobSerializer.Meta):
+        model = ProposalPipelineJob
+        fields = ParsingJobSerializer.Meta.fields + (
+            "file_hexhash",
+        )
+
+    # Fields we don't want user input for are listed below.
+    file_hexhash = serializers.CharField(max_length=32)
+
+
+class FileUploadSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = RegulationFile
+        fields = (
+            "contents",
+            "file",
+            "filename",
+            "hexhash",
+            "url"
+        )
+
+    contents = serializers.SerializerMethodField()
+    file = serializers.FileField()
+    filename = serializers.CharField(read_only=True)
+    hexhash = serializers.CharField(read_only=True)
+    url = serializers.URLField(read_only=True)
+
+    def get_contents(self, obj):
+        return "File contents not shown."

regparser/web/jobs/urls.py

@@ -3,8 +3,19 @@
 from rest_framework.urlpatterns import format_suffix_patterns
 
 urlpatterns = [
-    url(r'^job/$', views.JobViewList.as_view()),
-    url(r'^job/(?P<job_id>[-a-z0-9]+)/$',
+    url(r'^job(/)$', views.JobViewList.as_view()),
+    url(r'^job/pipeline(/)$', views.PipelineJobViewList.as_view()),
+    url(r'^job/pipeline(/)(?P<job_id>[-a-z0-9]+)(/)$',
+        views.PipelineJobViewInstance.as_view()),
+    url(r'^job/proposal-pipeline(/)$',
+        views.ProposalPipelineJobViewList.as_view()),
+    url(r'^job/proposal-pipeline(/)(?P<job_id>[-a-z0-9]+)(/)$',
+        views.ProposalPipelineJobViewInstance.as_view()),
+    url(r'^job/upload/(?P<hexhash>[a-z0-9]{32})(/)$',
+        views.FileUploadViewInstance.as_view()),
+    url(r'^job/upload(/)$',
+        views.FileUploadView.as_view()),
+    url(r'^job/(?P<job_id>[-a-z0-9]+)(/)$',
         views.JobViewInstance.as_view())
 ]