Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit e103b21

Browse files
cmc333333cmc333333
authored
Merge pull request #301 from tadhg-ohiggins/tadhg-api
Web API: file upload & proposal_pipeline support, tests, and some cleanup
2 parents 73fe5ed + 58f8246 commit e103b21

16 files changed

+990
-59
lines changed

‎regparser/web/jobs/migrations/0003_auto_20160810_0027.py

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-10 00:27
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0002_auto_20160728_2331'),
12+
]
13+
14+
operations = [
15+
migrations.CreateModel(
16+
name='PipelineJob',
17+
fields=[
18+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
19+
('created', models.DateTimeField(auto_now_add=True)),
20+
('clear_cache', models.BooleanField(default=False)),
21+
('destination', models.URLField(default=b'http://localhost:8888/api', max_length=2000)),
22+
('notification_email', models.EmailField(blank=b'True', max_length=254)),
23+
('job_id', models.UUIDField(default=None, null=True)),
24+
('use_uploaded_metadata', models.UUIDField(default=None, null=True)),
25+
('use_uploaded_regulation', models.UUIDField(default=None, null=True)),
26+
('parser_errors', models.TextField(blank=True)),
27+
('regulation_url', models.URLField(blank=True, max_length=2000)),
28+
('status', models.CharField(choices=[(b'received', b'received'), (b'in_progress', b'in_progress'), (b'failed', b'failed'), (b'complete', b'complete'), (b'complete_with_errors', b'complete_with_errors')], default=b'received', max_length=32)),
29+
('url', models.URLField(blank=True, max_length=2000)),
30+
('cfr_title', models.IntegerField()),
31+
('cfr_part', models.IntegerField()),
32+
('only_latest', models.BooleanField(default=False)),
33+
],
34+
options={
35+
'abstract': False,
36+
},
37+
),
38+
migrations.DeleteModel(
39+
name='ParsingJob',
40+
),
41+
]

‎regparser/web/jobs/migrations/0004_uploadedfile.py

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-11 00:19
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0003_auto_20160810_0027'),
12+
]
13+
14+
operations = [
15+
migrations.CreateModel(
16+
name='UploadedFile',
17+
fields=[
18+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
19+
('file', models.FileField(upload_to=b'')),
20+
],
21+
),
22+
]

‎regparser/web/jobs/migrations/0005_auto_20160811_2132.py

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-11 21:32
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0004_uploadedfile'),
12+
]
13+
14+
operations = [
15+
migrations.CreateModel(
16+
name='RegulationFile',
17+
fields=[
18+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
19+
('hexhash', models.CharField(default=None, max_length=32, null=True)),
20+
('filename', models.CharField(default=None, max_length=512, null=True)),
21+
('contents', models.BinaryField()),
22+
],
23+
),
24+
migrations.DeleteModel(
25+
name='UploadedFile',
26+
),
27+
]

‎regparser/web/jobs/migrations/0006_regulationfile_file.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-11 22:30
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0005_auto_20160811_2132'),
12+
]
13+
14+
operations = [
15+
migrations.AddField(
16+
model_name='regulationfile',
17+
name='file',
18+
field=models.FileField(null=True, upload_to=b''),
19+
),
20+
]

‎regparser/web/jobs/migrations/0007_regulationfile_url.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-19 04:28
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0006_regulationfile_file'),
12+
]
13+
14+
operations = [
15+
migrations.AddField(
16+
model_name='regulationfile',
17+
name='url',
18+
field=models.URLField(blank=True, max_length=2000),
19+
),
20+
]

‎regparser/web/jobs/migrations/0008_proposalpipelinejob.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-19 04:42
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0007_regulationfile_url'),
12+
]
13+
14+
operations = [
15+
migrations.CreateModel(
16+
name='ProposalPipelineJob',
17+
fields=[
18+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
19+
('created', models.DateTimeField(auto_now_add=True)),
20+
('clear_cache', models.BooleanField(default=False)),
21+
('destination', models.URLField(default=b'http://localhost:8888/api', max_length=2000)),
22+
('notification_email', models.EmailField(blank=b'True', max_length=254)),
23+
('job_id', models.UUIDField(default=None, null=True)),
24+
('use_uploaded_metadata', models.UUIDField(default=None, null=True)),
25+
('use_uploaded_regulation', models.UUIDField(default=None, null=True)),
26+
('parser_errors', models.TextField(blank=True)),
27+
('regulation_url', models.URLField(blank=True, max_length=2000)),
28+
('status', models.CharField(choices=[(b'received', b'received'), (b'in_progress', b'in_progress'), (b'failed', b'failed'), (b'complete', b'complete'), (b'complete_with_errors', b'complete_with_errors')], default=b'received', max_length=32)),
29+
('url', models.URLField(blank=True, max_length=2000)),
30+
('file_hexhash', models.CharField(max_length=32)),
31+
('only_latest', models.BooleanField(default=True)),
32+
],
33+
options={
34+
'abstract': False,
35+
},
36+
),
37+
]

‎regparser/web/jobs/migrations/0009_auto_20160824_2347.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.9.7 on 2016-08-24 23:47
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('jobs', '0008_proposalpipelinejob'),
12+
]
13+
14+
operations = [
15+
migrations.AlterField(
16+
model_name='pipelinejob',
17+
name='destination',
18+
field=models.URLField(max_length=2000),
19+
),
20+
migrations.AlterField(
21+
model_name='proposalpipelinejob',
22+
name='destination',
23+
field=models.URLField(max_length=2000),
24+
),
25+
]

‎regparser/web/jobs/models.py

+37-12
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,52 @@
11
from django.db import models
22

3+
job_status_pairs = (
4+
("complete", "complete"),
5+
("complete_with_errors", "complete_with_errors"),
6+
("failed", "failed"),
7+
("in_progress", "in_progress"),
8+
("received", "received")
9+
)
10+
job_status_values = tuple(j[0] for j in job_status_pairs)
11+
312

413
class ParsingJob(models.Model):
514

15+
class Meta:
16+
abstract = True
17+
618
created = models.DateTimeField(auto_now_add=True)
7-
cfr_title = models.IntegerField()
8-
cfr_part = models.IntegerField()
919
clear_cache = models.BooleanField(default=False)
10-
destination = models.URLField(default="http://fake-reg-site.gov/api",
11-
max_length=2000)
20+
destination = models.URLField(max_length=2000)
1221
notification_email = models.EmailField(blank="True", max_length=254)
13-
only_latest = models.BooleanField(default=False)
1422
job_id = models.UUIDField(default=None, null=True)
1523
use_uploaded_metadata = models.UUIDField(default=None, null=True)
1624
use_uploaded_regulation = models.UUIDField(default=None, null=True)
1725

1826
parser_errors = models.TextField(blank=True)
1927
regulation_url = models.URLField(blank=True, max_length=2000)
20-
status = models.CharField(max_length=32, choices=(
21-
("received", "received"),
22-
("in_progress", "in_progress"),
23-
("failed", "failed"),
24-
("complete", "complete"),
25-
("complete_with_errors", "complete_with_errors")
26-
), default="received")
28+
status = models.CharField(max_length=32, choices=job_status_pairs,
29+
default="received")
30+
url = models.URLField(blank=True, max_length=2000)
31+
32+
33+
class PipelineJob(ParsingJob):
34+
35+
cfr_title = models.IntegerField()
36+
cfr_part = models.IntegerField()
37+
only_latest = models.BooleanField(default=False)
38+
39+
40+
class ProposalPipelineJob(ParsingJob):
41+
42+
file_hexhash = models.CharField(max_length=32)
43+
only_latest = models.BooleanField(default=True)
44+
45+
46+
class RegulationFile(models.Model):
47+
48+
contents = models.BinaryField()
49+
file = models.FileField(null=True)
50+
filename = models.CharField(default=None, max_length=512, null=True)
51+
hexhash = models.CharField(default=None, max_length=32, null=True)
2752
url = models.URLField(blank=True, max_length=2000)

‎regparser/web/jobs/serializers.py

+51-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
1-
from regparser.web.jobs.models import ParsingJob
1+
from regparser.web.jobs.models import (
2+
ParsingJob,
3+
PipelineJob,
4+
ProposalPipelineJob,
5+
RegulationFile
6+
)
27
from rest_framework import serializers
38

49

510
class ParsingJobSerializer(serializers.ModelSerializer):
11+
612
class Meta:
713
model = ParsingJob
814
fields = (
9-
"cfr_title",
10-
"cfr_part",
1115
"clear_cache",
1216
"destination", # Unsure about whether this should accept user
1317
# input or be set by the system.
@@ -33,3 +37,47 @@ class Meta:
3337

3438
def save(self, **kwargs):
3539
super(ParsingJobSerializer, self).save(**kwargs)
40+
41+
42+
class PipelineJobSerializer(ParsingJobSerializer):
43+
44+
class Meta(ParsingJobSerializer.Meta):
45+
model = PipelineJob
46+
fields = ParsingJobSerializer.Meta.fields + (
47+
"cfr_title",
48+
"cfr_part"
49+
)
50+
51+
52+
class ProposalPipelineJobSerializer(ParsingJobSerializer):
53+
54+
class Meta(ParsingJobSerializer.Meta):
55+
model = ProposalPipelineJob
56+
fields = ParsingJobSerializer.Meta.fields + (
57+
"file_hexhash",
58+
)
59+
60+
# Fields we don't want user input for are listed below.
61+
file_hexhash = serializers.CharField(max_length=32)
62+
63+
64+
class FileUploadSerializer(serializers.ModelSerializer):
65+
66+
class Meta:
67+
model = RegulationFile
68+
fields = (
69+
"contents",
70+
"file",
71+
"filename",
72+
"hexhash",
73+
"url"
74+
)
75+
76+
contents = serializers.SerializerMethodField()
77+
file = serializers.FileField()
78+
filename = serializers.CharField(read_only=True)
79+
hexhash = serializers.CharField(read_only=True)
80+
url = serializers.URLField(read_only=True)
81+
82+
def get_contents(self, obj):
83+
return "File contents not shown."

‎regparser/web/jobs/urls.py

+13-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,19 @@
33
from rest_framework.urlpatterns import format_suffix_patterns
44

55
urlpatterns = [
6-
url(r'^job/$', views.JobViewList.as_view()),
7-
url(r'^job/(?P<job_id>[-a-z0-9]+)/$',
6+
url(r'^job(/)$', views.JobViewList.as_view()),
7+
url(r'^job/pipeline(/)$', views.PipelineJobViewList.as_view()),
8+
url(r'^job/pipeline(/)(?P<job_id>[-a-z0-9]+)(/)$',
9+
views.PipelineJobViewInstance.as_view()),
10+
url(r'^job/proposal-pipeline(/)$',
11+
views.ProposalPipelineJobViewList.as_view()),
12+
url(r'^job/proposal-pipeline(/)(?P<job_id>[-a-z0-9]+)(/)$',
13+
views.ProposalPipelineJobViewInstance.as_view()),
14+
url(r'^job/upload/(?P<hexhash>[a-z0-9]{32})(/)$',
15+
views.FileUploadViewInstance.as_view()),
16+
url(r'^job/upload(/)$',
17+
views.FileUploadView.as_view()),
18+
url(r'^job/(?P<job_id>[-a-z0-9]+)(/)$',
819
views.JobViewInstance.as_view())
920
]
1021

0 commit comments

Comments
 (0)