Add working code for the vilbert-multitask demo

.gitignore

@@ -0,0 +1,8 @@
+*/.vscode/*
+media/*
+*.pyc
+env/
+db.sqlite3
+uwsgi.ini
+vilbert_multitask_nginx.conf
+static/

demo/admin.py

@@ -0,0 +1,34 @@
+from django.contrib import admin
+
+from .models import Tasks, QuestionAnswer
+# from import_export.admin import ImportExportMixin
+
+
+class ImportExportTimeStampedAdmin(admin.ModelAdmin):
+    exclude = ("created_at", "modified_at")
+
+
+@admin.register(Tasks)
+class TaskAdmin(ImportExportTimeStampedAdmin):
+    readonly_fields = ("created_at",)
+    list_display = (
+        "unique_id",
+        "name",
+        "placeholder",
+        "example",
+        "num_of_images",
+        "description",
+    )
+
+
+@admin.register(QuestionAnswer)
+class QuestionAnswerAdmin(ImportExportTimeStampedAdmin):
+    readonly_fields = ("created_at",)
+    list_display = (
+        "task",
+        "input_text",
+        "input_images",
+        "answer_text",
+        "answer_images",
+        "socket_id",
+    )

demo/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class DemoConfig(AppConfig):
+    name = 'demo'

demo/constants.py

@@ -0,0 +1,22 @@
+from django.conf import settings
+import os
+
+
+COCO_IMAGES_PATH = os.path.join(settings.MEDIA_ROOT, "test2014")
+COCO_IMAGES_URL = os.path.join(settings.MEDIA_URL, "test2014")
+
+VILBERT_MULTITASK_CONFIG = {
+    "gpuid": 1,
+    "image_dir": os.path.join(settings.MEDIA_ROOT, "demo"),
+}
+
+
+SLACK_WEBHOOK_URL = ""
+BASE_VQA_DIR_PATH = ""
+COCO_PARTIAL_IMAGE_NAME = "COCO_test2014_"
+RABBITMQ_QUEUE_USERNAME = ""
+RABBITMQ_QUEUE_PASSWORD = ""
+RABBITMQ_HOST_SERVER = ""
+RABBITMQ_HOST_PORT = ""
+RABBITMQ_VIRTUAL_HOST = ""
+IMAGES_BASE_URL = ""

demo/consumers.py

@@ -0,0 +1,12 @@
+from channels import Group
+from .utils import log_to_terminal
+
+def ws_connect(message):
+    print("User connnected via Socket")
+
+
+def ws_message(message):
+    print("Message recieved from client side and the content is ", message.content['text'])
+    socketid = message.content['text']
+    Group(socketid).add(message.reply_channel)
+    log_to_terminal(socketid, {"info": "User added to the Channel Group"})

demo/migrations/0001_add_models_for_demo.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2020-03-28 01:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='QuestionAnswer',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('modified_at', models.DateTimeField(auto_now=True)),
+                ('input_text', models.TextField(blank=True, null=True)),
+                ('input_images', models.CharField(blank=True, max_length=10000, null=True)),
+                ('answer_text', models.TextField(blank=True, null=True)),
+                ('answer_images', models.CharField(blank=True, max_length=10000, null=True)),
+                ('socket_id', models.CharField(blank=True, max_length=1000, null=True)),
+            ],
+            options={
+                'db_table': 'questionanswer',
+            },
+        ),
+        migrations.CreateModel(
+            name='Tasks',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('modified_at', models.DateTimeField(auto_now=True)),
+                ('unique_id', models.PositiveIntegerField(unique=True)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('placeholder', models.TextField(blank=True, null=True)),
+                ('description', models.TextField(blank=True, null=True)),
+                ('num_of_images', models.PositiveIntegerField()),
+            ],
+            options={
+                'db_table': 'tasks',
+            },
+        ),
+        migrations.AddField(
+            model_name='questionanswer',
+            name='task',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='demo.Tasks'),
+        ),
+    ]

demo/migrations/0002_attachment.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.1 on 2020-03-30 05:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('demo', '0001_add_models_for_demo'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Attachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file', models.FileField(upload_to='attachments')),
+            ],
+        ),
+    ]

demo/migrations/0003_tasks_example.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.1 on 2020-04-05 10:07
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('demo', '0002_attachment'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tasks',
+            name='example',
+            field=models.CharField(blank=True, max_length=1000, null=True),
+        ),
+    ]

demo/models.py

@@ -0,0 +1,46 @@
+from django.db import models
+from django.utils.html import format_html
+
+class TimeStampedModel(models.Model):
+    """
+    An abstract base class model that provides self-managed `created_at` and
+    `modified_at` fields.
+    """
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    modified_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        abstract = True
+        app_label = "demo"
+
+
+class Tasks(TimeStampedModel):
+    unique_id = models.PositiveIntegerField(unique=True)
+    name = models.CharField(max_length=1000, blank=True, null=True)
+    placeholder = models.TextField(null=True, blank=True)
+    description = models.TextField(null=True, blank=True)
+    num_of_images = models.PositiveIntegerField()
+    example = models.CharField(max_length=1000, null=True, blank=True)
+
+    class Meta:
+        app_label = "demo"
+        db_table = "tasks"
+
+
+class QuestionAnswer(TimeStampedModel):
+    task = models.ForeignKey(Tasks)
+    input_text = models.TextField(null=True, blank=True)
+    input_images = models.CharField(max_length=10000, null=True, blank=True)
+    answer_text = models.TextField(null=True, blank=True)
+    answer_images = models.CharField(max_length=10000, null=True, blank=True)
+    socket_id = models.CharField(max_length=1000, null=True, blank=True)
+    class Meta:
+        app_label = "demo"
+        db_table = "questionanswer"
+
+    def img_url(self):
+        return format_html("<img src='{}' height='150px'>", self.image)
+
+class Attachment(models.Model):
+    file = models.FileField(upload_to='attachments')

demo/routers.py

@@ -0,0 +1,7 @@
+from channels.routing import route
+from .consumers import ws_message, ws_connect
+
+channel_routing = [
+    route("websocket.receive", ws_message),
+    route("websocket.connect", ws_connect),
+]

demo/sender.py

@@ -0,0 +1,35 @@
+from django.conf import settings
+from .utils import log_to_terminal
+
+import os
+import pika
+import sys
+import json
+
+
+def vilbert_task(image_path, question, task_id, socket_id):
+
+    connection = pika.BlockingConnection(pika.ConnectionParameters(
+        host='localhost',
+        port=5672,
+        socket_timeout=10000))
+    channel = connection.channel()
+    queue = "vilbert_multitask_queue"
+    channel.queue_declare(queue=queue, durable=True)
+    message = {
+        'image_path': image_path,
+        'question': question,
+        'socket_id': socket_id,
+        "task_id": task_id
+    }
+    log_to_terminal(socket_id, {"terminal": "Publishing job to ViLBERT Queue"})
+    channel.basic_publish(exchange='',
+                          routing_key=queue,
+                          body=json.dumps(message),
+                          properties=pika.BasicProperties(
+                          delivery_mode = 2, # make message persistent
+                      ))
+
+    print(" [x] Sent %r" % message)
+    log_to_terminal(socket_id, {"terminal": "Job published successfully"})
+    connection.close()

demo/templates/index.html

@@ -0,0 +1,25 @@
+<!Doctype html>
+{% load static %}
+<html>
+
+<head>
+    {% block head %} {%include "head.html"%} {%endblock%}
+</head>
+
+<body>
+    {%block header%} {%include "header.html"%} {% endblock %}
+
+    <div class="container">
+        <div class="row">
+            <div class="col-md-12">
+                {% block header_content%} {%include "header_content.html"%} {%endblock%}
+            </div>
+        </div>
+        <div class="row">
+            {%block demo_images %} {%include "demo_images.html"%} {%endblock%}
+        </div> 
+        {%block terminal %}
+        {% include "terminal.html" %}{% endblock %} {% block result%}{%include "result.html"%}
+        {%endblock%} {% block credits %} {%include "credits.html"%} {%endblock%}
+</body>
+</html>

demo/templates/vilbert_multitask/credits.html

@@ -0,0 +1,85 @@
+<script>
+    function scrollToElement(dstElement) {
+      try {
+        var scrollPixels = dstElement.offset().top - $(".navbar-fixed-top").height() - 30;
+        $('html, body').animate({
+          scrollTop: scrollPixels
+        }, 1500);
+      }
+      catch (err) {
+        console.log(err);
+      }
+    }
+
+    function submitForm(src, id) {
+      //Change the image to loading jpeg
+      console.log("Submitted question.");
+      $('#comments').prepend('<li>' + "Asking the question..." + '</li>');
+      console.log(src);
+
+      var txtArea = document.getElementById("txt" + id);
+
+      $.ajax({
+        type: 'POST', // define the type of HTTP verb we want to use (POST for our form)
+        url: '{% url 'demo:upload_image' %}', // the url where we want to POST
+        data: { 'src': src, } // our data object
+      })
+        .done(function (data) {
+          data = JSON.parse(data);
+          console.log(data);
+          $("#ResultDiv").addClass("");
+        });
+    }
+</script>
+
+<script>
+
+    $(document).ready(function () {
+      $("#show-demo-images-btn button").click(function () {
+        sampleImagesList = [];
+        if ($(this).text() == "Show Demo Images") {
+          $(this).text("Show Random Images");
+          $('#demoImages3').hide();
+          $('#demoImages4').hide();
+          $('#demoImages1').show();
+          $('#demoImages2').show();
+        } else if ($(this).text() == "Show Random Images") {
+          $(this).text("Show Demo Images");
+          $('#demoImages1').hide();
+          $('#demoImages2').hide();
+          $('#demoImages3').show();
+          $('#demoImages4').show();
+        } else {
+          alert("An error occured. We will fix it soon.");
+        }
+      });
+    });
+
+</script>
+
+<script type="text/javascript">
+$("#question").keypress(function (event) {
+    if (event.which == 13) {
+    if (($("#question").val() != "") && ($("#selected-task").val()!= null)) {
+        submitImageForMultitask();
+        console.log("submited the form");
+    }
+    }
+});
+</script>
+
+<div class="container">
+    <h2 class="page-header"> Credits </h2>
+    <div class="fs-20">
+        Built by <a href="https://rishabhjain.xyz" target="_blank">@rishabh jain</a>
+    </div>
+</div>
+<div class="container">
+  <h2 class="page-header"> Acknowledgements </h2>
+  <div class="fs-20">
+      We thank <a href="https://www.cc.gatech.edu/~jlu347/" target="_blank">@jiasen lu</a> for his help.
+  </div>
+  <br>
+  <br>
+</div>
+<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/reconnecting-websocket/1.0.0/reconnecting-websocket.min.js"></script>