добавил описания, пофиксил модель на сервере

Author: Iliaavilov

model.py

@@ -37,7 +37,11 @@ def forward(self, images):
 
 class Attention(nn.Module):
 
-    def __init__(self, word_embeddings_dim, attention_dim, encoded_image_size):
+    def __init__(self, word_embeddings_dim, attention_dim):
+        '''
+        :param word_embeddings_dim: length of word embedding
+        :param attention_dim: length of attention vector
+        '''
         super(Attention, self).__init__()
 
         # Attention layer for encoder
@@ -50,19 +54,35 @@ def __init__(self, word_embeddings_dim, attention_dim, encoded_image_size):
         self.relu = nn.ReLU()
 
     def forward(self, encoder_out, decoder_out):
+        '''
+        :param encoder_out: embedding of image
+        :param decoder_out: embedding of previous word
+        :return: weighted encoded image
+        '''
 
-        att_encoder_computed = self.att_encoder(encoder_out)  # (batch_size, encoded_image_size**2, attention_dim)
-        att_decoder_computed = self.att_decoder(decoder_out)  # (batch_size, attention_dim)
-        att = self.att_final(self.relu(att_encoder_computed + att_decoder_computed.unsqueeze(1))).squeeze(2)  # (batch_size, encoded_image_size**2)
-        att_weights = self.softmax(att)  # (batch_size, 2048)
-        encoder_weighted = (encoder_out * att_weights.unsqueeze(2)).sum(dim=1)  # (batch_size, encoder_dim)
+        # Attention vector for image
+        att_encoder_computed = self.att_encoder(encoder_out)
+        # Attention vector for previous word
+        att_decoder_computed = self.att_decoder(decoder_out)
+        # Combining 2 attentions
+        att = self.att_final(self.relu(att_encoder_computed + att_decoder_computed.unsqueeze(1))).squeeze(2)
+        # Weighting image parts based on attention
+        att_weights = self.softmax(att)
+        encoder_weighted = (encoder_out * att_weights.unsqueeze(2)).sum(dim=1)
 
         return encoder_weighted
 
 
 class Decoder(nn.Module):
 
     def __init__(self, vocab_size, word_embeddings_dim, attention_dim, decoder_hidden_size, encoded_image_size):
+        '''
+        :param vocab_size: number of words in corpus
+        :param word_embeddings_dim: length of word embedding
+        :param attention_dim: length of attention vector
+        :param decoder_hidden_size: hidden size of lstm
+        :param encoded_image_size: size of each encoded image channel
+        '''
         super(Decoder, self).__init__()
 
         self.encoded_image_size = encoded_image_size
@@ -73,7 +93,7 @@ def __init__(self, vocab_size, word_embeddings_dim, attention_dim, decoder_hidde
         self.LSTMCell = torch.nn.LSTMCell(2048 + word_embeddings_dim,
                                           hidden_size=decoder_hidden_size, bias = True)
         self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=word_embeddings_dim)
-        self.Attention = Attention(word_embeddings_dim, attention_dim, encoded_image_size)
+        self.Attention = Attention(word_embeddings_dim, attention_dim)
         self.linear = torch.nn.Linear(decoder_hidden_size, vocab_size)
         self.h_init = torch.nn.Linear(2048, decoder_hidden_size)
         self.c_init = torch.nn.Linear(2048, decoder_hidden_size)
@@ -83,51 +103,53 @@ def __init__(self, vocab_size, word_embeddings_dim, attention_dim, decoder_hidde
 
 
     def forward(self, captions, encoder_out, captions_lengths):
+        '''
+        :param captions: captions for images
+        :param encoder_out: encoded images
+        :param captions_lengths: lengths of captions
+        :return:
+        '''
 
-
-        # Размер батча (нужно для инициализации векторов)
+        # Initialising vectors of predictions
         batch_size = encoder_out.size()[0]
-        # Инициализирум вектор предсказаний размерности  # (batch_size, max(captions_length), vocab_size) \
-        # (то есть для каждого наблюдения имеет вектор, состоящий из векторов вероятности появления каждого слова на конкретном месте предложения)
         predictions = torch.zeros(batch_size, max(captions_lengths), self.vocab_size).to(device) # (batch_size, max(captions_length), vocab_size)
-        predictions[:, 0, 0] = 1 # ставим вероятность в 1 для первого слова
-        # Выравниваем каналы (то есть было 2048 матриц размерностями encoded_image_size, encoded_image_size, \
-        # а стало 2048 векторов размерностями encoded_image_size**2)
-        encoder_out = encoder_out.view(batch_size, -1, 2048) # (batch_size, max(captions_length), 2048)
-        # Сортируем наблюдения в порядке убывания длины предложения
+        # First word of each caption guruanteed to be <start>
+        predictions[:, 0, 0] = 1
+        # Falttening channels
+        encoder_out = encoder_out.view(batch_size, -1, 2048)
+        # Sort captions by their length (for faster loop)
         captions_lengths, sort_ind = captions_lengths.squeeze(1).sort(dim=0, descending=True)
-        encoder_out = encoder_out[sort_ind] # (batch_size, max(captions_length), 2048)
+        encoder_out = encoder_out[sort_ind]
         captions = captions[sort_ind]
-        # Делаем из слов эмбеддинги
-        embeddings = self.embedding(captions) # (batch_size, max(captions_length), word_embeddings_dim)
-        # Инициализируем вектора LSTM для первого слова (с помощью картинки)
+        # Embedding each word of captions
+        embeddings = self.embedding(captions)
+        # Initialising lstm vectors for first word
         h = self.h_init(encoder_out.mean(dim = 1)) # (batch_size, decoder_hidden_size)
         c = self.c_init(encoder_out.mean(dim = 1)) # (batch_size, decoder_hidden_size)
 
 
         for word_n in range(1, max(captions_lengths)):
-            # Количество наблюдений, для которых длина предложения больше заданной длины
+            # Number of captions with greater length
             batch_size_n = sum([length > word_n for length in captions_lengths])
 
-            # Выбираем эмбеддинг слова, стоящего на позиции word_n - 1 (то есть эмбеддинг предыдущего слова)
+            # Obtain embedding of previous word
             decoder_out = embeddings[:, (word_n - 1)] # (batch_size, word_embeddings_dim)
 
 
-            # Механизм внимания
-            encoder_weighted = self.Attention(batch_size = batch_size_n,
-                                              encoder_out = encoder_out[:batch_size_n],
-                                              decoder_out = decoder_out[:batch_size_n]) # (batch_size, encoded_image_size**2)
+            # Attention mechanism
+            encoder_weighted = self.Attention(encoder_out = encoder_out[:batch_size_n],
+                                              decoder_out = decoder_out[:batch_size_n])
 
-            gate = self.sigmoid(self.f_beta(h[:batch_size_n]))  # gating scalar, (batch_size_t, encoder_dim)
+            gate = self.sigmoid(self.f_beta(h[:batch_size_n]))
             encoder_weighted = gate * encoder_weighted
 
-            # Конкатенируем информцию из механизма внимания и информацию о предыдущем слове
-            decoder_in = torch.cat((encoder_weighted, decoder_out[:batch_size_n]), 1) # (batch_size, encoded_image_size**2 + word_embeddings_dim)
+            # Concatenating attention and previous word
+            decoder_in = torch.cat((encoder_weighted, decoder_out[:batch_size_n]), 1)
 
-            # Предсказываем вероятности появления слов на текущей позиции
-            h, c = self.LSTMCell(decoder_in, (h[:batch_size_n], c[:batch_size_n])) # (batch_size, decoder_hidden_size)
-            predictions_word = self.linear(h) # (batch_size, decoder_hidden_size)
-            # Записываем информацию о предсказанных вероятностях (еще не вероятностях) в вектор
+            # Obtaining probabilities (not exectaly, because no softmax on this step) of word appearing on this step
+            h, c = self.LSTMCell(decoder_in, (h[:batch_size_n], c[:batch_size_n]))
+            predictions_word = self.linear(h)
+            # Store probabilities (not exectaly, because no softmax on this step) in vector
             predictions[:batch_size_n, word_n, :] = predictions_word
 
         return predictions, captions, captions_lengths, sort_ind

server/.dockerignore

@@ -0,0 +1,4 @@
+*.png
+*.mp4
+*.jpg
+.idea

server/Dockerfile

@@ -1,10 +1,11 @@
 FROM python:3.7
 
+RUN apt-get update ##[edited]
+RUN apt-get install ffmpeg libsm6 libxext6  -y
+
 COPY requirements.txt requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-EXPOSE 5000
-
 CMD [ "python", "server.py" ]

server/docker-compose.yaml

@@ -0,0 +1,5 @@
+services:
+  image_captioning:
+    build: .
+    ports:
+      - 8888:1567