Merge pull request #5 from LEEYOONHYUNG/master

final

README.md

@@ -1,18 +1,15 @@
 # FastSpeech
 Implementation of ["FastSpeech: Fast, Robust and Controllable Text to Speech"](https://arxiv.org/abs/1905.09263)  
 
-(2020/02/10)  
-- I was able to finish the implementation of [Transformer-TTS](https://github.com/Deepest-Project/Transformer-TTS) by completing the Stop token prediction and remove the concatenation of inputs and outputs of multihead attention.  
-- However, the alignments of the implementation are less diagonal, so it can not generate proper alignments for fastspeech.  
-- As a result, I failed to train the fastspeech with the new transformer-tts implementation :(  
-
 ## Training  
 0. Set `data_path` in `hparams.py` as the LJSpeech folder  
 1. Set `teacher_dir` in `hparams.py` as the data directory where the alignments and melspectrogram targets are saved  
-2. Set `teacer_model` as the pre-trained [transfomer-tts model](https://github.com/Deepest-Project/Transformer-TTS) for pretrained_embedding
-3. `python train.py --gpu='0'`  
+2. Put checkpoint of the pre-trained transformer-tts (weights of the embedding/encoder layers are used)  
+3. `python train.py`  
 
-## Training curves (orange:  / blue: )  
+## Training curves (orange: character  / blue: phoneme)  
+##### The size of the train dataset is different because transformer-tts trained with phoneme shows more diagonal attention  
+##### train:val:test=8:1:1, total => character:1126 / phoneme:3412  
 <img src="figures/duration_loss.JPG" height="200">  
 <img src="figures/train_loss.JPG" height="200">  
 <img src="figures/val_loss.JPG" height="200">  

generate_samples.ipynb

@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2020-01-09T06:05:50.543482Z",
@@ -19,7 +19,7 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '2'\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '3'\n",
     "\n",
     "import warnings\n",
     "warnings.filterwarnings(\"ignore\")\n",
@@ -50,7 +50,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -89,7 +89,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -112,13 +112,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {
     "scrolled": false
    },
    "outputs": [],
    "source": [
-    "for data_type in ['char']:\n",
+    "for data_type in ['char', 'phone']:\n",
     "    for step in ['20000', '50000', '100000', '200000']:\n",
     "        checkpoint_path = f\"training_log/fastspeech-{data_type}/checkpoint_{step}\"\n",
     "        state_dict = {}\n",
@@ -131,7 +131,7 @@
     "\n",
     "        for i in [1, 6, 22]:\n",
     "            file_name, _, text = lines[i]\n",
-    "            sequence = np.array(text2seq(text,data_type))[None, :]\n",
+    "            sequence = np.array(text2seq(text, data_type))[None, :]\n",
     "            sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long()\n",
     "\n",
     "            for alpha in [0.8, 0.9, 1.0, 1.1, 1.2]:\n",

hparams.py

@@ -6,9 +6,9 @@
 seed=1234
 n_gpus=2
 output_directory = 'training_log'
-log_directory = 'fastspeech-char'
+log_directory = 'fastspeech-phone'
 data_path = '/media/disk1/lyh/LJSpeech-1.1/preprocessed'
-teacher_dir = '/media/disk1/lyh/fastspeech_char'
+teacher_dir = '/media/disk1/lyh/fastspeech_phone'
 
 training_files='filelists/ljs_audio_text_train_filelist.txt'
 validation_files='filelists/ljs_audio_text_val_filelist.txt'
@@ -30,7 +30,7 @@
 # Model Parameters             #
 ################################
 n_symbols=len(symbols)
-data_type='char_seq' # 'phone_seq'
+data_type='phone_seq' # 'phone_seq'
 symbols_embedding_dim=256
 hidden_dim=256
 dprenet_dim=256

index.html

@@ -8,7 +8,7 @@
 
 <body>
   <div>
-    <h2>LJ001-0029.wav</h2>
+    <h2>LJ001-0029.wav (character)</h2>
     <table>
       <thead>
         <tr>
@@ -76,7 +76,7 @@ <h2>LJ001-0029.wav</h2>
       </tbody>
     </table>
 
-    <h2>LJ001-0085.wav</h2>
+    <h2>LJ001-0085.wav (character)</h2>
     <table>
       <thead>
         <tr>
@@ -144,7 +144,7 @@ <h2>LJ001-0085.wav</h2>
       </tbody>
     </table>
 
-    <h2>LJ002-0106.wav</h2>
+    <h2>LJ002-0106.wav (character)</h2>
     <table>
       <thead>
         <tr>
@@ -213,6 +213,213 @@ <h2>LJ002-0106.wav</h2>
     </table>
 
   </div>
+
+  <div>
+    <h2>LJ001-0029.wav (phoneme)</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Steps</th>
+          <th>alpha=0.8</th>
+          <th>alpha=0.9</th>
+          <th>alpha=1.0</th>
+          <th>alpha=1.1</th>
+          <th>alpha=1.2</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Steps 20000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone20000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone20000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone20000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone20000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone20000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 50000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone50000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone50000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone50000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone50000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone50000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 100000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone100000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone100000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone100000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone100000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone100000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 200000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone200000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone200000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone200000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone200000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0029_phone200000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+      </tbody>
+    </table>
+
+    <h2>LJ001-0085.wav (phoneme)</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Steps</th>
+          <th>alpha=0.8</th>
+          <th>alpha=0.9</th>
+          <th>alpha=1.0</th>
+          <th>alpha=1.1</th>
+          <th>alpha=1.2</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Steps 20000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone20000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone20000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone20000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone20000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone20000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 50000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone50000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone50000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone50000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone50000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone50000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 100000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone100000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone100000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone100000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone100000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone100000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 200000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone200000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone200000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone200000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone200000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ001-0085_phone200000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+      </tbody>
+    </table>
+
+    <h2>LJ002-0106.wav (phoneme)</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Steps</th>
+          <th>alpha=0.8</th>
+          <th>alpha=0.9</th>
+          <th>alpha=1.0</th>
+          <th>alpha=1.1</th>
+          <th>alpha=1.2</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>Steps 20000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone20000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone20000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone20000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone20000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone20000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 50000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone50000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone50000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone50000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone50000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone50000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 100000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone100000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone100000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone100000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone100000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone100000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+        <tr>
+          <td>Steps 200000</td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone200000_8.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone200000_9.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone200000_10.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone200000_11.wav" type="audio/wav"></audio></td>
+          <td><audio controls class="audio-player" preload="metadata">
+              <source src="wavs/LJ002-0106_phone200000_12.wav" type="audio/wav"></audio></td>
+        </tr>
+      </tbody>
+    </table>
+
+  </div>
 </body>
 
 </html>

modules/model.py

@@ -160,7 +160,7 @@ def inference(self, text, alpha=1.0):
 
         ### Duration Predictor ###
         durations = self.Duration(hidden_states.permute(1,2,0))
-        hidden_states_expanded = self.LR(hidden_states, durations, alpha)
+        hidden_states_expanded = self.LR(hidden_states, durations, alpha, inference=True)
         hidden_states_expanded += self.alpha2*self.pe[:hidden_states_expanded.size(0)].unsqueeze(1)
         mel_mask = text.new_zeros(1, hidden_states_expanded.size(0)).to(torch.bool)
 
@@ -185,10 +185,11 @@ def align2duration(self, alignments, mel_mask):
         return durations
 
 
-    def LR(self, hidden_states, durations, alpha=1.0):
+    def LR(self, hidden_states, durations, alpha=1.0, inference=False):
         L, B, D = hidden_states.size()
         durations = torch.round(durations*alpha).to(torch.long)
-        durations[durations<=0]=1
+        if inference:
+            durations[durations<=0]=1
         T=int(torch.sum(durations, dim=-1).max().item())
         expanded = hidden_states.new_zeros(T, B, D)