try this

Author: r9y9

hparams.py

@@ -84,7 +84,7 @@
             "query_position_rate": 1.0,
             "key_position_rate": 1.385,
             "key_projection": True,
-            "value_projection": True,
+            "value_projection": False,
             "clip_thresh": 0.1,
             "initial_learning_rate": 5e-4,
         },
@@ -205,7 +205,7 @@
     # whether to rescale waveform or not.
     # Let x is an input waveform, rescaled waveform y is given by:
     # y = x / np.abs(x).max() * rescaling_max
-    rescaling=False,
+    rescaling=True,
     rescaling_max=0.999,
     # mel-spectrogram is normalized to [0, 1] for each utterance and clipping may
     # happen depends on min_level_db and ref_level_db, causing clipping noise.