frontend updates

Author: ikhovryak

Azure/correct_sound.wav

@@ -145,14 +145,14 @@ def feedback():
     else:
         threshold = len(correct_list)//2
 
-    path_to_save = os.path.join(app.root_path, "static/Sounds", "plots.png")
+    path_to_save = "static/Sounds/plots" + secrets.token_hex(6) + ".png"
     path_to_user = os.path.join(app.root_path, speech.user_audio_location)
     print("user_path -> ", path_to_user)
     path_to_correct = os.path.join(app.root_path, speech.correct_audio_filename)
     print("correct_path -> ", path_to_correct)
-    has_breaks = SoundComparison().compare_waves(path_to_user, path_to_correct, path_to_save)
+    long_breaks = SoundComparison().compare_waves(path_to_user, path_to_correct, path_to_save)
     # return_breaks("D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/input_soundd99ec5ce7675.wav", "D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/correct_sound145255b4ec90.wav", path_to_save)
-    return render_template('feedback.html',  pic_path="static/Sounds/plots.png", threshold=threshold, correct_list=correct_list, user_list=user_list, wrong_correct=wrong_correct, wrong_user=wrong_user, title="Reconnect - Feedback")
+    return render_template('feedback.html',  pic_path=path_to_save, threshold=threshold, correct_list=correct_list, user_list=user_list, wrong_correct=wrong_correct, wrong_user=wrong_user, title="Reconnect - Feedback")
 
 @app.route("/restart")
 def restart():

Azure/input_sound.wav

@@ -7,7 +7,12 @@
 
 class SoundComparison:
 
-    def compare_waves(self, speaker_sound, correct_sound, location_to_save):
+    def __init__(self):
+        self.result = {"too_little_breaks": False, "too_many_breaks": False,
+                       "short_breaks": [], "long_breaks": [],
+                       "short_pronunciation": [], "long_pronunciation": []}
+
+    def compare_waves(self, speaker_sound, correct_sound, location):
         speaker_rate, pre_speaker_data = scipy.io.wavfile.read(speaker_sound)
         correct_rate, correct_data = scipy.io.wavfile.read(correct_sound)
         speaker_data = self.stereo_to_mono(pre_speaker_data)
@@ -24,14 +29,14 @@ def compare_waves(self, speaker_sound, correct_sound, location_to_save):
         speaker_time = np.arange(0, len(speaker_data), 1) / speaker_rate
         correct_time = np.arange(0, len(correct_data), 1) / correct_rate
 
-        if self.check_for_long_breaks(speaker_data, speaker_rate) is not None:
-            self.plot_graphs(correct_time, correct_data, speaker_time, speaker_data, "#5cb85c", location_to_save)
-            return False
-        self.plot_graphs(correct_time, correct_data, speaker_time, speaker_data, "#5cb85c", location_to_save)
-        return self.check_sensibility_of_breaks(speaker_silence, correct_silence)
-        # if self.check_for_amplitude_inconsistencies(speaker_data, speaker_rate, correct_data, correct_rate) is not None:
-        #     return False
-        # return True
+        self.check_sensibility_of_breaks(speaker_silence, correct_silence)
+        long_breaks = self.result["long_breaks"]
+        if(len(long_breaks)==0):
+            self.plot_graphs(correct_time, correct_data, speaker_time, speaker_data, "#5cb85c", location)
+        else:
+            self.plot_graphs(correct_time, correct_data, speaker_time, speaker_data, "#f0ad4e", location)
+        print(long_breaks)
+        return long_breaks
 
     def plot_graphs(self, correct_time, correct_data, speaker_time, speaker_data, color, location):
          # plot amplitude (or loudness) over time
@@ -83,8 +88,10 @@ def find_audio_chunk_breaks(self, audio_data, rate, silence_amplitude):
     def check_sensibility_of_breaks(self, speaker_breaks, correct_breaks):
         #last_time_difference is used to track the interval between gaps so that past differences would not stack up
         last_time_difference = 0
-        if len(speaker_breaks) != len(correct_breaks):
-            return False
+        if len(speaker_breaks) > len(correct_breaks):
+            self.result["too_many_breaks"] = True
+        elif len(speaker_breaks) < len(correct_breaks):
+            self.result["too_little_breaks"] = False
         else:
             for i in range(len(speaker_breaks)):
                 speaker_start = speaker_breaks[i][0]
@@ -93,12 +100,17 @@ def check_sensibility_of_breaks(self, speaker_breaks, correct_breaks):
                 correct_start = correct_breaks[i][0]
                 correct_end = correct_breaks[i][1]
                 correct_break_time = correct_end - correct_start
-                if abs(speaker_break_time - correct_break_time) > 0.30:
-                    return False
-                if abs(speaker_start - correct_start) > (last_time_difference + 0.2):
-                    return False
+                if (speaker_end - speaker_start) > 2.2:
+                    self.result["long_breaks"].append(speaker_breaks[i])
+                elif (speaker_break_time - correct_break_time) > 0.30:
+                    self.result["long_breaks"].append(speaker_breaks[i])
+                elif (correct_break_time - speaker_break_time) > 0.30:
+                    self.result["short_breaks"].append(speaker_breaks[i])
+                if (speaker_start - correct_start) > (last_time_difference + 0.5):
+                    self.result["long_pronunciation"].append((speaker_breaks[i-1][1], speaker_breaks[i][0]))
+                elif (correct_start - speaker_start) > (last_time_difference + 0.5):
+                    self.result["short_pronunciation"].append((speaker_breaks[i-1][1], speaker_breaks[i][0]))
                 last_time_difference = abs(correct_end - speaker_end)
-            return True
 
     def remove_audio_wave_silence(self, audio_data, rate, min=None):
         start_counter = 0
@@ -114,18 +126,18 @@ def calculate_silent_amplitude(self, audio_data, rate, min=None):
         end = int(rate/3)
         return max(max(audio_data[-end:]), 0.15) if min is None else 0.1
 
-    def check_for_long_breaks(self, audio_data, rate):
-        counter = 0
-        duration = 0
-        while counter < len(audio_data):
-            if audio_data[counter] < self.calculate_silent_amplitude(audio_data, rate):
-                duration += 1
-                if duration > (2.2 * rate):
-                    return counter / rate
-            else:
-                duration = 0
-            counter += 1
-        return None
+    # def check_for_long_breaks(self, audio_data, rate):
+    #     counter = 0
+    #     duration = 0
+    #     while counter < len(audio_data):
+    #         if audio_data[counter] < self.calculate_silent_amplitude(audio_data, rate):
+    #             duration += 1
+    #             if duration > (2.2 * rate):
+    #                 return counter / rate
+    #         else:
+    #             duration = 0
+    #         counter += 1
+    #     return None
 
     # def check_for_amplitude_inconsistencies(self, audio_data1, rate1, audio_data2, rate2):
     #     chunk_audio1 = self.convert_audio_data_to_chunk_audio_data(audio_data1, rate1)
@@ -145,9 +157,6 @@ def convert_audio_data_to_chunk_audio_data(self, audio_data, rate):
             chunk_audio_data[i] = chunk_audio_data[i] / (int(rate) / 10)
         return chunk_audio_data, int(rate/10)
 
-def return_breaks(correct_loc, user_loc, location_to_save):
-    print(SoundComparison().compare_waves(user_loc, correct_loc, location_to_save))
-
 if __name__ == "__main__":
     # web_file="C:\Users\Samuel\PycharmProjects\speech_analysis\wave_comparison"
     #
@@ -173,5 +182,4 @@ def return_breaks(correct_loc, user_loc, location_to_save):
     #     string += str(abs(happy[i] / max))
     #     string += "|"
     # print(happy[10400:15000])
-
-    print(SoundComparison().compare_waves("D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/input_soundd99ec5ce7675.wav", "D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/correct_sound145255b4ec90.wav", "plots.png"))
+    print(SoundComparison().compare_waves("D:/Haverford/LocalHack/speech_analysis/audio_samples/no_pause1.wav", "D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/correct_sound09d546aba5a4.wav", "plots.png"))

audio_samples/break.m4a

@@ -73,6 +73,25 @@ <h5> Nice try! </h5>
   <hr>
   <strong><p class="lead"><span class="badge badge-info mr-2">2</span>Pacing</p></strong>
   <img src="{{ pic_path }}" />
+  <div class="row pl-4">
+    <div class="col">
+      {% if long_breaks|length == 0 %}
+        <div class="bd-callout bd-callout-success">
+          <h5> Great job! </h5>
+          <p>Congratulations! You pronounced the words with correct pace and you spoke naturally without significant breaks!</p>
+        </div>
+      {% else %}
+      <div class="bd-callout bd-callout-warning">
+        <h5> Nice try! </h5>
+        <p>Your speeking was good, however, you made some significant breaks while pronoucing the sentence. Here are the times of your breaks:</p>
+          <ol>
+          {% for break in long_breaks %}
+            <li>{{ break }}</li>
+          {% endfor %}
+          </ol>
+        <p> Prectice makes perfect! Try decreasing the breaks between the words.</p>
+      </div>
+      {% endif %}
     <div class="row ">
       <div class="col text-center">
         <a href="{{ url_for('restart') }}" class="btn btn-info">Try again!</a>

audio_samples/no_pause1.wav

@@ -28,20 +28,30 @@ def compare_waves(self, speaker_sound, correct_sound):
         correct_silence = self.find_audio_chunk_breaks(correct_data, correct_rate, correct_data_silence)
 
 
-        # plot amplitude (or loudness) over time
-        # speaker_time = np.arange(0, len(speaker_data), 1) / speaker_rate
-        # correct_time = np.arange(0, len(correct_data), 1) / correct_rate
-        # plt.figure(1)
-        # plt.subplot(211)
-        # plt.plot(speaker_time, speaker_data, linewidth=0.1, alpha=1, color='#000000')
-        # plt.xlabel('Time (s)')
-        # plt.ylabel('Amplitude')
-        # plt.subplot(212)
-        # plt.plot(correct_time, correct_data, linewidth=0.1, alpha=1, color='#000000')
-        # plt.show()
         self.check_sensibility_of_breaks(speaker_silence, correct_silence)
         return self.result
 
+    def plot_graphs(self, correct_time, correct_data, speaker_time, speaker_data, color, location):
+         # plot amplitude (or loudness) over time
+
+        plt.subplot(211)
+        plt.plot(correct_time, correct_data, linewidth=0.1, alpha=1, color=color)
+        plt.xlabel('Time (s)')
+        plt.ylabel('Amplitude')
+        plt.title('Correct sound amplitude')
+
+        plt.figure(1)
+
+        plt.subplot(212)
+        plt.plot(speaker_time, speaker_data, linewidth=0.1, alpha=1, color=color)
+        plt.xlabel('Time (s)')
+        plt.ylabel('Amplitude')
+        plt.title("Your recording's amplitude")
+
+        plt.subplots_adjust(hspace=0.7)
+        plt.savefig(location)
+        # plt.show()
+
     def stereo_to_mono(self, audio_data):
         audio_data = audio_data.astype(float)
         return audio_data.sum(axis=1)
@@ -165,5 +175,4 @@ def convert_audio_data_to_chunk_audio_data(self, audio_data, rate):
     #     string += str(abs(happy[i] / max))
     #     string += "|"
     # print(happy[10400:15000])
-    print(SoundComparison().compare_waves("C:/Users/Samuel/PycharmProjects/speech_analysis/wave_comparison/input_sound.wav", "C:/Users/Samuel/PycharmProjects/speech_analysis/wave_comparison/correct_sound.wav"))
-
+    print(SoundComparison().compare_waves("D:/Haverford/LocalHack/speech_analysis/audio_samples/no_pause1.wav", "D:/Haverford/LocalHack/speech_analysis/reconnect_app/static/Sounds/correct_sound09d546aba5a4.wav"))