bug fix and new feature: microphone

- add microphone recording feature
- fix the HTTPS security error for microphone permission
- add a function to allow recording preprocessing

melodytalk/cert.pem

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCDCCAvACCQCuNwx7H4ODqDANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJK
+UDESMBAGA1UECAwJSGFtYW1hdHN1MRIwEAYDVQQHDAlIYW1hbWF0c3UxDzANBgNV
+BAoMBllhbWFoYTAeFw0yMzA2MjMwODI0MzdaFw0yNDA2MjIwODI0MzdaMEYxCzAJ
+BgNVBAYTAkpQMRIwEAYDVQQIDAlIYW1hbWF0c3UxEjAQBgNVBAcMCUhhbWFtYXRz
+dTEPMA0GA1UECgwGWWFtYWhhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+AgEArkDceLDD0NcbzJ8YTkn2ium9Spd3gJgHbh1yzqzzdJ2oMqeX6MV8DIqg2DWm
+xqAWrxJLZqfOfWvuEPI+lzImDEDgIkLgfzqIzK4VsZT0ysINMXnU/QKZdfkV5tTt
+aOUyRauYabCaLGDNIIFqswTdCqm8nTlxOd7r+53QT3sf+P1E6c+SCAQD2oj0bpbN
+BZ1fAtOaZMldiPg1BppFG4kPd7w5gUrIneVKdAsbnGaf/7hco0bWXwLcz7KgGYc9
+/Sszy0/Ui/ZQ/FhikS77+aquaChYOwGb97mjUzzJ7OgXJrJQxMSUTqwHXkhN0dXD
+Mc+Ijpzawo72ng2QsmIRwr/eTA9c3SsUsA8kkJnANIdvqTpwdQxjomB/D2w+sHzo
+7cE+PMYTBNZtcP27lFQwhyUy0MpMX9hnkIe6VpCpdSkfvwhOJS1QEvgF4rTOFKrL
+rWpFZYbgmQhlWrSC2kd/VMIjWjxCsUrvNeWVvBSHgCjlZDCEKhV4qkr90P7vudB0
+A9Vs+2m147dfYirEs5biL758uWcvi8/oYsvUFPmiLVrowu6LBg14ERVsN8rQ0ZeB
+oVpjl/nDdH99eRqwn71RLmihO6crYOKzEmoUDd4oA0uIEP1Hc5mSUxLhXMJbB1GW
+A81MTVUi1W5phWT+5GSEIHqNW8TNbO4ovfP30ANH0CF1+50CAwEAATANBgkqhkiG
+9w0BAQsFAAOCAgEAE9qd2mrtROvRbyxUUZNQXgrAMWbRECnzHDS+ol/CYbIkYdcU
+zQcW5G3zsSDh4CumFNQl8F19J4J8uWr8LKykhk71McDYsMmR2mdSxmqbtNWxZKKq
+tlfQP3io7hdcyl2QEHKKM4SY9IGbDCkSX5jbTfPjGJDL9n7aZSoDDa/p1YS5Pn+T
+VkXoYIbhUs7xo/ZGnW4HppAl+SsZUdaVo5q+tRo997RnsBmDJfN7GgZspu6ul1Du
+TZ7Dou2Mpk0UjjpTMpnAcCIOBZlCrkASkB34GtvzWmm45xzq0Rj9JMrW3ulg5gqJ
+Qx/zLd2ZJQV12JK1HbQCp4FkizLbsFW8FigywSWjJDmfF2591sBWoI755hRLaebu
+x9zWLXufaRQYigtoV5wwpYZN4uSBO0Im+Ege5TUiogn3lxvzEQDxUHOZeeRxPOFk
+/cz7ngyfhPNa7JDL7KgkunleoI/rjNC2N+jZLt91ivPx66p5gV4xQg9tzAYZVcwC
+fLWUyi4N2/7XfDEy4fmMGx2WTq1pL9vD4JvnteS6ekTLMya+CnQuJb59mgDUyKqR
+wu6TrQN/zCKJwzyfrteciKM/fIFDM2Inq0M/yRwtIohMvc6Jqeghvk1Id6PwJ59m
+L/Xkr4sCZD2sDc3jbp1cXQ56IGHeSGh2ca/KGy0Rke/9dWVodQe1KqAseoc=
+-----END CERTIFICATE-----

melodytalk/key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCuQNx4sMPQ1xvM
+nxhOSfaK6b1Kl3eAmAduHXLOrPN0nagyp5foxXwMiqDYNabGoBavEktmp859a+4Q
+8j6XMiYMQOAiQuB/OojMrhWxlPTKwg0xedT9Apl1+RXm1O1o5TJFq5hpsJosYM0g
+gWqzBN0KqbydOXE53uv7ndBPex/4/UTpz5IIBAPaiPRuls0FnV8C05pkyV2I+DUG
+mkUbiQ93vDmBSsid5Up0CxucZp//uFyjRtZfAtzPsqAZhz39KzPLT9SL9lD8WGKR
+Lvv5qq5oKFg7AZv3uaNTPMns6BcmslDExJROrAdeSE3R1cMxz4iOnNrCjvaeDZCy
+YhHCv95MD1zdKxSwDySQmcA0h2+pOnB1DGOiYH8PbD6wfOjtwT48xhME1m1w/buU
+VDCHJTLQykxf2GeQh7pWkKl1KR+/CE4lLVAS+AXitM4UqsutakVlhuCZCGVatILa
+R39UwiNaPEKxSu815ZW8FIeAKOVkMIQqFXiqSv3Q/u+50HQD1Wz7abXjt19iKsSz
+luIvvny5Zy+Lz+hiy9QU+aItWujC7osGDXgRFWw3ytDRl4GhWmOX+cN0f315GrCf
+vVEuaKE7pytg4rMSahQN3igDS4gQ/UdzmZJTEuFcwlsHUZYDzUxNVSLVbmmFZP7k
+ZIQgeo1bxM1s7ii98/fQA0fQIXX7nQIDAQABAoICAEQxvha9luS0SnzaBnkdGQ9k
+SmBWxr3cwYummrenl9+bk6e/TwQZxWJjHm9/D7XMERSS5E9YfMc45gJby4MAbUN+
+YDkIUzN3hPIVbwgAc4ymROOH+oLJ8aTpxliAHClhCTua3BsAslWLzLtb6pYE4Ty7
+EMW3JUiwpUnjiRKR7QFUnROj2FKqv5dW4dIl0jbQT86bYWDwLVrdSGM9HK6aBSOm
+1hpzWe/xPCIN9tBOmW9Kh3ae0mZ8iE4AageTlf7uWWbH5zafPzKCpJCNAq/CzF6W
+63CsQI3CaNi6gYvliKnsPw/Cnwfq6zdKtKnaLnnf7mFhoXti18EjLe2h07GTxTr5
+k6HB74hWtbYAxUCXflYGXnIy37O4nTTzTb/HRRaO9pS7UhUrrFsjTd1EigHVc++G
+IREMD5ivrF6JTM/mkhYqU4BormjHoBPQWAUie2BLKHQEsNZJWsTpWJ995qDXVg1Z
+Kg+8hpI7WPmLtP7/ZKwCpFONrleAZCqeNClCOEMgS6h84he+SUREIyp/KnTEEWBm
+S9/NdzWo9/xV3FOu1Zgzpgd9NSzYSUK3HlL5+WPz2KDCs6rjPvRTGzVRohYUMAqn
+GqbxBkCFYuCK0sq0c2mPtTxjwVrYJPLVARGpXpLGCAHldz3ENFFYGkgF8moRUC8I
+bVKkeWJ5vp1xs0JzRONBAoIBAQDcEQ5Gxb22L6tP15WpwRnC3TDdjNYuDl4EUz8n
+NIE6wE982K71SXAjGd8brWvVhgOJ/I82cFQ9QWiha5y5rssTrw4bX9blfTr59Qzv
+fbbfumBVObpXu/GeRbzjP66LpuhF8+aTnpK2MfKHeDCPbFb183JvvIJ6OhSV9wtA
+k8ZJI9AFwlpqmeRGwR9UrW75R9baP4K/M7Q2yqTABKKK73PebT/7Fem8IUpNKqDw
+1stf6+Nqbmysp8VBEscaxbvYF2H5G2khd9pwBzq2ZbA4DL2iJu4EVdsXXsdJtRSc
+6P4cymp00R659jalE42s3aRV5wMxqvK3M+HXRjkxUXuxi4ffAoIBAQDKtMcrroE3
+mUKmGm78/m964mRt5tyQLNg1RIZde0XWXuh53lfReYjsG9+EdP1K/HFSq+P8MK4q
+KZN8yF+q/VzBQtXzSacC9FbB6ormlAs8rntmn7vGDRS5UU7+THAYO+EfHf6Nkfel
+xpfWYH3ebUebLketYbaZtg0x4ww+GBUO7tnbnwIwfo8qDGBVKTcfiSmKa2xOR7wn
+ql0VHgGEqYxLs8f0tmARXdJ/ZUA4B/ZD9sfuJmwL6MqByTNgWtJSg3lPJbTiKFvL
+EHBS2E7aAiH6M2fJ9bw9Twz812OChw1PJGHdCLLCARD7NjyvFP9hO9KAAcm6RR6t
+DVhfxPDg6RwDAoIBAQCi+3rATYjwWiOdMUQlgd7rY9fh7YFegMN+nozCI2+mLHFy
+PCRvg2Eqjemf3PNlIyiR1FvYbJ7m8v9jtYhQOLUsMdTXz0YflnzMTIG1CT0cOzNH
+S0bH3WjX2DxJbYt+4DFcRYO6RpHPn/RIAmqsfgd1hkR1nn5fuY4vdtR9kjQjTQOu
+kD9YffsDr9ndY8kxk9P6tVjG9Kr+fVxA3e3nyendjLn64kLGUz1tIyoo1FuxAcne
+tEOGJ4a+aALPwCwboU2tKXDWGabMm34PQgadrEnxwrU6vr9Rcp9T5mmKwEAa4Iqx
+U4NXJJNZ9L5TjZ89A6do7Htv1VvojJik/v0GBBmvAoIBACmwyrICPxNXUyzkjOU9
+02HbxDoSXFTO1F0ITfZWD0QK2xeNklREAdRF20/b2+ibSRduwQIyZO/JA0uLzplV
+aMla3LQf9tiVtm5FtfG8ix8c58Zwneca86M1xOBqijeBwUhsG9m0JtISqVHvIJQp
+t6uD9jwDt9GfbtFO2ErBGzuqBma0Kngc34cPapS5WFwFEuUkOO9Us68Yy/I27KsD
+UBaV0E9VXybTPCL1iSK9CHpr/CzP/DLq2xOcdROhOla8MawEnlJRaKe4JtHzKSyP
+ebuC7RrNjryxUIOELTRAYI1hdztFubZLD9oLCqob0HaIHkAznfDQnIGNGBsCX2Ry
+rU8CggEAW5eDedi/ymmvANXkqB0gUJW0ktBZKg0Y8vD2dh4D6dhgI0wWTe9Hd4+V
+q2+3KHAlcY8+E2pV4vz03mkhisoEWXkD2pfkyxVSQr0HO9LBKYiZwF5haG00W0Ml
+5fXf2yxXDI+cLrf8PABU/2Aso4F8gOoY3u+fzyMeWfq7aDKRCNtXTisSoyH98FBg
+LJKFgHRZT9tivnI44/uHBpnLs1NSs+k3INg1sT/BbnwKBoGb2U3/DVSVelTfY0O1
+vbDLLK8oYA6bX4xB1oEDGl/C0anJTXzXeva+UA9lgJIlSFhTzwKmBFkjNNLPT7SY
+BqrD5IG4DWv0gGZQLtywUM2cMhWcwA==
+-----END PRIVATE KEY-----

melodytalk/main.py

@@ -250,7 +250,7 @@ def init_agent(self, lang):
             agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
                           'suffix': SUFFIX}, )
         return gr.update(visible=True), gr.update(visible=False), gr.update(placeholder=place), gr.update(
-            value=label_clear)
+            value=label_clear), gr.update(visible=True)
 
     def run_text(self, text, state):
         # LangChain has changed its implementation, so we are not able to cut the dialogue history anymore.
@@ -268,7 +268,9 @@ def run_text(self, text, state):
     def run_audio(self, file, state, txt, lang):
         music_filename = os.path.join('music', str(uuid.uuid4())[0:8] + ".wav")
         print("Inputs:", file, state)
-        audio_load, sr = torchaudio.load(file.name)
+        if not isinstance(file, str):  # recording pass a path, while button pass a file object
+            file = file.name
+        audio_load, sr = torchaudio.load(file)
         audio_write(music_filename[:-4], audio_load, sr, strategy="loudness", loudness_compressor=True)
         # description = self.models['ImageCaptioning'].inference(image_filename)
         if lang == 'Chinese':
@@ -286,6 +288,13 @@ def run_audio(self, file, state, txt, lang):
               f"Current Memory: {self.agent.memory.buffer}")
         return state, state
 
+    def run_recording(self, file_path, state, txt, lang):
+
+        return self.run_audio(file_path, state, txt, lang)
+
+    def clear_input_audio(self):
+        return gr.Audio.update(value=None)
+
 if __name__ == '__main__':
     if not os.path.exists("checkpoints"):
         os.mkdir("checkpoints")
@@ -306,14 +315,26 @@ def run_audio(self, file, state, txt, lang):
                 clear = gr.Button("Clear")
             with gr.Column(scale=0.15, min_width=0):
                 btn = gr.UploadButton("Upload",file_types=["audio"])
-            # with gr.Column(scale=0.15, min_width=0):
-            #     rec = gr.A("Record",source="microphone",file_types=["audio"])
 
-        lang.change(bot.init_agent, [lang], [input_raws, lang, txt, clear])
+        with gr.Row(visible=False) as record_raws:
+            with gr.Column(scale=0.7):
+                rec_audio = gr.Audio(source='microphone', type='filepath', interactive=True)
+            with gr.Column(scale=0.15, min_width=0):
+                rec_clear = gr.Button("Re-recording")
+            with gr.Column(scale=0.15, min_width=0):
+                rec_submit = gr.Button("Submit")
+
+        lang.change(bot.init_agent, [lang], [input_raws, lang, txt, clear, record_raws])
         txt.submit(bot.run_text, [txt, state], [chatbot, state])
         txt.submit(lambda: "", None, txt)
         btn.upload(bot.run_audio, [btn, state, txt, lang], [chatbot, state])
+
+        rec_submit.click(bot.run_recording, [rec_audio, state, txt, lang], [chatbot, state])
+        rec_clear.click(bot.clear_input_audio, None, rec_audio)
+
         clear.click(bot.memory.clear)
         clear.click(lambda: [], None, chatbot)
         clear.click(lambda: [], None, state)
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+        clear.click(bot.clear_input_audio, None, rec_audio)
+    demo.launch(server_name="0.0.0.0", server_port=7860,
+                ssl_certfile="cert.pem", ssl_keyfile="key.pem", ssl_verify=False)