feat: Added ChatGPT batch translation feature (Beta). resolved #319

.github/workflows/unit-testing.yml

@@ -27,4 +27,4 @@ jobs:
           sudo -v && wget -nv -O- https://download.calibre-ebook.com/linux-installer.sh | sudo sh /dev/stdin
       - name: Test with calibre-debug
         run: |
-          calibre-customize -b .; calibre-debug test.py
+          export CALIBRE_OVERRIDE_LANG=en; calibre-customize -b .; calibre-debug test.py

advanced.py

@@ -725,7 +725,6 @@ def start_batch_translation():
             translator = get_translator(self.current_engine)
             translator.set_source_lang(self.ebook.source_lang)
             translator.set_target_lang(self.ebook.target_lang)
-            translator.disable_stream()
             batch_translator = ChatgptBatchTranslate(translator)
             batch = ChatgptBatchTranslationManager(
                 batch_translator, self.cache, self.table, self)

components/chatgpt.py

@@ -4,6 +4,8 @@
 
 from ..lib.utils import traceback_error
 
+from .alert import AlertMessage
+
 
 try:
     from qt.core import (
@@ -17,6 +19,7 @@
         QThread, QHBoxLayout, QPlainTextEdit, QEvent)
 
 load_translations()
+
 log = Log()
 
 
@@ -25,7 +28,7 @@ def request(func):
     def wrapper(self):
         try:
             func(self)
-        except Exception as e:
+        except Exception:
             self.show_information.emit(
                 'Oops, an error occurred!', traceback_error())
             self.stack_index.emit(3)
@@ -105,8 +108,11 @@ def check_details(self):
     def cancel_batch(self):
         self.process_tip.emit(_('canceling...'))
         self.stack_index.emit(1)
-        self._batch_translator.cancel(self._batch_id)
-        self._batch_translator.delete(self._file_id)
+        self._batch_info = self._batch_translator.check(self._batch_id)
+        if self._batch_info.get('status') not in (
+                'cancelling', 'cancelled', 'completed', 'failed'):
+            self._batch_translator.cancel(self._batch_id)
+            self._batch_translator.delete(self._file_id)
         self.remove_batch.emit()
         self.finished.emit()
 
@@ -142,6 +148,8 @@ def __init__(self, translator, cache, table, parent=None):
         self.cache = cache
         self.table = table
 
+        self.alert = AlertMessage(self)
+
         self.batch_worker = ChatgptBatchTranslationWorker(translator)
         self.batch_worker.moveToThread(self.batch_thread)
         self.batch_thread.finished.connect(self.batch_worker.deleteLater)
@@ -267,31 +275,40 @@ def layout_buttons(self):
         self.batch_worker.enable_apply_button.connect(apply.setEnabled)
 
         refresh.clicked.connect(self.batch_worker.check.emit)
-        cancel.clicked.connect(self.batch_worker.cancel.emit)
         apply.clicked.connect(lambda: self.batch_worker.apply.emit())
 
+        def cancel_batch_translation():
+            action = self.alert.ask(
+                _('Are you sure you want to cancel the batch translation?'))
+            if action == 'yes':
+                self.batch_worker.cancel.emit()
+        cancel.clicked.connect(cancel_batch_translation)
+
         return widget
 
     def layout_data(self):
         status = QLabel()
-        total = QLabel()
-        completed = QLabel()
-        failed = QLabel()
+        detail = QPlainTextEdit()
+        detail.setReadOnly(True)
 
         def set_details_data(data):
-            status.setText(str(data.get('status') or 'n/a'))
-            request_counts = data.get('request_counts')
-            total.setText(str(request_counts.get('total') or 'n/a'))
-            completed.setText(str(request_counts.get('completed') or 'n/a'))
-            failed.setText(str(request_counts.get('failed') or 'n/a'))
+            detail.clear()
+            batch_status = data.get('status')
+            status.setText(str(batch_status))
+            if batch_status == 'completed':
+                request_counts = data.get('request_counts')
+                detail.appendPlainText(str(request_counts))
+            else:
+                error_info = data.get('errors')
+                detail.appendPlainText(str(error_info))
         self.batch_worker.trans_details.connect(set_details_data)
 
         widget = QGroupBox(_('Batch translation details'))
         layout = QFormLayout(widget)
         layout.addRow(_('Status'), status)
-        layout.addRow(_('Total'), total)
-        layout.addRow(_('Completed'), completed)
-        layout.addRow(_('Failed'), failed)
+        layout.addRow(_('Detail'), detail)
+
+        self.set_form_layout_policy(layout)
 
         return widget
 
@@ -318,3 +335,10 @@ def changeEvent(self, event):
     def done(self, reason):
         QDialog.done(self, reason)
         self.parent().raise_()
+
+    def set_form_layout_policy(self, layout):
+        field_policy = getattr(
+            QFormLayout.FieldGrowthPolicy, 'AllNonFixedFieldsGrow', None) \
+            or QFormLayout.AllNonFixedFieldsGrow
+        layout.setFieldGrowthPolicy(field_policy)
+        layout.setLabelAlignment(Qt.AlignRight)

engines/base.py

@@ -91,9 +91,6 @@ def api_key_error_message(cls):
         return _('A correct key format "{}" is required.') \
             .format(cls.api_key_hint)
 
-    def disable_stream(self):
-        self.stream = False
-
     def get_api_key(self):
         if self.need_api_key and self.api_keys:
             return self.api_keys.pop(0)

engines/openai.py

@@ -153,12 +153,12 @@ class ChatgptBatchTranslate:
 
     def __init__(self, translator):
         self.translator = translator
-        self.translator.stream = True
+        self.translator.stream = False
 
         domain_name = '://'.join(
             urlsplit(self.translator.endpoint, 'https')[:2])
         self.file_endpoint = '%s/v1/files' % domain_name
-        self.batch_endpont = '%s/v1/batches' % domain_name
+        self.batch_endpoint = '%s/v1/batches' % domain_name
 
     def _create_multipart_form_data(self, body):
         """https://www.rfc-editor.org/rfc/rfc2046#section-5.1"""
@@ -195,12 +195,12 @@ def upload(self, paragraphs):
                 .format(self.translator.model))
         body = io.StringIO()
         for paragraph in paragraphs:
+            data = self.translator.get_body(paragraph.original)
             body.write(json.dumps({
                 "custom_id": paragraph.md5,
                 "method": "POST",
                 "url": "/v1/chat/completions",
-                "body": self.translator.get_body(paragraph.original)
-            }))
+                "body": json.loads(data)}))
             if paragraph != paragraphs[-1]:
                 body.write('\n')
         content_type = 'multipart/form-data; boundary="%s"' % self.boundary
@@ -231,8 +231,8 @@ def retrieve(self, output_file_id):
         headers = self.translator.get_headers()
         del headers['Content-Type']
         response = request(
-            '%s/%s/content' % (self.batch_endpont, output_file_id),
-            headers=headers)
+            '%s/%s/content' % (self.file_endpoint, output_file_id),
+            headers=headers, as_bytes=True)
 
         translations = {}
         for line in io.BytesIO(response):
@@ -251,14 +251,28 @@ def create(self, file_id):
         body = json.dumps({
             'input_file_id': file_id,
             'endpoint': '/v1/chat/completions',
-            'completion_window': '24h',
-        })
-        response = request(self.batch_endpont, body, headers, 'POST')
+            'completion_window': '24h'})
+        response = request(self.batch_endpoint, body, headers, 'POST')
         return json.loads(response).get('id')
 
     def check(self, batch_id):
         # time.sleep(2)
         # return {
+        #     'status': 'failed',
+        #     'output_file_id': 'xxxx',
+        #     'errors': {
+        #         'object': 'list',
+        #         'data': [
+        #             {
+        #                 'code': 'error-code',
+        #                 'message': 'error-message',
+        #                 'param': 'error-param',
+        #                 'line': 'error-line',
+        #             }
+        #         ]
+        #     },
+        # }
+        # return {
         #     'status': 'completed',
         #     'output_file_id': 'xxxx',
         #     'request_counts': {
@@ -269,7 +283,7 @@ def check(self, batch_id):
         # }
 
         response = request(
-            '%s/%s' % (self.batch_endpont, batch_id),
+            '%s/%s' % (self.batch_endpoint, batch_id),
             headers=self.translator.get_headers())
         return json.loads(response)
 
@@ -279,6 +293,7 @@ def cancel(self, batch_id):
 
         headers = self.translator.get_headers()
         response = request(
-            '%s/%s/cancel' % (self.batch_endpont, batch_id),
+            '%s/%s/cancel' % (self.batch_endpoint, batch_id),
             headers=headers, method='POST')
-        return json.loads(response).get('status') == 'cancelling'
+        return json.loads(response).get('status') in (
+            'cancelling', 'cancelled')

lib/utils.py

@@ -145,7 +145,7 @@ def traceback_error():
 
 def request(
         url, data=None, headers={}, method='GET', timeout=30, proxy_uri=None,
-        stream=False):
+        as_bytes=False, stream=False):
     br = Browser()
     br.set_handle_robots(False)
     # Do not verify SSL certificates
@@ -171,4 +171,6 @@ def request(
     response = br.response()
     if stream:
         return response
+    if as_bytes:
+        return response.read()
     return response.read().decode('utf-8').strip()

tests/test_convertion.py

@@ -6,8 +6,6 @@
 from ..lib.ebook import Ebooks
 
 
-load_translations()
-
 module_name = 'calibre_plugins.ebook_translator.lib.conversion'
 
 
@@ -37,7 +35,7 @@ def test_translate_done_job_failed_not_debug(self):
         with patch(module_name + '.DEBUG', False):
             self.worker.translate_done(self.job)
             self.gui.job_exception.assert_called_once_with(
-                self.job, dialog_title=_('Translation job failed'))
+                self.job, dialog_title='Translation job failed')
 
     @patch(module_name + '.os')
     @patch(module_name + '.open')
@@ -95,17 +93,16 @@ def test_translate_done_ebook_to_library(
         self.worker.api.format_abspath.assert_called_once_with(89, 'epub')
 
         self.worker.gui.status_bar.show_message.assert_called_once_with(
-            'test description ' + _('completed'), 5000)
+            'test description completed', 5000)
         arguments = self.worker.gui.proceed_question.mock_calls[0].args
         self.assertIsInstance(arguments[0], Callable)
         self.assertIs(self.worker.gui.job_manager.launch_gui_app, arguments[1])
         self.assertEqual('/path/to/log', arguments[2])
-        self.assertEqual(_('Ebook Translation Log'), arguments[3])
-        self.assertEqual(_('Translation Completed'), arguments[4])
-        self.assertEqual(_(
-            'The translation of "{}" was completed. '
-            'Do you want to open the book?')
-            .format('test custom title [German]'),
+        self.assertEqual('Ebook Translation Log', arguments[3])
+        self.assertEqual('Translation Completed', arguments[4])
+        self.assertEqual(
+            'The translation of "test custom title [German]" was completed. '
+            'Do you want to open the book?',
             arguments[5])
 
         mock_payload = Mock()
@@ -161,17 +158,16 @@ def test_translate_done_ebook_to_path(
             '/path/to/test.epub',
             '/path/to/test_ custom title_ [German].epub')
         self.worker.gui.status_bar.show_message.assert_called_once_with(
-            'test description ' + _('completed'), 5000)
+            'test description ' + 'completed', 5000)
         arguments = self.worker.gui.proceed_question.mock_calls[0].args
         self.assertIsInstance(arguments[0], Callable)
         self.assertIs(self.worker.gui.job_manager.launch_gui_app, arguments[1])
         self.assertEqual('/path/to/log', arguments[2])
-        self.assertEqual(_('Ebook Translation Log'), arguments[3])
-        self.assertEqual(_('Translation Completed'), arguments[4])
-        self.assertEqual(_(
-            'The translation of "{}" was completed. '
-            'Do you want to open the book?')
-            .format('test: custom title* [German]'),
+        self.assertEqual('Ebook Translation Log', arguments[3])
+        self.assertEqual('Translation Completed', arguments[4])
+        self.assertEqual(
+            'The translation of "test: custom title* [German]" was completed. '
+            'Do you want to open the book?',
             arguments[5])
 
         mock_payload = Mock()
@@ -225,19 +221,18 @@ def test_translate_done_other_to_library(
             .assert_called_once_with(1)
         self.worker.api.format_abspath.assert_called_once_with(90, 'srt')
         self.worker.gui.status_bar.show_message.assert_called_once_with(
-            'test description ' + _('completed'), 5000)
+            'test description ' + 'completed', 5000)
         self.assertEqual('test custom title [German]', metadata.title)
 
         arguments = self.worker.gui.proceed_question.mock_calls[0].args
         self.assertIsInstance(arguments[0], Callable)
         self.assertIs(self.worker.gui.job_manager.launch_gui_app, arguments[1])
         self.assertEqual('C:\\path\\to\\log', arguments[2])
-        self.assertEqual(_('Ebook Translation Log'), arguments[3])
-        self.assertEqual(_('Translation Completed'), arguments[4])
-        self.assertEqual(_(
-            'The translation of "{}" was completed. '
-            'Do you want to open the book?')
-            .format('test custom title [German]'),
+        self.assertEqual('Ebook Translation Log', arguments[3])
+        self.assertEqual('Translation Completed', arguments[4])
+        self.assertEqual(
+            'The translation of "test custom title [German]" was completed. '
+            'Do you want to open the book?',
             arguments[5])
 
         mock_payload = Mock()
@@ -281,17 +276,16 @@ def test_translate_done_other_to_path(
             '/path/to/test.srt',
             '/path/to/test_ custom title_ [German].srt')
         self.worker.gui.status_bar.show_message.assert_called_once_with(
-            'test description ' + _('completed'), 5000)
+            'test description ' + 'completed', 5000)
         arguments = self.worker.gui.proceed_question.mock_calls[0].args
         self.assertIsInstance(arguments[0], Callable)
         self.assertIs(self.worker.gui.job_manager.launch_gui_app, arguments[1])
         self.assertEqual('/path/to/log', arguments[2])
-        self.assertEqual(_('Ebook Translation Log'), arguments[3])
-        self.assertEqual(_('Translation Completed'), arguments[4])
-        self.assertEqual(_(
-            'The translation of "{}" was completed. '
-            'Do you want to open the book?')
-            .format('test: custom title* [German]'),
+        self.assertEqual('Ebook Translation Log', arguments[3])
+        self.assertEqual('Translation Completed', arguments[4])
+        self.assertEqual(
+            'The translation of "test: custom title* [German]" was completed. '
+            'Do you want to open the book?',
             arguments[5])
 
         mock_payload = Mock()