Merge branch 'Byaidu:main' into main

README.md

@@ -23,6 +23,8 @@ English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md)
     <img src="https://img.shields.io/badge/ModelScope-Demo-blue"></a>
   <a href="https://github.com/Byaidu/PDFMathTranslate/pulls">
     <img src="https://img.shields.io/badge/contributions-welcome-green"></a>
+  <a href="https://gitcode.com/Byaidu/PDFMathTranslate/overview">
+    <img src="https://gitcode.com/Byaidu/PDFMathTranslate/star/badge.svg"></a>
   <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
     <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"></a>
 </p>
@@ -172,7 +174,8 @@ In the following table, we list all advanced options for reference:
 | `-o`  | Output dir | `pdf2zh example.pdf -o output` |
 | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` |
 | `--share` | [Get gradio public link] | `pdf2zh -i --share` |
-| `-a` | [add authorization and custom login page] | `pdf2zh -i -a users.txt [auth.html]` |
+| `--authorized` | [add authorization and custom login page] | `pdf2zh -i --authorized users.txt [auth.html]` |
+| `--prompt` | [custom llm prompt] | `pdf2zh --prompt [prompt.txt]` |
 
 <h3 id="partial">Full / partial document translation</h3>
 
@@ -252,7 +255,34 @@ Use `-t` to specify how many threads to use in translation:
 ```bash
 pdf2zh example.pdf -t 1
 ```
+<h3 id="prompt">custom prompt</h3>
+Use `--prompt` to specify which prompt to use in llm:
+```bash
+pdf2zh example.pdf -pr prompt.txt
+```
+
+
+example prompt.txt
+```
+[
+    {
+        "role": "system",
+        "content": "You are a professional,authentic machine translation engine.",
+    },
+    {
+        "role": "user",
+        "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:",
+    },
+]
+```
+
 
+In custom prompt file, there are three variables can be used.
+|**variables**|**comment**|
+|-|-|
+|`lang_in`|input language|
+|`lang_out`|output language|
+|`text`|text need to be translated|
 <h2 id="todo">API</h2>
 
 ### Python

README_ja-JP.md

@@ -23,6 +23,8 @@
     <img src="https://img.shields.io/badge/ModelScope-Demo-blue"></a>
   <a href="https://github.com/Byaidu/PDFMathTranslate/pulls">
     <img src="https://img.shields.io/badge/contributions-welcome-green"/></a>
+  <a href="https://gitcode.com/Byaidu/PDFMathTranslate/overview">
+    <img src="https://gitcode.com/Byaidu/PDFMathTranslate/star/badge.svg"></a>
   <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
     <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"/></a>
 </p>
@@ -173,7 +175,8 @@ Python環境を事前にインストールする必要はありません
 | `-o`  | 出力ディレクトリ | `pdf2zh example.pdf -o output` |
 | `-f`, `-c` | [例外](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` |
 | `--share` | [gradio公開リンクを取得] | `pdf2zh -i --share` |
-| `-a` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i -a users.txt [auth.html]` |
+| `--authorized` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i --authorized users.txt [auth.html]` |
+| `--prompt` | [カスタムビッグモデルのプロンプトを使用する] | `pdf2zh --prompt [prompt.txt]` |
 
 <h3 id="partial">全文または部分的なドキュメント翻訳</h3>
 
@@ -254,6 +257,35 @@ pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|
 pdf2zh example.pdf -t 1
 ```
 
+<h3 id="prompt">custom prompt</h3>
+(need Japenese translation)
+Use `--prompt` to specify which prompt to use in llm:
+```bash
+pdf2zh example.pdf -pr prompt.txt
+```
+
+
+example prompt.txt
+```
+[
+    {
+        "role": "system",
+        "content": "You are a professional,authentic machine translation engine.",
+    },
+    {
+        "role": "user",
+        "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:",
+    },
+]
+```
+
+
+In custom prompt file, there are three variables can be used.
+|**variables**|**comment**|
+|-|-|
+|`lang_in`|input language|
+|`lang_out`|output language|
+|`text`|text need to be translated|
 <h2 id="todo">API</h2>
 
 ### Python

README_zh-CN.md

@@ -23,6 +23,8 @@
     <img src="https://img.shields.io/badge/ModelScope-Demo-blue"></a>
   <a href="https://github.com/Byaidu/PDFMathTranslate/pulls">
     <img src="https://img.shields.io/badge/contributions-welcome-green"/></a>
+  <a href="https://gitcode.com/Byaidu/PDFMathTranslate/overview">
+    <img src="https://gitcode.com/Byaidu/PDFMathTranslate/star/badge.svg"></a>
   <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
     <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"/></a>
 </p>
@@ -172,7 +174,8 @@ USE_MODELSCOPE=1 pdf2zh
 | `-o`  | 输出目录 | `pdf2zh example.pdf -o output` |
 | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` |
 | `--share` | [获取 gradio 公开链接] | `pdf2zh -i --share` |
-| `-a` | [添加网页认证和自定义认证页] | `pdf2zh -i -a users.txt [auth.html]` |
+| `--authorized` | [添加网页认证和自定义认证页] | `pdf2zh -i --authorized users.txt [auth.html]` |
+| `--prompt` | [使用自定义的大模型prompt] | `pdf2zh --prompt [prompt.txt]` |
 
 <h3 id="partial">全文或部分文档翻译</h3>
 
@@ -252,6 +255,37 @@ pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|
 ```bash
 pdf2zh example.pdf -t 1
 ```
+<h3 id="prompt">自定义大模型prompt</h3>
+
+使用 `--prompt` 指定使用大模型翻译时使用的 Prompt 文件。
+
+```bash
+pdf2zh example.pdf -pr prompt.txt
+```
+
+
+示例 `prompt.txt` 文件
+
+```
+[
+    {
+        "role": "system",
+        "content": "You are a professional,authentic machine translation engine.",
+    },
+    {
+        "role": "user",
+        "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:",
+    },
+]
+```
+
+
+自定义 Prompt 文件中，可以使用三个内置变量用来传递参数。
+|**变量名**|**说明**|
+|-|-|
+|`lang_in`|输入的语言|
+|`lang_out`|输出的语言|
+|`text`|需要翻译的文本|
 
 <h2 id="todo">API</h2>
 

pdf2zh/converter.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, List
 
 from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager
 from pdfminer.pdffont import PDFCIDFont
@@ -136,6 +136,7 @@ def __init__(
         resfont: str = "",
         noto: Font = None,
         envs: Dict = None,
+        prompt: List = None,
     ) -> None:
         super().__init__(rsrcmgr)
         self.vfont = vfont
@@ -151,7 +152,7 @@ def __init__(
         for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator,
                            OpenAITranslator, ZhipuTranslator, ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]:
             if service_name == translator.name:
-                self.translator = translator(lang_in, lang_out, service_model, envs=envs)
+                self.translator = translator(lang_in, lang_out, service_model, envs=envs, prompt=prompt)
         if not self.translator:
             raise ValueError("Unsupported translation service")
 

pdf2zh/high_level.py

@@ -103,6 +103,7 @@ def translate_patch(
         resfont,
         noto,
         kwarg.get("envs", {}),
+        kwarg.get("prompt", []),
     )
 
     assert device is not None
@@ -229,7 +230,7 @@ def translate_stream(
 
     fp = io.BytesIO()
     doc_zh.save(fp)
-    obj_patch: dict = translate_patch(fp, envs=kwarg["envs"], **locals())
+    obj_patch: dict = translate_patch(fp, prompt=kwarg["prompt"], **locals())
 
     for obj_id, ops_new in obj_patch.items():
         # ops_old=doc_en.xref_stream(obj_id)
@@ -295,7 +296,9 @@ def translate(
 
         doc_raw = open(file, "rb")
         s_raw = doc_raw.read()
-        s_mono, s_dual = translate_stream(s_raw, envs=kwarg.get('envs'), **locals())
+        s_mono, s_dual = translate_stream(
+            s_raw, envs=kwarg.get("envs"), prompt=kwarg["prompt"], **locals()
+        )
         file_mono = Path(output) / f"{filename}-mono.pdf"
         file_dual = Path(output) / f"{filename}-dual.pdf"
         doc_mono = open(file_mono, "wb")

pdf2zh/pdf2zh.py

@@ -11,6 +11,7 @@
 from typing import List, Optional
 from pdf2zh import __version__, log
 from pdf2zh.high_level import translate
+from string import Template
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -117,12 +118,15 @@ def create_parser() -> argparse.ArgumentParser:
     )
     parse_params.add_argument(
         "--authorized",
-        "-a",
         type=str,
         nargs="+",
-        default=["./users.txt", "./auth.html"],
         help="user name and password.",
     )
+    parse_params.add_argument(
+        "--prompt",
+        type=str,
+        help="user custom prompt.",
+    )
 
     return parser
 
@@ -169,6 +173,14 @@ def main(args: Optional[List[str]] = None) -> int:
         celery_app.start(argv=sys.argv[2:])
         return 0
 
+    if parsed_args.prompt:
+        try:
+            with open(parsed_args.prompt, "r", encoding="utf-8") as file:
+                content = file.read()
+            parsed_args.prompt = Template(content)
+        except Exception:
+            raise ValueError("prompt error.")
+
     translate(**vars(parsed_args))
     return 0