From dd32516d8bc56d0a655a9563329b64066ca23906 Mon Sep 17 00:00:00 2001 From: Fang Yuan Date: Tue, 24 Dec 2024 19:12:29 +0800 Subject: [PATCH] feat: Add new book sources and remove invalid ones! Signed-off-by: Fang Yuan --- frontend/src/components/ViewConfig.tsx | 1 - internal/config/config.go | 58 ++++++++++++-- internal/config/default_config.yaml | 2 +- internal/source/rule/rule1.json | 27 +++---- internal/source/rule/rule2.json | 93 ++++++++++------------- internal/source/rule/rule3.json | 46 ----------- internal/tools/concurrency/concurrency.go | 2 +- 7 files changed, 108 insertions(+), 121 deletions(-) delete mode 100644 internal/source/rule/rule3.json diff --git a/frontend/src/components/ViewConfig.tsx b/frontend/src/components/ViewConfig.tsx index 28b66c2..06a3306 100644 --- a/frontend/src/components/ViewConfig.tsx +++ b/frontend/src/components/ViewConfig.tsx @@ -109,7 +109,6 @@ const ViewConfig: React.FC = () => { options={[ { value: 1, label: '1' }, { value: 2, label: '2' }, - { value: 3, label: '3' } ]} style={{ width: '100%' }} /> diff --git a/internal/config/config.go b/internal/config/config.go index e7efc33..bacee09 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -106,30 +106,72 @@ func GetConf() Info { func SetConf(conf string) error { var newConf Info - // Parsing a JSON string + currentConf := GetConf() + + // Parse the input JSON string if err := json.Unmarshal([]byte(conf), &newConf); err != nil { return fmt.Errorf("failed to unmarshal config: %w", err) } - // Updating the in-memory configuration - confValue.Store(newConf) + // Compare and update configuration + updated := false + + // Update Base fields + if newConf.Base.SourceID != 0 && newConf.Base.SourceID != currentConf.Base.SourceID { + currentConf.Base.SourceID = newConf.Base.SourceID + updated = true + } + if newConf.Base.DownloadPath != "" && + newConf.Base.DownloadPath != currentConf.Base.DownloadPath { + currentConf.Base.DownloadPath = newConf.Base.DownloadPath + updated = true + } + if newConf.Base.Extname != "" && newConf.Base.Extname != currentConf.Base.Extname { + currentConf.Base.Extname = newConf.Base.Extname + updated = true + } + if newConf.Base.LogLevel != "" && newConf.Base.LogLevel != currentConf.Base.LogLevel { + currentConf.Base.LogLevel = newConf.Base.LogLevel + updated = true + } + + // Update Crawl fields + if newConf.Crawl.Threads != 0 && newConf.Crawl.Threads != currentConf.Crawl.Threads { + currentConf.Crawl.Threads = newConf.Crawl.Threads + updated = true + } + + // Update Retry fields + if newConf.Retry.MaxAttempts != 0 && + newConf.Retry.MaxAttempts != currentConf.Retry.MaxAttempts { + currentConf.Retry.MaxAttempts = newConf.Retry.MaxAttempts + updated = true + } + + // If no updates, return early + if !updated { + return nil + } + + // Update the in-memory configuration + confValue.Store(currentConf) - // Write the new configuration to a user-defined configuration file + // Write the new configuration to the user-defined configuration file expandedPath := os.ExpandEnv(customConfigPath) - // Make sure the catalog exists + // Ensure the directory exists dir := filepath.Dir(expandedPath) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create config directory: %w", err) } - // Converting Configuration to JSON - jsonData, err := json.MarshalIndent(newConf, "", " ") + // Convert configuration to JSON + jsonData, err := json.MarshalIndent(currentConf, "", " ") if err != nil { return fmt.Errorf("failed to marshal config to JSON: %w", err) } - // write to a file + // Write to file if err := os.WriteFile(expandedPath, jsonData, 0644); err != nil { return fmt.Errorf("failed to write config file: %w", err) } diff --git a/internal/config/default_config.yaml b/internal/config/default_config.yaml index a80ac2e..c352d43 100644 --- a/internal/config/default_config.yaml +++ b/internal/config/default_config.yaml @@ -1,5 +1,5 @@ base: - # 启用书源 (只能选一个, 当前可选值：1、2、3) + # 启用书源 (只能选一个, 当前可选值：1、2) source-id: 3 # 下载路径, 绝对相对均可 (Windows 路径分隔符不要用 \ , 用 / 或 \) download-path: "downloads" diff --git a/internal/source/rule/rule1.json b/internal/source/rule/rule1.json index f45ca0f..3f93072 100644 --- a/internal/source/rule/rule1.json +++ b/internal/source/rule/rule1.json @@ -1,25 +1,26 @@ { "id": "1", - "url": "http://www.xbiqugu.net", - "name": "香书小说", + "url": "http://www.mcmssc.la/", + "name": "梦书中文", "comment": "无限流", "type": "html", "search": { - "url": "http://www.xbiqugu.net/modules/article/waps.php", + "url": "http://www.mcmssc.la/search.html", "method": "post", "body": { - "kw": "searchkey" + "kw": "name" }, "cookies": {}, "pagination": false, - "result": "#checkform > table > tbody > tr", - "bookName": "td.even > a", - "author": "td:nth-of-type(3)", - "latestChapter": "td.odd > a", - "update": "td:nth-of-type(4)" + "nextPage": "", + "result": ".novelslist2 > ul > li", + "bookName": "span.s2.wid > a", + "author": "span.s4.wid > a", + "latestChapter": "span.s3.wid3 > a", + "update": "span.s6.wid6" }, "book": { - "url": "http://www.xbiqugu.net/0/%s/", + "url": "http://www.mcmssc.la/%s/", "bookName": "meta[property=\"og:novel:book_name\"]", "author": "meta[property=\"og:novel:author\"]", "intro": "meta[property=\"og:description\"]", @@ -29,17 +30,17 @@ "latestUpdate": "", "isEnd": "", "catalog": "#list > dl > dd > a", - "catalogOffset": 0 + "catalogOffset": 12 }, "chapter": { - "url": "http://www.xbiqugu.net/0/%s/%s.html", + "url": "http://www.mcmssc.la/%s/%s.html", "pagination": false, "chapterNo": 0, "title": "", "content": "#content", "paragraphTagClosed": false, "paragraphTag": "

", - "filterTxt": "一秒记住【文学巴士 】，精彩无弹窗免费阅读！|(www.xbiquge.la 新笔趣阁)，高速全文字在线阅读！", + "filterTxt": "天才一秒记住本站地址：[梦书中文] .+最快更新！无广告！|(www.xbiquge.la 新笔趣阁)，高速全文字在线阅读！", "filterTag": "div p script" } } \ No newline at end of file diff --git a/internal/source/rule/rule2.json b/internal/source/rule/rule2.json index 2a2c50c..0959d86 100644 --- a/internal/source/rule/rule2.json +++ b/internal/source/rule/rule2.json @@ -1,51 +1,42 @@ -{ - "id": "2", - "url": "https://www.shuhaige.net/", - "name": "书海阁小说网", - "comment": "搜索有反爬机制，爬取过快会丢包", - "type": "html", - "search": { - "url": "https://www.shuhaige.net/search.html", - "method": "post", - "body": { - "kw": "searchkey", - "searchtype": "all" - }, - "cookies": { - "waf_sc": "", - "HMACCOUNT": "" - }, - "pagination": true, - "nextPage": "#pagelink > a:not(.pgroup):not(.ngroup)", - "result": "#sitembox > dl", - "bookName": "dd > h3 > a", - "author": "dd:nth-child(3) > span:nth-child(1)", - "latestChapter": "dd:nth-child(5) > a", - "update": "dd:nth-child(5) > span" - }, - "book": { - "url": "https://www.shuhaige.net/%s/", - "bookName": "meta[property=\"og:novel:book_name\"]", - "author": "meta[property=\"og:novel:author\"]", - "intro": "meta[property=\"og:description\"]", - "category": "meta[property=\"og:novel:category\"]", - "coverUrl": "#fmimg > img", - "latestChapter": "", - "latestUpdate": "", - "isEnd": "", - "catalog": "#list > dl > dd > a", - "catalogOffset": 12 - }, - "chapter": { - "url": "https://www.shuhaige.net/%s/%s.html", - "pagination": true, - "nextPage": "#pager_next", - "chapterNo": 0, - "title": ".bookname > h1", - "content": "#content", - "paragraphTagClosed": true, - "paragraphTag": "p", - "filterTxt": "本小章还未完，请点击下一页继续阅读后面精彩内容！|小主，这个章节后面还有哦，请点击下一页继续阅读，后面更精彩！|这章没有结束，请点击下一页继续阅读！|喜欢.*?请大家收藏：(.*?).*?书海阁小说网更新速度全网最快。", - "filterTag": "" - } -} \ No newline at end of file +{ + "id": "2", + "url": "http://www.99xs.info/", + "name": "鸟书网", + "comment": "书很全，有15w+；无反爬", + "type": "html", + "search": { + "url": "http://www.99xs.info/read/search/", + "method": "post", + "body": {"kw": "searchkey"}, + "cookies": {}, + "pagination": false, + "result": "div.wrap > div > div > div", + "bookName": "div.bookinfo > h4 > a", + "author": "div.bookinfo > div.author", + "latestChapter": "div.bookinfo > div.update > a" + }, + "book": { + "url": "http://www.99xs.info/tag/%s/", + "bookName": "meta[property=\"og:novel:book_name\"]", + "author": "meta[property=\"og:novel:author\"]", + "intro": "meta[property=\"og:description\"]", + "category": "meta[property=\"og:novel:category\"]", + "coverUrl": "div.cover > img", + "latestChapter": "", + "latestUpdate": "", + "isEnd": "", + "catalog": "div.listmain > dl > dd > a", + "catalogOffset": 12 + }, + "chapter": { + "url": "http://www.99xs.info/tag/%s/%s.html", + "pagination": false, + "chapterNo": 0, + "title": "", + "content": "#content", + "paragraphTagClosed": false, + "paragraphTag": "

", + "filterTxt": "请记住本书首发域名：.+。鸟书网手机版阅读网址：.+|7017k", + "filterTag": "" + } + } \ No newline at end of file diff --git a/internal/source/rule/rule3.json b/internal/source/rule/rule3.json deleted file mode 100644 index 736810d..0000000 --- a/internal/source/rule/rule3.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "3", - "url": "http://www.mcmssc.la/", - "name": "梦书中文", - "comment": "无限流", - "type": "html", - "search": { - "url": "http://www.mcmssc.la/search.html", - "method": "post", - "body": { - "kw": "name" - }, - "cookies": {}, - "pagination": false, - "nextPage": "", - "result": ".novelslist2 > ul > li", - "bookName": "span.s2.wid > a", - "author": "span.s4.wid > a", - "latestChapter": "span.s3.wid3 > a", - "update": "span.s6.wid6" - }, - "book": { - "url": "http://www.mcmssc.la/%s/", - "bookName": "meta[property=\"og:novel:book_name\"]", - "author": "meta[property=\"og:novel:author\"]", - "intro": "meta[property=\"og:description\"]", - "category": "meta[property=\"og:novel:category\"]", - "coverUrl": "#fmimg > img", - "latestChapter": "", - "latestUpdate": "", - "isEnd": "", - "catalog": "#list > dl > dd > a", - "catalogOffset": 12 - }, - "chapter": { - "url": "http://www.mcmssc.la/%s/%s.html", - "pagination": false, - "chapterNo": 0, - "title": "", - "content": "#content", - "paragraphTagClosed": false, - "paragraphTag": "

", - "filterTxt": "天才一秒记住本站地址：[梦书中文] .+最快更新！无广告！|(www.xbiquge.la 新笔趣阁)，高速全文字在线阅读！", - "filterTag": "div p script" - } -} \ No newline at end of file diff --git a/internal/tools/concurrency/concurrency.go b/internal/tools/concurrency/concurrency.go index a6c31e5..66be5bb 100644 --- a/internal/tools/concurrency/concurrency.go +++ b/internal/tools/concurrency/concurrency.go @@ -16,7 +16,7 @@ func GetConcurrencyNum(target int) int { once.Do(func() { defaultConcurrency = runtime.NumCPU() }) - if target == -1 { + if target == -1 || target == 0 { target = defaultConcurrency * 2 } if target >= maxThreads {