diff --git a/.gitignore b/.gitignore
index 620b7a8..a9f3b61 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+.DS_Store
+
.idea/
.vscode/
bin/
@@ -6,3 +8,4 @@ config/dev.yaml
go.sum
dist/
+
diff --git a/example/main.go b/example/main.go
index 2720a8a..86ea7c9 100644
--- a/example/main.go
+++ b/example/main.go
@@ -3,6 +3,10 @@ package main
import (
"encoding/json"
"fmt"
+ "log"
+ "strings"
+
+ md "github.com/JohannesKaufmann/html-to-markdown"
)
type Node struct {
@@ -18,7 +22,10 @@ func main() {
{"type":"ul","children":[{"type":"li","children":[{"type":"lic","listStyleType":"","indent":0,"children":[{"text":"编辑: 鹿沐"}]}]},
{"type":"li","children":[{"type":"lic","listStyleType":"","indent":0,"children":[{"text":"订阅新闻: "},{"type":"a","url":"http://tinyletter.com/gocn","target":"_blank","children":[{"text":"http://tinyletter.com/gocn"}]},{"text":""}]}]},
{"type":"li","children":[{"type":"lic","listStyleType":"","indent":0,"children":[{"text":"招聘专区: "},{"type":"a","url":"https://gocn.vip/jobs","target":"_blank","children":[{"text":"https://gocn.vip/jobs"}]},{"text":""}]}]}]}]`
- //data := `[{"type":"ol","children":[{"type":"li","children":[{"children":[{"text":"了解 Go 中的指针的一页 "},{"type":"a","url":"https://medium.com/@Lekia/a-one-pager-to-understanding-pointers-in-go-ad6cbfac3afc","children":[{"text":"https://medium.com/@Lekia/a-one-pager-to-understanding-pointers-in-go-ad6cbfac3afc"}]},{"text":""}],"type":"lic"}]},{"type":"li","children":[{"type":"lic","children":[{"text":"GO-select 的实现原理 "},{"type":"a","url":"https://juejin.cn/post/7201423410168741946","children":[{"text":"https://juejin.cn/post/7201423410168741946"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Golang：使用同步包将性能提高 10 倍并减少内存占用 "},{"type":"a","url":"https://medium.com/@aryehlevklein/golang-using-sync-package-to-10x-performance-and-reduce-memory-footprint-a1ed4ee14931","children":[{"text":"https://medium.com/@aryehlevklein/golang-using-sync-package-to-10x-performance-and-reduce-memory-footprint-a1ed4ee14931"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"云原生系列Go语言篇-错误处理 "},{"type":"a","url":"https://juejin.cn/post/7201509055713427513","children":[{"text":"https://juejin.cn/post/7201509055713427513"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"终极 2023 Web 服务器基准测试：NodeJS vs Java vs Rust vs Go "},{"type":"a","url":"https://medium.com/@alexeynovikov_89393/ultimate-2023-web-server-benchmark-nodejs-vs-java-vs-rust-vs-go-e367d932f699","children":[{"text":"https://medium.com/@alexeynovikov_89393/ultimate-2023-web-server-benchmark-nodejs-vs-java-vs-rust-vs-go-e367d932f699"}]},{"text":"","strikethrough":true}]}]}]},{"type":"p","children":[{"text":""}]},{"type":"ul","children":[{"type":"li","children":[{"type":"lic","children":[{"text":"编辑: zsr228"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"订阅新闻: http://tinyletter.com/gocn"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"招聘专区: https://gocn.vip/jobs"}]}]}]}]`
+ //data = `[{"type":"ol","children":[{"type":"li","children":[{"children":[{"text":"了解 Go 中的指针的一页 "},{"type":"a","url":"https://medium.com/@Lekia/a-one-pager-to-understanding-pointers-in-go-ad6cbfac3afc","children":[{"text":"https://medium.com/@Lekia/a-one-pager-to-understanding-pointers-in-go-ad6cbfac3afc"}]},{"text":""}],"type":"lic"}]},{"type":"li","children":[{"type":"lic","children":[{"text":"GO-select 的实现原理 "},{"type":"a","url":"https://juejin.cn/post/7201423410168741946","children":[{"text":"https://juejin.cn/post/7201423410168741946"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Golang：使用同步包将性能提高 10 倍并减少内存占用 "},{"type":"a","url":"https://medium.com/@aryehlevklein/golang-using-sync-package-to-10x-performance-and-reduce-memory-footprint-a1ed4ee14931","children":[{"text":"https://medium.com/@aryehlevklein/golang-using-sync-package-to-10x-performance-and-reduce-memory-footprint-a1ed4ee14931"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"云原生系列Go语言篇-错误处理 "},{"type":"a","url":"https://juejin.cn/post/7201509055713427513","children":[{"text":"https://juejin.cn/post/7201509055713427513"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"终极 2023 Web 服务器基准测试：NodeJS vs Java vs Rust vs Go "},{"type":"a","url":"https://medium.com/@alexeynovikov_89393/ultimate-2023-web-server-benchmark-nodejs-vs-java-vs-rust-vs-go-e367d932f699","children":[{"text":"https://medium.com/@alexeynovikov_89393/ultimate-2023-web-server-benchmark-nodejs-vs-java-vs-rust-vs-go-e367d932f699"}]},{"text":"","strikethrough":true}]}]}]},{"type":"p","children":[{"text":""}]},{"type":"ul","children":[{"type":"li","children":[{"type":"lic","children":[{"text":"编辑: zsr228"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"订阅新闻: http://tinyletter.com/gocn"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"招聘专区: https://gocn.vip/jobs"}]}]}]}]`
+ //data = `[{"type":"ol","children":[{"type":"li","children":[{"children":[{"text":"ServiceWeaver：一个编写分布式应用程序的框架 https://opensource.googleblog.com/2023/03/introducing-service-weaver-framework-for-writing-distributed-applications.html"}],"type":"lic"}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Go1.20 arena 能手动管理内存了，怎么用？ https://mp.weixin.qq.com/s/mwWMOwLsiY8EtODpyEoTIg"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Go 语言性能剖析利器：pprof 实战 https://toutiao.io/posts/ye9g2eb"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Go中gin框架中Session详解 https://juejin.cn/post/7205016004925423653"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Go-Benchmark入门-基础篇（上） "},{"type":"a","url":"https://juejin.cn/post/7205764215222403132","children":[{"text":"https://juejin.cn/post/7205764215222403132"}]},{"text":""}]}]}]},{"type":"p","children":[{"text":""}]},{"type":"ul","children":[{"type":"li","children":[{"type":"lic","children":[{"text":"编辑: flint"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"订阅新闻: http://tinyletter.com/gocn"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"招聘专区: https://gocn.vip/jobs"}]}]}]}]`
+ //data = `[{"type":"code_block","children":[{"type":"code_line","children":[{"text":"## GoCN 每日新闻 (2023-02-28)"}]},{"type":"code_line","children":[{"text":"1. [K8S] client-go 的正确打开方式 https://juejin.cn/post/7203690731276517432"}]},{"type":"code_line","children":[{"text":"2. 优化time.After后，性能提升34%，内存减少67% https://juejin.cn/post/7203274235426324536"}]},{"type":"code_line","children":[{"text":"3. 如何在Golang中进行热重载和优雅的关闭 https://medium.com/@ramseyjiang_22278/how-to-do-hot-reload-and-graceful-shutdown-in-golang-3f84a9f17d79"}]},{"type":"code_line","children":[{"text":"4. 带单位测试的Golang的代理设计模式 https://medium.com/@ramseyjiang_22278/proxy-design-pattern-in-golang-with-unit-tests-d9c6c0d01d49"}]},{"type":"code_line","children":[{"text":"5. 跨平台的Golang GUI库, 核心绑定自 Lazarus 创建的通用跨平台GUI库 liblcl https://github.com/ying32/govcl"}]},{"type":"code_line","children":[{"text":""}]},{"type":"code_line","children":[{"text":"- 编辑: Rolle"}]},{"type":"code_line","children":[{"text":"- 订阅新闻: http://tinyletter.com/gocn"}]},{"type":"code_line","children":[{"text":"- 招聘专区: https://gocn.vip/jobs"}]},{"type":"code_line","children":[{"text":""}]}],"lang":"markdown"},{"type":"p","children":[{"text":""}]}]`
+ data = `[{"type":"h2","children":[{"text":"GoCN 每日新闻 (2023-02-26)"}]},{"type":"ol","children":[{"type":"li","children":[{"type":"lic","children":[{"text":"内存对齐这个事儿只能自己搞明白 "},{"type":"a","url":"https://mp.weixin.qq.com/s/TphkrCag_Wr6uD9thPMwcg","children":[{"text":"https://mp.weixin.qq.com/s/TphkrCag_Wr6uD9thPMwcg"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"紧急下班：修炼内功---内存模型和垃圾回收，不焦虑打工指南 "},{"type":"a","url":"https://mp.weixin.qq.com/s/qF5eiWYkcwTnZ2tFwpzx2Q","children":[{"text":"https://mp.weixin.qq.com/s/qF5eiWYkcwTnZ2tFwpzx2Q"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Go 语言跨平台文件监听库 fsnotify 怎么使用？ "},{"type":"a","url":"https://mp.weixin.qq.com/s/tJ1LvDf14EKg-qQlJUQapQ","children":[{"text":"https://mp.weixin.qq.com/s/tJ1LvDf14EKg-qQlJUQapQ"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"解析 Golang 网络 IO 模型之 EPOLL "},{"type":"a","url":"https://mp.weixin.qq.com/s/xt0Elppc_OaDFnTI_tW3hg","children":[{"text":"https://mp.weixin.qq.com/s/xt0Elppc_OaDFnTI_tW3hg"}]},{"text":""}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"Golang HTTP 标准库实现原理 "},{"type":"a","url":"https://mp.weixin.qq.com/s/zFG6_o0IKjXh4RxKmPTt4g","children":[{"text":"https://mp.weixin.qq.com/s/zFG6_o0IKjXh4RxKmPTt4g"}]},{"text":""}]}]}]},{"type":"p","children":[{"text":""}]},{"type":"ul","children":[{"type":"li","children":[{"type":"lic","children":[{"text":"编辑: 鱼雷"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"订阅新闻: http://tinyletter.com/gocn"}]}]},{"type":"li","children":[{"type":"lic","children":[{"text":"招聘专区: https://gocn.vip/jobs"}]}]}]}]`
var nodes []*Node
err := json.Unmarshal([]byte(data), &nodes)
@@ -29,17 +36,34 @@ func main() {
var texts []string
for _, node := range nodes {
nodeTexts := getText(node)
- fmt.Println("node texts is ", nodeTexts, len(nodeTexts))
+ //fmt.Println("node texts is ", nodeTexts, len(nodeTexts))
texts = append(texts, nodeTexts...)
}
- fmt.Println(texts)
+ //fmt.Println(texts)
+ info := hello(texts)
+
+ converter := md.NewConverter("", true, nil)
+
+ markdown, err := converter.ConvertString(info)
+ if err != nil {
+ // 转换失败直接赋值
+ log.Printf("convert err: %v\n", err)
+ markdown = info
+ }
+
+ fmt.Println(markdown)
+
}
func getText(node *Node) []string {
var texts []string
- if node.Text != "" {
- texts = append(texts, node.Text)
+ if node.Type != "" {
+ texts = append(texts, node.Type)
+ }
+ text := strings.TrimSpace(strings.Trim(node.Text, "\n"))
+ if text != "" {
+ texts = append(texts, text)
}
for _, child := range node.Children {
@@ -49,3 +73,40 @@ func getText(node *Node) []string {
return texts
}
+
+func hello(data []string) string {
+
+ var result strings.Builder
+ var stack []string
+ var stackText []string
+
+ for i := 0; i < len(data); i++ {
+ item := data[i]
+ switch item {
+ case "li", "ol", "ul", "h1", "h2":
+ if len(stack) > 0 && len(stackText) > 0 {
+ result.WriteString(fmt.Sprintf("\n", stack[len(stack)-1]))
+ stack = stack[:len(stack)-1]
+ stackText = stackText[:len(stackText)-1]
+ }
+ result.WriteString(fmt.Sprintf("<%s>", item))
+ stack = append(stack, item)
+ case "lic", "p", "code_line", "code_block": // pass
+ case "a":
+ result.WriteString(fmt.Sprintf("%s", data[i+1], data[i+1]))
+ i = i + 1
+ default:
+ result.WriteString(item)
+ stackText = append(stackText, item)
+ }
+ }
+
+ //fmt.Println(stackText)
+ for i := len(stack) - 1; i >= 0; i-- {
+ result.WriteString(fmt.Sprintf("", stack[i]))
+ }
+
+ //fmt.Println(result.String())
+
+ return result.String()
+}
diff --git a/go.mod b/go.mod
index 1fc9574..99ab143 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.17
require (
github.com/JohannesKaufmann/html-to-markdown v1.3.3
- github.com/cloud-org/msgpush v0.0.2
+ github.com/cloud-org/msgpush v0.0.5
github.com/go-resty/resty/v2 v2.7.0
github.com/gocolly/colly v1.2.0
github.com/robfig/cron/v3 v3.0.1
diff --git a/server/gocn_new_2023.go b/server/gocn_new_2023.go
index 6cd9c03..144d94a 100644
--- a/server/gocn_new_2023.go
+++ b/server/gocn_new_2023.go
@@ -1,13 +1,13 @@
package server
import (
- "bytes"
"encoding/json"
"fmt"
"log"
"strings"
"time"
+ md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/go-resty/resty/v2"
)
@@ -119,50 +119,26 @@ func (g *GoCnNew2023) parseContent(title string, data string) (*string, error) {
return nil, err
}
- var news bytes.Buffer
- news.WriteString(title + "\n\n")
+ var texts []string
for _, node := range nodes {
nodeTexts := getText(node)
- if len(nodeTexts) <= 1 { // 长度为 0 或者是标题
- continue
- }
- if len(nodeTexts)&1 == 0 && len(nodeTexts) >= 8 { // 偶数 正文
- count := 1
- for i := 0; i < len(nodeTexts); i = i + 2 {
- record := fmt.Sprintf("%d. %s %s\n", count, nodeTexts[i], nodeTexts[i+1])
- if strings.HasPrefix(nodeTexts[i], fmt.Sprintf("%d", count)) {
- record = fmt.Sprintf("%s %s\n", nodeTexts[i], nodeTexts[i+1])
- }
- news.WriteString(record)
- count++
- }
- news.WriteString("\n")
- }
- if len(nodeTexts)&1 == 0 && len(nodeTexts) <= 4 { // 偶数 可能是宣传链接之类
- for i := 0; i < len(nodeTexts); i = i + 2 {
- news.WriteString(fmt.Sprintf("%s %s\n", nodeTexts[i], nodeTexts[i+1]))
- }
- news.WriteString("\n")
- }
- if len(nodeTexts)&1 == 1 { //奇数 编辑信息
- news.WriteString(nodeTexts[0] + "\n")
- // 适配奇怪的渲染 有些是 text 和 url 分开 有些则不是
- if len(nodeTexts) <= 3 {
- for i := 1; i < len(nodeTexts); i++ {
- news.WriteString(fmt.Sprintf("%s\n", nodeTexts[i]))
- }
- }
- if len(nodeTexts) > 3 {
- for i := 1; i < len(nodeTexts); i = i + 2 {
- news.WriteString(fmt.Sprintf("%s %s\n", nodeTexts[i], nodeTexts[i+1]))
- }
- }
- }
+ //fmt.Println("node texts is ", nodeTexts, len(nodeTexts))
+ texts = append(texts, nodeTexts...)
}
- res := news.String()
+ //fmt.Println(texts)
+ info := buildMarkdown(texts)
+
+ converter := md.NewConverter("", true, nil)
+
+ markdown, err := converter.ConvertString(info)
+ if err != nil {
+ // 转换失败直接赋值
+ log.Printf("convert err: %v\n", err)
+ markdown = info
+ }
- return &res, nil
+ return &markdown, nil
}
type Node struct {
@@ -174,6 +150,9 @@ type Node struct {
//getText 递归获取 texts
func getText(node *Node) []string {
var texts []string
+ if node.Type != "" {
+ texts = append(texts, node.Type)
+ }
text := strings.TrimSpace(strings.Trim(node.Text, "\n"))
if text != "" {
texts = append(texts, text)
@@ -186,3 +165,37 @@ func getText(node *Node) []string {
return texts
}
+
+func buildMarkdown(data []string) string {
+
+ var result strings.Builder
+ var stack []string
+ var stackText []string
+
+ for i := 0; i < len(data); i++ {
+ item := data[i]
+ switch item {
+ case "li", "ol", "ul", "h1", "h2":
+ if len(stack) > 0 && len(stackText) > 0 {
+ result.WriteString(fmt.Sprintf("\n", stack[len(stack)-1]))
+ stack = stack[:len(stack)-1]
+ stackText = stackText[:len(stackText)-1]
+ }
+ result.WriteString(fmt.Sprintf("<%s>", item))
+ stack = append(stack, item)
+ case "lic", "p", "code_line", "code_block": // pass
+ case "a":
+ result.WriteString(fmt.Sprintf("%s", data[i+1], data[i+1]))
+ i = i + 1
+ default:
+ result.WriteString(item)
+ stackText = append(stackText, item)
+ }
+ }
+
+ for i := len(stack) - 1; i >= 0; i-- {
+ result.WriteString(fmt.Sprintf("", stack[i]))
+ }
+
+ return result.String()
+}
diff --git a/server/push.go b/server/push.go
index 193c884..4d77e87 100644
--- a/server/push.go
+++ b/server/push.go
@@ -57,13 +57,6 @@ func (n *NewsPush) Push() {
}
content := strings.Join(contents, "")
for i := 0; i < len(n.Notifys); i++ {
- if n.Notifys[i].String() == "wecom" { // send text
- w := n.Notifys[i].(*msgpush.WeCom)
- if err = w.SendText(content); err != nil {
- log.Printf("wecom 推送失败, err: %v\n", err)
- }
- continue
- }
if err = n.Notifys[i].Send(content); err != nil {
log.Printf("%s 推送发生错误, err: %v\n", n.Notifys[i].String(), err)
continue