diff --git a/cmd/fake-server/main.go b/cmd/fake-server/main.go index 684f6cc..44e5728 100644 --- a/cmd/fake-server/main.go +++ b/cmd/fake-server/main.go @@ -24,6 +24,13 @@ var akAnnoTests = [6]string{ } var akAnnoIdx = 0 +var sirenTests = [6]string{ + "tests/siren/00.json", + "tests/siren/01.json", + "tests/siren/750450.json", +} +var sirenIdx = 0 + func weiboHandler(w http.ResponseWriter, r *http.Request) { data, _ := ioutil.ReadFile(weiboTests[weiboIdx]) log.Printf("Deliverd %v\n", weiboTests[weiboIdx]) @@ -38,10 +45,18 @@ func akAnnoHandler(w http.ResponseWriter, r *http.Request) { w.Write(data) } +func sirenHandler(w http.ResponseWriter, r *http.Request) { + data, _ := ioutil.ReadFile(sirenTests[sirenIdx]) + log.Printf("Deliverd %v\n", sirenTests[sirenIdx]) + sirenIdx = (sirenIdx + 1) % 3 + w.Write(data) +} + func main() { listenAddr := ":8088" http.HandleFunc("/weibo", weiboHandler) http.HandleFunc("/akanno", akAnnoHandler) + http.HandleFunc("/siren", sirenHandler) log.Printf("Listen at %v\n", listenAddr) log.Fatal(http.ListenAndServe(listenAddr, nil)) diff --git a/dr-feeder.go b/dr-feeder.go index 584fb53..9b55edd 100644 --- a/dr-feeder.go +++ b/dr-feeder.go @@ -14,7 +14,7 @@ import ( ) // Version is current `git describe --tags` infomation. -var Version string = "v2.2.0" +var Version string = "v2.3.0" func consume(ch chan common.NotifyPayload, notifiers []notifier.Notifier) { for { diff --git a/tests/siren/00.json b/tests/siren/00.json new file mode 100644 index 0000000..0daf689 --- /dev/null +++ b/tests/siren/00.json @@ -0,0 +1 @@ +{"code":0,"msg":"","data":{"list":[{"cid":"750451","title":"Loyal to the beat正式上架","cate":1,"date":"2021-04-02"},{"cid":"241306","title":"#EMPEROR","cate":8,"date":"2021-04-01"},{"cid":"992679","title":"#4 主流音乐","cate":5,"date":"2021-03-29"},{"cid":"605964","title":"#3 混音技巧","cate":5,"date":"2021-03-28"},{"cid":"336215","title":"#2 音色偏好","cate":5,"date":"2021-03-27"},{"cid":"578837","title":"歌曲上线公告","cate":1,"date":"2021-03-26"},{"cid":"863542","title":"#1 试音环节","cate":5,"date":"2021-03-26"},{"cid":"114093","title":"官方致辞","cate":5,"date":"2021-03-23"}],"end":true}} diff --git a/tests/siren/01.json b/tests/siren/01.json new file mode 100644 index 0000000..b3962d5 --- /dev/null +++ b/tests/siren/01.json @@ -0,0 +1 @@ +{"code":0,"msg":"","data":{"list":[{"cid":"750450","title":"#1","cate":7,"date":"2021-04-02"},{"cid":"750451","title":"Loyal to the beat正式上架","cate":1,"date":"2021-04-02"},{"cid":"241306","title":"#EMPEROR","cate":8,"date":"2021-04-01"},{"cid":"992679","title":"#4 主流音乐","cate":5,"date":"2021-03-29"},{"cid":"605964","title":"#3 混音技巧","cate":5,"date":"2021-03-28"},{"cid":"336215","title":"#2 音色偏好","cate":5,"date":"2021-03-27"},{"cid":"578837","title":"歌曲上线公告","cate":1,"date":"2021-03-26"},{"cid":"863542","title":"#1 试音环节","cate":5,"date":"2021-03-26"},{"cid":"114093","title":"官方致辞","cate":5,"date":"2021-03-23"}],"end":true}} diff --git a/tests/siren/750450.json b/tests/siren/750450.json new file mode 100644 index 0000000..d7a3e59 --- /dev/null +++ b/tests/siren/750450.json @@ -0,0 +1 @@ +{"code":0,"msg":"","data":{"cid":"750450","title":"#1","cate":7,"author":"塞壬唱片","content":"

乐章裹暮色，约定在次月的余晖里。

待黑夜浸入大海，幕布开启，故人自远方归来。

使黑白染上彩色的，是与风浪合唱的，启航时的歌声。

\"4.2.2-showcase交响预告.jpg\"

","date":"2021-04-02"}} \ No newline at end of file diff --git a/watcher/siren.go b/watcher/siren.go new file mode 100644 index 0000000..36420b7 --- /dev/null +++ b/watcher/siren.go @@ -0,0 +1,186 @@ +package watcher + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + "github.com/antchfx/htmlquery" + "github.com/gocolly/colly/v2" + "github.com/hguandl/dr-feeder/v2/common" + "github.com/mitchellh/mapstructure" + + bolt "go.etcd.io/bbolt" +) + +type sirenWatcher struct { + name string + latestNews sirenNewsData + debugURL string + db *bolt.DB +} + +// NewSirenWatcher creates a Watcher of news from Monster Siren. +func NewSirenWatcher(dbPath string, debugURL string) (Watcher, error) { + var err error = nil + + watcher := new(sirenWatcher) + watcher.name = "塞壬唱片" + watcher.debugURL = debugURL + + watcher.db, err = bolt.Open(dbPath, 0666, nil) + if err != nil { + return watcher, err + } + + err = watcher.setup() + return watcher, err +} + +func (watcher sirenWatcher) apiURL(newsID string) string { + if watcher.debugURL != "" { + return watcher.debugURL + } + return fmt.Sprintf("%s%s", + "https://monster-siren.hypergryph.com/api/news/", + newsID, + ) +} + +func (watcher sirenWatcher) fetchAPI(newID string) (sirenAPIPayload, error) { + var err error = nil + var data sirenAPIPayload + + c := colly.NewCollector( + colly.UserAgent(safariUA), + ) + + c.OnError(func(_ *colly.Response, e error) { + err = e + }) + + c.OnResponse(func(r *colly.Response) { + err = json.Unmarshal(r.Body, &data) + }) + + c.Visit(watcher.apiURL(newID)) + c.Wait() + + return data, err +} + +func (watcher sirenWatcher) getNewsList() ([]sirenNewsData, error) { + data, err := watcher.fetchAPI("") + if err != nil { + return nil, err + } + + var content sirenListData + err = mapstructure.Decode(data.Data, &content) + if err != nil { + return nil, err + } + + return content.List, nil +} + +func (watcher sirenWatcher) getNews(newsID string) (sirenNewsData, error) { + var content sirenNewsData + data, err := watcher.fetchAPI(newsID) + if err != nil { + return content, err + } + + err = mapstructure.Decode(data.Data, &content) + if err != nil { + return content, err + } + + return content, nil +} + +func (watcher *sirenWatcher) setup() error { + newsList, err := watcher.getNewsList() + if err != nil { + return err + } + + watcher.storeNews(newsList) + + return nil +} + +func (watcher *sirenWatcher) storeNews(newsList []sirenNewsData) error { + err := watcher.db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("Siren")) + for _, news := range newsList { + err = b.Put([]byte(news.Cid), []byte(news.Title)) + } + return err + }) + + return err +} + +func (watcher *sirenWatcher) update() bool { + newsList, err := watcher.getNewsList() + if err != nil { + log.Println(err) + return false + } + + ret := false + err = watcher.db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("Siren")) + for _, news := range newsList { + v := b.Get([]byte(news.Cid)) + if v == nil { + watcher.latestNews = news + ret = true + err = b.Put([]byte(news.Cid), []byte(news.Title)) + break + } + } + return err + }) + + if err != nil { + log.Println(err) + return false + } + + return ret +} + +func (watcher sirenWatcher) parseContent() common.NotifyPayload { + news := watcher.latestNews + fullNews, err := watcher.getNews(news.Cid) + texts := news.Title + "\n" + if err == nil { + doc, _ := htmlquery.Parse( + strings.NewReader(fullNews.Content), + ) + nodes, _ := htmlquery.QueryAll(doc, "//text()") + + for _, node := range nodes { + texts += "\n" + texts += strings.Trim(node.Data, " \n") + } + } + + return common.NotifyPayload{ + Title: watcher.name, + Body: texts, + URL: fmt.Sprintf("%s%s", "https://monster-siren.hypergryph.com/info/", news.Cid), + } +} + +func (watcher *sirenWatcher) Produce(ch chan common.NotifyPayload) { + if watcher.update() { + log.Printf("New post from \"%s\"...\n", watcher.name) + ch <- watcher.parseContent() + } else { + log.Printf("Waiting for post \"%s\"...\n", watcher.name) + } +} diff --git a/watcher/sirenmodels.go b/watcher/sirenmodels.go new file mode 100644 index 0000000..52e3c47 --- /dev/null +++ b/watcher/sirenmodels.go @@ -0,0 +1,21 @@ +package watcher + +type sirenAPIPayload struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data map[string]interface{} `json:"data"` +} + +type sirenListData struct { + List []sirenNewsData `mapstructure:"list"` + End bool `mapstructure:"end"` +} + +type sirenNewsData struct { + Cid string `mapstructure:"cid"` + Title string `mapstructure:"title"` + Cate int `mapstructure:"cate"` + Author string `mapstructure:"author,omitempty"` + Content string `mapstructure:"content,omitempty"` + Date string `mapstructure:"date"` +} diff --git a/watcher/watcher.go b/watcher/watcher.go index 7b0047f..daea95c 100644 --- a/watcher/watcher.go +++ b/watcher/watcher.go @@ -24,6 +24,10 @@ type akAnnoConfig struct { DebugURL string `mapstructure:"debug_url"` } +type sirenConfig struct { + DebugURL string `mapstructure:"debug_url"` +} + func wrapDebug(debugURL string, debugMode bool) string { if debugURL != "" { if debugMode { @@ -65,6 +69,12 @@ func ParseWatchers(configs []map[string]interface{}, dataPath string, debugMode break } ret[idx], err = NewAkAnnounceWatcher(path.Join(dataPath, "akanno.db"), wrapDebug(akConfig.DebugURL, debugMode)) + case "siren": + var akConfig sirenConfig + if err = mapstructure.Decode(config, &akConfig); err != nil { + break + } + ret[idx], err = NewSirenWatcher(path.Join(dataPath, "siren.db"), wrapDebug(akConfig.DebugURL, debugMode)) default: err = fmt.Errorf("unknown watcher #%d with type \"%s\"", idx, watcherType) }