Skip to content

Commit

Permalink
feat: add weak word filter
Browse files Browse the repository at this point in the history
  • Loading branch information
JingYiJun committed Nov 10, 2023
1 parent e56c036 commit 7089bb5
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 6 deletions.
19 changes: 18 additions & 1 deletion data/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ var NamesMapping map[string]string

var SensitiveWordFilter *sensitive.Filter

var WeakSensitiveWordFilter *sensitive.Filter

func init() {
err := initNamesMapping()
if err != nil {
Expand All @@ -29,6 +31,11 @@ func init() {
if err != nil {
log.Err(err).Msg("could not init sensitive words")
}

err = initWeakSensitiveWords()
if err != nil {
log.Err(err).Msg("could not init weak sensitive words")
}
}

func initNamesMapping() error {
Expand All @@ -42,7 +49,17 @@ func initNamesMapping() error {

func initSensitiveWords() error {
SensitiveWordFilter = sensitive.New()
err := SensitiveWordFilter.LoadWordDict("data/sensitive_words.txt")
err := WeakSensitiveWordFilter.LoadWordDict("data/sensitive_words.txt")
if err != nil {
WeakSensitiveWordFilter = nil
return err
}
return nil
}

func initWeakSensitiveWords() error {
WeakSensitiveWordFilter = sensitive.New()
err := SensitiveWordFilter.LoadWordDict("data/weak_sensitive_words.txt")
if err != nil {
SensitiveWordFilter = nil
return err
Expand Down
3 changes: 2 additions & 1 deletion models/floor.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,8 @@ func (floor *Floor) SensitiveCheck(tx *gorm.DB, hole *Hole) (err error) {
hasZZMGTag = true
}
}
if hasZZMGTag && utils.IsSensitive(floor.Content) {

if utils.IsSensitive(floor.Content, !hasZZMGTag) {
floor.IsSensitive = true
floor.IsActualSensitive = nil
}
Expand Down
16 changes: 12 additions & 4 deletions utils/sensitive.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,22 @@ import (
"treehole_next/data"
)

func IsSensitive(content string) bool {
func IsSensitive(content string, weak bool) bool {
if !config.Config.OpenSensitiveCheck {
return false
}

if data.SensitiveWordFilter != nil {
in, _ := data.SensitiveWordFilter.FindIn(content)
return in
if weak {
if data.WeakSensitiveWordFilter != nil {
in, _ := data.WeakSensitiveWordFilter.FindIn(content)
return in
}
} else {
if data.SensitiveWordFilter != nil {
in, _ := data.SensitiveWordFilter.FindIn(content)
return in
}
}

return true
}

0 comments on commit 7089bb5

Please sign in to comment.