Merge pull request #135 from OpenTreeHole/feat_sensitive_check_api

Feat sensitive check api

apis/floor/apis.go

@@ -2,11 +2,11 @@ package floor
 
 import (
 	"fmt"
-	"slices"
-	"time"
-
 	"github.com/opentreehole/go-common"
 	"github.com/rs/zerolog/log"
+	"slices"
+	"time"
+	"treehole_next/utils/sensitive"
 
 	. "treehole_next/models"
 	. "treehole_next/utils"
@@ -139,8 +139,8 @@ func CreateFloor(c *fiber.Ctx) error {
 		return err
 	}
 
-	if len([]rune(body.Content)) > 15000 {
-		return common.BadRequest("文本限制 15000 字")
+	if len([]rune(body.Content)) > 10000 {
+		return common.BadRequest("文本限制 10000 字")
 	}
 
 	holeID, err := c.ParamsInt("id")
@@ -211,8 +211,8 @@ func CreateFloorOld(c *fiber.Ctx) error {
 		return err
 	}
 
-	if len([]rune(body.Content)) > 15000 {
-		return common.BadRequest("文本限制 15000 字")
+	if len([]rune(body.Content)) > 10000 {
+		return common.BadRequest("文本限制 10000 字")
 	}
 
 	// get hole to check DivisionID and Locked
@@ -286,8 +286,8 @@ func ModifyFloor(c *fiber.Ctx) error {
 		return common.BadRequest("无效请求")
 	}
 
-	if body.Content != nil && len([]rune(*body.Content)) > 15000 {
-		return common.BadRequest("文本限制 15000 字")
+	if body.Content != nil && len([]rune(*body.Content)) > 10000 {
+		return common.BadRequest("文本限制 10000 字")
 	}
 
 	// parse floor_id
@@ -343,11 +343,17 @@ func ModifyFloor(c *fiber.Ctx) error {
 			floor.Content = *body.Content
 
 			// sensitive check
-			err = floor.SensitiveCheck(tx, &hole)
+
+			sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{
+				Content:  *body.Content,
+				Id:       time.Now().UnixNano(),
+				TypeName: sensitive.TypeFloor,
+			})
 			if err != nil {
 				return err
 			}
 
+			floor.IsSensitive = !sensitiveResp.Pass
 			// update floor.mention after update floor.content
 			err = tx.Where("floor_id = ?", floorID).Delete(&FloorMention{}).Error
 			if err != nil {
@@ -738,6 +744,8 @@ func RestoreFloor(c *fiber.Ctx) error {
 	}
 	floor.Deleted = false
 	floor.Content = floorHistory.Content
+	floor.IsSensitive = floorHistory.IsSensitive
+	floor.IsActualSensitive = floorHistory.IsActualSensitive
 	DB.Save(&floor)
 
 	go FloorIndex(FloorModel{

apis/hole/apis.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"slices"
 	"strconv"
+	"time"
+	"treehole_next/utils/sensitive"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/opentreehole/go-common"
@@ -248,8 +250,8 @@ func CreateHole(c *fiber.Ctx) error {
 		return err
 	}
 
-	if len([]rune(body.Content)) > 15000 {
-		return common.BadRequest("文本限制 15000 字")
+	if len([]rune(body.Content)) > 10000 {
+		return common.BadRequest("文本限制 10000 字")
 	}
 
 	divisionID, err := c.ParamsInt("id")
@@ -275,12 +277,22 @@ func CreateHole(c *fiber.Ctx) error {
 		body.SpecialTag = user.DefaultSpecialTag
 	}
 
+	sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{
+		Content:  body.Content,
+		Id:       time.Now().UnixNano(),
+		TypeName: sensitive.TypeFloor,
+	})
+	if err != nil {
+		return err
+	}
+
 	hole := Hole{
 		Floors: Floors{{
-			UserID:     user.ID,
-			Content:    body.Content,
-			SpecialTag: body.SpecialTag,
-			IsMe:       true,
+			UserID:      user.ID,
+			Content:     body.Content,
+			SpecialTag:  body.SpecialTag,
+			IsMe:        true,
+			IsSensitive: !sensitiveResp.Pass,
 		}},
 		UserID:     user.ID,
 		DivisionID: divisionID,
@@ -310,8 +322,8 @@ func CreateHoleOld(c *fiber.Ctx) error {
 		return err
 	}
 
-	if len([]rune(body.Content)) > 15000 {
-		return common.BadRequest("文本限制 15000 字")
+	if len([]rune(body.Content)) > 10000 {
+		return common.BadRequest("文本限制 10000 字")
 	}
 
 	// get user from auth
@@ -332,13 +344,23 @@ func CreateHoleOld(c *fiber.Ctx) error {
 		body.SpecialTag = user.DefaultSpecialTag
 	}
 
+	sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{
+		Content:  body.Content,
+		Id:       time.Now().UnixNano(),
+		TypeName: sensitive.TypeFloor,
+	})
+	if err != nil {
+		return err
+	}
+
 	// create hole
 	hole := Hole{
 		Floors: Floors{{
-			UserID:     user.ID,
-			Content:    body.Content,
-			SpecialTag: body.SpecialTag,
-			IsMe:       true,
+			UserID:      user.ID,
+			Content:     body.Content,
+			SpecialTag:  body.SpecialTag,
+			IsMe:        true,
+			IsSensitive: !sensitiveResp.Pass,
 		}},
 		UserID:     user.ID,
 		DivisionID: body.DivisionID,

apis/tag/apis.go

@@ -2,6 +2,8 @@ package tag
 
 import (
 	"strings"
+	"time"
+	"treehole_next/utils/sensitive"
 
 	"github.com/opentreehole/go-common"
 	"gorm.io/plugin/dbresolver"
@@ -38,15 +40,15 @@ func ListTags(c *fiber.Ctx) error {
 				return err
 			}
 			go UpdateTagCache(tags)
-			return c.JSON(&tags)
+			return Serialize(c, &tags)
 		}
 	}
 	err = DB.Where("name LIKE ?", "%"+query.Search+"%").
 		Order("temperature DESC").Find(&tags).Error
 	if err != nil {
 		return err
 	}
-	return c.JSON(&tags)
+	return Serialize(c, &tags)
 }
 
 // GetTag
@@ -66,7 +68,7 @@ func GetTag(c *fiber.Ctx) error {
 	if result.Error != nil {
 		return result.Error
 	}
-	return c.JSON(&tag)
+	return Serialize(c, &tag)
 }
 
 // CreateTag
@@ -99,12 +101,26 @@ func CreateTag(c *fiber.Ctx) error {
 		if strings.HasPrefix(body.Name, "@") {
 			return common.BadRequest("只有管理员才能创建 @ 开头的 tag")
 		}
+		if strings.HasPrefix(tag.Name, "*") {
+			return common.BadRequest("只有管理员才能创建 * 开头的 tag")
+		}
+	}
+
+	sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{
+		Content:  body.Name,
+		Id:       time.Now().UnixNano(),
+		TypeName: sensitive.TypeTag,
+	})
+	if err != nil {
+		return err
 	}
+	tag.IsSensitive = !sensitiveResp.Pass
 
 	// bind and create tag
 	body.Name = strings.TrimSpace(body.Name)
 	tag.Name = body.Name
 	result := DB.Where("name = ?", body.Name).FirstOrCreate(&tag)
+
 	if result.RowsAffected == 0 {
 		c.Status(200)
 	} else {
@@ -149,6 +165,17 @@ func ModifyTag(c *fiber.Ctx) error {
 	DB.Find(&tag, id)
 	tag.Name = body.Name
 	tag.Temperature = body.Temperature
+
+	sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{
+		Content:  body.Name,
+		Id:       time.Now().UnixNano(),
+		TypeName: sensitive.TypeTag,
+	})
+	if err != nil {
+		return err
+	}
+	tag.IsSensitive = !sensitiveResp.Pass
+
 	DB.Save(&tag)
 
 	// log

config/config.go

@@ -35,6 +35,12 @@ var Config struct {
 	HolePurgeDivisions []int    `env:"HOLE_PURGE_DIVISIONS" envDefault:"2"`
 	HolePurgeDays      int      `env:"HOLE_PURGE_DAYS" envDefault:"30"`
 	OpenSensitiveCheck bool     `env:"OPEN_SENSITIVE_CHECK" envDefault:"true"`
+
+	YiDunBusinessIdText  string   `env:"YI_DUN_BUSINESS_ID_TEXT" envDefault:""`
+	YiDunBusinessIdImage string   `env:"YI_DUN_BUSINESS_ID_IMAGE" envDefault:""`
+	YiDunSecretId        string   `env:"YI_DUN_SECRET_KEY" envDefault:""`
+	YiDunSecretKey       string   `env:"YI_DUN_SECRET_KEY" envDefault:""`
+	ValidImageUrl        []string `env:"VALID_IMAGE_URL"`
 }
 
 var DynamicConfig struct {

data/data.go

@@ -5,7 +5,6 @@ import (
 	"os"
 
 	"github.com/goccy/go-json"
-	"github.com/importcjj/sensitive"
 	"github.com/rs/zerolog/log"
 )
 
@@ -17,25 +16,11 @@ var MetaFile []byte
 
 var NamesMapping map[string]string
 
-var SensitiveWordFilter *sensitive.Filter
-
-var WeakSensitiveWordFilter *sensitive.Filter
-
 func init() {
 	err := initNamesMapping()
 	if err != nil {
 		log.Err(err).Msg("could not init names mapping")
 	}
-
-	err = initSensitiveWords()
-	if err != nil {
-		log.Err(err).Msg("could not init sensitive words")
-	}
-
-	err = initWeakSensitiveWords()
-	if err != nil {
-		log.Err(err).Msg("could not init weak sensitive words")
-	}
 }
 
 func initNamesMapping() error {
@@ -46,23 +31,3 @@ func initNamesMapping() error {
 
 	return json.Unmarshal(NamesMappingData, &NamesMapping)
 }
-
-func initSensitiveWords() error {
-	SensitiveWordFilter = sensitive.New()
-	err := SensitiveWordFilter.LoadWordDict("data/sensitive_words.txt")
-	if err != nil {
-		SensitiveWordFilter = nil
-		return err
-	}
-	return nil
-}
-
-func initWeakSensitiveWords() error {
-	WeakSensitiveWordFilter = sensitive.New()
-	err := WeakSensitiveWordFilter.LoadWordDict("data/weak_sensitive_words.txt")
-	if err != nil {
-		WeakSensitiveWordFilter = nil
-		return err
-	}
-	return nil
-}

docs/docs.go

@@ -3055,7 +3055,7 @@ const docTemplate = `{
                     "type": "string"
                 },
                 "content": {
-                    "description": "content of the floor, no more than 15000",
+                    "description": "content of the floor, no more than 10000",
                     "type": "string"
                 },
                 "deleted": {

docs/swagger.json

@@ -3048,7 +3048,7 @@
                     "type": "string"
                 },
                 "content": {
-                    "description": "content of the floor, no more than 15000",
+                    "description": "content of the floor, no more than 10000",
                     "type": "string"
                 },
                 "deleted": {

docs/swagger.yaml

@@ -357,7 +357,7 @@ definitions:
         description: a random username
         type: string
       content:
-        description: content of the floor, no more than 15000
+        description: content of the floor, no more than 10000
         type: string
       deleted:
         description: whether the floor is deleted

go.mod

@@ -11,19 +11,20 @@ require (
 	github.com/goccy/go-json v0.10.2
 	github.com/gofiber/fiber/v2 v2.52.1
 	github.com/hetiansu5/urlquery v1.2.7
-	github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b
 	github.com/opentreehole/go-common v0.1.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/redis/go-redis/v9 v9.3.1
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/fiber-swagger v1.3.0
 	github.com/swaggo/swag v1.16.2
+	github.com/yidun/yidun-golang-sdk v1.0.5
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
 	gorm.io/driver/mysql v1.5.2
 	gorm.io/driver/sqlite v1.5.4
 	gorm.io/gorm v1.25.5
 	gorm.io/plugin/dbresolver v1.5.0
+	mvdan.cc/xurls/v2 v2.5.0
 )
 
 require (
@@ -65,6 +66,7 @@ require (
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
+	github.com/tjfoc/gmsm v1.4.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.51.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect