diff --git a/apis/floor/apis.go b/apis/floor/apis.go index 6632041..2a13848 100644 --- a/apis/floor/apis.go +++ b/apis/floor/apis.go @@ -2,11 +2,11 @@ package floor import ( "fmt" - "slices" - "time" - "github.com/opentreehole/go-common" "github.com/rs/zerolog/log" + "slices" + "time" + "treehole_next/utils/sensitive" . "treehole_next/models" . "treehole_next/utils" @@ -139,8 +139,8 @@ func CreateFloor(c *fiber.Ctx) error { return err } - if len([]rune(body.Content)) > 15000 { - return common.BadRequest("文本限制 15000 字") + if len([]rune(body.Content)) > 10000 { + return common.BadRequest("文本限制 10000 字") } holeID, err := c.ParamsInt("id") @@ -211,8 +211,8 @@ func CreateFloorOld(c *fiber.Ctx) error { return err } - if len([]rune(body.Content)) > 15000 { - return common.BadRequest("文本限制 15000 字") + if len([]rune(body.Content)) > 10000 { + return common.BadRequest("文本限制 10000 字") } // get hole to check DivisionID and Locked @@ -286,8 +286,8 @@ func ModifyFloor(c *fiber.Ctx) error { return common.BadRequest("无效请求") } - if body.Content != nil && len([]rune(*body.Content)) > 15000 { - return common.BadRequest("文本限制 15000 字") + if body.Content != nil && len([]rune(*body.Content)) > 10000 { + return common.BadRequest("文本限制 10000 字") } // parse floor_id @@ -343,11 +343,17 @@ func ModifyFloor(c *fiber.Ctx) error { floor.Content = *body.Content // sensitive check - err = floor.SensitiveCheck(tx, &hole) + + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: *body.Content, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeFloor, + }) if err != nil { return err } + floor.IsSensitive = !sensitiveResp.Pass // update floor.mention after update floor.content err = tx.Where("floor_id = ?", floorID).Delete(&FloorMention{}).Error if err != nil { @@ -738,6 +744,8 @@ func RestoreFloor(c *fiber.Ctx) error { } floor.Deleted = false floor.Content = floorHistory.Content + floor.IsSensitive = floorHistory.IsSensitive + floor.IsActualSensitive = floorHistory.IsActualSensitive DB.Save(&floor) go FloorIndex(FloorModel{ diff --git a/apis/hole/apis.go b/apis/hole/apis.go index 5a9d6cc..ded6181 100644 --- a/apis/hole/apis.go +++ b/apis/hole/apis.go @@ -4,6 +4,8 @@ import ( "fmt" "slices" "strconv" + "time" + "treehole_next/utils/sensitive" "github.com/gofiber/fiber/v2" "github.com/opentreehole/go-common" @@ -248,8 +250,8 @@ func CreateHole(c *fiber.Ctx) error { return err } - if len([]rune(body.Content)) > 15000 { - return common.BadRequest("文本限制 15000 字") + if len([]rune(body.Content)) > 10000 { + return common.BadRequest("文本限制 10000 字") } divisionID, err := c.ParamsInt("id") @@ -275,12 +277,22 @@ func CreateHole(c *fiber.Ctx) error { body.SpecialTag = user.DefaultSpecialTag } + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: body.Content, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeFloor, + }) + if err != nil { + return err + } + hole := Hole{ Floors: Floors{{ - UserID: user.ID, - Content: body.Content, - SpecialTag: body.SpecialTag, - IsMe: true, + UserID: user.ID, + Content: body.Content, + SpecialTag: body.SpecialTag, + IsMe: true, + IsSensitive: !sensitiveResp.Pass, }}, UserID: user.ID, DivisionID: divisionID, @@ -310,8 +322,8 @@ func CreateHoleOld(c *fiber.Ctx) error { return err } - if len([]rune(body.Content)) > 15000 { - return common.BadRequest("文本限制 15000 字") + if len([]rune(body.Content)) > 10000 { + return common.BadRequest("文本限制 10000 字") } // get user from auth @@ -332,13 +344,23 @@ func CreateHoleOld(c *fiber.Ctx) error { body.SpecialTag = user.DefaultSpecialTag } + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: body.Content, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeFloor, + }) + if err != nil { + return err + } + // create hole hole := Hole{ Floors: Floors{{ - UserID: user.ID, - Content: body.Content, - SpecialTag: body.SpecialTag, - IsMe: true, + UserID: user.ID, + Content: body.Content, + SpecialTag: body.SpecialTag, + IsMe: true, + IsSensitive: !sensitiveResp.Pass, }}, UserID: user.ID, DivisionID: body.DivisionID, diff --git a/apis/tag/apis.go b/apis/tag/apis.go index 8107abd..de68e64 100644 --- a/apis/tag/apis.go +++ b/apis/tag/apis.go @@ -2,6 +2,8 @@ package tag import ( "strings" + "time" + "treehole_next/utils/sensitive" "github.com/opentreehole/go-common" "gorm.io/plugin/dbresolver" @@ -38,7 +40,7 @@ func ListTags(c *fiber.Ctx) error { return err } go UpdateTagCache(tags) - return c.JSON(&tags) + return Serialize(c, &tags) } } err = DB.Where("name LIKE ?", "%"+query.Search+"%"). @@ -46,7 +48,7 @@ func ListTags(c *fiber.Ctx) error { if err != nil { return err } - return c.JSON(&tags) + return Serialize(c, &tags) } // GetTag @@ -66,7 +68,7 @@ func GetTag(c *fiber.Ctx) error { if result.Error != nil { return result.Error } - return c.JSON(&tag) + return Serialize(c, &tag) } // CreateTag @@ -99,12 +101,26 @@ func CreateTag(c *fiber.Ctx) error { if strings.HasPrefix(body.Name, "@") { return common.BadRequest("只有管理员才能创建 @ 开头的 tag") } + if strings.HasPrefix(tag.Name, "*") { + return common.BadRequest("只有管理员才能创建 * 开头的 tag") + } + } + + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: body.Name, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeTag, + }) + if err != nil { + return err } + tag.IsSensitive = !sensitiveResp.Pass // bind and create tag body.Name = strings.TrimSpace(body.Name) tag.Name = body.Name result := DB.Where("name = ?", body.Name).FirstOrCreate(&tag) + if result.RowsAffected == 0 { c.Status(200) } else { @@ -149,6 +165,17 @@ func ModifyTag(c *fiber.Ctx) error { DB.Find(&tag, id) tag.Name = body.Name tag.Temperature = body.Temperature + + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: body.Name, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeTag, + }) + if err != nil { + return err + } + tag.IsSensitive = !sensitiveResp.Pass + DB.Save(&tag) // log diff --git a/config/config.go b/config/config.go index 2e7d445..e068095 100644 --- a/config/config.go +++ b/config/config.go @@ -35,6 +35,12 @@ var Config struct { HolePurgeDivisions []int `env:"HOLE_PURGE_DIVISIONS" envDefault:"2"` HolePurgeDays int `env:"HOLE_PURGE_DAYS" envDefault:"30"` OpenSensitiveCheck bool `env:"OPEN_SENSITIVE_CHECK" envDefault:"true"` + + YiDunBusinessIdText string `env:"YI_DUN_BUSINESS_ID_TEXT" envDefault:""` + YiDunBusinessIdImage string `env:"YI_DUN_BUSINESS_ID_IMAGE" envDefault:""` + YiDunSecretId string `env:"YI_DUN_SECRET_KEY" envDefault:""` + YiDunSecretKey string `env:"YI_DUN_SECRET_KEY" envDefault:""` + ValidImageUrl []string `env:"VALID_IMAGE_URL"` } var DynamicConfig struct { diff --git a/data/data.go b/data/data.go index a1c85ce..6189df1 100644 --- a/data/data.go +++ b/data/data.go @@ -5,7 +5,6 @@ import ( "os" "github.com/goccy/go-json" - "github.com/importcjj/sensitive" "github.com/rs/zerolog/log" ) @@ -17,25 +16,11 @@ var MetaFile []byte var NamesMapping map[string]string -var SensitiveWordFilter *sensitive.Filter - -var WeakSensitiveWordFilter *sensitive.Filter - func init() { err := initNamesMapping() if err != nil { log.Err(err).Msg("could not init names mapping") } - - err = initSensitiveWords() - if err != nil { - log.Err(err).Msg("could not init sensitive words") - } - - err = initWeakSensitiveWords() - if err != nil { - log.Err(err).Msg("could not init weak sensitive words") - } } func initNamesMapping() error { @@ -46,23 +31,3 @@ func initNamesMapping() error { return json.Unmarshal(NamesMappingData, &NamesMapping) } - -func initSensitiveWords() error { - SensitiveWordFilter = sensitive.New() - err := SensitiveWordFilter.LoadWordDict("data/sensitive_words.txt") - if err != nil { - SensitiveWordFilter = nil - return err - } - return nil -} - -func initWeakSensitiveWords() error { - WeakSensitiveWordFilter = sensitive.New() - err := WeakSensitiveWordFilter.LoadWordDict("data/weak_sensitive_words.txt") - if err != nil { - WeakSensitiveWordFilter = nil - return err - } - return nil -} diff --git a/docs/docs.go b/docs/docs.go index bbe8532..92227d8 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3055,7 +3055,7 @@ const docTemplate = `{ "type": "string" }, "content": { - "description": "content of the floor, no more than 15000", + "description": "content of the floor, no more than 10000", "type": "string" }, "deleted": { diff --git a/docs/swagger.json b/docs/swagger.json index 27cb6f2..13655b2 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3048,7 +3048,7 @@ "type": "string" }, "content": { - "description": "content of the floor, no more than 15000", + "description": "content of the floor, no more than 10000", "type": "string" }, "deleted": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 0ff3eec..6eb9b15 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -357,7 +357,7 @@ definitions: description: a random username type: string content: - description: content of the floor, no more than 15000 + description: content of the floor, no more than 10000 type: string deleted: description: whether the floor is deleted diff --git a/go.mod b/go.mod index 90db65f..c0f353e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/goccy/go-json v0.10.2 github.com/gofiber/fiber/v2 v2.52.1 github.com/hetiansu5/urlquery v1.2.7 - github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b github.com/opentreehole/go-common v0.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/redis/go-redis/v9 v9.3.1 @@ -19,11 +18,13 @@ require ( github.com/stretchr/testify v1.8.4 github.com/swaggo/fiber-swagger v1.3.0 github.com/swaggo/swag v1.16.2 + github.com/yidun/yidun-golang-sdk v1.0.5 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc gorm.io/driver/mysql v1.5.2 gorm.io/driver/sqlite v1.5.4 gorm.io/gorm v1.25.5 gorm.io/plugin/dbresolver v1.5.0 + mvdan.cc/xurls/v2 v2.5.0 ) require ( @@ -65,6 +66,7 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/swaggo/files v1.0.1 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0ce5348..81ed17b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= @@ -15,8 +16,11 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -37,6 +41,9 @@ github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWIO github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch/v8 v8.11.1 h1:1VgTgUTbpqQZ4uE+cPjkOvy/8aw1ZvKcU0ZUE5Cn1mc= github.com/elastic/go-elasticsearch/v8 v8.11.1/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -72,12 +79,26 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofiber/fiber/v2 v2.32.0/go.mod h1:CMy5ZLiXkn6qwthrl03YMyW1NLfj0rhxz2LKl4t7ZTY= github.com/gofiber/fiber/v2 v2.52.1 h1:1RoU2NS+b98o1L77sdl5mboGPiW+0Ypsi5oLmcYlgHI= github.com/gofiber/fiber/v2 v2.52.1/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -86,8 +107,6 @@ github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hetiansu5/urlquery v1.2.7 h1:jn0h+9pIRqUziSPnRdK/gJK8S5TCnk+HZZx5fRHf8K0= github.com/hetiansu5/urlquery v1.2.7/go.mod h1:wFpZdTHRdwt7mk0EM/DdZEWtEN4xf8HJoH/BLXm/PG0= -github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b h1:9hudrgWUhyfR4FRMOfL9KB1uYw48DUdHkkgr9ODOw7Y= -github.com/importcjj/sensitive v0.0.0-20200106142752-42d1c505be7b/go.mod h1:zLVdX6Ed2SvCbEamKmve16U0E03UkdJo4ls1TBfmc8Q= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -140,6 +159,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= @@ -179,6 +199,8 @@ github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2u github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ= github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -188,24 +210,36 @@ github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1S github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/yidun/yidun-golang-sdk v1.0.5 h1:qjes0b9OAZ12P2rckOlhwsgRlK8muMUd+EGThFV4NRU= +github.com/yidun/yidun-golang-sdk v1.0.5/go.mod h1:+JGdWbkUvLi9uKTtHI+nrxajulfZKA7BXDPlzt1RCsU= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= @@ -216,12 +250,17 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -250,7 +289,11 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= @@ -261,6 +304,20 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= @@ -289,3 +346,7 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/plugin/dbresolver v1.5.0 h1:XVHLxh775eP0CqVh3vcfJtYqja3uFl5Wr3cKlY8jgDY= gorm.io/plugin/dbresolver v1.5.0/go.mod h1:l4Cn87EHLEYuqUncpEeTC2tTJQkjngPSD+lo8hIvcT0= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8= +mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE= diff --git a/models/floor.go b/models/floor.go index 2aa6517..148db0c 100644 --- a/models/floor.go +++ b/models/floor.go @@ -3,6 +3,7 @@ package models import ( "fmt" "time" + "treehole_next/utils/sensitive" "github.com/opentreehole/go-common" "github.com/rs/zerolog/log" @@ -24,8 +25,8 @@ type Floor struct { /// base info - // content of the floor, no more than 15000 - Content string `json:"content" gorm:"not null;size:15000"` + // content of the floor, no more than 10000 + Content string `json:"content" gorm:"not null;size:10000"` // a random username Anonyname string `json:"anonyname" gorm:"not null;size:32"` @@ -121,6 +122,9 @@ func (floors Floors) Preprocess(c *fiber.Ctx) error { floorIDs := make([]int, len(floors)) IDFloorMapping := make(map[int]*Floor) for i, floor := range floors { + if floor.Sensitive() { + floors[i].Content = "" + } floors[i].IsMe = userID == floor.UserID floorIDs[i] = floor.ID IDFloorMapping[floor.ID] = floors[i] @@ -195,10 +199,12 @@ Create func (floor *Floor) Create(tx *gorm.DB, hole *Hole) (err error) { // sensitive check - err = floor.SensitiveCheck(tx, hole) - if err != nil { - return err - } + sensitiveCheckResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: floor.Content, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeFloor, + }) + floor.IsSensitive = !sensitiveCheckResp.Pass // load floor mention, in another session floor.Mention, err = LoadFloorMentions(DB, floor.Content) @@ -253,7 +259,7 @@ func (floor *Floor) Create(tx *gorm.DB, hole *Hole) (err error) { // return err // only for test } - if !hole.Hidden && !floor.IsSensitive { + if !hole.Hidden && !floor.Sensitive() { // insert into Elasticsearch go FloorIndex(FloorModel{ ID: floor.ID, @@ -268,6 +274,16 @@ func (floor *Floor) Create(tx *gorm.DB, hole *Hole) (err error) { return utils.DeleteCache(hole.CacheName()) } +func (floor *Floor) Sensitive() bool { + if floor == nil { + return false + } + if floor.IsActualSensitive != nil { + return *floor.IsActualSensitive + } + return floor.IsSensitive +} + func (floor *Floor) AfterFind(_ *gorm.DB) (err error) { floor.FloorID = floor.ID return nil @@ -290,10 +306,12 @@ func (floor *Floor) AfterCreate(tx *gorm.DB) (err error) { // Backup Update and Modify func (floor *Floor) Backup(tx *gorm.DB, userID int, reason string) error { history := FloorHistory{ - Content: floor.Content, - Reason: reason, - FloorID: floor.ID, - UserID: userID, + Content: floor.Content, + Reason: reason, + FloorID: floor.ID, + UserID: userID, + IsSensitive: floor.IsSensitive, + IsActualSensitive: floor.IsActualSensitive, } return tx.Create(&history).Error } @@ -341,31 +359,6 @@ func (floor *Floor) ModifyLike(tx *gorm.DB, userID int, likeOption int8) (err er return nil } -func (floor *Floor) SensitiveCheck(tx *gorm.DB, hole *Hole) (err error) { - var tags Tags - if hole.Tags != nil { - tags = hole.Tags - } else { - err = tx.Model(&Tag{}).Joins("JOIN hole_tags ON hole_tags.tag_id = tag.id"). - Where("hole_tags.hole_id = ?", floor.HoleID).Find(&tags).Error - if err != nil { - return err - } - } - - hasZZMGTag := hole.DivisionID == 2 - for _, tag := range tags { - if tag.IsZZMG { - hasZZMGTag = true - } - } - - floor.IsSensitive = utils.IsSensitive(floor.Content, !hasZZMGTag) - floor.IsActualSensitive = nil - - return nil -} - /*************************** Send Notifications ******************/ diff --git a/models/floor_history.go b/models/floor_history.go index ae095dc..91b9028 100644 --- a/models/floor_history.go +++ b/models/floor_history.go @@ -7,9 +7,14 @@ type FloorHistory struct { ID int `json:"id" gorm:"primaryKey"` CreatedAt time.Time `json:"time_created"` UpdatedAt time.Time `json:"time_updated"` - Content string `json:"content" gorm:"size:15000"` + Content string `json:"content" gorm:"size:10000"` Reason string `json:"reason"` FloorID int `json:"floor_id"` + // auto sensitive check + IsSensitive bool `json:"is_sensitive"` + + // manual sensitive check + IsActualSensitive *bool `json:"is_actual_sensitive"` // The one who modified the floor UserID int `json:"user_id"` } diff --git a/models/hole.go b/models/hole.go index 00add8f..0785970 100644 --- a/models/hole.go +++ b/models/hole.go @@ -234,7 +234,11 @@ func (holes Holes) Preprocess(c *fiber.Ctx) error { // only admin can see hole is hidden if !user.IsAdmin { - for _, hole := range holes { + for i, hole := range holes { + err = holes[i].Floors.Preprocess(c) + if err != nil { + return err + } hole.Hidden = false } } @@ -317,10 +321,6 @@ func (hole *Hole) Create(tx *gorm.DB, user *User, tagNames []string) (err error) } var firstFloor = hole.Floors[0] - err = firstFloor.SensitiveCheck(tx, hole) - if err != nil { - return err - } // Find floor.Mentions, in different sql session firstFloor.Mention, err = LoadFloorMentions(tx, firstFloor.Content) diff --git a/models/tag.go b/models/tag.go index e92c715..f5b3283 100644 --- a/models/tag.go +++ b/models/tag.go @@ -1,8 +1,11 @@ package models import ( + "github.com/gofiber/fiber/v2" "strings" + "sync" "time" + "treehole_next/utils/sensitive" "github.com/opentreehole/go-common" "github.com/rs/zerolog/log" @@ -27,7 +30,11 @@ type Tag struct { /// association info, should add foreign key Holes Holes `json:"-" gorm:"many2many:hole_tags;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` + // auto sensitive check + IsSensitive bool `json:"is_sensitive" gorm:"index:idx_tag_actual_sensitive,priority:1"` + // manual sensitive check + IsActualSensitive *bool `json:"is_actual_sensitive" gorm:"index:idx_tag_actual_sensitive,priority:2"` /// generated field TagID int `json:"tag_id" gorm:"-:all"` } @@ -73,7 +80,6 @@ func FindOrCreateTags(tx *gorm.DB, user *User, names []string) (Tags, error) { if len(newTags) == 0 { return tags, nil } - for _, tag := range newTags { if strings.HasPrefix(tag.Name, "#") { if !user.IsAdmin { @@ -92,6 +98,24 @@ func FindOrCreateTags(tx *gorm.DB, user *User, names []string) (Tags, error) { } } + var wg sync.WaitGroup + for _, tag := range newTags { + wg.Add(1) + go func(tag *Tag) { + sensitiveResp, err := sensitive.CheckSensitive(sensitive.ParamsForCheck{ + Content: tag.Name, + Id: time.Now().UnixNano(), + TypeName: sensitive.TypeTag, + }) + if err != nil { + return + } + tag.IsSensitive = !sensitiveResp.Pass + wg.Done() + }(tag) + } + wg.Wait() + err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&newTags).Error go UpdateTagCache(nil) @@ -112,3 +136,30 @@ func UpdateTagCache(tags Tags) { log.Printf("update tag cache error: %s", err) } } + +func (tag *Tag) Preprocess(c *fiber.Ctx) error { + return Tags{tag}.Preprocess(c) +} + +func (tags Tags) Preprocess(c *fiber.Ctx) error { + tagIDs := make([]int, len(tags)) + IdTagMapping := make(map[int]*Tag) + for i, tag := range tags { + if tags[i].Sensitive() { + tags[i].Name = "" + } + tagIDs[i] = tag.ID + IdTagMapping[tag.ID] = tags[i] + } + return nil +} + +func (tag *Tag) Sensitive() bool { + if tag == nil { + return false + } + if tag.IsActualSensitive != nil { + return *tag.IsActualSensitive + } + return tag.IsSensitive +} diff --git a/tests/hole_test.go b/tests/hole_test.go index 6279e00..62b4a78 100644 --- a/tests/hole_test.go +++ b/tests/hole_test.go @@ -63,7 +63,7 @@ func TestCreateHole(t *testing.T) { content = strings.Repeat("~", 15001) data = Map{"content": content, "tags": []Map{{"name": "a"}, {"name": "ab"}, {"name": "abc"}}} - testAPI(t, "post", "/api/divisions/1/holes", 400, data) // data no more than 15000 + testAPI(t, "post", "/api/divisions/1/holes", 400, data) // data no more than 10000 tags := make([]Map, 11) for i := range tags { diff --git a/utils/log.go b/utils/log.go index cba6ff1..4a83e54 100644 --- a/utils/log.go +++ b/utils/log.go @@ -25,3 +25,10 @@ func MyLog(model string, action string, objectID, userID int, role Role, msg ... Str("role", string(role)). Msg(message) } + +func RequestLog(msg string, TypeName string, Id int64, ans bool) { + log.Info().Str("TypeName", TypeName). + Int64("Id", Id). + Bool("CheckAnswer", ans). + Msg(msg) +} diff --git a/utils/sensitive.go b/utils/sensitive.go deleted file mode 100644 index 85f4403..0000000 --- a/utils/sensitive.go +++ /dev/null @@ -1,26 +0,0 @@ -package utils - -import ( - "treehole_next/config" - "treehole_next/data" -) - -func IsSensitive(content string, weak bool) bool { - if !config.Config.OpenSensitiveCheck { - return false - } - - if weak { - if data.WeakSensitiveWordFilter != nil { - in, _ := data.WeakSensitiveWordFilter.FindIn(content) - return in - } - } else { - if data.SensitiveWordFilter != nil { - in, _ := data.SensitiveWordFilter.FindIn(content) - return in - } - } - - return true -} diff --git a/utils/sensitive/api.go b/utils/sensitive/api.go new file mode 100644 index 0000000..7cf1a79 --- /dev/null +++ b/utils/sensitive/api.go @@ -0,0 +1,162 @@ +package sensitive + +import ( + "fmt" + "github.com/yidun/yidun-golang-sdk/yidun/service/antispam/image/v5" + "github.com/yidun/yidun-golang-sdk/yidun/service/antispam/image/v5/check" + "strconv" + "time" + "treehole_next/config" + "treehole_next/utils" + + v5 "github.com/yidun/yidun-golang-sdk/yidun/service/antispam/text" + "github.com/yidun/yidun-golang-sdk/yidun/service/antispam/text/v5/check/sync/single" +) + +const ( + TypeHole = "Hole" + TypeFloor = "Floor" + TypeTag = "Tag" + TypeImage = "Image" +) + +type ParamsForCheck struct { + Content string + Id int64 + TypeName string +} + +type ResponseForCheck struct { + Pass bool + Labels []int +} + +func CheckSensitive(params ParamsForCheck) (resp *ResponseForCheck, err error) { + images := detect(params.Content) + if len(images) != 0 { + for _, img := range images { + pass, err := checkValidUrl(img) + if err != nil { + return nil, err + } + if !pass { + return &ResponseForCheck{ + Pass: false, + Labels: nil, + }, nil + } + + ret, err := checkSensitiveImage(ParamsForCheck{ + Content: img, + Id: time.Now().UnixNano(), + TypeName: TypeImage, + }) + if err != nil { + return nil, err + } + if !ret.Pass { + return ret, nil + } + } + } + + params.Content = deleteImagesInMarkdown(params.Content) + if hasTextUrl(params.Content) { + return &ResponseForCheck{ + Pass: false, + Labels: nil, + }, nil + } + + return CheckSensitiveText(params) +} + +func CheckSensitiveText(params ParamsForCheck) (resp *ResponseForCheck, err error) { + if !checkType(params) { + return nil, fmt.Errorf("invalid type for sensitive check") + } + + request := single.NewTextCheckRequest(config.Config.YiDunBusinessIdText) + textCheckClient := v5.NewTextClientWithAccessKey(config.Config.YiDunSecretId, config.Config.YiDunSecretKey) + + request.SetDataID(strconv.FormatInt(params.Id, 10) + "_" + params.TypeName) + request.SetContent(params.Content) + request.SetTimestamp(time.Now().UnixMilli()) + + response, err := textCheckClient.SyncCheckText(request) + if err != nil { + // 处理错误并打印日志 + utils.RequestLog(fmt.Sprintf("sync request error:%+v", err.Error()), params.TypeName, params.Id, false) + resp = nil + } + + resp = &ResponseForCheck{} + if response.GetCode() == 200 { + + if *response.Result.Antispam.Suggestion == 0 { + utils.RequestLog("Sensitive text check response code is 200", params.TypeName, params.Id, true) + resp.Pass = true + return + } + + utils.RequestLog("Sensitive text check response code is 200", params.TypeName, params.Id, false) + resp.Pass = false + for _, label := range response.Result.Antispam.Labels { + resp.Labels = append(resp.Labels, *label.Label) + } + return + } + + utils.RequestLog("Sensitive text check http response code is not 200", params.TypeName, params.Id, false) + resp.Pass = false + return +} + +func checkSensitiveImage(params ParamsForCheck) (resp *ResponseForCheck, err error) { + // 设置易盾内容安全分配的businessId + url := params.Content + + request := check.NewImageV5CheckRequest(config.Config.YiDunBusinessIdImage) + + // 实例化一个textClient,入参需要传入易盾内容安全分配的secretId,secretKey + imageCheckClient := image.NewImageClientWithAccessKey(config.Config.YiDunSecretId, config.Config.YiDunSecretKey) + + imageInst := check.NewImageBeanRequest() + imageInst.SetData(url) + imageInst.SetName(strconv.FormatInt(params.Id, 10) + "_" + params.TypeName) + // 设置图片数据的类型,1:图片URL,2:图片BASE64 + imageInst.SetType(1) + + imageBeans := []check.ImageBeanRequest{*imageInst} + request.SetImages(imageBeans) + + response, err := imageCheckClient.ImageSyncCheck(request) + if err != nil { + // 处理错误并打印日志 + utils.RequestLog(fmt.Sprintf("sync request error:%+v", err.Error()), params.TypeName, params.Id, false) + resp = nil + } + + if response.GetCode() == 200 { + if len(*response.Result) == 0 { + return nil, fmt.Errorf("sensitive image check returns empty response") + } + + if *((*response.Result)[0].Antispam.Suggestion) == 0 { + utils.RequestLog("Sensitive image check response code is 200", params.TypeName, params.Id, true) + resp.Pass = true + return + } + + utils.RequestLog("Sensitive image check response code is 200", params.TypeName, params.Id, false) + resp.Pass = false + for _, label := range *((*response.Result)[0].Antispam.Labels) { + resp.Labels = append(resp.Labels, *label.Label) + } + return + } + + utils.RequestLog("Sensitive image check http response code is not 200", params.TypeName, params.Id, false) + resp.Pass = false + return +} diff --git a/utils/sensitive/utils.go b/utils/sensitive/utils.go new file mode 100644 index 0000000..49e7d73 --- /dev/null +++ b/utils/sensitive/utils.go @@ -0,0 +1,79 @@ +package sensitive + +import ( + "golang.org/x/exp/slices" + "mvdan.cc/xurls/v2" + imageUrl "net/url" + "regexp" + "treehole_next/config" +) + +var imageRegex = regexp.MustCompile( + `!\[.*?\]\(([^" )]*)`, +) +var deleteImageRegex = regexp.MustCompile( + `!\[(.*?)\]\(([^" ]*)( ".*")?\)`, +) + +// findImagesInMarkdown 从Markdown文本中查找所有图片链接 +func findImagesInMarkdown(markdown string) []string { + + matches := imageRegex.FindAllStringSubmatch(markdown, -1) + images := make([]string, 0, len(matches)) + for _, match := range matches { + images = append(images, match[1]) + } + return images +} + +func detect(markdownText string) []string { + + var ret []string + images := findImagesInMarkdown(markdownText) + for _, image := range images { + ret = append(ret, image) + } + + return ret +} + +func checkType(params ParamsForCheck) bool { + if params.TypeName != TypeTag && params.TypeName != TypeHole && params.TypeName != TypeFloor { + return true + } + return false +} + +func hasTextUrl(content string) bool { + xurlsRelaxed := xurls.Relaxed() + output := xurlsRelaxed.FindAllString(content, -1) + if len(output) == 0 { + return false + } + return true +} + +func checkValidUrl(input string) (bool, error) { + url, err := imageUrl.Parse(input) + if err != nil { + return false, err + } + if slices.Contains(config.Config.ValidImageUrl, url.Hostname()) { + return false, nil + } + return true, nil +} + +func deleteImagesInMarkdown(markdown string) string { + + return imageRegex.ReplaceAllStringFunc(markdown, func(s string) string { + submatches := deleteImageRegex.FindStringSubmatch(s) + altText := submatches[1] + if len(submatches) > 3 && submatches[3] != "" { + // If there is a title, return it along with the alt text + return altText + " " + submatches[3][2:len(submatches[3])-1] + } + // If there is no title, return the alt text + return altText + }) +} diff --git a/utils/sensitive_test.go b/utils/sensitive_test.go deleted file mode 100644 index 72da3ab..0000000 --- a/utils/sensitive_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package utils - -import ( - "testing" - - "github.com/importcjj/sensitive" - "github.com/stretchr/testify/assert" - - "treehole_next/config" - "treehole_next/data" -) - -func TestIsSensitive(t *testing.T) { - data.SensitiveWordFilter = sensitive.New() - data.SensitiveWordFilter.AddWord("天气", "小明", "123") - config.Config.OpenSensitiveCheck = true - type args struct { - content string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "test1", - args: args{content: "今天天气真好"}, - want: true, - }, - { - name: "test2", - args: args{content: "昨天吃了什么"}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, IsSensitive(tt.args.content), "IsSensitive(%v)", tt.args.content) - }) - } -}