Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Keep top n decisions to get around cf limits #80

Merged
merged 3 commits into from
Feb 28, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 69 additions & 16 deletions cloudflare.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (

const CallsPerSecondLimit uint32 = 4

var TotalIPListCapacity int = 10000

var CloudflareActionByDecisionType = map[string]string{
"captcha": "challenge",
"ban": "block",
Expand All @@ -49,9 +51,13 @@ type ZoneLock struct {
ZoneID string
}

type IPSetItem struct {
CreatedAt time.Time
}

type IPListState struct {
IPList *cloudflare.IPList
IPSet map[string]struct{}
IPSet map[string]IPSetItem
}

// one firewall rule per state.
Expand Down Expand Up @@ -90,7 +96,7 @@ func allZonesHaveAction(zones []ZoneConfig, action string) bool {
return allSupport
}

func calculateSetDiff(setA map[string]struct{}, setB map[string]struct{}) (int, int) {
func calculateIPSetDiff(setA map[string]IPSetItem, setB map[string]IPSetItem) (int, int) {
exclusiveToA := 0
exclusiveToB := 0
for item := range setA {
Expand Down Expand Up @@ -400,7 +406,7 @@ func (worker *CloudflareWorker) setUpIPList() error {
return err
}
*worker.CFStateByAction[action].IPListState.IPList = tmp
worker.CFStateByAction[action].IPListState.IPSet = make(map[string]struct{})
worker.CFStateByAction[action].IPListState.IPSet = make(map[string]IPSetItem)
worker.CFStateByAction[action].UpdateExpr()

}
Expand Down Expand Up @@ -430,7 +436,7 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
// IP decisions are applied at account level
newDecisonsByAction := dedupAndClassifyDecisionsByAction(worker.NewIPDecisions)
expiredDecisonsByAction := dedupAndClassifyDecisionsByAction(worker.ExpiredIPDecisions)
newIPSetByAction := make(map[string]map[string]struct{})
newIPSetByAction := make(map[string]map[string]IPSetItem)

for action, decisions := range newDecisonsByAction {
// In case some zones support this action and others don't, we put this in account's default action.
Expand All @@ -442,18 +448,21 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
action = worker.Account.DefaultAction
worker.Logger.Debugf("ip action defaulted to %s", action)
}
for ip := range worker.CFStateByAction[action].IPListState.IPSet {
for ip, item := range worker.CFStateByAction[action].IPListState.IPSet {
if _, ok := newIPSetByAction[action]; !ok {
newIPSetByAction[action] = make(map[string]struct{})
newIPSetByAction[action] = make(map[string]IPSetItem)
}
newIPSetByAction[action][ip] = struct{}{}
newIPSetByAction[action][ip] = item
}

for _, decision := range decisions {
if _, ok := newIPSetByAction[action]; !ok {
newIPSetByAction[action] = make(map[string]struct{})
newIPSetByAction[action] = make(map[string]IPSetItem)
}
if _, ok := newIPSetByAction[action][*decision.Value]; !ok {
newIPSetByAction[action][*decision.Value] = struct{}{}
newIPSetByAction[action][*decision.Value] = IPSetItem{
buixor marked this conversation as resolved.
Show resolved Hide resolved
CreatedAt: time.Now(),
}
}
}
}
Expand All @@ -469,9 +478,9 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
worker.Logger.Debugf("ip action defaulted to %s", action)
}
if _, ok := newIPSetByAction[action]; !ok {
newIPSetByAction[action] = make(map[string]struct{})
for ip := range worker.CFStateByAction[action].IPListState.IPSet {
newIPSetByAction[action][ip] = struct{}{}
newIPSetByAction[action] = make(map[string]IPSetItem)
for ip, item := range worker.CFStateByAction[action].IPListState.IPSet {
newIPSetByAction[action][ip] = item
}
}
for _, decision := range decisions {
Expand All @@ -481,6 +490,14 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
}
}

for action := range worker.CFStateByAction {
var dropCount int
newIPSetByAction[action], dropCount = keepLatestNIPSetItems(newIPSetByAction[action], *worker.Account.TotalIPListCapacity)
if dropCount > 0 {
worker.Logger.Warnf("%d IPs would be dropped to avoid exceeding IP list limit", dropCount)
}
}

for action, set := range newIPSetByAction {
if reflect.DeepEqual(worker.CFStateByAction[action].IPListState.IPSet, set) {
log.Info("no changes to IP rules ")
Expand All @@ -492,7 +509,9 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
// in the set and continue as usual, and end up with 1 item in the IP list. Then the
// defer call takes care of cleaning up the extra IP.
worker.Logger.Warningf("emptying IP list for %s action", action)
set["10.0.0.1"] = struct{}{}
set["10.0.0.1"] = IPSetItem{
CreatedAt: time.Now(),
}
defer func(action string) {
ipListId := worker.CFStateByAction[action].IPListState.IPList.ID
items, err := worker.getAPI().ListIPListItems(worker.Ctx, ipListId)
Expand All @@ -510,7 +529,7 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
if err != nil {
worker.Logger.Error(err)
}
worker.CFStateByAction[action].IPListState.IPSet = make(map[string]struct{})
worker.CFStateByAction[action].IPListState.IPSet = make(map[string]IPSetItem)
worker.CFStateByAction[action].IPListState.IPList.NumItems = 0
worker.UpdatedState <- worker.CFStateByAction
worker.Logger.Infof("emptied IP list for %s action", action)
Expand Down Expand Up @@ -544,7 +563,7 @@ func (worker *CloudflareWorker) UpdateIPLists() error {
}
time.Sleep(time.Second)
}
newItemCount, deletedItemCount := calculateSetDiff(set, worker.CFStateByAction[action].IPListState.IPSet)
newItemCount, deletedItemCount := calculateIPSetDiff(set, worker.CFStateByAction[action].IPListState.IPSet)
log.Infof("added %d new IPs and deleted %d IPs", newItemCount, deletedItemCount)
worker.CFStateByAction[action].IPListState.IPSet = set
worker.CFStateByAction[action].IPListState.IPList.NumItems = len(set)
Expand Down Expand Up @@ -679,7 +698,7 @@ func (worker *CloudflareWorker) Init() error {
worker.CFStateByAction[action] = &CloudflareState{
AccountID: worker.Account.ID,
Action: action,
IPListState: IPListState{IPList: &cloudflare.IPList{Name: listName}, IPSet: make(map[string]struct{})},
IPListState: IPListState{IPList: &cloudflare.IPList{Name: listName}, IPSet: make(map[string]IPSetItem)},
}
worker.CFStateByAction[action].FilterIDByZoneID = make(map[string]string)
worker.CFStateByAction[action].CountrySet = make(map[string]struct{})
Expand Down Expand Up @@ -776,6 +795,40 @@ func (worker *CloudflareWorker) DeleteASBans() error {
return nil
}

func keepLatestNIPSetItems(set map[string]IPSetItem, n int) (map[string]IPSetItem, int) {
currentItems := len(set)
if currentItems <= n {
return set, 0
}
newSet := make(map[string]IPSetItem)
itemsCreationTime := make([]time.Time, len(set))
i := 0
for _, val := range set {
itemsCreationTime[i] = val.CreatedAt
i++
}
// We use this to find the cutoff duration. This can be improved using more
// sophisticated algo at cost of more code.
sort.Slice(itemsCreationTime, func(i, j int) bool {
return itemsCreationTime[i].After(itemsCreationTime[j])
})
dropCount := 0
tc := 0
for ip, item := range set {
if item.CreatedAt.After(itemsCreationTime[n-1]) || item.CreatedAt.Equal(itemsCreationTime[n-1]) {
newSet[ip] = item
tc++
} else {
dropCount++
}
if tc == n {
break
}
}

return newSet, dropCount
}

func (worker *CloudflareWorker) normalizeActionForZone(action string, zoneCfg ZoneConfig) string {
zoneLogger := worker.Logger.WithFields(log.Fields{"zone_id": zoneCfg.ID})
if _, spAction := zoneCfg.ActionSet[action]; action == "defaulted" || !spAction {
Expand Down
102 changes: 90 additions & 12 deletions cloudflare_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strconv"
"sync"
"testing"
"time"

"github.com/cloudflare/cloudflare-go"
"github.com/crowdsecurity/crowdsec/pkg/models"
Expand Down Expand Up @@ -167,8 +168,9 @@ var dummyCFAccount AccountConfig = AccountConfig{
Actions: []string{"block"},
},
},
IPListPrefix: "crowdsec",
DefaultAction: "block",
IPListPrefix: "crowdsec",
DefaultAction: "block",
TotalIPListCapacity: &TotalIPListCapacity,
}

var mockCfAPI cloudflareAPI = &mockCloudflareAPI{
Expand Down Expand Up @@ -869,7 +871,7 @@ func TestCloudflareWorker_AddNewIPs(t *testing.T) {
"block": {
AccountID: dummyCFAccount.ID,
IPListState: IPListState{
IPSet: make(map[string]struct{}),
IPSet: make(map[string]IPSetItem),
IPList: &cloudflare.IPList{},
},
},
Expand All @@ -884,7 +886,7 @@ func TestCloudflareWorker_AddNewIPs(t *testing.T) {
tests := []struct {
name string
fields fields
want map[string]struct{}
want map[string]IPSetItem
}{
{
name: "supported ip decision",
Expand All @@ -896,7 +898,7 @@ func TestCloudflareWorker_AddNewIPs(t *testing.T) {
},
API: mockCfAPI,
},
want: map[string]struct{}{
want: map[string]IPSetItem{
"1.2.3.4": {},
},
},
Expand All @@ -910,7 +912,7 @@ func TestCloudflareWorker_AddNewIPs(t *testing.T) {
},
API: mockCfAPI,
},
want: map[string]struct{}{
want: map[string]IPSetItem{
"1.2.3.4": {},
},
},
Expand All @@ -932,7 +934,7 @@ func TestCloudflareWorker_AddNewIPs(t *testing.T) {
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(tt.want, worker.CFStateByAction["block"].IPListState.IPSet) {
if !IPSetsAreEqual(tt.want, worker.CFStateByAction["block"].IPListState.IPSet) {
t.Errorf("want=%+v, found=%+v", tt.want, worker.CFStateByAction["block"].IPListState.IPSet)
}
})
Expand All @@ -950,7 +952,7 @@ func TestCloudflareWorker_DeleteIPs(t *testing.T) {
"block": {
AccountID: dummyCFAccount.ID,
IPListState: IPListState{
IPSet: map[string]struct{}{
IPSet: map[string]IPSetItem{
"1.2.3.4": {},
"1.2.3.5": {},
},
Expand All @@ -968,7 +970,7 @@ func TestCloudflareWorker_DeleteIPs(t *testing.T) {
tests := []struct {
name string
fields fields
want map[string]struct{}
want map[string]IPSetItem
}{
{
name: "supported ip decision",
Expand All @@ -980,7 +982,7 @@ func TestCloudflareWorker_DeleteIPs(t *testing.T) {
},
API: mockCfAPI,
},
want: map[string]struct{}{
want: map[string]IPSetItem{
"1.2.3.5": {},
},
},
Expand All @@ -994,7 +996,7 @@ func TestCloudflareWorker_DeleteIPs(t *testing.T) {
},
API: mockCfAPI,
},
want: map[string]struct{}{
want: map[string]IPSetItem{
"1.2.3.5": {},
},
},
Expand All @@ -1016,9 +1018,85 @@ func TestCloudflareWorker_DeleteIPs(t *testing.T) {
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(tt.want, worker.CFStateByAction["block"].IPListState.IPSet) {
if !IPSetsAreEqual(tt.want, worker.CFStateByAction["block"].IPListState.IPSet) {
t.Errorf("want=%+v, found=%+v", tt.want, worker.CFStateByAction["block"].IPListState.IPSet)
}
})
}
}

func timeForMonth(month time.Month) time.Time {
return time.Date(2000, month, 1, 1, 1, 1, 1, time.UTC)
}

func Test_keepLatestNIPSetItems(t *testing.T) {
type args struct {
set map[string]IPSetItem
n int
}
tests := []struct {
name string
args args
want map[string]IPSetItem
}{
{
name: "regular",
args: args{
set: map[string]IPSetItem{
"1.2.3.5": {CreatedAt: timeForMonth(time.May)},
"1.2.3.4": {CreatedAt: timeForMonth(time.April)},
"1.2.3.6": {CreatedAt: timeForMonth(time.March)},
},
n: 2,
},
want: map[string]IPSetItem{
"1.2.3.5": {CreatedAt: timeForMonth(time.May)},
"1.2.3.4": {CreatedAt: timeForMonth(time.April)},
},
},
{
name: "no items to drop",
args: args{
set: map[string]IPSetItem{
"1.2.3.5": {CreatedAt: timeForMonth(time.May)},
"1.2.3.4": {CreatedAt: timeForMonth(time.April)},
"1.2.3.6": {CreatedAt: timeForMonth(time.March)},
},
n: 3,
},
want: map[string]IPSetItem{
"1.2.3.5": {CreatedAt: timeForMonth(time.May)},
"1.2.3.4": {CreatedAt: timeForMonth(time.April)},
"1.2.3.6": {CreatedAt: timeForMonth(time.March)},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got, _ := keepLatestNIPSetItems(tt.args.set, tt.args.n); !reflect.DeepEqual(got, tt.want) {
t.Errorf("keepLatestNIPSetItems() = %v, want %v", got, tt.want)
}
})
}
}

func Test_keepLatestNIPSetItemsBackwardCompat(t *testing.T) {

arg := map[string]IPSetItem{
"1.2.3.5": {CreatedAt: timeForMonth(time.May)},
"1.2.3.4": {CreatedAt: timeForMonth(time.May)},
"1.2.3.6": {CreatedAt: timeForMonth(time.May)},
}

for n := 1; n <= len(arg); n++ {
res, _ := keepLatestNIPSetItems(arg, n)
if len(res) != n {
t.Errorf("expected len(res)=%d, Got=%d", n, len(res))
}
}
}

func IPSetsAreEqual(a map[string]IPSetItem, b map[string]IPSetItem) bool {
aOnly, bOnly := calculateIPSetDiff(a, b)
return aOnly == 0 && bOnly == 0
}
Loading