-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathiblocklist-loader.sh
202 lines (190 loc) · 14.1 KB
/
iblocklist-loader.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
#!/bin/sh
# Generic iblocklist.com ipset loader for ipset v4 and v6 (Original version)
# Author: redhat27, Version 1.1
# snbforums thread: https://www.snbforums.com/threads/iblocklist-com-generic-ipset-loader-for-ipset-v6-and-v4.37976/
# credits for v6 implementation: http://www.unix.com/shell-programming-and-scripting/233825-convert-ip-ranges-cidr-netblocks.html
# Available free lists from [https://www.iblocklist.com/lists] Format:
# ------List name-----------Maintainer--Download URL-------------------------------------------------------------------------
List01="Pedophiles I-Blocklist http://list.iblocklist.com/?list=dufcxgnbjsdwmwctgfuj&fileformat=p2p&archiveformat=gz"
List02="level1 Bluetack http://list.iblocklist.com/?list=ydxerpxkpcfqjaybcssw&fileformat=p2p&archiveformat=gz"
List03="level2 Bluetack http://list.iblocklist.com/?list=gyisgnzbhppbvsphucsw&fileformat=p2p&archiveformat=gz"
List04="level3 Bluetack http://list.iblocklist.com/?list=uwnukjqktoggdknzrhgh&fileformat=p2p&archiveformat=gz"
List05="edu Bluetack http://list.iblocklist.com/?list=imlmncgrkbnacgcwfjvh&fileformat=p2p&archiveformat=gz"
List06="rangetest Bluetack http://list.iblocklist.com/?list=plkehquoahljmyxjixpu&fileformat=p2p&archiveformat=gz"
List07="bogon Bluetack http://list.iblocklist.com/?list=gihxqmhyunbxhbmgqrla&fileformat=p2p&archiveformat=gz"
List08="ads Bluetack http://list.iblocklist.com/?list=dgxtneitpuvgqqcpfulq&fileformat=p2p&archiveformat=gz"
List09="spyware Bluetack http://list.iblocklist.com/?list=llvtlsjyoyiczbkjsxpf&fileformat=p2p&archiveformat=gz"
List10="proxy Bluetack http://list.iblocklist.com/?list=xoebmbyexwuiogmbyprb&fileformat=p2p&archiveformat=gz"
List11="badpeers Bluetack http://list.iblocklist.com/?list=cwworuawihqvocglcoss&fileformat=p2p&archiveformat=gz"
List12="Microsoft Bluetack http://list.iblocklist.com/?list=xshktygkujudfnjfioro&fileformat=p2p&archiveformat=gz"
List13="spider Bluetack http://list.iblocklist.com/?list=mcvxsnihddgutbjfbghy&fileformat=p2p&archiveformat=gz"
List14="hijacked Bluetack http://list.iblocklist.com/?list=usrcshglbiilevmyfhse&fileformat=p2p&archiveformat=gz"
List15="dshield Bluetack http://list.iblocklist.com/?list=xpbqleszmajjesnzddhv&fileformat=p2p&archiveformat=gz"
List16="forumspam Bluetack http://list.iblocklist.com/?list=ficutxiwawokxlcyoeye&fileformat=p2p&archiveformat=gz"
List17="webexploit Bluetack http://list.iblocklist.com/?list=ghlzqtqxnzctvvajwwag&fileformat=p2p&archiveformat=gz"
List18="iana-reserved Bluetack http://list.iblocklist.com/?list=bcoepfyewziejvcqyhqo&fileformat=p2p&archiveformat=gz"
List19="iana-private Bluetack http://list.iblocklist.com/?list=cslpybexmxyuacbyuvib&fileformat=p2p&archiveformat=gz"
List20="iana-multicast Bluetack http://list.iblocklist.com/?list=pwqnlynprfgtjbgqoizj&fileformat=p2p&archiveformat=gz"
List21="NonLanComputers Bluetack http://list.iblocklist.com/?list=jhaoawihmfxgnvmaqffp&fileformat=p2p&archiveformat=gz"
List22="exclusions Bluetack http://list.iblocklist.com/?list=mtxmiireqmjzazcsoiem&fileformat=p2p&archiveformat=gz"
List23="DROP Spamhaus http://list.iblocklist.com/?list=zbdlwrqkabxbcppvrnos&fileformat=p2p&archiveformat=gz"
List24="ZeuS abuse http://list.iblocklist.com/?list=ynkdjqsjyfmilsgbogqf&fileformat=p2p&archiveformat=gz"
List25="SpyEye abuse http://list.iblocklist.com/?list=zvjxsfuvdhoxktpeiokq&fileformat=p2p&archiveformat=gz"
List26="Palevo abuse http://list.iblocklist.com/?list=erqajhwrxiuvjxqrrwfj&fileformat=p2p&archiveformat=gz"
List27="Malicious CI-Army http://list.iblocklist.com/?list=npkuuhuxcsllnhoamkvm&fileformat=p2p&archiveformat=gz"
List28="malc0de malc0de http://list.iblocklist.com/?list=pbqcylkejciyhmwttify&fileformat=p2p&archiveformat=gz"
List29="adservers Yoyo http://list.iblocklist.com/?list=zhogegszwduurnvsyhdf&fileformat=p2p&archiveformat=gz"
List30="bogon cidr-report http://list.iblocklist.com/?list=lujdnbasfaaixitgmxpp&fileformat=p2p&archiveformat=gz"
List31="cruzit-web-attacks CruzIT http://list.iblocklist.com/?list=czvaehmjpsnwwttrdoyl&fileformat=p2p&archiveformat=gz"
List32="Business-ISPs TBG http://list.iblocklist.com/?list=jcjfaxgyyshvdbceroxf&fileformat=p2p&archiveformat=gz"
List33="Primary-Threats TBG http://list.iblocklist.com/?list=ijfqtofzixtwayqovmxn&fileformat=p2p&archiveformat=gz"
List34="Hijacked TBG http://list.iblocklist.com/?list=tbnuqfclfkemqivekikv&fileformat=p2p&archiveformat=gz"
List35="Bogon TBG http://list.iblocklist.com/?list=ewqglwibdgjttwttrinl&fileformat=p2p&archiveformat=gz"
List36="Search-Engines TBG http://list.iblocklist.com/?list=pfefqteoxlfzopecdtyw&fileformat=p2p&archiveformat=gz"
List37="Corporate-Ranges TBG http://list.iblocklist.com/?list=ecqbsykllnadihkdirsh&fileformat=p2p&archiveformat=gz"
BLOCKLIST_INDEXES="13 15 27" # Can be any combination of above list indexes, e.g "15 13", "1", "7 24 8 29 31" etc. [Example: PeerGuardian implementation would be "2 11"]
# Your favorite domain blocked after your chosen blocklist(s) are active? You can specify domains to whitelist in a local file
WHITELIST_DOMAINS_FILE="/jffs/ipset_lists/whitelist-domains.txt" # One line per domain, comments (starting with the '#' character) allowed, even inline comments
# Use locally cached ipset data or download on each run
USE_LOCAL_CACHE=Y # [Y|N]
# Re-download blocklist data if locally saved files are older than this many days [Needed mostly for USE_LOCAL_CACHE=Y]
LISTS_SAVE_DAYS=10
# Use DROP or REJECT target for iptable rule. Briefly, for DROP, attacker (or IP being blocked) will get no response and timeout,
# and REJECT will send immediate response of destination-unreachable (Attacker will know your IP is actively rejecting requests)
# See: http://www.chiark.greenend.org.uk/~peterb/network/drop-vs-reject and http://serverfault.com/questions/157375/reject-vs-drop-when-using-iptables
# or from our own RMerlin: https://www.snbforums.com/threads/ip-tables-confusion.30373/#post-237738
IPTABLES_RULE_TARGET=DROP # [DROP|REJECT]
# Folder to cache downloaded files [Needed for USE_LOCAL_CACHE=Y or storing the file for posterity]
IPSET_LISTS_DIR=/jffs/ipset_lists
# *** No settings to modify from here on down ***
[ -d "$IPSET_LISTS_DIR" ] || mkdir -p $IPSET_LISTS_DIR
# Wait if this is run early on (before the router has internet connectivity) [Needed by wget to download files]
while ! ping -q -c 1 google.com &>/dev/null; do
sleep 1
WaitSeconds=$((WaitSeconds+1))
[ $WaitSeconds -gt 300 ] && logger -t Firewall "$0: Router not online: attempting to use cached files if they exist" && USE_LOCAL_CACHE=Y
done
GetSetDetails () {
index=$1
[ ${#index} -eq 1 ] && index="0${index}"
SetName=$(eval echo \$$(eval echo List${index}) | awk '{ print toupper(substr($2,1,1)) substr($2,2) toupper(substr($1,1,1)) substr($1,2) }')
Url=$(eval echo \$$(eval echo List${index}) | awk '{ print $3 }')
[ ! -s "$IPSET_LISTS_DIR/${SetName}.gz" -o -n "$(find $IPSET_LISTS_DIR/${SetName}.gz -mtime +$LISTS_SAVE_DAYS -print 2>/dev/null)" ] && wget -q -O $IPSET_LISTS_DIR/${SetName}.gz ${Url}
[ "$USE_LOCAL_CACHE" = "Y" ] && GetCommand="cat $IPSET_LISTS_DIR/${SetName}.gz" || GetCommand="wget -q -O - \"${Url}\""
}
# Different routers got different iptables and ipset syntax, also ipset v6.x did away with iptreemap.
# That resulted in a totally different way of parsing the large IP ranges, (hash:ip cannot handle large sets of sometimes 8M+ IPs)
# For ipset v6.x, the script converts IP ranges to CIDR. It creates 2 sets: One for single IPs, and one for CIDRs.
# For ipset v4.x, the original implementaion of using iptreemap is retained.
case $(ipset -v | grep -o "v[4,6]") in
v6)
# Loading ipset modules
lsmod | grep -q "xt_set" || \
for module in ip_set ip_set_hash_net ip_set_hash_ip xt_set; do
modprobe $module
done;
MATCH_SET='--match-set'; CREATE='create'; DESTROY='destroy'; ADD='add'; IPHASH='hash:ip'
ipset destroy tIP 2>/dev/null; ipset destroy tNet 2>/dev/null # Recover if previous run aborted
for index in $BLOCKLIST_INDEXES; do
GetSetDetails $index
# Create the sets if they do not exist
$(ipset swap ${SetName}Single ${SetName}Single 2>&1 | grep -q "name does not exist") && ipset n ${SetName}Single hash:ip hashsize 2048 maxelem 1048576
$(ipset swap ${SetName}CIDR ${SetName}CIDR 2>&1 | grep -q "name does not exist") && ipset n ${SetName}CIDR hash:net hashsize 4096 maxelem 4194304
if ! $(iptables-save | grep -q ${SetName}) || [ "$USE_LOCAL_CACHE" = "N" ]; then
logger -t Firewall "$0: Started processing ${SetName} blocklist"
( echo -e "n tIP -exist hash:ip hashsize 2048 maxelem 1048576\nn tNet -exist hash:net hashsize 4096 maxelem 4194304"
eval $GetCommand | gunzip | sed -n '/0.0.0.0/d;s/^.*://p' | \
nice -n 15 awk '
# convert dotted quads to long decimal ip. Ex: int ip2dec("192.168.0.15")
function ip2dec(ip, slice) {
split(ip, slice, ".")
return (slice[1] * 2^24) + (slice[2] * 2^16) + (slice[3] * 2^8) + slice[4]
}
# convert decimal long ip to dotted quads. Ex: str dec2ip(1171259392)
function dec2ip(dec, ip, quad) {
for (i=3; i>=1; i--) { quad = 256^i; ip = ip int(dec/quad) "."; dec = dec%quad }
return ip dec
}
# convert ip ranges to CIDR notation. Ex: str range2cidr(ip2dec("192.168.0.15"), ip2dec("192.168.5.115"))
function range2cidr(ipStart, ipEnd, bits, mask, newip) {
bits = 1; mask = 1
while (bits < 32) {
newip = or(ipStart, mask)
if ((newip>ipEnd) || ((lshift(rshift(ipStart,bits),bits)) != ipStart)) { bits--; mask = rshift(mask,1); break }
bits++; mask = lshift(mask,1)+1
}
newip = or(ipStart, mask); bits = 32 - bits
# ipset cannot handle single IP via /32 [https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=583079]
if (bits==32) return "add tIP " dec2ip(ipStart)
else result = dec2ip(ipStart) "/" bits
if (newip < ipEnd) result = result "\n" range2cidr(newip + 1, ipEnd)
return "add tNet " result
}
BEGIN { FS="-" }
$1==$2 { print "add tIP " $1 }
$1!=$2 { print range2cidr(ip2dec($1), ip2dec($2)) }
'
) > /tmp/${SetName}.txt
(grep " tIP " /tmp/${SetName}.txt; echo "COMMIT") | nice -n 15 ipset restore
(grep " tNet " /tmp/${SetName}.txt; echo "COMMIT") | nice -n 15 ipset restore
rm -f /tmp/${SetName}.txt
ipset swap tIP ${SetName}Single
ipset swap tNet ${SetName}CIDR
ipset destroy tIP; ipset destroy tNet
logger -t Firewall "$0: Loaded ${SetName}Single blocklist with $(ipset -L ${SetName}Single | wc -l | awk '{print $1-7}') entries"
logger -t Firewall "$0: Loaded ${SetName}CIDR blocklist with $(ipset -L ${SetName}CIDR | wc -l | awk '{print $1-7}') entries"
else
logger -t Firewall "$0: Skipped loading ${SetName} blocklists as they are already loaded. To force reloading, set USE_LOCAL_CACHE=N"
iptables -D PREROUTING -t raw -m set --match-set ${SetName}Single src -j $IPTABLES_RULE_TARGET
iptables -D PREROUTING -t raw -m set --match-set ${SetName}CIDR src -j $IPTABLES_RULE_TARGET
fi
iptables -I PREROUTING -t raw -m set --match-set ${SetName}Single src -j $IPTABLES_RULE_TARGET
iptables -I PREROUTING -t raw -m set --match-set ${SetName}CIDR src -j $IPTABLES_RULE_TARGET
done;;
v4)
# Loading ipset modules
lsmod | grep -q "ipt_set" || \
for module in ip_set ip_set_iptreemap ipt_set; do
modprobe $module
done;
MATCH_SET='--set'; CREATE='--create'; DESTROY='--destroy'; ADD='--add'; IPHASH='iphash'
ipset --destroy iBTmp 2>/dev/null # Recover if previous run aborted
for index in $BLOCKLIST_INDEXES; do
GetSetDetails $index
# Create the set if it does not exist
$(ipset --swap ${SetName} ${SetName} 2>&1 | grep -q "Unknown set") && ipset -N ${SetName} iptreemap
if ! $(iptables-save | grep -q ${SetName}) || [ "$USE_LOCAL_CACHE" = "N" ]; then
logger -t Firewall "$0: Started processing ${SetName} blocklist"
( echo "-N iBTmp iptreemap"
eval $GetCommand | gunzip | nice -n 15 sed -n '/0.0.0.0/d;s/^.*:/-A iBTmp /p'
echo -e "COMMIT"
) | nice -n 15 ipset --restore
ipset --swap iBTmp ${SetName}
ipset --destroy iBTmp
logger -t Firewall "$0: Loaded ${SetName} blocklist with $(ipset -L ${SetName} | wc -l | awk '{print $1-6}') entries"
else
logger -t Firewall "$0: Skipped loading ${SetName} blocklist as it's already loaded. To force reloading, set USE_LOCAL_CACHE=N"
iptables -D PREROUTING -t raw -m set --set ${SetName} src -j $IPTABLES_RULE_TARGET
fi
iptables -I PREROUTING -t raw -m set --set ${SetName} src -j $IPTABLES_RULE_TARGET
done;;
*)
logger -t Firewall "$0: Unknown ipset version. Exiting."
exit 1;;
esac
if [ -s "$WHITELIST_DOMAINS_FILE" ]; then
iptables-save | grep -q WhitelistDomains && iptables -D PREROUTING -t raw -m set $MATCH_SET WhitelistDomains src,dst -j ACCEPT
ipset $DESTROY WhitelistDomains &>/dev/null # Destroy *if* existing (It will exist if this script is run more than once, e.g. scheduled in cron)
ipset $CREATE WhitelistDomains $IPHASH
[ $? -eq 0 ] && entryCount=0
while read line; do
if [ -n "${line%%#*}" ]; then
for ip in $(nslookup ${line%%#*} | sed -n '/^$/,$ s/^A.*: //p' | cut -d' ' -f1 | grep -v ":"); do
ipset $ADD WhitelistDomains $ip
[ $? -eq 0 ] && entryCount=$((entryCount+1))
done
fi
done < $WHITELIST_DOMAINS_FILE
logger -t Firewall "$0: Added WhitelistDomains ($entryCount entries)"
iptables-save | grep -q WhitelistDomains || iptables -I PREROUTING -t raw -m set $MATCH_SET WhitelistDomains src,dst -j ACCEPT
fi