#!/bin/bash
#
# usage updatethreatblock.sh <configuration file>
# eg: updatethreatblock.sh /etc/ipset-threatblock/ipset-threatblock.conf
#
function exists() { command -v "$1" >/dev/null 2>&1 ; }
if [[ -z "$1" ]]; then
echo "Error: please specify a configuration file, e.g. $0 /etc/ipset-threatblock/ipset-threatblock.conf"
exit 1
fi
# shellcheck source=ipset-threatblock.conf
if ! source "$1"; then
echo "Error: can't load configuration file $1"
exit 1
fi
if ! exists curl && exists egrep && exists grep && exists ipset && exists iptables && exists sed && exists sort && exists wc ; then
echo >&2 "Error: searching PATH fails to find executables among: curl egrep grep ipset iptables sed sort wc"
exit 1
fi
DO_OPTIMIZE_CIDR=no
if exists iprange && [[ ${OPTIMIZE_CIDR:-yes} != no ]]; then
DO_OPTIMIZE_CIDR=yes
fi
if [[ ! -d $(dirname "$IP_BLACKLIST") || ! -d $(dirname "$IP_BLACKLIST_RESTORE") ]]; then
echo >&2 "Error: missing directory(s): $(dirname "$IP_BLACKLIST" "$IP_BLACKLIST_RESTORE"|sort -u)"
exit 1
fi
if [ -f "$IP_BLACKLIST_EXCEPTIONS" ]; then
EXCEPTIONS_TMP=$(mktemp)
for exception in $(sed -r -e 's/\s*#.*$//;/^$/d;/^(0.0.0.0|10.|127.|172.1[6-9].|172.2[0-9].|172.3[0-1].|192.168.|22[4-9].|23[0-9].)/d' "$IP_BLACKLIST_EXCEPTIONS")
do
exception_array+=( "$exception" )
echo $exception >> $EXCEPTIONS_TMP
done
fi
# create the ipset if needed (or abort if does not exists and FORCE=no)
if ! ipset list -n|command grep -q "$IPSET_BLACKLIST_NAME"; then
if [[ ${FORCE:-no} != yes ]]; then
echo >&2 "Error: ipset does not exist yet, add it using:"
echo >&2 "# ipset create $IPSET_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}"
exit 1
fi
if ! ipset create "$IPSET_BLACKLIST_NAME" -exist hash:net family inet hashsize "${HASHSIZE:-16384}" maxelem "${MAXELEM:-65536}"; then
echo >&2 "Error: while creating the initial ipset"
exit 1
fi
fi
# create the iptables binding if needed (or abort if does not exists and FORCE=no)
if ! iptables -nvL INPUT|command grep -q "match-set $IPSET_BLACKLIST_NAME"; then
# we may also have assumed that INPUT rule n°1 is about packets statistics (traffic monitoring)
if [[ ${FORCE:-no} != yes ]]; then
echo >&2 "Error: iptables does not have the needed ipset INPUT rule, add it using:"
echo >&2 "# iptables -I INPUT ${IPTABLES_IPSET_RULE_NUMBER:-1} -m set --match-set $IPSET_BLACKLIST_NAME src -j DROP"
exit 1
fi
if ! iptables -I INPUT "${IPTABLES_IPSET_RULE_NUMBER:-1}" -m set --match-set "$IPSET_BLACKLIST_NAME" src -j DROP; then
echo >&2 "Error: while adding the --match-set ipset rule to iptables"
exit 1
fi
fi
IP_BLACKLIST_TMP=$(mktemp)
for i in "${BLACKLISTS[@]}"
do
IP_TMP=$(mktemp)
(( HTTP_RC=$(curl -L -A "blacklist-update/script/github" --connect-timeout 10 --max-time 10 -o "$IP_TMP" -s -w "%{http_code}" "$i") ))
if (( HTTP_RC == 200 || HTTP_RC == 302 || HTTP_RC == 0 )); then # "0" because file:/// returns 000
command grep -Po '^(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?' "$IP_TMP" | sed -r 's/^0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)$/\1.\2.\3.\4/' >> "$IP_BLACKLIST_TMP"
[[ ${VERBOSE:-yes} == yes ]] && echo -n "."
elif (( HTTP_RC == 503 )); then
echo -e "\\nUnavailable (${HTTP_RC}): $i"
else
echo >&2 -e "\\nWarning: curl returned HTTP response code $HTTP_RC for URL $i"
fi
rm -f "$IP_TMP"
done
# sort -nu does not work as expected
sed -r -e '/^(0\.0\.0\.0|10\.|127\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|192\.168\.|22[4-9]\.|23[0-9]\.)/d' "$IP_BLACKLIST_TMP"|sort -n|sort -mu >| "$IP_BLACKLIST"
if [[ ${DO_OPTIMIZE_CIDR} == yes ]]; then
if [[ ${VERBOSE:-no} == yes ]]; then
echo -e "\\nAddresses before CIDR optimization: $(wc -l "$IP_BLACKLIST" | cut -d' ' -f1)"
fi
< "$IP_BLACKLIST" iprange --optimize - > "$IP_BLACKLIST_TMP" 2>/dev/null
if [[ ${#exception_array[@]} > 0 ]]; then
echo "Allowing for ${#exception_array[@]} exclusions from blacklist"
echo "Addresses before removing exclusions: $(wc -l "$IP_BLACKLIST_TMP" | cut -d' ' -f1)"
IP_BLACKLIST_WITH_EXCEPT_TMP=$(mktemp)
iprange "$IP_BLACKLIST_TMP" --except "$EXCEPTIONS_TMP" > "$IP_BLACKLIST_WITH_EXCEPT_TMP" 2>/dev/null
cp "$IP_BLACKLIST_WITH_EXCEPT_TMP" "$IP_BLACKLIST_TMP"
fi
if [[ ${VERBOSE:-no} == yes ]]; then
echo "Addresses after CIDR optimization: $(wc -l "$IP_BLACKLIST_TMP" | cut -d' ' -f1)"
fi
cp "$IP_BLACKLIST_TMP" "$IP_BLACKLIST"
fi
rm -f "$IP_BLACKLIST_TMP"
# family = inet for IPv4 only
cat >| "$IP_BLACKLIST_RESTORE" <<EOF
create $IPSET_TMP_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}
create $IPSET_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}
EOF
# can be IPv4 including netmask notation
# IPv6 ? -e "s/^([0-9a-f:./]+).*/add $IPSET_TMP_BLACKLIST_NAME \1/p" \ IPv6
sed -rn -e '/^#|^$/d' \
-e "s/^([0-9./]+).*/add $IPSET_TMP_BLACKLIST_NAME \\1/p" "$IP_BLACKLIST" >> "$IP_BLACKLIST_RESTORE"
cat >> "$IP_BLACKLIST_RESTORE" <<EOF
swap $IPSET_BLACKLIST_NAME $IPSET_TMP_BLACKLIST_NAME
destroy $IPSET_TMP_BLACKLIST_NAME
EOF
ipset -file "$IP_BLACKLIST_RESTORE" restore
if [[ ${VERBOSE:-no} == yes ]]; then
echo
echo "Threatblock addresses found: $(wc -l "$IP_BLACKLIST" | cut -d' ' -f1)"
fi
There I was shooting the breeze with an old mate. The conversation turned to why Madge Networks which I wrote about here went titsup. My analysis is that Madge Networks had a solution and decided to go out and find a problem. They deferred to more incorrect strategic technology choices. The truth of the matter is that when something goes titsup, its not because of one reason only, but a myriad of them all contributing to the negative consequence. There are the immediate or visual ones, which are underpinned by intermediate ones and finally after digging right down, there are the root causes. There is never a singular root cause for anything but I'll present my opinion and encourage everyone else to chip in. All of them together are more likely the reason the company went titsup. As far as technology brainfarts go there is no better example than Kodak . They invented the digital camera that killed them. However, they were so focused on milking people in their leg
Comments
Post a Comment