#!/bin/bash # reducer.sh v 0.2 # Scot Hacker, birdhouse.org/blog # Harvests bad IP addresses from multiple sources and adds them to the # CSF firewall (see configserver.com). This version works with WordPress # and Movable Type weblogs, and optionally the exim ACL deny list. # Future versions will scan other sources for bad IPs as well. # Can be adapted easily to work with any # publishing system that marks spam and tracks IP addresses. # Systems that use the Akismet plugin (akismet.com) work excellently, # since comments land in the database already rated by the global brain. # Requires: # * Configured CSF firewall on a WHM/cPanel hosting system. # * One or more heavily hit Movable Type or WordPress blogs on the server. # (Consider setting up a "honeypot" blog with comments wide open to catch spam IPs) # * Optionally the ConfigServer exim ACL system to catch IPs of mail dictionary attackers # * Run by root crontab at regular intervals (hourly is good - this is not resource-intensive). # This script contains your server's root database password. # It is ESSENTIAL that this file not be readable by anyone but root. # chmod 700 !!! ################### # Configuration # Root database password root_db_pass='thepass' # Array of Movable Type database names to scan, one per line. # Important: Only list MT dbs here for MT installs where the # comments table includes the comment_junk_status field, i.e. MT version > 3.1 # Script will fail if field is missing. # Best results by far if the user has Akismet for MT installed. mt_dbs=( username_mt mary_blog joe_mt ) # Array of WordPress database names to scan, one per line. # Important: Only list WP dbs here for WP installs where the # comments table includes the comment_approved field # (not sure which version of WP this is). Script will fail if field is missing. # Best results by far if the user has Akismet for WP installed. wp_dbs=( elmo_wp my_wpblog fred_wp ) # Path to CSF firewall binary. This default should work on most systems. # Use "which csf" to be sure. csf_path='/usr/sbin/csf' # Path to CSF deny rules deny_rules='/etc/csf/csf.deny' # The default settings for the options below work well for me; season to taste. # Threshold sets max number of rejected messages allowed by a given IP # This is safety for lightly hit databases where the top returns from # the query might be low, i.e. might be legit IPs. Lower this number # carefully if you're not getting enough results. threshold=10 # Number of most recently added comments to check. If you run this script # frequently, this number can be kept low. If run infrequently, raise this # number. Suggestion: Raise this to 500 for first script run, then turn down to 100. db_limit=100 # The SQL queries will return hundreds of IPs, in order of the number of comments they submitted, # from highest to lowest. Since we can't put *everything* in the firewall, trim off # the top of each result set. How many lines you use here depends on how "rich" the databases are # with spammy IPs. If a site is lightly trafficked, you might lower this number. lines_to_keep=15 # If using ConfigServer's exim email dictionary attack ACL system: # http://www.configserver.com/free/eximdeny.html # this script can cram the IPs it stores into the firewall as well. # Advantage: email spammers may also be blog spammers, and you can block # them from server completely. # Disadvantages: Risk of banning legit users on shared IPs, and the ACL # generates high numbers of IPs, which means you're rotating through # the bad IPs list more quickly. use_exim_deny="0" # Send mail to admin announcing newly added IPs? mail_admin="1" # Admin email address admin_email="you@domain.com" # Path to temp files used by reducer. Probably no reason to change these. # ipcut stores sorted bad IPs with number of occurrences of each IP (ranks) # badips stores raw IPs without ranks, which we'll post-process # and insert into firewall. reducer.msg stores data for the email that # may be sent to the administrator ipcut='/tmp/ipcut' badips='/tmp/badips' reducer_msg='/tmp/reducer.msg' ############################## # Do not edit below this line ############################## # Blank out temp files from previous runs, if exist if [ -f $ipcut ] then rm "$ipcut" fi if [ -f $badips ] then rm "$badips" fi if [ -f $reducer_msg ] then rm "$reducer_msg" fi # Query string for Movable Type databases mtquery=" select count(comment_ip) as comm_count, comment_ip from mt_comment where comment_junk_status < '0' group by comment_ip order by comm_count desc limit $db_limit; " # Query string for WordPress databases # Here we can allow for comment_approved = "0" (no akismet) # or comment_approved = "spam" (has akismet). wpquery=" select count(comment_author_IP) as comm_count, comment_author_IP from wp_comments where comment_approved = 'spam' or comment_approved = '0' group by comment_author_IP order by comm_count desc limit $db_limit; " # The following two blocks really should be a single function, but I'm not # having any luck getting around a "bad substitution" error where: # numdbs=${#mt_dbs[@]} should become something like numdbs=${#$1[@]} # Let me know if you know how to solve this one. ########################### # Process Movable Type dbs # Count number of databases in MT db array numdbs=${#mt_dbs[@]} # Loop through array of databases, performing query for each # and writing log of most spammy IPs to temp file x=0 while [ "$x" != $numdbs ] do dbname="${mt_dbs[$x]}" mysql -u root -e "$mtquery" -p$root_db_pass $dbname --disable-column-names | head -$lines_to_keep >> $ipcut let x+=1 done ########################### # Process WordPress dbs # Count number of databases in WP db array numdbs=${#wp_dbs[@]} # Loop through arrays of databases, performing query for each # and writing log of most spammy IPs to temp file x=0 while [ "$x" != $numdbs ] do dbname="${wp_dbs[$x]}" mysql -u root -e "$wpquery" -p$root_db_pass $dbname --disable-column-names | head -$lines_to_keep >> $ipcut let x+=1 done ############################ # Sanity - post-process ipcut file and spit out badips file: # 1) Remove lines with number of attempts lower than the threshold # 2) Remove duplicate lines # 3) Check whether IP is already in the firewall echo echo "Qualified IPs:" echo cat $ipcut | while read line; do # rank is the first field in each row of the ipcut temp file # and represents the number of occurrences of that IP in the log. rank=$(echo "$line" | cut -f1) ip=$(echo "$line" | cut -f2) # Does rank of this IP exceed the threshold? if [ "$rank" -gt "$threshold" ]; then # Is this IP already in the deny rules? (More efficient to # let grep check here than to make CSF try and fail to stuff # already existing rules into a running firewall) if ! grep "$ip" "$deny_rules" ; then # Add bad IP to temp file echo "$ip" >> $badips fi fi done # If using the exim ACL deny system option: if [ $use_exim_deny == "1" ]; then cat /etc/exim_deny | while read ip; do # Is this IP already in the deny rules? (More efficient to # let grep check here than to make CSF try and fail to stuff # already existing rules into a running firewall) if ! grep "$ip" "$deny_rules" ; then # Add bad IP to temp file echo "$ip" >> $badips fi done fi # Even though CSF will automatically weed out duplicate IPs, # more efficient to do it before insertion attempt. If no new IPs # were found, $badips will not exist, so check. if [ -f $badips ]; then sort $badips | uniq > $badips.tmp mv -f $badips.tmp $badips fi # Write entries in the final bad IPs file to firewall deny rules. # The CSF -d flag allows IP to be added immediately - no need to restart firewall. # CSF will elegantly rotate out old IPs to make room for newly added ones, # so no need to worry about over-stuffing the turkey. If no new bad IPs # found ($badips file does not exist), do nothing. if [ -f $badips ]; then echo "Bad IPs added to CSF firewall by reducer:" > $reducer_msg counter=0 cat $badips | while read ip; do let counter++ echo -n "Adding rule #$counter: " >> $reducer_msg # Here comes the money shot... $csf_path -d $ip >> $reducer_msg done echo "All other IPs were already present in firewall." >> $reducer_msg # Send summary to admin if [ $mail_admin == "1" ]; then admin_msg=$(/bin/cat $reducer_msg) echo "$admin_msg" | /bin/mail -s "IPs added to firewall by reducer" $admin_email fi else echo echo "All qualifying IPs already in firewall; nothing to add." fi