#! /bin/bash
#
# This is the ModSecurity CRS Whitelist Generator Script version 16/03/2014
# Not affiliated with the Modsecurity project or the CRS
# Script by Sam Hobbs https://samhobbs.co.uk
# Licence: public domain
# Please let me know how you get on so that I can make improvements: leave a
# comment on my blog or email me at: sam at samhobbs dot co dot uk

# Every installation of Apache with Modsecurity will probably have false positives.
# This script aims to make writing a whitelist file quicker than doing it manually.
# Required input is as many error logs as you have available from the relevant virutalhost,
# with modsecurity set to detect only and all of the CRS enabled 

#====================================== CHANGELOG =========================================#

# Script version 
VERSION="27/05/2014"

# 16/03/2014 changed default <LocationMatch> regex in whitelist file to .* from *
# 12/05/2014 	changed defaults for drupal
#		egrep used instead of grep to provide better regex
#		now checks to see if locationmatch would be empty before writing it

#===================================== USER INPUT =========================================#

# This script works by assuming that all traffic from a friendly IP address is legitimate
# and that the resulting errors are false positives.

# Define one or more friendly IP address, separating each IP with a space. If you are
# hosting at home, a good choice is your router's LAN IP address, since your server sees
# all traffic from your LAN as originating here.
# You might also like to add IP addresses of users you know weren't abusing the site, for
# example people who left legitimate comments. Wordpress tells you the IP address used to
# post each comment on the comment moderation GUI.
FRIENDLY_IP="192.168.1.1"

# Define a list of special locations. The matching process uses regex. A LocationMatch
# statement will be created for each one of these locations, which will be populated with
# rule IDs for all the locations that match that regex. Leave a space between each location
# that you enter.
# There is no need to start each location with "^"; the script adds the character for you.
# If you don't end the location with a "$" then the script will automatically add an
# asterisk to the end of the location in the LocationMatch statement so that it will match
# all files beginning with that path, i.e.
# <LocationMatch "^/wordpress/wp-admin/*"> matches /wordpress/wp-admin/wp-login.php, but
# <LocationMatch "^/wordpress/wp-admin/$"> does not.

SPECIAL_LOCATIONS="\
/authorize.php \
/admin/config$ \
/admin/config/content/mollom \
/admin/config/content/syntaxhighlighter \
/admin/config/people \
/admin/config/search \
/admin/config/system/actions \
/admin/content \
/admin/reports \
/admin/structure/menu \
/admin/structure/types \
/admin/modules \
/admin/appearance/settings \
/admin/people/permissions \
/comment/reply \
/comment/[0-9]+$ \
/comment/[0-9]+/edit \
/comment/[0-9]+/approve \
/file/ajax/field_image/und/0/ \
/index.php$ \
/node/[0-9]+/delete$ \
/node/[0-9]+/edit$ \
/node/add/article \
/sites/default/files/css/ \
/sites/default/files/js \
/token/tree \
/user/[0-9]+/edit$ \
.*.(png|jpg|JPG|gif|ico)$"

# Define directory holding the Apache error log files to be processed
LOG_DIR=~/errors-new

# Define directory for output files:
OUTPUT_DIR=~/modsec-whitelist-samhobbs-new-1 

#===================================== KNOWN BUGS =========================================#

# SecRuleRemoveById 891143 doesn't work because the rule has its own whitelist built in
# see:   /usr/share/modsecurity-crs/optional_rules/modsecurity_crs_43_csrf_protection.conf

#==================================== HOUSEKEEPING ========================================#

# Create the output directory if it doesn't exist:
if [ ! -d $OUTPUT_DIR ]
then
  mkdir $OUTPUT_DIR
  echo "Output directory has been created at $OUTPUT_DIR"
  echo ""
fi

# Delete the previous output files if they exist
if [ -f $OUTPUT_DIR/whitelist ]
then
  echo -n "Old output files detected, deleting..."
  rm $OUTPUT_DIR/*
  echo "...done"
  echo ""
fi

# Some files:
COMBINED_LOG=$OUTPUT_DIR/combined_log
PROBLEM_LOCATIONS=$OUTPUT_DIR/problem_locations
ROOT_LOG=$OUTPUT_DIR/root_log
PROBLEM_LOCATIONS_REMAINING=$OUTPUT_DIR/problem_locations_remaining
ROOT_IDS=$OUTPUT_DIR/root_ids
GROUPED_LOCATIONS=$OUTPUT_DIR/grouped_locations
WHITELIST_FILE=$OUTPUT_DIR/whitelist

# Rules tripped on the root domain will be added as generic exceptions for the whole virtualhost:
echo "Rules tripped on the root domain will be added as generic exceptions for the whole virtualhost"
echo ""
echo "In addition to this, you have selected the following special locations to be grouped into whitelist statements"
for variable in $SPECIAL_LOCATIONS; do
echo "> $variable";
done
echo ""

# Perform work in temorary files
TEMPFILE1=$(mktemp)
TEMPFILE2=$(mktemp)
TEMPFILE3=$(mktemp)
TEMPFILE4=$(mktemp)

#==================================== LOG PROCESSING ======================================#

# Read all files in the log file directory and combine them into one long file:
echo "Processing error logs..."
for f in $LOG_DIR/*; do
  if [[ "$f" =~ \.gz$ ]]; then
    echo "> reading $f"
    zcat $f >> $TEMPFILE1
  elif [[ "$f" =~ \.log ]]; then
    echo "> reading $f"
    cat $f >> $TEMPFILE1
  else
    echo "File $f not recognised as a log file, skipping"
  fi
done
echo ""

echo "Converting logs into a useful format:"
# Remove any log entries from the file that are not generated by ModSecurity
echo -n "> Removing entries that were not generated by ModSecurity..."
grep ModSecurity $TEMPFILE1 > $TEMPFILE2
echo "...done."
#echo ""

# Remove log entries from traffic that is not from friendly IP addresses
echo -n "> Removing errors that were not from friendly IP addresses..."
for ip in $FRIENDLY_IP; do
grep "client $ip" $TEMPFILE2 >> $TEMPFILE4
done
echo "...done."
#echo ""

# Write combined log file:
echo -n "> Generating combined log file..."
cp $TEMPFILE4 $COMBINED_LOG
echo "...done ($COMBINED_LOG)."
#echo ""

# Filter out rules that match the root regex:
echo -n "> Separating entries that match the root location"
cat $COMBINED_LOG | grep uri.\"/\" > $ROOT_LOG
echo "...done ($ROOT_LOG)."
echo ""


# Now generate a list of locations with problems:
echo -n "Generating a list of all problem locations"
awk ' { print $(NF-2) }' $COMBINED_LOG | cut -d '"' -f 2 | sort | uniq > $PROBLEM_LOCATIONS
echo "...done ($PROBLEM_LOCATIONS)."
echo ""

#==================================== WHITELIST HEADER ====================================#

echo "# This file was created using the ModSecurity CRS Whitelist Generator script, version $VERSION" >> $WHITELIST_FILE
echo "# Save this file to /etc/modsecurity/whitelists/domainname.conf and include it in the relevant" >> $WHITELIST_FILE
echo "# VirtualHost configuration with \"Include /etc/modsecurity/whitelists/domain.conf\"" >> $WHITELIST_FILE
echo "# See https://samhobbs.co.uk for more information" >> $WHITELIST_FILE
echo "" >> $WHITELIST_FILE
echo "" >> $WHITELIST_FILE

#==================================== MATCH EVERYWHERE ====================================#
# Rule matches for the site's root will be whitelisted for the whole site:

echo "Now working on rules to whitelist everywhere"
# Generate a list of IDs that match the root regex
echo -n "> Generating a list of rule IDs for the root location"
echo "##" > $ROOT_IDS
cat $ROOT_LOG | grep -o 'id \"......\"' | cut -d '"' -f 2 | sort -u >> $ROOT_IDS
echo "...done."
echo -n "> Writing LocationMatch statement to whitelist file..."
cat $ROOT_IDS | while read line; do sed 's/^/SecRuleRemoveById /'>> $WHITELIST_FILE; done
echo "" >> $WHITELIST_FILE
echo "...done."
echo ""

#=================================== SPECIAL LOCATIONS ====================================#

COUNT=1
for LOCATION in $SPECIAL_LOCATIONS; do
  echo "Now working on the following location: $LOCATION"
  # Generate a list of problem locations for $LOCATION
  echo -n "> Generating a list of matching locations"
  eval "echo "$OUTPUT_DIR/location_${COUNT}" >/dev/null"
  TEMPLOCATION="$OUTPUT_DIR/location_${COUNT}"
  cat $PROBLEM_LOCATIONS | egrep "^$LOCATION" >> $TEMPLOCATION
  echo "...done."

  # Generate list of rule IDs for $LOCATION
  echo -n "> Generating a list of rule IDs for this location"
  echo "#" > $TEMPFILE3 # just to clear tempfile3
  cat $TEMPLOCATION | while read LINE; do
  grep $LINE $COMBINED_LOG | grep -o 'id \"......\"' | cut -d '"' -f 2 >> $TEMPFILE3
  done
  eval "echo "$OUTPUT_DIR/id_${COUNT}" >/dev/null"
  TEMPIDFILE="$OUTPUT_DIR/id_${COUNT}"
  cat $TEMPFILE3 | sort -u | grep -vxF -f $ROOT_IDS > $TEMPIDFILE
  echo "...done."
  let COUNT=COUNT+1
  
  # Add $LOCATION to whitelist file
  echo -n "> Writing LocationMatch statement to whitelist file"
  # If defined location ends in $ don't add *, if not then do add a *
  if [[ "$LOCATION" =~ \$$ ]]; then
    echo "<LocationMatch \"^$LOCATION\">" >> $WHITELIST_FILE
  else
    echo "<LocationMatch \"^$LOCATION.*\">" >> $WHITELIST_FILE
  fi
  cat $TEMPIDFILE | while read line; do sed 's/^/SecRuleRemoveById /'>> $WHITELIST_FILE; done
  echo "</LocationMatch>" >> $WHITELIST_FILE
  echo "" >> $WHITELIST_FILE
  echo "...done."
  echo ""
done

#================================ REMAINING LOCATIONS LIST ================================#

# Generate a list of all locations that are covered by the group statements
echo -n "Find locations already covered by the group statements"
for file in $OUTPUT_DIR/*; do
  if [[ "$file" =~ location_[0-9] ]]; then
  cat $file >> $GROUPED_LOCATIONS
  fi
done
echo "...done ($GROUPED_LOCATIONS)."

# Now remove those locations from the master list of problem locations to leave a list of remaining problem locations. Also remove root, since this is dealt with separately.
echo -n "Remove these locations from the master list"
grep -vxF -f $GROUPED_LOCATIONS $PROBLEM_LOCATIONS | grep -v ^/$ > $PROBLEM_LOCATIONS_REMAINING
echo "...done ($PROBLEM_LOCATIONS_REMAINING)."

#================================== REMAINING LOCATIONS ===================================#

# Now write the remaining locations to the whitelist file
echo -n "Now writing the remaining problem locations to the whitelist file"
cat $PROBLEM_LOCATIONS_REMAINING | while read line; do
  # list all the rule IDs that match this location and don't match the rule IDs we have removed globally
  grep $line $COMBINED_LOG | grep -o 'id \"......\"' | cut -d '"' -f 2 | sort -u | grep -vxF -f $ROOT_IDS | sed 's/^/SecRuleRemoveById /'> $TEMPFILE4
  # count the number of rule IDs - if there are none then we don't want to write an empty locationmatch statement
  LINES=$(wc -l $TEMPFILE4 | cut -f1 -d ' ')
  if [[ $LINES != 0 ]] ; then
    # since some of these are .php scripts, .php?foo=bar needs to match, so don't add $
    echo "<LocationMatch \"^$line.*\">" >> $WHITELIST_FILE
    cat $TEMPFILE4 >> $WHITELIST_FILE
    echo "</LocationMatch>" >> $WHITELIST_FILE
    echo "" >> $WHITELIST_FILE
  fi
done
echo "...done."
echo ""
echo "Your whitelist file has been created at $WHITELIST_FILE"
echo ""

#====================================== CLEAN UP ==========================================#

# Clean up temporary files
echo -n "Cleaning up..."
rm -f $TEMPFILE1 $TEMPFILE2 $TEMPFILE3 $TEMPFILE4
echo "...done"