throw away bad referrers; and don't log them in access.log

src/core/http.py

@@ -188,6 +188,8 @@ class HttpRequestHandlerMixin:
         virtualHost = environs.getVar("virtualHost", default = None)
         if virtualHost and virtualHost.accessLogFile:
             referer = self.headers.get("Referer", "-")
+            if referer == "BAD":
+                return
             useragent = self.headers.get("User-agent", "-")
             virtualHost.accessLogFile.write("%s - - [%s] %s \"%s\" \"%s\"\n" %
                 (self.address_string(), self.log_date_time_string(), format%args,

src/core/main.py

@@ -37,6 +37,7 @@ import logging
 from optparse import OptionParser
 import os
 import pwd
+import re
 import sys
 import urllib
 
@@ -219,6 +220,15 @@ class Application(object):
             environs.setVar("rootStation", rootDataHolder)
             environs.setVar("currentStation", rootDataHolder)
 
+            # Throw away bad referers
+            bad_referrers = rootDataHolder.getConfigList("yep:badReferrers/yep:host", default = [])
+            referer = httpRequestHandler.headers.get("Referer", "")
+            if referer:
+                for bad in bad_referrers:
+                    if bad == referer or re.findall(bad, referer):
+                        httpRequestHandler.headers["Referer"] = "BAD"
+                        return httpRequestHandler.outputErrorAccessForbidden("")
+
             # Get query, headers and form variables in a submission object.
             submission = submissions.readSubmission(httpRequestHandler)
             environs.setVar("submission", submission)