MoinMoin: attachment:cache-blacklist.patch of MoinMoinBugs/SlowAntispam

   1 * looking for arch@arch.thinkmo.de--2003-archives/moin--main--1.3--patch-633 to compare with
   2 * comparing to arch@arch.thinkmo.de--2003-archives/moin--main--1.3--patch-633
   3 M  MoinMoin/util/antispam.py
   4 
   5 * modified files
   6 
   7 --- orig/MoinMoin/util/antispam.py
   8 +++ mod/MoinMoin/util/antispam.py
   9 @@ -21,6 +21,8 @@
  10  from MoinMoin.security import Permissions
  11  from MoinMoin import caching, wikiutil
  12  
  13 +BLACKLISTPAGES = ["BadContent", "LocalBadContent"]
  14 +_cache = {}
  15  
  16  # Errors ---------------------------------------------------------------
  17  
  18 @@ -53,14 +55,22 @@
  19  
  20  
  21  def makelist(text):
  22 -    """ Split text into lines, strip them, skip # comments """
  23 +    """ Split text into lines, strip them, skip # comments
  24 +
  25 +    Retunr list of re objects.
  26 +    """
  27      lines = text.splitlines()
  28      list = []
  29      for line in lines:
  30          line = line.split(' # ', 1)[0] # rest of line comment
  31          line = line.strip()
  32          if line and not line.startswith('#'):
  33 -            list.append(line)
  34 +            try:
  35 +                scanner = re.compile(line, re.I)
  36 +                list.append(scanner)
  37 +            except re.error, err:
  38 +                dprint("Error in regex '%s': %s. Please check the pages %s." % (
  39 +                    line, str(err), ', '.join(BLACKLISTPAGES)))
  40      return list
  41  
  42  
  43 @@ -72,6 +82,8 @@
  44      @rtype: list
  45      @return: list of blacklisted regular expressions
  46      """
  47 +    global _cache
  48 +    
  49      from MoinMoin.PageEditor import PageEditor
  50      p = PageEditor(request, pagename, uid_override="Antispam subsystem")
  51      if do_update:
  52 @@ -122,6 +134,8 @@
  53                          raise WikirpcError("failed to get BadContent data",
  54                                             response)
  55                      p._write_file(response)
  56 +                    # Delete cache for this page
  57 +                    _cache[pagename] = None
  58  
  59              except (timeoutsocket.Timeout, timeoutsocket.error), err:
  60                  # Log the error
  61 @@ -138,34 +152,37 @@
  62  
  63              # set back socket timeout
  64              timeoutsocket.setDefaultSocketTimeout(old_timeout)
  65 -                
  66 -    blacklist = p.get_raw_body()
  67 -    return makelist(blacklist)
  68 +            
  69 +    # Return cached blacklist or create new
  70 +    if not pagename in _cache:
  71 +        blacklist = p.get_raw_body()
  72 +        _cache[pagename] = makelist(blacklist)
  73 +
  74 +    return _cache[pagename]
  75  
  76  
  77  class SecurityPolicy(Permissions):
  78      """ Extend the default security policy with antispam feature """
  79      
  80      def save(self, editor, newtext, rev, **kw):
  81 -        BLACKLISTPAGES = ["BadContent", "LocalBadContent"]
  82          if not editor.page_name in BLACKLISTPAGES:
  83              request = editor.request
  84  
  85              # Start timing of antispam operation
  86              request.clock.start('antispam')
  87 -            
  88 +
  89 +            request.clock.start('antispam-get-blacklist')
  90              blacklist = []
  91              for pn in BLACKLISTPAGES:
  92                  do_update = (pn != "LocalBadContent")
  93                  blacklist += getblacklist(request, pn, do_update)
  94 +            request.clock.stop('antispam-get-blacklist')
  95              if blacklist:
  96 -                for blacklist_re in blacklist:
  97 -                    try:
  98 -                        match = re.search(blacklist_re, newtext, re.I)
  99 -                    except re.error, err:
 100 -                        dprint("Error in regex '%s': %s. Please check the pages %s." % (blacklist_re, str(err), ', '.join(BLACKLISTPAGES)))
 101 -                        continue
 102 +                request.clock.start('antispam-match')
 103 +                for scanner in blacklist:
 104 +                    match = scanner.search(newtext)
 105                      if match:
 106 +                        request.clock.stop('antispam-match')
 107                          # Log error and raise SaveError, PageEditor
 108                          # should handle this.
 109                          _ = editor.request.getText
 110 @@ -175,6 +192,7 @@
 111                              }
 112                          dprint(msg)
 113                          raise editor.SaveError(msg)
 114 +                request.clock.stop('antispam-match')
 115              request.clock.stop('antispam')
 116              
 117          # No problem to save if my base class agree