Allowing user to specify file containing regex

Author: Hainish

README.md

@@ -22,6 +22,20 @@ Here are some example CustomLog lines for your Apache config files:
 
 Notice that if you're using the `-c` option, you need to escape spaces in the command you're running with three backslashes.
 
+## Custom Regex File
+
+You can use the `-r` option to specify a file containing some regex describing the log entry format. The tokens `*IPV6*` and `*IPV4*` can be used as a subsitute for the entire regex of those respective protocols. The group `IP` will be anonymized, so for instance if the file contains the following:
+
+(?P<IP>*IPV6*)(, )(?P<OTHER>.*)
+
+Then the entry
+
+::ffff:8.8.8.8, 10.10.10.10 - - [14/Oct/2015:17:32:51 -0700] "GET /some/url HTTP/1.1" 200 13160
+
+will be anonymized as
+
+d68qCQ 10.10.10.10 - - [14/Oct/2015:17:32:51 -0700] "GET /some/url HTTP/1.1" 200 13160
+
 ## Requirements
 
 - Python 2.7

cryptolog.py

@@ -133,6 +133,9 @@ def EncryptField(self, field, hashed_size):
 
 if__name__=="__main__":
 parser=ArgumentParser(description='A program to encrypt the IP addresses in web server logs, to be used within an Apache CustomLog line. It assumes that the IP address is the first space-separated field in the log line. Input comes in the form of log lines from stdin.')
+parser.add_argument('-r',
+dest='regex',
+help='file providing regex for log format')
 parser.add_argument('-w',
 dest='write', 
 help='filename to write logs to')
@@ -160,7 +163,13 @@ def EncryptField(self, field, hashed_size):
 entities=args.entities.split(',')
 ipv6_exp='([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
 ipv4_exp='\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?'
-regex=re.compile(r'(?P<IP>'+ipv4_exp+'|'+ipv6_exp+')( )(?P<OTHER>.*)')
+ifargs.regex:
+withopen(args.regex, 'r') asregex_file:
+expression=regex_file.read().replace('*IPV6*', ipv6_exp).replace('*IPV4*', ipv4_exp)
+regex=re.compile(expression)
+else:
+regex=re.compile(r'(?P<IP>'+ipv4_exp+'|'+ipv6_exp+')( )(?P<OTHER>.*)')
+
 # todo:dta improve this regex for common log format
 apache_regex=re.compile(r'(?P<IP>'+ipv4_exp+'|'+ipv6_exp+') (?P<SAVE1>-) (?P<SAVE2>-) (?P<DATETIME>\[.*\]) (?P<REQUEST>".*") (?P<SAVE3>\d*|\-) (?P<SAVE4>\d*|\-) (?P<OTHER>.*)')
 delete_list= []