adding redmine tools

pykat/tools/redmine.py

+"""
+This file contains various functions for processing the Redmine download data for Finesse.
+"""
+
+import fileinput
+from dateutil.parser import parse
+import json
+import urllib
+
+def processRedmineProductionLog(log, output):
+    """
+    This function parses the Redmine production log for all Finesse
+    downloads. The output is a text file that contains the file
+    downloaded, the IP address and the date.
+    
+    Production log file is typically big (e.g. 10GB), this takes awhile
+    to run.
+    
+    log: path and filename to log file
+    output: path and filename of output text file to create
+    """
+    found = -1
+
+    with open(output, "w") as ofile:
+        for line in fileinput.input(log_filename):
+
+            if found > 0 and found+1 == fileinput.lineno():
+                try:
+                    file = line.lstrip().split()[3]
+                    file = file.split("=>")[1][1:-2]
+                    found = -2
+                except:
+                    print("Couldn't parse download, probably a traceback")
+                    print(line)
+                    
+                    found = -1
+    
+            elif line.startswith("Processing AttachmentsController#download"):
+                found = fileinput.lineno()
+    
+                ip = line.split()[3]
+                date = line.split()[5]
+
+            if found == -2:
+                ext = file.split(".")[-1]
+    
+                if ext == "zip" or ext == "tar" or ext == "pdf":
+                    ofile.write("%s %s %s\n" %(file, ip, date))
+        
+                    print file, ip, date
+        
+                found = -1
+                
+
+def readFinesseDownloadData(data_filename, unique=True, geo_filename=None):
+    """
+    Reads the data files generated by processRedmineProductionLog
+    and getFinesseDownloadIPGeoData.
+    
+    File download data is returned in a dictionary with filenames as the
+    key. This then contains another dictionary with the date of downloads
+    and number of downloads that day, and a list of IP addresses that
+    downloaded it.
+    
+    If Geo data filename is provided another dictionary will be returned.
+    The keys of which are the IP addresses and the latitude and longitude
+    values.
+    
+    data_filename: filename and path of processRedmineProductionLog output file
+    unique: If true only unique IP addresses for each file download are read
+    geo_filename: filename and path of getFinesseDownloadIPGeoData output file
+    """
+    files = {}
+    IPGeo = {}
+
+    with open(data_filename, "r") as data:
+        for line in data:
+            split = line.split()
+            file = split[0].lower()
+            ip = split[1]
+            date = parse(split[2])
+        
+            if split[0] not in files:
+                files[split[0]] = ([], {})
+       
+            if unique and ip in files[split[0]][0]:
+                continue
+            
+            files[split[0]][0].append(ip)
+        
+            if date not in files[split[0]][1]:
+                files[split[0]][1][date] = 0
+            
+            files[split[0]][1][date] += 1
+        
+    if geo_filename != None:
+        with open(geo_filename, "r") as data:
+            for line in data:
+                split = line.split()
+                IPGeo[split[0]] = (split[1], split[2])
+        
+        return files, IPGeo
+    else:
+        return files
+    
+def getFinesseDownloadIPGeoData(input_file, output_file):
+    """
+    Using the information outputted to the file by the function processRedmineProductionLog
+    this will function will call various free IP to Latitude/Longitude web services.
+    The file outputted will contain a list of unique IP addresses and it's coordinates.
+    
+    Note: This takes a long time to run! Free webservice may kick you off too if too many
+    requests are made per hour.
+    
+    input_file: filename and path to processRedmineProductionLog output file
+    output_file: Output filename and path
+    """
+    
+    IPGeo = {}
+
+    with open(input_file, "r") as data:
+        for line in data:
+            split = line.split()
+            file = split[0].lower()
+            ip = split[1]
+            date = parse(split[2])
+                
+            if ip not in IPGeo:
+                IPGeo[ip] = ([], [])
+            
+                response = urllib.urlopen('http://api.hostip.info/get_html.php?ip=%s&position=true' % ip).read()
+        
+                lat = response.split('\n')[3].split()
+                lon = response.split('\n')[4].split()
+            
+                if len(lat) == 2 and len(lon) == 2:
+                    IPGeo[ip][0].append(float(lat[1]))
+                    IPGeo[ip][1].append(float(lon[1]))
+                else:
+                    response = json.loads(urllib.urlopen('https://freegeoip.net/json/%s' % ip).read())
+        
+                    if 'latitude' in response:
+                        IPGeo[ip][0].append(float(response['latitude']))
+                        IPGeo[ip][1].append(float(response['longitude']))
+            
+                print response        
+
+    with open(output_file, "w") as w:
+        for ip in IPGeo.keys():
+            w.write("%s %g %g\n" % (ip, IPGeo[ip][0],  IPGeo[ip][1]))
+        
\ No newline at end of file