Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
finesse
pykat
Commits
066d6dbe
Commit
066d6dbe
authored
Feb 07, 2015
by
Daniel Brown
Browse files
adding redmine tools
parent
15650200
Changes
1
Hide whitespace changes
Inline
Side-by-side
pykat/tools/redmine.py
0 → 100644
View file @
066d6dbe
"""
This file contains various functions for processing the Redmine download data for Finesse.
"""
import
fileinput
from
dateutil.parser
import
parse
import
json
import
urllib
def
processRedmineProductionLog
(
log
,
output
):
"""
This function parses the Redmine production log for all Finesse
downloads. The output is a text file that contains the file
downloaded, the IP address and the date.
Production log file is typically big (e.g. 10GB), this takes awhile
to run.
log: path and filename to log file
output: path and filename of output text file to create
"""
found
=
-
1
with
open
(
output
,
"w"
)
as
ofile
:
for
line
in
fileinput
.
input
(
log_filename
):
if
found
>
0
and
found
+
1
==
fileinput
.
lineno
():
try
:
file
=
line
.
lstrip
().
split
()[
3
]
file
=
file
.
split
(
"=>"
)[
1
][
1
:
-
2
]
found
=
-
2
except
:
print
(
"Couldn't parse download, probably a traceback"
)
print
(
line
)
found
=
-
1
elif
line
.
startswith
(
"Processing AttachmentsController#download"
):
found
=
fileinput
.
lineno
()
ip
=
line
.
split
()[
3
]
date
=
line
.
split
()[
5
]
if
found
==
-
2
:
ext
=
file
.
split
(
"."
)[
-
1
]
if
ext
==
"zip"
or
ext
==
"tar"
or
ext
==
"pdf"
:
ofile
.
write
(
"%s %s %s
\n
"
%
(
file
,
ip
,
date
))
print
file
,
ip
,
date
found
=
-
1
def
readFinesseDownloadData
(
data_filename
,
unique
=
True
,
geo_filename
=
None
):
"""
Reads the data files generated by processRedmineProductionLog
and getFinesseDownloadIPGeoData.
File download data is returned in a dictionary with filenames as the
key. This then contains another dictionary with the date of downloads
and number of downloads that day, and a list of IP addresses that
downloaded it.
If Geo data filename is provided another dictionary will be returned.
The keys of which are the IP addresses and the latitude and longitude
values.
data_filename: filename and path of processRedmineProductionLog output file
unique: If true only unique IP addresses for each file download are read
geo_filename: filename and path of getFinesseDownloadIPGeoData output file
"""
files
=
{}
IPGeo
=
{}
with
open
(
data_filename
,
"r"
)
as
data
:
for
line
in
data
:
split
=
line
.
split
()
file
=
split
[
0
].
lower
()
ip
=
split
[
1
]
date
=
parse
(
split
[
2
])
if
split
[
0
]
not
in
files
:
files
[
split
[
0
]]
=
([],
{})
if
unique
and
ip
in
files
[
split
[
0
]][
0
]:
continue
files
[
split
[
0
]][
0
].
append
(
ip
)
if
date
not
in
files
[
split
[
0
]][
1
]:
files
[
split
[
0
]][
1
][
date
]
=
0
files
[
split
[
0
]][
1
][
date
]
+=
1
if
geo_filename
!=
None
:
with
open
(
geo_filename
,
"r"
)
as
data
:
for
line
in
data
:
split
=
line
.
split
()
IPGeo
[
split
[
0
]]
=
(
split
[
1
],
split
[
2
])
return
files
,
IPGeo
else
:
return
files
def
getFinesseDownloadIPGeoData
(
input_file
,
output_file
):
"""
Using the information outputted to the file by the function processRedmineProductionLog
this will function will call various free IP to Latitude/Longitude web services.
The file outputted will contain a list of unique IP addresses and it's coordinates.
Note: This takes a long time to run! Free webservice may kick you off too if too many
requests are made per hour.
input_file: filename and path to processRedmineProductionLog output file
output_file: Output filename and path
"""
IPGeo
=
{}
with
open
(
input_file
,
"r"
)
as
data
:
for
line
in
data
:
split
=
line
.
split
()
file
=
split
[
0
].
lower
()
ip
=
split
[
1
]
date
=
parse
(
split
[
2
])
if
ip
not
in
IPGeo
:
IPGeo
[
ip
]
=
([],
[])
response
=
urllib
.
urlopen
(
'http://api.hostip.info/get_html.php?ip=%s&position=true'
%
ip
).
read
()
lat
=
response
.
split
(
'
\n
'
)[
3
].
split
()
lon
=
response
.
split
(
'
\n
'
)[
4
].
split
()
if
len
(
lat
)
==
2
and
len
(
lon
)
==
2
:
IPGeo
[
ip
][
0
].
append
(
float
(
lat
[
1
]))
IPGeo
[
ip
][
1
].
append
(
float
(
lon
[
1
]))
else
:
response
=
json
.
loads
(
urllib
.
urlopen
(
'https://freegeoip.net/json/%s'
%
ip
).
read
())
if
'latitude'
in
response
:
IPGeo
[
ip
][
0
].
append
(
float
(
response
[
'latitude'
]))
IPGeo
[
ip
][
1
].
append
(
float
(
response
[
'longitude'
]))
print
response
with
open
(
output_file
,
"w"
)
as
w
:
for
ip
in
IPGeo
.
keys
():
w
.
write
(
"%s %g %g
\n
"
%
(
ip
,
IPGeo
[
ip
][
0
],
IPGeo
[
ip
][
1
]))
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment