add web url stat

web/url_stats.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+        line_arr = line.split(" ")
+        try:
+            host = line_arr[-1]
+            host = host[1:]
+            host = host[:-1]
+            # CHANGE HERE
+            url = line_arr[6]
+            #url = ""
+            t = float(line_arr[-2])
+#            print host, url, t
+ 
+            try:
+                urls[host + url] = (urls[host + url][0] + t, urls[host + url][1] + 1)
+            except KeyError, e:
+                urls[host + url] = (t, 1)
+        except ValueError, e:
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,0), urls[k][0], urls[k][1], round (urls[k][0] * 1000 / urls[k][1], 0),  k
+        i += 1
+        if i > 100: break

web/url_stats.sh

+#!/bin/sh
+
+FILE=/var/log/nginx/access.log
+test -z "$1" || FILE="$1"
+
+echo "=== Requests which took most of the time (from $FILE) ===" > url_report.txt
+echo "percent - overall time - number of requests - average time - url" >> url_report.txt
+
+cat "$FILE" | ./url_stats.py >> url_report.txt
+#grep "\[25/Dec/2010:" "$FILE" | ./url_stats.py >> url_report.txt
+#cat /tmp/report.txt | mail -s "url performance report" root