update nginx stat scripts

nginx/config.example

 
 FILE="/var/log/nginx/*access.log"
 
+FILEERROR="/var/log/nginx/*error.log"
+
 FILE404="/var/log/nginx/error_404.log"

nginx/url_stats_count.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+	#print line
+        line_arr = line.split(" ")
+        try:
+	    if len(line_arr) <= 13:
+	        continue
+	    #print line_arr
+	    #print len(line_arr)
+            host = line_arr[13]
+            host = host[1:]
+            host = host[:-1]
+	    #print host
+            # CHANGE HERE
+            #url = line_arr[6]
+            #sp = url.split("/")
+            #if len(sp) < 2:
+            #    continue
+            #url = "/" + sp[1];
+            #url = ""
+            key = line_arr[0]
+            #t = float(line_arr[12])
+            t = 0.001
+            #print host, url, t
+ 
+            try:
+                urls[key] = (urls[key][0] + t, urls[key][1] + 1)
+            except KeyError, e:
+                urls[key] = (t, 1)
+        except ValueError, e:
+	    #print "Exception"
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,1), urls[k][0], urls[k][1], round (urls[k][0] * 1000 / urls[k][1], 0),  k
+        i += 1
+        if i > 100: break

nginx/url_stats_count.sh

+#!/bin/sh
+
+. ./config
+FILE=$FILEERROR
+test -z "$1" || FILE="$1"
+REPORT=url_stats_count.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+cat $FILE | ./url_stats_count.py >> $REPORT

nginx/url_stats_error404.sh

 #!/bin/sh
 
 . ./config
-test -z "$1" || FILE404="$1"
 FILE=$FILE404
+test -z "$1" || FILE="$1"
 REPORT=url_stats_error404.report
 
 echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT

nginx/url_stats_ip.py

@@ -12,7 +12,7 @@ try:
         try:
 	    if len(line_arr) <= 13:
 	        continue
-	    #print line_arr
+	    print line_arr
 	    #print len(line_arr)
             host = line_arr[13]
             host = host[1:]

nginx/url_stats_size.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+	#print line
+        line_arr = line.split(" ")
+        try:
+	    if len(line_arr) <= 13:
+	        continue
+	    #print line_arr
+	    #print len(line_arr)
+            host = line_arr[13]
+            host = host[1:]
+            host = host[:-1]
+	    #print host
+            # CHANGE HERE
+            #url = line_arr[6]
+            #sp = url.split("/")
+            #if len(sp) < 2:
+            #    continue
+            #url = "/" + sp[1];
+            #url = ""
+            key = line_arr[0]
+            t = float(line_arr[9])
+            #print host, url, t
+ 
+            try:
+                urls[key] = (urls[key][0] + t, urls[key][1] + 1)
+            except KeyError, e:
+                urls[key] = (t, 1)
+        except ValueError, e:
+	    #print "Exception"
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,1), round(urls[k][0] / 1024,0), urls[k][1], round (urls[k][0] / urls[k][1] / 1024, 0),  k
+        i += 1
+        if i > 100: break

nginx/url_stats_size.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE="$1"
+REPORT=url_stats_size.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average size - url" >> $REPORT
+
+cat $FILE | ./url_stats_size.py >> $REPORT