nginx stat script

nginx/config.example

+
+FILE="/var/log/nginx/*access.log"
+
+FILE404="/var/log/nginx/error_404.log"

nginx/url_stats.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+	#print line
+        line_arr = line.split(" ")
+        try:
+	    if len(line_arr) <= 13:
+	        continue
+	    #print line_arr
+	    #print len(line_arr)
+            host = line_arr[13]
+            host = host[1:]
+            host = host[:-1]
+	    #print host
+            # CHANGE HERE
+            url = line_arr[6]
+            #url = "/" + url.split("/");
+            #url = ""
+            t = float(line_arr[12])
+            #print host, url, t
+ 
+            try:
+                urls[host + url] = (urls[host + url][0] + t, urls[host + url][1] + 1)
+            except KeyError, e:
+                urls[host + url] = (t, 1)
+        except ValueError, e:
+	    #print "Exception"
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,1), urls[k][0], urls[k][1], round (urls[k][0] * 1000 / urls[k][1], 0),  k
+        i += 1
+        if i > 100: break

nginx/url_stats.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE="$1"
+REPORT=url_stats.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+cat $FILE | ./url_stats.py >> $REPORT

nginx/url_stats_404.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE="$1"
+REPORT=url_stats_com.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+cat $FILE | ./url_stats_com.py >> $REPORT

nginx/url_stats_com.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+	#print line
+        line_arr = line.split(" ")
+        try:
+	    if len(line_arr) <= 13:
+	        continue
+	    #print line_arr
+	    #print len(line_arr)
+            host = line_arr[13]
+            host = host[1:]
+            host = host[:-1]
+	    #print host
+            # CHANGE HERE
+            url = line_arr[6]
+            sp = url.split("/")
+            if len(sp) < 2:
+                continue
+            url = "/" + sp[1];
+            #url = ""
+            t = float(line_arr[12])
+            #print host, url, t
+ 
+            try:
+                urls[host + url] = (urls[host + url][0] + t, urls[host + url][1] + 1)
+            except KeyError, e:
+                urls[host + url] = (t, 1)
+        except ValueError, e:
+	    #print "Exception"
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,1), urls[k][0], urls[k][1], round (urls[k][0] * 1000 / urls[k][1], 0),  k
+        i += 1
+        if i > 100: break

nginx/url_stats_com.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE="$1"
+REPORT=url_stats_com.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+cat $FILE | ./url_stats_com.py >> $REPORT

nginx/url_stats_error404.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE404="$1"
+FILE=$FILE404
+REPORT=url_stats_error404.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+grep 'HTTP/1.1\" 404' $FILE | ./url_stats.py >> $REPORT

nginx/url_stats_ip.py

+#!/usr/bin/env python
+ 
+import sys
+ 
+urls = {}
+ 
+try:
+    while 1:
+        line = raw_input()
+	#print line
+        line_arr = line.split(" ")
+        try:
+	    if len(line_arr) <= 13:
+	        continue
+	    #print line_arr
+	    #print len(line_arr)
+            host = line_arr[13]
+            host = host[1:]
+            host = host[:-1]
+	    #print host
+            # CHANGE HERE
+            #url = line_arr[6]
+            #sp = url.split("/")
+            #if len(sp) < 2:
+            #    continue
+            #url = "/" + sp[1];
+            #url = ""
+            key = line_arr[0]
+            t = float(line_arr[12])
+            #print host, url, t
+ 
+            try:
+                urls[key] = (urls[key][0] + t, urls[key][1] + 1)
+            except KeyError, e:
+                urls[key] = (t, 1)
+        except ValueError, e:
+	    #print "Exception"
+            pass
+ 
+ 
+except EOFError, e:
+   pass
+ 
+def sort_by_value(d):
+    """ Returns the keys of dictionary d sorted by their values """
+    items=d.items()
+    backitems=[ [v[1],v[0]] for v in items]
+    backitems.sort(reverse=True)
+    return [backitems[i][1] for i in range(0,len(backitems))]
+ 
+if (len(sys.argv) > 1):
+    f = open(sys.argv[1], 'r')
+    for k in f.readlines():
+        k = k.strip()
+        try:
+             print urls[k][0], urls[k][1], urls[k][0] / urls[k][1], k
+        except:
+             print 0, 0, k
+else:
+    i = 0
+    alltime = 0
+    # FIXME
+    # reduce(lambda v1, v2 : v1[0] + v2[0], a)
+    for k in sort_by_value(urls):
+        alltime = alltime + urls[k][0]
+        i += 1
+        if i > 100: break
+
+    i = 0
+    for k in sort_by_value(urls):
+        print round(urls[k][0]*100/alltime,1), urls[k][0], urls[k][1], round (urls[k][0] * 1000 / urls[k][1], 0),  k
+        i += 1
+        if i > 100: break

nginx/url_stats_ip.sh

+#!/bin/sh
+
+. ./config
+test -z "$1" || FILE="$1"
+REPORT=url_stats_ip.report
+
+echo "=== Requests which took most of the time (from $FILE) ===" > $REPORT
+echo "percent - overall time - number of requests - average time - url" >> $REPORT
+
+cat $FILE | ./url_stats_ip.py >> $REPORT