update mail scripts

mail/check_blacklist.sh

+#!/bin/bash
+
+# TODO: skip <blacklist (in e-mail addresses)
+LOGFILE=/var/log/mail/all
+grep -q blacklist $LOGFILE || exit
+
+grep blacklist $LOGFILE | mutt -s "Blacklist result for mail.etersoft.ru" lav@etersoft.ru
+
+
+

mail/check_web_letters.sh

@@ -7,11 +7,14 @@
 # Sep 14 04:11:10 server postfix/smtpd[22127]: NOQUEUE: reject: RCPT from mail.mtml.ru[95.131.27.100]: 450 4.1.7 <noreply@mirtesen.ru>: Sender address rejected: unverified address: connect to mirtesen.ru[95.131.27.20]:25: Connection refused; from=<noreply@mirtesen.ru> to=<lav@etersoft.ru> proto=ESMTP helo=<mailgate1.mtml.ru>
 # Sep 14 22:46:17 server postfix/smtpd[20894]: NOQUEUE: reject: RCPT from mail.us.archive.org[207.241.224.6]: 450 4.1.7 <noreply@openlibrary.org>: Sender address rejected: unverified address: host mail.archive.org[207.241.224.6] said: 550 5.1.1 <noreply@openlibrary.org>: Recipient address rejected: User unknown in virtual mailbox table (in reply to RCPT TO command); from=<noreply@openlibrary.org> to=<lav@etersoft.ru> proto=ESMTP helo=<mail.archive.org>
 
-cat /var/log/mail/all | grep "reject: RCPT" | grep "Sender address rejected: unverified address" \
+if [ "$1" = "-f" ] ; then
+skip
+cat /var/log/mail/all | grep "reject: RCPT" | egrep "(Sender address rejected: unverified address|Helo command rejected: Host not found)" \
 	> $0.out
 #	sed "s!.*RCPT from \(.*\)\[.*from=<\(.*\)> to=<\(.*\)>.*!host=\1 from=\2 to=\3!g" 
+fi
 
 echo
 echo "==============="
 date
-$(dirname $0)/parse_mail_log.py
+$(dirname $0)/parse_mail_log.py $@

mail/copy_to_as.sh

 #!/bin/sh
 
+if false ; then
 echo ham
 SRCDIR="/var/spool/imap/domain/o/office.etersoft.ru/s/user/spam/&BD0ENQ- &BEEEPwQwBDw-"
 #ls -l "$SRCDIR"
@@ -7,7 +8,7 @@ ssh as mkdir -p /tmp/ham/
 scp "$SRCDIR"/*. as:/tmp/ham/ || exit
 ssh as sa-learn --ham /tmp/ham/ --dir --dbpath /var/spool/spamassassin/
 ssh as rm -f /tmp/ham/*
-
+fi
 
 echo spam
 SRCDIR="/var/spool/imap/domain/o/office.etersoft.ru/l/user/lav/spam"

mail/mymail/byregexp.sh

+#!/bin/sh
+
+fatal()
+{
+	echo "$@"
+	exit 1
+}
+
+if [ -z "$1" ] ; then
+	echo "Use: $0 <email> <regexp> <status> [period]"
+	echo "regexp - email or domain or regexp for sender address"
+	echo "status - rejected, "
+	exit
+fi
+
+EMAIL="$1"
+REGEXP="$2"
+STATUS="$3"
+PERIOD="$4"
+
+if [ "$PERIOD" = "all" ] ; then
+	CATCMD="ercat /var/log/mail/all $(ls /var/log/mail/all.*.bz2 | sort -n -t. -k2)"
+else
+	CATCMD="cat /var/log/mail/all"
+fi
+
+case $STATUS in
+	rejected)
+		STATUS="NOQUEUE: reject"
+		;;
+	"")
+		;;
+	*)
+		echo "unknow status $STATUS"
+		fatal
+		;;
+esac
+
+$CATCMD | grep -- $EMAIL | grep -- "$REGEXP" | grep -- "$STATUS"
+
+# TODO: записи нужно распознавать и приводить в читаемый вид.
+
+

mail/parse_mail_log.py

@@ -5,9 +5,21 @@ from operator import truth
 
 import datetime
 import httplib2
+import dns.resolver
 
 # TODO: переделать, чтобы можно было подключить в fail2ban: тогда будет оперативно действовать.
 
+# Возвращает true, если IP у доменов совпадают
+# FIXME: не учитывает количество IP на домене
+def cmp_domains(d1,d2):
+	print "Check " + d1 + " and " + d2;
+	try:
+		rdata1 = dns.resolver.query(d1, 'A')
+		rdata2 = dns.resolver.query(d2, 'A')
+		return not rdata1[0]._cmp(rdata2[0])
+	except:
+		return False
+
 def check_http(url):
 	h = httplib2.Http()
 	try:
@@ -48,30 +60,57 @@ def check_if_already(ip):
 	return ("#"+ip+" OK\n" in exists_ip) or (ip+" OK\n" in exists_ip)
 
 
+def check_line(line):
+	url = 'http://'+line['host']
+	print
+	print line
+	#print 'url=', url
+	if check_http(url):
+		if not check_if_already(line['ip']):
+			append_ip(line['host'], line['ip'], line['from'])
+		else:
+			print url, 'already exists'
+	else:
+		print 'Host ', url, 'are not accessed'
+
+
 test_string = "Sep 14 18:23:28 server postfix/smtpd[20585]: NOQUEUE: reject: RCPT from wiki.openvz.org[199.115.105.169]: 450 4.1.7 <apache@wiki.openvz.org>: Sender address rejected: unverified address: connect to wiki.openvz.org[199.115.105.169]:25: Connection refused; from=<apache@wiki.openvz.org> to=<lav@etersoft.ru> proto=ESMTP helo=<wiki.openvz.org>"
-tpl = re.compile(".*NOQUEUE: reject: RCPT from (.*?)\[(.*?)\].*from=<(.*?)>.*to=<(.*?)>.*")
+test_string1 = "Apr  3 15:02:14 server postfix/smtpd[26568]: NOQUEUE: reject: RCPT from u20325.netangels.ru[91.201.53.204]: 450 4.7.1 <u20325.localdomain>: Helo command rejected: Host not found; from=<info@zelenei.ru> to=<lav@etersoft.ru> proto=ESMTP helo=<u20325.localdomain>"
+
+tpl = re.compile("(.+? \d+? [\d:]+?) .* .*: NOQUEUE: reject: RCPT from (.*?)\[(.*?)\]: (\d*).*?: (.*?);.*from=<(.*?)>.*to=<(.*?)>.*proto=.*helo=<(.*?)>")
 #tpl = re.compile(".*RCPT from (.*?)\[(.*?)\].*")
 
 def parse_row(row):
 		parsed = tpl.search(row)
 		if not truth(parsed):
 			return
-		res = {"host":parsed.group(1),
-			 "ip":parsed.group(2),
-			"from":parsed.group(3),
-			 "to":parsed.group(4),
+		res = {
+			   "date" : parsed.group(1),
+			   "host" : parsed.group(2),
+			     "ip" : parsed.group(3),
+			 "status" : parsed.group(4),
+			   "text" : parsed.group(5),
+			   "from" : parsed.group(6),
+			     "to" : parsed.group(7),
+			   "helo" : parsed.group(8),
 			}
 		return res
 
 if __name__== "__main__":
-	#print test_string
-	#r1 = parse_row (test_string)
-	#print r1
 
 	list_only = 0
 	if len(sys.argv) > 1:
 		if sys.argv[1] == "--list":
 			list_only = 1
+		if sys.argv[1] == "--test":
+			print "Test string: " + test_string
+			r1 = parse_row (test_string)
+			print r1
+			print "Test string: " + test_string1
+			r1 = parse_row (test_string1)
+			print r1
+			print "Exiting..."
+			sys.exit()
 
 	if not list_only:
 		read_exists_ip()
@@ -80,7 +119,7 @@ if __name__== "__main__":
 	result = []
 	totip = []
 	for row in f.readlines():
-		#print row
+		print row
 		line = parse_row(row)
 		if line == False:
 			continue
@@ -92,19 +131,25 @@ if __name__== "__main__":
 				print line
 				continue
 
+			if not line['from']:
+				continue
+
 			sp = string.split(line['from'],'@')
+
+			if len(sp) <> 2:
+				continue
+
+			maildomain=sp[1]
+
+			# если IP домена в почте совпадает с IP хоста отправителя
+			if cmp_domains(maildomain, line['host']):
+				line['host'] = maildomain
+				# TODO: нужно проверять helo, если из-за него, то ничего не получится (его надо только в accesslist, а там же любой мусор может быть)
+				check_line(line)
+				continue
+
 			# если домен почты совпадает с хостом отправителя
-			if sp[1] == line['host']:
-				url = 'http://'+line['host']
-				print
-				print line
-				#print 'url=', url
-				if check_http(url):
-					if not check_if_already(line['ip']):
-						append_ip(line['host'], line['ip'], line['from'])
-					else:
-						print url, 'already exists'
-				else:
-					print 'Host ', url, 'are not accessed'
+			if maildomain == line['host']:
+				check_line(line)
 		#exit()
 	f.close()