Add htaccess parser && converter to map-format

redirector/generators.py

@@ -73,6 +73,7 @@ class ConfigGenerator:
 
     def generate_config(self, project_name, options):
         data = (self.start_line % project_name) + (self.rewrite_line % (project_name, project_name, "break")) + "}\n"
+
         for code in options:
             if code == "permanent":
                 data += (self.opt_start_line % (project_name, "permanent")) + (self.opt_rewrite_line % (project_name, project_name, "permanent", "permanent")) + "}\n"

redirector/htaccess_parser.py

+import argparse as ap
+import os
+
+class HtaccessParser:
+    def __init__(self, logger=None):
+        self.redirect_types = ['RewriteRule', 'RedirectMatch', 'Redirect']
+        self.logger = logger
+
+    def parse_line(self, line):
+        res_line = ''
+
+        line_parts = line.split()
+        line_type = line_parts[0]
+
+        if line_type in self.redirect_types:
+            
+            if line_type == 'Redirect':
+                res_line = '{} {} [status={}]'.format(line_parts[2], line_parts[3], line_parts[1])
+
+            elif line_type == 'RewriteRule':
+                res_line = '{} {}'.format(line_parts[1], line_parts[2])
+
+            elif line_type == 'RedirectMatch':
+                res_line = '{} {}'.format(line_parts[2], line_parts[3])
+        #print('\n')
+        #print('######')
+        #print(line)
+        #print(res_line)
+        #print('######')
+        return res_line
+
+
+class HtaccessReader:
+    def __init__(self, logger=None):
+        self.line_parser = HtaccessParser(logger=logger)
+        if logger:
+            self.logger = logger
+
+    def read_htaccess(self, htaccess_file, map_name):
+        abs_path = os.path.abspath(htaccess_file)
+        res = []
+
+        f = open(abs_path, 'r')
+        for line in f:
+            res.append('\n')
+            res.append('# {}'.format(line)) # for tests
+            res.append(self.line_parser.parse_line(line))
+            res.append('\n')
+        f.close()
+
+        self.write_map(res, map_name)
+
+
+    def write_map(self, map_content, map_name):
+        abs_path = os.path.abspath(map_name)
+
+        f = open(abs_path, 'w')
+        for line in map_content:
+            f.write(line)
+        f.close()
+
+
+def main(args=None):
+    parser = ap.ArgumentParser(description='Redirector: .htaccess converter')
+    parser.add_argument('htaccess_file', metavar='H', type=str, nargs=1, help='.htaccess file that defines redirect rules')
+    parser.add_argument('map_name', metavar='M', type=str, nargs=1, help='Output map name')
+
+    reader = HtaccessReader()
+
+    if not args:
+        args = parser.parse_args()
+
+    reader.read_htaccess(args.htaccess_file[0], args.map_name[0])
+
+
+if __name__ == "__main__":
+    main()
+

redirector/parser.py

@@ -38,6 +38,7 @@ class MapLineParser:
             return_code = 1
             options = [option[1:-1] for option in line[2:]]
         print(line[0])
+
         if line[0].startswith("~"):
             line[0] = "~" + prefix + line[0][1:]            
 
@@ -140,7 +141,6 @@ class ConfigReader:
                                     else:
                                         res[1][opt_] = [(request_url, redirect_url)]
 
-
             return res, res_prefix
         except YAMLError as e:
             self.logger.log("Error occurred while reading %s" % yaml_file + str(e))

redirector/redirector.py

@@ -23,6 +23,7 @@ class Redirector:
         # FIXME: what is the better way to do so?
         except Exception as e:
             raise self.RedirectorParserError("Can\'t parse .map file %s" % map_file, e)
+
         self.generator.generate(data, project_name)
 
     class RedirectorParserError(Exception):

tests/.htaccess

+RewriteRule    ^$                                               /cgi-bin/index.cgi [L]
+RewriteRule    ([0-9A-Za-z]{12})-del-([0-9A-Za-z]+)/.+$         /cgi-bin/index.cgi?del=$1-$2 [L]
+RedirectMatch 301 ^/manual/(.*)$ http://www.php.net/manual/$1
+RewriteRule    ^([0-9A-Za-z]{12})(\.html?|$)$                   /cgi-bin/index.cgi?op=download1&id=$1 [L]
+RewriteRule    ^([0-9A-Za-z]{12})(\/.+|\.html?|$)               /cgi-bin/index.cgi?op=download1&id=$1&fname=$2 [L]
+RedirectMatch 301 /java/jwarehouse/org.eclipse.(.*)/(.*) http://www.devdaily.com/java/jwarehouse/eclipse/org.eclipse.$1/$2
+RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})$   /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2 [L]
+RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})(\/.+|\.html?|$)        /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2&fname=$3 [L]
+RewriteRule    ^checkfiles\.html$                                       /cgi-bin/index.cgi?op=checkfiles [L]
+Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+RewriteRule    ^contact\.html$                                          /cgi-bin/index.cgi?op=contact [L]
+RewriteRule    ^premium\.html$                                          /cgi-bin/index.cgi?op=payments [L]
+RedirectMatch 301 ^/(.*)/htaccesselite-ultimate-htaccess-article.html(.*) /htaccess/htaccess.html
+RewriteRule    ^login\.html$                                            /cgi-bin/index.cgi?op=login [L]
+Redirect 301 /2007/phpbb/sending-post-form-data-with-php-curl.html /htaccess/sending-post-form-data-with-php-curl.html
+RewriteRule    ^catalogue(.*)\.html$                                    /cgi-bin/index.cgi?op=catalogue&date=$1 [L]
+RewriteRule    ^news([0-9]*)\.html$                                     /cgi-bin/index.cgi?op=news&page=$1 [L]
+Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+RewriteRule    ^n([0-9]+)-.*\.html$                                     /cgi-bin/index.cgi?op=news_details&news_id=$1 [L]
+RewriteRule    ^free([0-9]+)\.html$                     /cgi-bin/index.cgi?op=registration&aff_id=$1 [L]
+RewriteRule    ^users/([0-9A-Za-z\-_]{4,64})/?([0-9]+|$)        /cgi-bin/index.cgi?op=user_public&usr_login=$1&fld_id=$2 [L]
+RedirectMatch 301 ^/200([0-9])/([^01])(.*)$ /$2$3
+RewriteRule    ^embedmp3-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp3_embed&file_code=$1 [L]
+RewriteRule    ^embedmp4-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp32_embed&file_code=$1 [L]
+RewriteRule    ^box$                                    /cgi-bin/index_box.cgi [L]

tests/test.yaml

 projects:
-  - map: /home/kantegory/pvt/nginx-redirector/tests/test.map
+  - map: ./test.map
+    prefix: /test
+
+  - map: /home/kantegory/pvt/nginx-redirector/tests/test2.map
     prefix: /test
 

tests/test2.map

+
+# RewriteRule    ^$                                               /cgi-bin/index.cgi [L]
+^$ /cgi-bin/index.cgi
+
+# RewriteRule    ([0-9A-Za-z]{12})-del-([0-9A-Za-z]+)/.+$         /cgi-bin/index.cgi?del=$1-$2 [L]
+([0-9A-Za-z]{12})-del-([0-9A-Za-z]+)/.+$ /cgi-bin/index.cgi?del=$1-$2
+
+# RedirectMatch 301 ^/manual/(.*)$ http://www.php.net/manual/$1
+^/manual/(.*)$ http://www.php.net/manual/$1
+
+# RewriteRule    ^([0-9A-Za-z]{12})(\.html?|$)$                   /cgi-bin/index.cgi?op=download1&id=$1 [L]
+^([0-9A-Za-z]{12})(\.html?|$)$ /cgi-bin/index.cgi?op=download1&id=$1
+
+# RewriteRule    ^([0-9A-Za-z]{12})(\/.+|\.html?|$)               /cgi-bin/index.cgi?op=download1&id=$1&fname=$2 [L]
+^([0-9A-Za-z]{12})(\/.+|\.html?|$) /cgi-bin/index.cgi?op=download1&id=$1&fname=$2
+
+# RedirectMatch 301 /java/jwarehouse/org.eclipse.(.*)/(.*) http://www.devdaily.com/java/jwarehouse/eclipse/org.eclipse.$1/$2
+/java/jwarehouse/org.eclipse.(.*)/(.*) http://www.devdaily.com/java/jwarehouse/eclipse/org.eclipse.$1/$2
+
+# RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})$   /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2 [L]
+^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})$ /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2
+
+# RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})(\/.+|\.html?|$)        /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2&fname=$3 [L]
+^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})(\/.+|\.html?|$) /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2&fname=$3
+
+# RewriteRule    ^checkfiles\.html$                                       /cgi-bin/index.cgi?op=checkfiles [L]
+^checkfiles\.html$ /cgi-bin/index.cgi?op=checkfiles
+
+# Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+/2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html [status=301]
+
+# RewriteRule    ^contact\.html$                                          /cgi-bin/index.cgi?op=contact [L]
+^contact\.html$ /cgi-bin/index.cgi?op=contact
+
+# RewriteRule    ^premium\.html$                                          /cgi-bin/index.cgi?op=payments [L]
+^premium\.html$ /cgi-bin/index.cgi?op=payments
+
+# RedirectMatch 301 ^/(.*)/htaccesselite-ultimate-htaccess-article.html(.*) /htaccess/htaccess.html
+^/(.*)/htaccesselite-ultimate-htaccess-article.html(.*) /htaccess/htaccess.html
+
+# RewriteRule    ^login\.html$                                            /cgi-bin/index.cgi?op=login [L]
+^login\.html$ /cgi-bin/index.cgi?op=login
+
+# Redirect 301 /2007/phpbb/sending-post-form-data-with-php-curl.html /htaccess/sending-post-form-data-with-php-curl.html
+/2007/phpbb/sending-post-form-data-with-php-curl.html /htaccess/sending-post-form-data-with-php-curl.html [status=301]
+
+# RewriteRule    ^catalogue(.*)\.html$                                    /cgi-bin/index.cgi?op=catalogue&date=$1 [L]
+^catalogue(.*)\.html$ /cgi-bin/index.cgi?op=catalogue&date=$1
+
+# RewriteRule    ^news([0-9]*)\.html$                                     /cgi-bin/index.cgi?op=news&page=$1 [L]
+^news([0-9]*)\.html$ /cgi-bin/index.cgi?op=news&page=$1
+
+# Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+/2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html [status=301]
+
+# RewriteRule    ^n([0-9]+)-.*\.html$                                     /cgi-bin/index.cgi?op=news_details&news_id=$1 [L]
+^n([0-9]+)-.*\.html$ /cgi-bin/index.cgi?op=news_details&news_id=$1
+
+# RewriteRule    ^free([0-9]+)\.html$                     /cgi-bin/index.cgi?op=registration&aff_id=$1 [L]
+^free([0-9]+)\.html$ /cgi-bin/index.cgi?op=registration&aff_id=$1
+
+# RewriteRule    ^users/([0-9A-Za-z\-_]{4,64})/?([0-9]+|$)        /cgi-bin/index.cgi?op=user_public&usr_login=$1&fld_id=$2 [L]
+^users/([0-9A-Za-z\-_]{4,64})/?([0-9]+|$) /cgi-bin/index.cgi?op=user_public&usr_login=$1&fld_id=$2
+
+# RedirectMatch 301 ^/200([0-9])/([^01])(.*)$ /$2$3
+^/200([0-9])/([^01])(.*)$ /$2$3
+
+# RewriteRule    ^embedmp3-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp3_embed&file_code=$1 [L]
+^embedmp3-([0-9A-Za-z]{12})\.html$ /cgi-bin/index.cgi?op=mp3_embed&file_code=$1
+
+# RewriteRule    ^embedmp4-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp32_embed&file_code=$1 [L]
+^embedmp4-([0-9A-Za-z]{12})\.html$ /cgi-bin/index.cgi?op=mp32_embed&file_code=$1
+
+# RewriteRule    ^box$                                    /cgi-bin/index_box.cgi [L]
+^box$ /cgi-bin/index_box.cgi

tests/test3.map

+
+# RewriteRule    ^$                                               /cgi-bin/index.cgi [L]
+^$ /cgi-bin/index.cgi
+
+# RewriteRule    ([0-9A-Za-z]{12})-del-([0-9A-Za-z]+)/.+$         /cgi-bin/index.cgi?del=$1-$2 [L]
+([0-9A-Za-z]{12})-del-([0-9A-Za-z]+)/.+$ /cgi-bin/index.cgi?del=$1-$2
+
+# RedirectMatch 301 ^/manual/(.*)$ http://www.php.net/manual/$1
+^/manual/(.*)$ http://www.php.net/manual/$1
+
+# RewriteRule    ^([0-9A-Za-z]{12})(\.html?|$)$                   /cgi-bin/index.cgi?op=download1&id=$1 [L]
+^([0-9A-Za-z]{12})(\.html?|$)$ /cgi-bin/index.cgi?op=download1&id=$1
+
+# RewriteRule    ^([0-9A-Za-z]{12})(\/.+|\.html?|$)               /cgi-bin/index.cgi?op=download1&id=$1&fname=$2 [L]
+^([0-9A-Za-z]{12})(\/.+|\.html?|$) /cgi-bin/index.cgi?op=download1&id=$1&fname=$2
+
+# RedirectMatch 301 /java/jwarehouse/org.eclipse.(.*)/(.*) http://www.devdaily.com/java/jwarehouse/eclipse/org.eclipse.$1/$2
+/java/jwarehouse/org.eclipse.(.*)/(.*) http://www.devdaily.com/java/jwarehouse/eclipse/org.eclipse.$1/$2
+
+# RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})$   /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2 [L]
+^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})$ /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2
+
+# RewriteRule    ^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})(\/.+|\.html?|$)        /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2&fname=$3 [L]
+^([0-9A-Za-z\-_]{4,64})/([0-9A-Za-z]{12})(\/.+|\.html?|$) /cgi-bin/index.cgi?op=download1&usr_login=$1&id=$2&fname=$3
+
+# RewriteRule    ^checkfiles\.html$                                       /cgi-bin/index.cgi?op=checkfiles [L]
+^checkfiles\.html$ /cgi-bin/index.cgi?op=checkfiles
+
+# Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+/2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html [status=301]
+
+# RewriteRule    ^contact\.html$                                          /cgi-bin/index.cgi?op=contact [L]
+^contact\.html$ /cgi-bin/index.cgi?op=contact
+
+# RewriteRule    ^premium\.html$                                          /cgi-bin/index.cgi?op=payments [L]
+^premium\.html$ /cgi-bin/index.cgi?op=payments
+
+# RedirectMatch 301 ^/(.*)/htaccesselite-ultimate-htaccess-article.html(.*) /htaccess/htaccess.html
+^/(.*)/htaccesselite-ultimate-htaccess-article.html(.*) /htaccess/htaccess.html
+
+# RewriteRule    ^login\.html$                                            /cgi-bin/index.cgi?op=login [L]
+^login\.html$ /cgi-bin/index.cgi?op=login
+
+# Redirect 301 /2007/phpbb/sending-post-form-data-with-php-curl.html /htaccess/sending-post-form-data-with-php-curl.html
+/2007/phpbb/sending-post-form-data-with-php-curl.html /htaccess/sending-post-form-data-with-php-curl.html [status=301]
+
+# RewriteRule    ^catalogue(.*)\.html$                                    /cgi-bin/index.cgi?op=catalogue&date=$1 [L]
+^catalogue(.*)\.html$ /cgi-bin/index.cgi?op=catalogue&date=$1
+
+# RewriteRule    ^news([0-9]*)\.html$                                     /cgi-bin/index.cgi?op=news&page=$1 [L]
+^news([0-9]*)\.html$ /cgi-bin/index.cgi?op=news&page=$1
+
+# Redirect 301 /2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html
+/2007/webmaster/custom-phpini-with-fastcgi-on-dreamhost.html /dreamhost/custom-phpini-with-fastcgi-on-dreamhost.html [status=301]
+
+# RewriteRule    ^n([0-9]+)-.*\.html$                                     /cgi-bin/index.cgi?op=news_details&news_id=$1 [L]
+^n([0-9]+)-.*\.html$ /cgi-bin/index.cgi?op=news_details&news_id=$1
+
+# RewriteRule    ^free([0-9]+)\.html$                     /cgi-bin/index.cgi?op=registration&aff_id=$1 [L]
+^free([0-9]+)\.html$ /cgi-bin/index.cgi?op=registration&aff_id=$1
+
+# RewriteRule    ^users/([0-9A-Za-z\-_]{4,64})/?([0-9]+|$)        /cgi-bin/index.cgi?op=user_public&usr_login=$1&fld_id=$2 [L]
+^users/([0-9A-Za-z\-_]{4,64})/?([0-9]+|$) /cgi-bin/index.cgi?op=user_public&usr_login=$1&fld_id=$2
+
+# RedirectMatch 301 ^/200([0-9])/([^01])(.*)$ /$2$3
+^/200([0-9])/([^01])(.*)$ /$2$3
+
+# RewriteRule    ^embedmp3-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp3_embed&file_code=$1 [L]
+^embedmp3-([0-9A-Za-z]{12})\.html$ /cgi-bin/index.cgi?op=mp3_embed&file_code=$1
+
+# RewriteRule    ^embedmp4-([0-9A-Za-z]{12})\.html$   /cgi-bin/index.cgi?op=mp32_embed&file_code=$1 [L]
+^embedmp4-([0-9A-Za-z]{12})\.html$ /cgi-bin/index.cgi?op=mp32_embed&file_code=$1
+
+# RewriteRule    ^box$                                    /cgi-bin/index_box.cgi [L]
+^box$ /cgi-bin/index_box.cgi