Created multiprocess version of RedirectorWatch to watch for different .yaml files

dev/__init__.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-

dev/const.py

+NAME = "redirector"
+VERSION = "1.0"
+AUTHOR = "Nikita Yefremov"
+EMAIL = "enk@etersoft.ru"
+PYTHON_VERSION = "3.6"
+VERSION_STATUS = "alpha"
+
+REDIRECTOR_DIR = "/root/redirector" # /home/enk
+CONFIG_YAML = '/etc/redirector/config.yaml'
+MAPS_DIR = REDIRECTOR_DIR + "/maps"
+CONFIG_DIR = REDIRECTOR_DIR + "/location-includes"

dev/generators.py

+
+class MapGenerator:
+    def __init__(self):
+        self.start_lines = [
+            "map $uri $%s-redirect {",
+            "\nmap $uri $%s-option-redirect {",
+            "\nmap $uri $%s-option {"
+        ]
+        self.endline = "\n}"
+
+    def generate_map(self, redirects_sorted, project_name):
+        data = ""
+        for i, map_type in enumerate(self.start_lines):
+            data += map_type % project_name
+            for arg1, arg2 in redirects_sorted[i]:
+                data += "\n\t%s\t%s;" % (arg1, arg2)
+            data += self.endline
+        return data
+
+
+class ConfigGenerator:
+    def __init__(self):
+        self.start_line = "if($%s-redirect) {\n"
+        self.rewrite_line = "\trewrite ^/%s/(.^)$ $%s-redirect redirect;\n}"
+
+    def generate_conf(self, project_name):
+        return (self.start_line % project_name) + (self.rewrite_line % (project_name, project_name))
+    
+    
+

dev/logger.py

+import logging
+
+
+class Logger:
+    def __init__(self, file_=None):
+        if file_:
+            self.file_ = file_
+            logging.basicConfig(filename=file_)
+
+    def log(self, message, level="critical"):
+        if self.file_:
+            if level == "critical":
+                logging.critical(message)
+            elif level == "debug":
+                logging.debug(message)
+            elif level == "info":
+                logging.info(message)
+        else:
+            print(level.upper() + ":\n" + message)

dev/parser.py

+import os
+from yaml import load, YAMLError
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
+import re
+
+
+class MapLineParser:
+    def __init__(self, logger=None):
+        self.url_regexp = re.compile(r'(\/[(\w)-:]+)+\/?(\.\w+)?')
+        self.url_absolute = re.compile(
+            r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$")
+        self.logger = logger
+
+    def parse_line(self, line, prefix, i):
+        line = line.strip()
+        return_code, options = 0, []
+
+        if line.startswith("#") or len(line) == 0:
+            return -1, None, None
+
+        line = line.split()
+
+        if len(line) < 2:
+            raise self.ParseLineError("Error on %s:{line};\n"
+                                      "Not enough arguments to parse!".format(line=i), None)
+        elif len(line) > 2:
+            return_code = 1
+            options = [option[1:-1] for option in line[2:]]
+
+        if not self.url_regexp.fullmatch(line[0]):  # not url - regexp
+            try:
+                re.compile(line[0])
+                re.compile(line[1])
+            except re.error:
+                raise self.RegexpTestError("Can\'t compile regular expressions {expression1} {expression2} in "
+                                           "%s:{line_num}".format(expression1=line[0], expression2=line[1],
+                                                                  line_num=i), re.error)
+
+        elif line[0].startswith("//"):  # if new URI relative to the root
+            line[0] = line[0][1:]  # cutting out extra '/' at the beginning
+        elif line[1].startswith("//"):
+            line[1] = line[1][1:]
+
+        elif self.url_absolute.fullmatch(line[1]):
+            pass
+
+        else:  # default url
+            line[0] = prefix + line[0]
+            line[1] = prefix + line[1]
+
+        return return_code, line[0], line[1], options
+
+    class ParseLineError(Exception):
+        """Raised when line contains not enough arguments
+
+        Attributes:
+            message -- expanation where in .yaml config file aprser didn't find enough arguments
+        """
+
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.message = message
+            self.errors = errors
+
+    class RegexpTestError(Exception):
+        """Raised when parser fails to compile url containing regular expression
+
+        Attributes:
+            message -- explanation what regexp failed to compile and where
+        """
+
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.message = message
+            self.errors = errors
+
+
+class ConfigReader:
+    def __init__(self, logger=None):
+        self.line_parser = MapLineParser(logger=logger)
+        if logger:
+            self.logger = logger
+
+    @staticmethod
+    def parse_yaml(file_dir):
+        return_list = []
+        with open(file_dir, 'r') as stream:
+            data = load(stream, Loader=Loader)
+            for project in data.get("projects"):
+                map_path = project.get("map")
+                project_prefix = project.get("prefix")
+                return_list.append((map_path, project_prefix))
+        return return_list
+
+    def parse_map(self, map_file, yaml_file):
+        res = [[], [], []]
+        res_prefix = None
+        try:
+            for map_path, prefix in self.parse_yaml(yaml_file):
+                if map_path == map_file or os.getcwd() + "/" + map_path == map_file:
+                    res_prefix = prefix.split("/")[-1]  # ???: Is the last directory of map_path a project's name?
+                    with open(map_file, "r") as file:
+                        for i, line in enumerate(file):
+                            request_url, redirect_url, options = None, None, []
+                            try:
+                                return_code, request_url, redirect_url, *options = \
+                                    self.line_parser.parse_line(line, prefix, i)
+                            except MapLineParser.RegexpTestError as e:
+                                self.logger.log(e.message % map_file)
+                            except MapLineParser.ParseLineError as e:
+                                self.logger.log(e.message % map_file)
+
+                            if not request_url or not redirect_url or return_code == -1:
+                                continue
+                            else:
+                                if return_code == 0:
+                                    res[0].append((request_url, redirect_url))
+                                elif return_code == 1:
+                                    res[1].append((request_url, redirect_url))
+                                    res[2].append((request_url, options))
+            return res, res_prefix
+        except YAMLError as e:
+            self.logger.log("Error occurred while reading %s" % yaml_file + str(e))
+

dev/redirector.py

+import sys
+import os
+import pyinotify
+from multiprocessing import Queue, Lock, Process, Event
+import argparse as ap
+
+from dev import generators
+from dev import logger
+from dev import parser
+from dev import const
+
+redirector_watch = None
+    
+class Redirector:
+    def __init__(self, logger=None):
+        self.parser = parser.ConfigReader(logger=logger)
+        self.map_generator = generators.MapGenerator()
+        self.config_generator = generators.ConfigGenerator()
+
+    def generate(self, yaml_file, map_file):
+        import pdb
+        pdb.set_trace()
+        project_name = "Error"
+        try:
+            data, project_name = self.parser.parse_map(map_file, yaml_file)
+            if not project_name:
+                raise self.RedirectorInputDataError("Can\'t properly parse .map or .yaml files", Exception())
+        # FIXME: what is the better way to do so?
+        except Exception as e:
+            raise self.RedirectorParserError("Can\'t parse .map file %s" % map_file, e)
+        try:
+            with open(const.MAPS_DIR + "/%s.map" % project_name, "w") as map_file:
+                map_file.write(self.map_generator.generate_map(data, project_name))
+            with open(const.CONFIG_DIR + "/%s.conf" % project_name, "w") as conf_file:
+                conf_file.write(self.config_generator.generate_conf(project_name))
+        except Exception as e:
+            print(e)
+            raise self.RedirectorGenerationError("Can\'t generate new .conf or new .map file \
+                 from %s .map file for %s project" % (map_file, project_name), e)
+
+    class RedirectorParserError(Exception):
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.errors = errors
+
+    class RedirectorGenerationError(Exception):
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.errors = errors
+
+    class RedirectorInputDataError(Exception):
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.errors = errors
+
+
+class RedirectorWatch:
+    def __init__(self):
+        # multiprocessing.Process.__init__(self)
+        self.logger = logger.Logger("redirector-watch.log")
+        self.redirector = Redirector()
+        self.parser = parser.ConfigReader(logger=self.logger)
+        self.mask = pyinotify.IN_MODIFY 
+        self.pool = {}
+
+    def watch(self, yaml_files, remove):
+        if not yaml_files:
+            raise self.RedirectorWatchException("yaml_file list is empty", Exception)
+        yaml_files = yaml_files if isinstance(yaml_files, list) else [yaml_files]
+
+        for yfile in yaml_files:
+            if remove:
+                try:
+                    self.pool[yfile][1].stop()
+                except RuntimeError as e:
+                    print("Thread stopped at:\n%s" % str(e))
+                redirector_watch.pool[yfile] = None
+                continue
+            else:
+                Watch = self.create_watch_class(yfile)
+                try:
+                    self.set_inotify(yfile, Watch)
+                except pyinotify.PyinotifyError as e:
+                    raise self.RedirectorWatchException("Can\'t set inotify for yamll file %s" % yfile, e)
+
+    # creating class Watch with custom static 'yaml_file' argument
+    def create_watch_class(self, yaml_file):
+        global redirector_watch
+        class Watch(pyinotify.ProcessEvent):
+            def __init__(self, *args, **kwargs):
+                self.redirector = kwargs.pop("redirector")
+                super().__init__(*args, **kwargs)
+
+            def process_IN_MODIFY(self, event):
+                try:
+                    queue = Queue()
+                    redirector_watch = queue.get()
+                    redirector_watch.pool[yaml_file][1].stop()
+                except RuntimeError as e:
+                    print("Thread stopped at:\n%s" % str(e))
+
+                redirector_watch.pool[yaml_file] = None
+                self.redirector.generate(yaml_file, event.pathname)
+                with open('redirector_watch.log', 'a') as file_:
+                    file_.write(yaml_file + " has been stopped?\n")
+        return Watch
+
+    # we suggest that self.watch_manager already exists
+    def set_inotify(self, yaml_file, Watch_class):
+        wm = pyinotify.WatchManager()
+        self.add_files(yaml_file, wm)
+        notifier = pyinotify.ThreadedNotifier(wm, Watch_class(redirector=self.redirector))
+        notifier.start()
+        self.pool[yaml_file] = [wm, notifier]
+
+    def add_files(self, file_, wm):
+        file_list = self.parser.parse_yaml(file_)
+        for map_path, _ in file_list:
+            cwd = os.getcwd()
+            wdd = wm.add_watch(cwd + "/" + map_path, self.mask, rec=True)
+ 
+    class RedirectorWatchException(Exception):
+        def __init__(self, message, errors):
+            super().__init__(message)
+            self.errors = errors
+
+
+def main(args=None):
+    parser = ap.ArgumentParser(description='Redirector: nginx config generator')
+    parser.add_argument('yaml_file', metavar='Y', type=str, nargs=1,
+                        help='.yaml file that defines project and it\'s content')
+    parser.add_argument('map_file', metavar='M', type=str, nargs=1,
+                        help='.map file that defines redirects for project')
+    if not args:
+        args = parser.parse_args()
+        
+    redirector = Redirector()
+    try:
+        redirector.generate(args.yaml_file[0], args.map_file[0])
+    except (Redirector.RedirectorGenerationError, redirector.RedirectorParserError) as e:
+        print("CRITICAL:\n" + str(e))
+
+
+def add_watch(args=None):
+    parser = ap.ArgumentParser(description='Redirector watchdog utility')
+    parser.add_argument('filename', metavar='Y', type=str, nargs="+",
+                        help='Sets watch on the content of .yaml file')
+    parser.add_argument('-r', '--remove', action='store_true', dest='remove', help='Option for for removing watch on .yaml file')
+    if not args:
+        args = parser.parse_args()
+
+    queue = Queue()
+    redirector_watch = queue.get()
+    
+    redirector_watch.watch(args.filename, args.remove)
+
+def watch_main(args=None):
+    parser = ap.ArgumentParser(description='Redirector watchdog utility')
+    parser.add_argument('filename', metavar='Y', type=str, nargs="+",
+                        help='Sets watch on the content of .yaml file')
+    parser.add_argument('-r', '--remove', action='store_true', dest='remove', help='Option for for removing watch on .yaml file')
+    if not args:
+        args = parser.parse_args()
+    
+    import pdb
+    pdb.set_trace()
+    queue = Queue()
+    redirector_watch = RedirectorWatch()
+    redirector_watch.watch(args.filename, args.remove)
+    queue.put(redirector_watch)
+
+print('Correct execution ended')

dev/utils.py

+from pyinotify import ThreadedNotifier
+
+class CustomThreadedNotifier(ThreadedNotifier):
+	def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None):
+		super().__init__(watch_manager, default_proc_fun, read_freq, threshold, timeout)
+		
+	def exit(self):
+		try:
+			self.stop()
+		except RuntimeError:
+			pass
+                 
+		

setup.py

-import os
-
 from setuptools import setup, find_packages
-from core import const
+from dev import const
 
 classifiers_list = [
     "Development Status :: 4 - Beta",
@@ -24,10 +22,10 @@ setup(
     packages=find_packages(),
     entry_points={
         'console_scripts': [
-            'redirector = core.redirector:main',
-            'redirector-watch = core.redirector:watch'
+            'redirector = dev.redirector:main',
+            'redirector-watch = dev.redirector:add_watch',
+			'redirector-watch-start = dev.redirector:watch_main'
         ]
     },
     classifiers=classifiers_list
 )
-