From e72663be081b8ef534b4ef43b90807c55d15b8fb Mon Sep 17 00:00:00 2001 From: MatthewShao Date: Tue, 12 Jan 2016 11:45:03 +0800 Subject: [PATCH 1/8] Fix script reloader on OS X. --- libmproxy/flow.py | 8 ++++---- libmproxy/script/reloader.py | 22 ++++++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index f02b57678..5e4ca5f61 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -666,7 +666,7 @@ class FlowMaster(controller.Master): script.reloader.unwatch(script_obj) self.scripts.remove(script_obj) - def load_script(self, command, use_reloader=False): + def load_script(self, command, use_reloader=True): """ Loads a script. Returns an error description if something went wrong. @@ -1040,14 +1040,14 @@ class FlowMaster(controller.Master): s.unload() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') try: s.load() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') else: - self.add_event('"{}" reloaded.'.format(s.filename)) + self.add_event('"{}" reloaded.'.format(s.filename), 'info') return ok def shutdown(self): diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 26691fa3e..ad6303a68 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -1,4 +1,5 @@ import os +import fnmatch from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer @@ -24,17 +25,30 @@ def unwatch(script): class _ScriptModificationHandler(PatternMatchingEventHandler): - def __init__(self, callback): + def __init__(self, callback, pattern='*.py'): # We could enumerate all relevant *.py files (as werkzeug does it), # but our case looks like it isn't as simple as enumerating sys.modules. # This should be good enough for now. super(_ScriptModificationHandler, self).__init__( ignore_directories=True, - patterns=["*.py"] ) self.callback = callback + self.pattern = pattern def on_modified(self, event): - self.callback() + super(_ScriptModificationHandler, self).on_modified(event) + if event.is_directory: + files_in_dir = [event.src_path + "/" + \ + f for f in os.listdir(event.src_path)] + if len(files_in_dir) > 0: + modifiedFilename = max(files_in_dir, key=os.path.getmtime) + else: + return + else: + modifiedFilename = event.src_path + + if fnmatch.fnmatch(os.path.basename(modifiedFilename), self.pattern): + self.callback() + +__all__ = ["watch", "unwatch"] -__all__ = ["watch", "unwatch"] \ No newline at end of file From 89679bc386fe2f1fc739e12525d87888264fd88b Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Tue, 12 Jan 2016 20:50:33 +0800 Subject: [PATCH 2/8] Fix observer in same dir situation. --- libmproxy/script/reloader.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index ad6303a68..d8b5ed4c8 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -10,7 +10,8 @@ def watch(script, callback): if script in _observers: raise RuntimeError("Script already observed") script_dir = os.path.dirname(os.path.abspath(script.args[0])) - event_handler = _ScriptModificationHandler(callback) + script_name = os.path.basename(script.args[0]) + event_handler = _ScriptModificationHandler(callback, filename=script_name) observer = Observer() observer.schedule(event_handler, script_dir) observer.start() @@ -25,7 +26,7 @@ def unwatch(script): class _ScriptModificationHandler(PatternMatchingEventHandler): - def __init__(self, callback, pattern='*.py'): + def __init__(self, callback, filename='*'): # We could enumerate all relevant *.py files (as werkzeug does it), # but our case looks like it isn't as simple as enumerating sys.modules. # This should be good enough for now. @@ -33,21 +34,21 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): ignore_directories=True, ) self.callback = callback - self.pattern = pattern + self.filename = filename def on_modified(self, event): - super(_ScriptModificationHandler, self).on_modified(event) + # super(_ScriptModificationHandler, self).on_modified(event) if event.is_directory: files_in_dir = [event.src_path + "/" + \ f for f in os.listdir(event.src_path)] if len(files_in_dir) > 0: - modifiedFilename = max(files_in_dir, key=os.path.getmtime) + modified_filepath = max(files_in_dir, key=os.path.getmtime) else: return else: - modifiedFilename = event.src_path + modified_filepath = event.src_path - if fnmatch.fnmatch(os.path.basename(modifiedFilename), self.pattern): + if fnmatch.fnmatch(os.path.basename(modifiedFilename), self.filename): self.callback() __all__ = ["watch", "unwatch"] From 201fdea6e51624e104273eba760b7c5e02848c89 Mon Sep 17 00:00:00 2001 From: MatthewShao Date: Tue, 12 Jan 2016 21:36:59 +0800 Subject: [PATCH 3/8] Change Observer() to PollingObserver(). --- libmproxy/script/reloader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index d8b5ed4c8..f824d838a 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -1,7 +1,7 @@ import os import fnmatch from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer +from watchdog.observers.polling import PollingObserver _observers = {} @@ -12,7 +12,7 @@ def watch(script, callback): script_dir = os.path.dirname(os.path.abspath(script.args[0])) script_name = os.path.basename(script.args[0]) event_handler = _ScriptModificationHandler(callback, filename=script_name) - observer = Observer() + observer = PollingObserver() observer.schedule(event_handler, script_dir) observer.start() _observers[script] = observer @@ -37,7 +37,6 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): self.filename = filename def on_modified(self, event): - # super(_ScriptModificationHandler, self).on_modified(event) if event.is_directory: files_in_dir = [event.src_path + "/" + \ f for f in os.listdir(event.src_path)] @@ -48,7 +47,7 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): else: modified_filepath = event.src_path - if fnmatch.fnmatch(os.path.basename(modifiedFilename), self.filename): + if fnmatch.fnmatch(os.path.basename(modified_filepath), self.filename): self.callback() __all__ = ["watch", "unwatch"] From 181c2973e6d1d27cb58c7622991b6c3f7f41c22c Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Tue, 12 Jan 2016 22:24:18 +0800 Subject: [PATCH 4/8] PollingObserver() fixed on Linux and OS X. --- libmproxy/script/reloader.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index f824d838a..6f3c69ed1 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -27,9 +27,7 @@ def unwatch(script): class _ScriptModificationHandler(PatternMatchingEventHandler): def __init__(self, callback, filename='*'): - # We could enumerate all relevant *.py files (as werkzeug does it), - # but our case looks like it isn't as simple as enumerating sys.modules. - # This should be good enough for now. + super(_ScriptModificationHandler, self).__init__( ignore_directories=True, ) @@ -38,7 +36,7 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): def on_modified(self, event): if event.is_directory: - files_in_dir = [event.src_path + "/" + \ + files_in_dir = [event.src_path + "/" + f for f in os.listdir(event.src_path)] if len(files_in_dir) > 0: modified_filepath = max(files_in_dir, key=os.path.getmtime) @@ -50,5 +48,9 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): if fnmatch.fnmatch(os.path.basename(modified_filepath), self.filename): self.callback() + def on_created(self, event): + if fnmatch.fnmatch(os.path.basename(event.src_path), self.filename): + self.callback() + __all__ = ["watch", "unwatch"] From 5b442a8947df332be6f79e3cdde2a80d1f3b00c6 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 14 Jan 2016 09:30:06 +0800 Subject: [PATCH 5/8] Use PollingObserver() only on OS X. --- libmproxy/script/reloader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 6f3c69ed1..390ea3432 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -1,7 +1,12 @@ import os import fnmatch +import sys from watchdog.events import PatternMatchingEventHandler -from watchdog.observers.polling import PollingObserver +if sys.platform == 'darwin': + from watchdog.observers.polling import PollingObserver as Observer +else: + from watchdog.observers import Observer +# Use PollingObserver on OS X, and natvie Observer on Linux and Windows _observers = {} @@ -12,7 +17,7 @@ def watch(script, callback): script_dir = os.path.dirname(os.path.abspath(script.args[0])) script_name = os.path.basename(script.args[0]) event_handler = _ScriptModificationHandler(callback, filename=script_name) - observer = PollingObserver() + observer = Observer() observer.schedule(event_handler, script_dir) observer.start() _observers[script] = observer @@ -48,9 +53,5 @@ class _ScriptModificationHandler(PatternMatchingEventHandler): if fnmatch.fnmatch(os.path.basename(modified_filepath), self.filename): self.callback() - def on_created(self, event): - if fnmatch.fnmatch(os.path.basename(event.src_path), self.filename): - self.callback() - __all__ = ["watch", "unwatch"] From 7f2b17d0ef4c118ddb83a7a0f3e73e65cae116d2 Mon Sep 17 00:00:00 2001 From: MatthewShao Date: Mon, 18 Jan 2016 09:57:58 +0800 Subject: [PATCH 6/8] Use RegexMatchingEventHandler() instead. --- libmproxy/script/reloader.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 390ea3432..367a59dd1 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -1,12 +1,12 @@ import os -import fnmatch import sys -from watchdog.events import PatternMatchingEventHandler +from watchdog.events import RegexMatchingEventHandler if sys.platform == 'darwin': from watchdog.observers.polling import PollingObserver as Observer else: from watchdog.observers import Observer -# Use PollingObserver on OS X, and natvie Observer on Linux and Windows +# The OSX reloader is watchdog 0.8.3 breaks when unobserving paths. +# We use the PollingObserver instead. _observers = {} @@ -30,28 +30,18 @@ def unwatch(script): observer.join() -class _ScriptModificationHandler(PatternMatchingEventHandler): - def __init__(self, callback, filename='*'): +class _ScriptModificationHandler(RegexMatchingEventHandler): + def __init__(self, callback, filename='.*'): super(_ScriptModificationHandler, self).__init__( ignore_directories=True, + regexes=['.*'+filename] ) self.callback = callback self.filename = filename def on_modified(self, event): - if event.is_directory: - files_in_dir = [event.src_path + "/" + - f for f in os.listdir(event.src_path)] - if len(files_in_dir) > 0: - modified_filepath = max(files_in_dir, key=os.path.getmtime) - else: - return - else: - modified_filepath = event.src_path - - if fnmatch.fnmatch(os.path.basename(modified_filepath), self.filename): - self.callback() + self.callback() __all__ = ["watch", "unwatch"] From 98cee0e14713fc91a48539c6ffbb6e22fc01f9f8 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Mon, 18 Jan 2016 10:15:09 +0800 Subject: [PATCH 7/8] Update reloader.py --- libmproxy/script/reloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 367a59dd1..1ad6e29af 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -5,7 +5,7 @@ if sys.platform == 'darwin': from watchdog.observers.polling import PollingObserver as Observer else: from watchdog.observers import Observer -# The OSX reloader is watchdog 0.8.3 breaks when unobserving paths. +# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths. # We use the PollingObserver instead. _observers = {} From 8aeed2032d369fa7e818d3a4cb46d440ec3569a4 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Mon, 18 Jan 2016 19:12:43 +0800 Subject: [PATCH 8/8] Update reloader.py --- libmproxy/script/reloader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 1ad6e29af..e81bdef68 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -38,7 +38,6 @@ class _ScriptModificationHandler(RegexMatchingEventHandler): regexes=['.*'+filename] ) self.callback = callback - self.filename = filename def on_modified(self, event): self.callback()