Replace configargparse with argparse, enable new options loading

This commit ditches configargparse and enables the new config file format. The
default location is ~/.mitmproxy/config.yaml. Unifying all of the various
Options objects will follow in the next patch.

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-*/tmp
+**/tmp
 /venv*
 *.py[cdo]
 *.swp

mitmproxy/optmanager.py

@@ -238,6 +238,7 @@ class OptManager(metaclass=_DefaultsMeta):
             OptionsError.
         """
         for p in paths:
+            p = os.path.expanduser(p)
             if os.path.exists(p) and os.path.isfile(p):
                 txt = open(p, "r").read()
                 self.load(txt)

mitmproxy/platform/windows.py

@@ -7,7 +7,7 @@ import struct
 import threading
 import time
 
-import configargparse
+import argparse
 import pydivert
 import pydivert.consts
 import pickle
@@ -386,8 +386,9 @@ class TransparentProxy:
 
 
 if __name__ == "__main__":
-    parser = configargparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-        description="Windows Transparent Proxy")
+        description="Windows Transparent Proxy"
+    )
     parser.add_argument(
         '--mode',
         choices=[

mitmproxy/tools/cmdline.py

@@ -1,6 +1,7 @@
-import configargparse
+import argparse
-import os
 import re
+import os
+
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
 from mitmproxy import options
@@ -11,6 +12,9 @@ from mitmproxy import version
 from mitmproxy.addons import view
 
 
+CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml")
+
+
 class ParseException(Exception):
     pass
 
@@ -113,13 +117,13 @@ def get_common_options(args):
         stream_large_bodies = human.parse_size(stream_large_bodies)
 
     reps = []
-    for i in args.replace:
+    for i in args.replace or []:
         try:
             p = parse_replace_hook(i)
         except ParseException as e:
             raise exceptions.OptionsError(e)
         reps.append(p)
-    for i in args.replace_file:
+    for i in args.replace_file or []:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException as e:
@@ -133,7 +137,7 @@ def get_common_options(args):
         reps.append((patt, rex, v))
 
     setheaders = []
-    for i in args.setheader:
+    for i in args.setheader or []:
         try:
             p = parse_setheader(i)
         except ParseException as e:
@@ -154,7 +158,7 @@ def get_common_options(args):
 
     # Proxy config
     certs = []
-    for i in args.certs:
+    for i in args.certs or []:
         parts = i.split("=", 1)
         if len(parts) == 1:
             parts = ["*", parts[0]]
@@ -287,8 +291,7 @@ def basic_options(parser):
     )
     parser.add_argument(
         "--anticache",
-        action="store_true", dest="anticache", default=False,
+        action="store_true", dest="anticache",
-
         help="""
             Strip out request headers that might cause the server to return
             304-not-modified.
@@ -296,12 +299,12 @@ def basic_options(parser):
     )
     parser.add_argument(
         "--cadir",
-        action="store", type=str, dest="cadir", default=options.CA_DIR,
+        action="store", type=str, dest="cadir",
         help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR
     )
     parser.add_argument(
         "--host",
-        action="store_true", dest="showhost", default=False,
+        action="store_true", dest="showhost",
         help="Use the Host header to construct URLs for display."
     )
     parser.add_argument(
@@ -311,12 +314,12 @@ def basic_options(parser):
     )
     parser.add_argument(
         "-r", "--read-flows",
-        action="store", dest="rfile", default=None,
+        action="store", dest="rfile",
         help="Read flows from file."
     )
     parser.add_argument(
         "-s", "--script",
-        action="append", type=str, dest="scripts", default=[],
+        action="append", type=str, dest="scripts",
         metavar='"script.py --bar"',
         help="""
             Run a script. Surround with quotes to pass script arguments. Can be
@@ -327,18 +330,17 @@ def basic_options(parser):
         "-t", "--stickycookie",
         action="store",
         dest="stickycookie_filt",
-        default=None,
         metavar="FILTER",
         help="Set sticky cookie filter. Matched against requests."
     )
     parser.add_argument(
         "-u", "--stickyauth",
-        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
+        action="store", dest="stickyauth_filt", metavar="FILTER",
         help="Set sticky auth filter. Matched against requests."
     )
     parser.add_argument(
         "-v", "--verbose",
-        action="store_const", dest="verbose", default=2, const=3,
+        action="store_const", dest="verbose", const=3,
         help="Increase log verbosity."
     )
     streamfile = parser.add_mutually_exclusive_group()
@@ -354,19 +356,19 @@ def basic_options(parser):
     )
     parser.add_argument(
         "-z", "--anticomp",
-        action="store_true", dest="anticomp", default=False,
+        action="store_true", dest="anticomp",
         help="Try to convince servers to send us un-compressed data."
     )
     parser.add_argument(
         "-Z", "--body-size-limit",
-        action="store", dest="body_size_limit", default=None,
+        action="store", dest="body_size_limit",
         metavar="SIZE",
         help="Byte size limit of HTTP request and response bodies."
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
     parser.add_argument(
         "--stream",
-        action="store", dest="stream_large_bodies", default=None,
+        action="store", dest="stream_large_bodies",
         metavar="SIZE",
         help="""
             Stream data to the client if response body exceeds the given
@@ -383,7 +385,6 @@ def proxy_modes(parser):
         action="store",
         type=str,
         dest="reverse_proxy",
-        default=None,
         help="""
             Forward all requests to upstream HTTP server:
             http[s]://host[:port]. Clients can always connect both
@@ -393,12 +394,12 @@ def proxy_modes(parser):
     )
     group.add_argument(
         "--socks",
-        action="store_true", dest="socks_proxy", default=False,
+        action="store_true", dest="socks_proxy",
         help="Set SOCKS5 proxy mode."
     )
     group.add_argument(
         "-T", "--transparent",
-        action="store_true", dest="transparent_proxy", default=False,
+        action="store_true", dest="transparent_proxy",
         help="Set transparent proxy mode."
     )
     group.add_argument(
@@ -406,7 +407,6 @@ def proxy_modes(parser):
         action="store",
         type=str,
         dest="upstream_proxy",
-        default=None,
         help="Forward all requests to upstream proxy server: http://host[:port]"
     )
 
@@ -415,12 +415,12 @@ def proxy_options(parser):
     group = parser.add_argument_group("Proxy Options")
     group.add_argument(
         "-b", "--bind-address",
-        action="store", type=str, dest="addr", default='',
+        action="store", type=str, dest="addr",
         help="Address to bind proxy to (defaults to all interfaces)"
     )
     group.add_argument(
         "-I", "--ignore",
-        action="append", type=str, dest="ignore_hosts", default=[],
+        action="append", type=str, dest="ignore_hosts",
         metavar="HOST",
         help="""
             Ignore host and forward all traffic without processing it. In
@@ -433,7 +433,7 @@ def proxy_options(parser):
     )
     group.add_argument(
         "--tcp",
-        action="append", type=str, dest="tcp_hosts", default=[],
+        action="append", type=str, dest="tcp_hosts",
         metavar="HOST",
         help="""
             Generic TCP SSL proxy mode for all hosts that match the pattern.
@@ -448,7 +448,7 @@ def proxy_options(parser):
     )
     group.add_argument(
         "-p", "--port",
-        action="store", type=int, dest="port", default=options.LISTEN_PORT,
+        action="store", type=int, dest="port",
         help="Proxy service port."
     )
     group.add_argument(
@@ -467,7 +467,7 @@ def proxy_options(parser):
 
     parser.add_argument(
         "--upstream-auth",
-        action="store", dest="upstream_auth", default=None,
+        action="store", dest="upstream_auth",
         type=str,
         help="""
             Add HTTP Basic authentcation to upstream proxy and reverse proxy
@@ -491,7 +491,7 @@ def proxy_options(parser):
     )
     group.add_argument(
         "--upstream-bind-address",
-        action="store", type=str, dest="upstream_bind_address", default='',
+        action="store", type=str, dest="upstream_bind_address",
         help="Address to bind upstream requests to (defaults to none)"
     )
 
@@ -502,7 +502,6 @@ def proxy_ssl_options(parser):
     group.add_argument(
         "--cert",
         dest='certs',
-        default=[],
         type=str,
         metavar="SPEC",
         action="append",
@@ -514,56 +513,55 @@ def proxy_ssl_options(parser):
              'as the first entry. Can be passed multiple times.')
     group.add_argument(
         "--ciphers-client", action="store",
-        type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS,
         help="Set supported ciphers for client connections. (OpenSSL Syntax)"
     )
     group.add_argument(
         "--ciphers-server", action="store",
-        type=str, dest="ciphers_server", default=None,
+        type=str, dest="ciphers_server",
         help="Set supported ciphers for server connections. (OpenSSL Syntax)"
     )
     group.add_argument(
         "--client-certs", action="store",
-        type=str, dest="clientcerts", default=None,
+        type=str, dest="clientcerts",
         help="Client certificate file or directory."
     )
     group.add_argument(
-        "--no-upstream-cert", default=False,
+        "--no-upstream-cert",
         action="store_true", dest="no_upstream_cert",
         help="Don't connect to upstream server to look up certificate details."
     )
     group.add_argument(
-        "--add-upstream-certs-to-client-chain", default=False,
+        "--add-upstream-certs-to-client-chain",
         action="store_true", dest="add_upstream_certs_to_client_chain",
         help="Add all certificates of the upstream server to the certificate chain "
              "that will be served to the proxy client, as extras."
     )
     group.add_argument(
-        "--insecure", default=False,
+        "--insecure",
         action="store_true", dest="ssl_insecure",
         help="Do not verify upstream server SSL/TLS certificates."
     )
     group.add_argument(
-        "--upstream-trusted-cadir", default=None, action="store",
+        "--upstream-trusted-cadir", action="store",
         dest="ssl_verify_upstream_trusted_cadir",
         help="Path to a directory of trusted CA certificates for upstream "
              "server verification prepared using the c_rehash tool."
     )
     group.add_argument(
-        "--upstream-trusted-ca", default=None, action="store",
+        "--upstream-trusted-ca", action="store",
         dest="ssl_verify_upstream_trusted_ca",
         help="Path to a PEM formatted trusted CA certificate."
     )
     group.add_argument(
         "--ssl-version-client", dest="ssl_version_client",
-        default="secure", action="store",
+        action="store",
         choices=tcp.sslversion_choices.keys(),
         help="Set supported SSL/TLS versions for client connections. "
              "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
     )
     group.add_argument(
         "--ssl-version-server", dest="ssl_version_server",
-        default="secure", action="store",
+        action="store",
         choices=tcp.sslversion_choices.keys(),
         help="Set supported SSL/TLS versions for server connections. "
              "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
@@ -574,12 +572,12 @@ def onboarding_app(parser):
     group = parser.add_argument_group("Onboarding App")
     group.add_argument(
         "--noapp",
-        action="store_false", dest="app", default=True,
+        action="store_false", dest="app",
         help="Disable the mitmproxy onboarding app."
     )
     group.add_argument(
         "--app-host",
-        action="store", dest="app_host", default=options.APP_HOST, metavar="host",
+        action="store", dest="app_host",
         help="""
             Domain to serve the onboarding app from. For transparent mode, use
             an IP when a DNS entry for the app domain is not present. Default:
@@ -590,7 +588,6 @@ def onboarding_app(parser):
         "--app-port",
         action="store",
         dest="app_port",
-        default=options.APP_PORT,
         type=int,
         metavar="80",
         help="Port to serve the onboarding app from."
@@ -601,7 +598,7 @@ def client_replay(parser):
     group = parser.add_argument_group("Client Replay")
     group.add_argument(
         "-c", "--client-replay",
-        action="append", dest="client_replay", default=[], metavar="PATH",
+        action="append", dest="client_replay", metavar="PATH",
         help="Replay client requests from a saved file."
     )
 
@@ -610,23 +607,23 @@ def server_replay(parser):
     group = parser.add_argument_group("Server Replay")
     group.add_argument(
         "-S", "--server-replay",
-        action="append", dest="server_replay", default=[], metavar="PATH",
+        action="append", dest="server_replay", metavar="PATH",
         help="Replay server responses from a saved file."
     )
     group.add_argument(
         "-k", "--replay-kill-extra",
-        action="store_true", dest="replay_kill_extra", default=False,
+        action="store_true", dest="replay_kill_extra",
         help="Kill extra requests during replay."
     )
     group.add_argument(
         "--server-replay-use-header",
-        action="append", dest="server_replay_use_headers", type=str, default=[],
+        action="append", dest="server_replay_use_headers", type=str,
         help="Request headers to be considered during replay. "
              "Can be passed multiple times."
     )
     group.add_argument(
         "--norefresh",
-        action="store_true", dest="norefresh", default=False,
+        action="store_true", dest="norefresh",
         help="""
             Disable response refresh, which updates times in cookies and headers
             for replayed responses.
@@ -634,21 +631,21 @@ def server_replay(parser):
     )
     group.add_argument(
         "--no-pop",
-        action="store_true", dest="server_replay_nopop", default=False,
+        action="store_true", dest="server_replay_nopop",
         help="Disable response pop from response flow. "
              "This makes it possible to replay same response multiple times."
     )
     payload = group.add_mutually_exclusive_group()
     payload.add_argument(
         "--replay-ignore-content",
-        action="store_true", dest="server_replay_ignore_content", default=False,
+        action="store_true", dest="server_replay_ignore_content",
         help="""
             Ignore request's content while searching for a saved flow to replay
         """
     )
     payload.add_argument(
         "--replay-ignore-payload-param",
-        action="append", dest="server_replay_ignore_payload_params", type=str, default=[],
+        action="append", dest="server_replay_ignore_payload_params", type=str,
         help="""
             Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to
             be ignored while searching for a saved flow to replay.
@@ -658,7 +655,7 @@ def server_replay(parser):
 
     group.add_argument(
         "--replay-ignore-param",
-        action="append", dest="server_replay_ignore_params", type=str, default=[],
+        action="append", dest="server_replay_ignore_params", type=str,
         help="""
             Request's parameters to be ignored while searching for a saved flow
             to replay. Can be passed multiple times.
@@ -668,7 +665,6 @@ def server_replay(parser):
         "--replay-ignore-host",
         action="store_true",
         dest="server_replay_ignore_host",
-        default=False,
         help="Ignore request's destination host while searching for a saved flow to replay")
 
 
@@ -683,13 +679,13 @@ def replacements(parser):
     )
     group.add_argument(
         "--replace",
-        action="append", type=str, dest="replace", default=[],
+        action="append", type=str, dest="replace",
         metavar="PATTERN",
         help="Replacement pattern."
     )
     group.add_argument(
         "--replace-from-file",
-        action="append", type=str, dest="replace_file", default=[],
+        action="append", type=str, dest="replace_file",
         metavar="PATH",
         help="""
             Replacement pattern, where the replacement clause is a path to a
@@ -709,7 +705,7 @@ def set_headers(parser):
     )
     group.add_argument(
         "--setheader",
-        action="append", type=str, dest="setheader", default=[],
+        action="append", type=str, dest="setheader",
         metavar="PATTERN",
         help="Header set pattern."
     )
@@ -747,6 +743,15 @@ def proxy_authentication(parser):
 
 
 def common_options(parser):
+    parser.add_argument(
+        "--conf",
+        type=str, dest="conf", default=CONFIG_PATH,
+        metavar="PATH",
+        help="""
+            Configuration file
+        """
+    )
+
     basic_options(parser)
     proxy_modes(parser)
     proxy_options(parser)
@@ -764,26 +769,17 @@ def mitmproxy():
     # platforms.
     from .console import palettes
 
-    parser = configargparse.ArgumentParser(
+    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
-        usage="%(prog)s [options]",
-        args_for_setting_config_path=["--conf"],
-        default_config_files=[
-            os.path.join(options.CA_DIR, "common.conf"),
-            os.path.join(options.CA_DIR, "mitmproxy.conf")
-        ],
-        add_config_file_help=True,
-        add_env_var_help=True
-    )
     common_options(parser)
     parser.add_argument(
-        "--palette", type=str, default=palettes.DEFAULT,
+        "--palette", type=str,
         action="store", dest="palette",
         choices=sorted(palettes.palettes.keys()),
         help="Select color palette: " + ", ".join(palettes.palettes.keys())
     )
     parser.add_argument(
         "--palette-transparent",
-        action="store_true", dest="palette_transparent", default=False,
+        action="store_true", dest="palette_transparent",
         help="Set transparent background for palette."
     )
     parser.add_argument(
@@ -798,7 +794,7 @@ def mitmproxy():
     )
     parser.add_argument(
         "--order",
-        type=str, dest="order", default=None,
+        type=str, dest="order",
         choices=[o[1] for o in view.orders],
         help="Flow sort order."
     )
@@ -813,33 +809,24 @@ def mitmproxy():
     )
     group.add_argument(
         "-i", "--intercept", action="store",
-        type=str, dest="intercept", default=None,
+        type=str, dest="intercept",
         help="Intercept filter expression."
     )
     group.add_argument(
         "-f", "--filter", action="store",
-        type=str, dest="filter", default=None,
+        type=str, dest="filter",
         help="Filter view expression."
     )
     return parser
 
 
 def mitmdump():
-    parser = configargparse.ArgumentParser(
+    parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
-        usage="%(prog)s [options] [filter]",
-        args_for_setting_config_path=["--conf"],
-        default_config_files=[
-            os.path.join(options.CA_DIR, "common.conf"),
-            os.path.join(options.CA_DIR, "mitmdump.conf")
-        ],
-        add_config_file_help=True,
-        add_env_var_help=True
-    )
 
     common_options(parser)
     parser.add_argument(
         "--keepserving",
-        action="store_true", dest="keepserving", default=False,
+        action="store_true", dest="keepserving",
         help="""
             Continue serving after client playback or file read. We exit by
             default.
@@ -847,7 +834,7 @@ def mitmdump():
     )
     parser.add_argument(
         "-d", "--detail",
-        action="count", dest="flow_detail", default=1,
+        action="count", dest="flow_detail",
         help="Increase flow detail display level. Can be passed multiple times."
     )
     parser.add_argument(
@@ -862,16 +849,7 @@ def mitmdump():
 
 
 def mitmweb():
-    parser = configargparse.ArgumentParser(
+    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
-        usage="%(prog)s [options]",
-        args_for_setting_config_path=["--conf"],
-        default_config_files=[
-            os.path.join(options.CA_DIR, "common.conf"),
-            os.path.join(options.CA_DIR, "mitmweb.conf")
-        ],
-        add_config_file_help=True,
-        add_env_var_help=True
-    )
 
     group = parser.add_argument_group("Mitmweb")
     group.add_argument(
@@ -881,13 +859,13 @@ def mitmweb():
     )
     group.add_argument(
         "--wport",
-        action="store", type=int, dest="wport", default=8081,
+        action="store", type=int, dest="wport",
         metavar="PORT",
         help="Mitmweb port."
     )
     group.add_argument(
         "--wiface",
-        action="store", dest="wiface", default="127.0.0.1",
+        action="store", dest="wiface",
         metavar="IFACE",
         help="Mitmweb interface."
     )
@@ -904,7 +882,7 @@ def mitmweb():
     )
     group.add_argument(
         "-i", "--intercept", action="store",
-        type=str, dest="intercept", default=None,
+        type=str, dest="intercept",
         help="Intercept filter expression."
     )
     return parser

mitmproxy/tools/main.py

@@ -17,6 +17,14 @@ from mitmproxy.utils import version_check  # noqa
 from mitmproxy.utils import debug  # noqa
 
 
+def notnone(d):
+    ret = {}
+    for k, v in d.items():
+        if v is not None:
+            ret[k] = v
+    return ret
+
+
 def assert_utf8_env():
     spec = ""
     for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
@@ -63,17 +71,21 @@ def mitmproxy(args=None):  # pragma: no cover
     args = parser.parse_args(args)
 
     try:
-        console_options = console.master.Options(
+        console_options = console.master.Options()
-            **cmdline.get_common_options(args)
+        console_options.load_paths(args.conf)
+        console_options.update(**notnone(cmdline.get_common_options(args)))
+        console_options.update(
+            **notnone(dict(
+                palette = args.palette,
+                palette_transparent = args.palette_transparent,
+                eventlog = args.eventlog,
+                focus_follow = args.focus_follow,
+                intercept = args.intercept,
+                filter = args.filter,
+                no_mouse = args.no_mouse,
+                order = args.order,
+            ))
         )
-        console_options.palette = args.palette
-        console_options.palette_transparent = args.palette_transparent
-        console_options.eventlog = args.eventlog
-        console_options.focus_follow = args.focus_follow
-        console_options.intercept = args.intercept
-        console_options.filter = args.filter
-        console_options.no_mouse = args.no_mouse
-        console_options.order = args.order
 
         server = process_options(parser, console_options, args)
         m = console.master.ConsoleMaster(console_options, server)
@@ -98,10 +110,17 @@ def mitmdump(args=None):  # pragma: no cover
 
     master = None
     try:
-        dump_options = dump.Options(**cmdline.get_common_options(args))
+        dump_options = dump.Options()
-        dump_options.flow_detail = args.flow_detail
+        dump_options.load_paths(args.conf)
-        dump_options.keepserving = args.keepserving
+        dump_options.update(**notnone(cmdline.get_common_options(args)))
-        dump_options.filtstr = " ".join(args.filter) if args.filter else None
+        dump_options.update(
+            **notnone(dict(
+                flow_detail = args.flow_detail,
+                keepserving = args.keepserving,
+                filtstr = " ".join(args.filter) if args.filter else None,
+            ))
+        )
+
         server = process_options(parser, dump_options, args)
         master = dump.DumpMaster(dump_options, server)
 
@@ -130,13 +149,18 @@ def mitmweb(args=None):  # pragma: no cover
     args = parser.parse_args(args)
 
     try:
-        web_options = web.master.Options(**cmdline.get_common_options(args))
+        web_options = web.master.Options()
-        web_options.intercept = args.intercept
+        web_options.load_paths(args.conf)
-        web_options.open_browser = args.open_browser
+        web_options.update(**notnone(cmdline.get_common_options(args)))
-        web_options.wdebug = args.wdebug
+        web_options.update(
-        web_options.wiface = args.wiface
+            **notnone(dict(
-        web_options.wport = args.wport
+                intercept = args.intercept,
-
+                open_browser = args.open_browser,
+                wdebug = args.wdebug,
+                wiface = args.wiface,
+                wport = args.wport,
+            ))
+        )
         server = process_options(parser, web_options, args)
         m = web.master.WebMaster(web_options, server)
     except exceptions.OptionsError as e:

setup.py

@@ -62,7 +62,6 @@ setup(
         "blinker>=1.4, <1.5",
         "click>=6.2, <7.0",
         "certifi>=2015.11.20.1",  # no semver here - this should always be on the last release!
-        "configargparse>=0.10, <0.12",
         "construct>=2.8, <2.9",
         "cryptography>=1.3, <1.7",
         "cssutils>=1.0.1, <1.1",

test/mitmproxy/test_proxy.py

@@ -5,6 +5,7 @@ import argparse
 from OpenSSL import SSL
 
 from mitmproxy.tools import cmdline
+from mitmproxy.tools import main
 from mitmproxy import options
 from mitmproxy.proxy import ProxyConfig
 from mitmproxy import connections
@@ -76,7 +77,7 @@ class TestProcessProxyOptions:
         cmdline.common_options(parser)
         args = parser.parse_args(args=args)
         opts = cmdline.get_common_options(args)
-        pconf = config.ProxyConfig(options.Options(**opts))
+        pconf = config.ProxyConfig(options.Options(**main.notnone(opts)))
         return parser, pconf
 
     def assert_err(self, err, *args):