Merge branch 'upstream-cert'

libmproxy/cmdline.py

@@ -141,6 +141,17 @@ def common_options(parser):
         help="Byte size limit of HTTP request and response bodies."\
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
+    parser.add_option(
+        "--cert-wait-time", type="float",
+        action="store", dest="cert_wait_time", default=0,
+        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
+    )
+    parser.add_option(
+        "--upstream-cn-lookup", default=False,
+        action="store_true", dest="upstream_cn_lookup",
+        help="Connect to upstream server to look up certificate Common Name."
+    )
+
     group = optparse.OptionGroup(parser, "Client Replay")
     group.add_option(
         "-c",
@@ -149,12 +160,6 @@ def common_options(parser):
     )
     parser.add_option_group(group)
 
-    parser.add_option(
-        "--cert-wait-time", type="float",
-        action="store", dest="cert_wait_time", default=0,
-        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
-    )
-
     group = optparse.OptionGroup(parser, "Server Replay")
     group.add_option(
         "-S",

libmproxy/proxy.py

@@ -35,12 +35,13 @@ class ProxyError(Exception):
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, body_size_limit = None, reverse_proxy=None):
+    def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, upstream_cn_lookup=False, body_size_limit = None, reverse_proxy=None):
         self.certfile = certfile
         self.ciphers = ciphers
         self.cacert = cacert
         self.certdir = None
         self.cert_wait_time = cert_wait_time
+        self.upstream_cn_lookup = upstream_cn_lookup
         self.body_size_limit = body_size_limit
         self.reverse_proxy = reverse_proxy
 
@@ -343,11 +344,14 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
         if server:
             server.terminate()
 
-    def find_cert(self, host):
+    def find_cert(self, host, port):
         if self.config.certfile:
             return self.config.certfile
         else:
-            ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host)
+            sans = []
+            if self.config.upstream_cn_lookup:
+                host, sans = utils.get_remote_cn(host, port)
+            ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host, sans)
             time.sleep(self.config.cert_wait_time)
             if not ret:
                 raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.")
@@ -374,7 +378,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
                         )
             self.wfile.flush()
             kwargs = dict(
-                certfile = self.find_cert(host),
+                certfile = self.find_cert(host, port),
                 keyfile = self.config.certfile or self.config.cacert,
                 server_side = True,
                 ssl_version = ssl.PROTOCOL_SSLv23,
@@ -538,5 +542,6 @@ def process_proxy_options(parser, options):
         ciphers = options.ciphers,
         cert_wait_time = options.cert_wait_time,
         body_size_limit = body_size_limit,
+        upstream_cn_lookup = options.upstream_cn_lookup,
         reverse_proxy = rp
     )

libmproxy/resources/cert.cnf

@@ -27,4 +27,7 @@ nsCertType = server
 basicConstraints = CA:false
 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
 nsCertType = server
+%(altnames)s
 
+[ alt_names ]
+%(sans)s

libmproxy/utils.py

@@ -12,8 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import re, os, subprocess, datetime, urlparse, string, urllib
+import re, os, subprocess, datetime, urlparse, string, urllib, socket
-import time, functools, cgi, textwrap, hashlib
+import time, functools, cgi, textwrap, hashlib, ssl, tempfile
 import json
 
 CERT_SLEEP_TIME = 1
@@ -276,7 +276,7 @@ def dummy_ca(path):
     return True
 
 
-def dummy_cert(certdir, ca, commonname):
+def dummy_cert(certdir, ca, commonname, sans):
     """
         certdir: Certificate directory.
         ca: Path to the certificate authority file, or None.
@@ -293,8 +293,22 @@ def dummy_cert(certdir, ca, commonname):
     reqpath = os.path.join(certdir, namehash + ".req")
 
     template = open(pkg_data.path("resources/cert.cnf")).read()
+
+    ss = []
+    for i, v in enumerate(sans):
+        ss.append("DNS.%s = %s"%(i+1, v))
+    ss = "\n".join(ss)
+
     f = open(confpath, "w")
-    f.write(template%(dict(commonname=commonname)))
+    f.write(
+        template%(
+            dict(
+                commonname=commonname,
+                sans=ss,
+                altnames="subjectAltName = @alt_names" if ss else ""
+            )
+        )
+    )
     f.close()
 
     if ca:
@@ -324,7 +338,7 @@ def dummy_cert(certdir, ca, commonname):
             "-CA", ca,
             "-CAcreateserial",
             "-extfile", confpath,
-            "-extensions", "v3_cert",
+            "-extensions", "v3_cert_req",
         ]
         ret = subprocess.call(
             cmd,
@@ -483,3 +497,60 @@ def parse_size(s):
         return int(s) * mult
     except ValueError:
         raise ValueError("Invalid size specification: %s"%s)
+
+
+def get_remote_cn(host, port):
+    addr = socket.gethostbyname(host)
+    s = ssl.get_server_certificate((addr, port))
+    f = tempfile.NamedTemporaryFile()
+    f.write(s)
+    f.flush()
+    p = subprocess.Popen(
+        [
+            "openssl",
+            "x509",
+            "-in", f.name,
+            "-text",
+            "-noout"
+        ],
+        stdout = subprocess.PIPE
+    )
+    out, _ = p.communicate()
+    return parse_text_cert(out)
+
+
+CNRE = re.compile(
+    r"""
+        Subject:.*CN=([^ \t\n\r\f\v/]*)
+    """,
+    re.VERBOSE|re.MULTILINE
+)
+SANRE = re.compile(
+    r"""
+        X509v3\ Subject\ Alternative\ Name:\s*
+        (.*)$
+    """,
+    re.VERBOSE|re.MULTILINE
+)
+def parse_text_cert(txt):
+    """
+        Returns a (common name, [subject alternative names]) tuple.
+    """
+    r = re.search(CNRE, txt)
+    if r:
+        cn = r.group(1)
+    else:
+        return None
+
+    r = re.search(SANRE, txt)
+    san = []
+    if r:
+        for i in r.group(1).split(","):
+            i = i.strip()
+            k, v = i.split(":")
+            if k == "DNS":
+                san.append(v)
+    else:
+        san = []
+    return (cn, san)
+

test/data/text_cert_2

@@ -0,0 +1,83 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 67908 (0x10944)
+        Signature Algorithm: sha1WithRSAEncryption
+        Issuer: C=IL, O=StartCom Ltd., OU=Secure Digital Certificate Signing, CN=StartCom Class 1 Primary Intermediate Server CA
+        Validity
+            Not Before: Jan 11 19:27:36 2010 GMT
+            Not After : Jan 12 09:14:55 2011 GMT
+        Subject: description=126832-MCxLsY6Tn1gm7o90, C=NZ, O=Persona Not Validated, OU=StartCom Free Certificate Member, CN=www.inode.co.nz/emailAddress=jim@inode.co.nz
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+            RSA Public Key: (2048 bit)
+                Modulus (2048 bit):
+                    00:be:a0:85:69:46:86:a8:3e:57:43:f9:f1:1d:d2:
+                    bc:b4:62:3b:d3:ab:75:e9:03:c7:39:90:6c:ac:10:
+                    71:cd:39:7e:b9:c5:e5:9f:6f:4f:a8:1a:ed:83:ae:
+                    5d:57:97:c6:16:02:02:8d:27:af:a2:88:fb:2f:cb:
+                    c7:e4:07:bc:fe:56:d2:72:9b:b3:eb:2f:cd:44:04:
+                    ae:ad:99:64:5c:21:87:57:b5:2d:b5:5c:72:b3:70:
+                    90:75:33:28:ea:b4:4d:36:95:75:4e:24:f2:40:d0:
+                    cf:36:05:f6:d4:d9:6f:e1:b9:1e:fd:41:26:3a:70:
+                    63:07:bc:3f:87:00:88:f2:4f:ac:51:4e:29:3c:92:
+                    cc:7a:a2:62:c5:e5:c7:eb:d1:d6:5d:5e:df:32:9a:
+                    be:17:7e:2d:54:a8:2d:65:97:6f:5d:f2:28:e0:ea:
+                    ea:f9:ee:88:78:d4:25:79:7c:09:f5:65:06:1a:06:
+                    a2:d5:d4:e0:94:79:71:bb:86:a1:6f:fc:fa:b9:30:
+                    11:d5:fe:62:80:af:54:2a:a0:77:1f:48:91:11:41:
+                    ea:e5:9f:37:aa:1c:52:21:6f:84:0d:1e:92:73:be:
+                    7b:0b:95:7d:12:e2:21:a4:83:07:ca:4d:c9:45:95:
+                    aa:ee:27:80:55:ad:58:ed:4e:61:98:34:23:fd:f6:
+                    06:47
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            X509v3 Key Usage: 
+                Digital Signature, Key Encipherment, Key Agreement
+            X509v3 Extended Key Usage: 
+                TLS Web Server Authentication
+            X509v3 Subject Key Identifier: 
+                1F:68:BD:91:8F:AA:FC:89:19:41:4E:0A:69:80:4E:D9:67:95:93:53
+            X509v3 Authority Key Identifier: 
+                keyid:EB:42:34:D0:98:B0:AB:9F:F4:1B:6B:08:F7:CC:64:2E:EF:0E:2C:45
+
+            X509v3 Subject Alternative Name: 
+                DNS:www.inode.co.nz, DNS:inode.co.nz
+            X509v3 Certificate Policies: 
+                Policy: 1.3.6.1.4.1.23223.1.2.1
+                  CPS: http://www.startssl.com/policy.pdf
+                  CPS: http://www.startssl.com/intermediate.pdf
+                  User Notice:
+                    Organization: StartCom Ltd.
+                    Number: 1
+                    Explicit Text: Limited Liability, see section *Legal Limitations* of the StartCom Certification Authority Policy available at http://www.startssl.com/policy.pdf
+
+            X509v3 CRL Distribution Points: 
+                URI:http://www.startssl.com/crt1-crl.crl
+                URI:http://crl.startssl.com/crt1-crl.crl
+
+            Authority Information Access: 
+                OCSP - URI:http://ocsp.startssl.com/sub/class1/server/ca
+                CA Issuers - URI:http://www.startssl.com/certs/sub.class1.server.ca.crt
+
+            X509v3 Issuer Alternative Name: 
+                URI:http://www.startssl.com/
+    Signature Algorithm: sha1WithRSAEncryption
+        8a:f5:88:0f:42:93:f2:ad:44:cd:6c:f2:f8:17:9c:b0:5a:f2:
+        85:08:6e:2c:57:c0:b0:f9:07:aa:93:5d:f4:e7:e4:45:2b:46:
+        06:e2:65:2d:7f:bd:0d:0f:53:ce:3a:ae:b5:d6:36:6e:e3:41:
+        b8:39:fb:d5:bb:99:2b:e6:39:7e:68:eb:db:ca:c0:4f:af:96:
+        31:df:8a:9c:29:79:58:b4:f9:20:e4:ba:67:db:37:71:e7:39:
+        4c:54:1b:98:e1:a5:7a:94:92:f5:0e:93:b3:d2:ac:5c:e1:c7:
+        42:f0:e4:41:e0:ea:39:5d:94:a9:d8:17:20:0c:85:9b:97:9f:
+        48:bf:49:5c:f5:e7:b5:16:ae:b0:db:b6:b3:c1:76:07:6a:d1:
+        25:ee:23:3d:f9:d4:cb:62:ef:63:c3:96:d5:8b:e3:50:0f:be:
+        be:97:56:45:d7:d4:6c:94:29:e7:99:91:7d:a6:0c:3d:19:f2:
+        52:6d:f4:94:02:36:ed:06:b1:81:03:c5:3a:a0:de:65:49:b9:
+        96:56:f9:78:51:bf:07:77:fc:ee:e8:60:f9:8d:68:13:29:1b:
+        12:f6:6b:20:53:78:86:2b:61:8a:84:34:b5:bf:b9:ec:52:a4:
+        90:6d:ac:5b:7b:9e:98:d8:90:c3:6a:ab:12:8f:c6:00:c6:f5:
+        70:8e:74:46
+

test/test_utils.py

@@ -149,14 +149,17 @@ class udummy_cert(libpry.AutoTree):
         p = utils.dummy_cert(
             os.path.join(d, "foo"),
             cacert,
-            "foo.com"
+            "foo.com",
+            ["one.com", "two.com", "*.three.com"]
         )
         assert os.path.exists(p)
+        
         # Short-circuit
         assert utils.dummy_cert(
             os.path.join(d, "foo"),
             cacert,
-            "foo.com"
+            "foo.com",
+            []
         )
 
     def test_no_ca(self):
@@ -164,7 +167,8 @@ class udummy_cert(libpry.AutoTree):
         p = utils.dummy_cert(
             d,
             None,
-            "foo.com"
+            "foo.com",
+            []
         )
         assert os.path.exists(p)
 
@@ -255,7 +259,22 @@ class u_parse_size(libpry.AutoTree):
         libpry.raises(ValueError, utils.parse_size, "ak")
 
 
+class uparse_text_cert(libpry.AutoTree):
+    def test_simple(self):
+        c = file("data/text_cert", "r").read()
+        cn, san = utils.parse_text_cert(c)
+        assert cn == "google.com"
+        assert len(san) == 436
+
+        c = file("data/text_cert_2", "r").read()
+        cn, san = utils.parse_text_cert(c)
+        assert cn == "www.inode.co.nz"
+        assert len(san) == 2
+
+
+
 tests = [
+    uparse_text_cert(),
     uformat_timestamp(),
     uisBin(),
     uisXML(),

test/tools/getcert

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+import sys
+sys.path.insert(0, "../..")
+import socket, tempfile, ssl, subprocess
+
+addr = socket.gethostbyname(sys.argv[1])
+s = ssl.get_server_certificate((addr, 443))
+f = tempfile.NamedTemporaryFile()
+f.write(s)
+f.flush()
+p = subprocess.Popen(
+    [
+        "openssl",
+        "x509",
+        "-in", f.name,
+        "-text",
+        "-noout"
+    ],
+    stdout = subprocess.PIPE
+)
+out, _ = p.communicate()
+print out
+
+

test/tools/getcn

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import sys
+sys.path.insert(0, "../..")
+from libmproxy import utils
+
+cn, san = utils.get_remote_cn(sys.argv[1], 443)
+print cn
+if san:
+    for i in san:
+        print "\t", i
+
+