add pathod

pathod/.appveyor.yml

@@ -0,0 +1,11 @@
+version: '{build}'
+shallow_clone: true
+environment:
+  matrix:
+    - PYTHON: "C:\\Python27"
+install:
+  - "%PYTHON%\\Scripts\\pip install --src . -r requirements.txt"
+  - "%PYTHON%\\python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
+build: off  # Not a C# project
+test_script:
+  - "%PYTHON%\\Scripts\\py.test -n 4"

pathod/.coveragerc

@@ -0,0 +1,10 @@
+[run]
+branch = True
+
+[report]
+show_missing = True
+include = *libpathod*
+exclude_lines =
+    pragma: nocover
+    pragma: no cover
+    raise NotImplementedError()

pathod/.env

@@ -0,0 +1,6 @@
+DIR="$( dirname "${BASH_SOURCE[0]}" )"
+ACTIVATE_DIR="$(if [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate"
+fi

pathod/.gitignore

@@ -0,0 +1,15 @@
+# Python object files
+*.py[cd]
+MANIFEST
+/build
+/dist
+# Vim swap files
+*.swp
+/doc
+.coverage
+.noseids
+netlib
+venv
+.idea/
+pathod.egg-info/
+.cache/

pathod/.jsbeautifyrc

@@ -0,0 +1,22 @@
+{
+    "indent_size": 4,
+    "indent_char": " ",
+    "eol": "\n",
+    "indent_level": 0,
+    "indent_with_tabs": false,
+    "preserve_newlines": true,
+    "max_preserve_newlines": 10,
+    "jslint_happy": false,
+    "space_after_anon_function": false,
+    "brace_style": "collapse",
+    "keep_array_indentation": false,
+    "keep_function_indentation": false,
+    "space_before_conditional": true,
+    "break_chained_methods": false,
+    "eval_code": false,
+    "unescape_strings": false,
+    "wrap_line_length": 80,
+    "wrap_attributes": "auto",
+    "wrap_attributes_indent_size": 4,
+    "end_with_newline": true
+}

pathod/.landscape.yml

@@ -0,0 +1,16 @@
+max-line-length: 120
+pylint:
+  options:
+    dummy-variables-rgx: _$|.+_$|dummy_.+
+
+  disable:
+    - missing-docstring
+    - protected-access
+    - too-few-public-methods
+    - too-many-arguments
+    - too-many-instance-attributes
+    - too-many-locals
+    - too-many-public-methods
+    - too-many-return-statements
+    - too-many-statements
+    - unpacking-non-sequence

pathod/.sources/bootswatch.less

@@ -0,0 +1,171 @@
+// Bootswatch.less
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// TYPOGRAPHY
+// -----------------------------------------------------
+
+@import url('https://fonts.googleapis.com/css?family=Open+Sans:400,700');
+
+h1, h2, h3, h4, h5, h6, .navbar .brand {
+	font-weight: 700;
+}
+
+// SCAFFOLDING
+// -----------------------------------------------------
+
+a {
+	text-decoration: none;
+}
+
+.nav a, .navbar .brand, .subnav a, a.btn, .dropdown-menu a {
+	text-decoration: none;
+}
+
+// NAVBAR
+// -----------------------------------------------------
+
+.navbar {
+
+	.navbar-inner {
+		@shadow: 0 2px 4px rgba(0,0,0,.25), inset 0 -1px 0 rgba(0,0,0,.1);
+		.box-shadow(@shadow);
+		border-top: 1px solid #E5E5E5;
+		.border-radius(0);
+	}
+
+	.brand {
+		text-shadow: none;
+
+		&:hover {
+			background-color: #EEEEEE;
+		}
+	}
+
+	.navbar-text {
+		line-height: 68px;
+	}
+
+	.nav > li > a {
+		text-shadow: none;
+	}
+
+	.dropdown-menu {
+		.border-radius(0);
+	}
+
+	.nav li.dropdown.active > .dropdown-toggle,
+	.nav li.dropdown.active > .dropdown-toggle:hover,
+	.nav li.dropdown.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle:hover {
+		background-color: @grayLighter;
+		color: @linkColor;
+	}
+
+	.nav li.dropdown .dropdown-toggle .caret,
+	.nav .open .caret,
+	.nav .open .dropdown-toggle:hover .caret {
+		border-top-color: @black;
+		opacity: 1;
+	}
+
+	.nav-collapse.in .nav li > a:hover {
+		background-color: @grayLighter;
+	}
+
+	.nav-collapse .nav li > a {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav-collapse .navbar-form,
+	.nav-collapse .navbar-search {
+		border-color: transparent;
+	}
+
+	.navbar-search .search-query,
+	.navbar-search .search-query:hover {
+		border: 1px solid @grayLighter;
+		color: @textColor;
+		.placeholder(@gray);
+	}
+}
+
+div.subnav {
+	background-color: @bodyBackground;
+	background-image: none;
+	@shadow: 0 1px 2px rgba(0,0,0,.25);
+	.box-shadow(@shadow);
+	.border-radius(0);
+
+	&.subnav-fixed {
+		top: @navbarHeight;
+	}
+
+	.nav > li > a:hover,
+	.nav > .active > a,
+	.nav > .active > a:hover {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav > li:first-child > a,
+	.nav > li:first-child > a:hover {
+		.border-radius(0);
+	}
+}
+
+// BUTTONS
+// -----------------------------------------------------
+
+.btn-primary {
+  .buttonBackground(lighten(@linkColor, 5%), @linkColor);
+}
+
+[class^="icon-"], [class*=" icon-"] {
+	vertical-align: -2px;
+}
+
+// MODALS
+// -----------------------------------------------------
+
+.modal {
+ 	.border-radius(0px);
+	background: @bodyBackground;
+}
+
+.modal-header {
+	border-bottom: none;
+}
+
+.modal-header .close {
+	text-decoration: none;
+}
+
+.modal-footer {
+	background: transparent;
+	.box-shadow(none);
+	border-top: none;
+}
+
+
+// MISC
+// -----------------------------------------------------
+
+code, pre, pre.prettyprint, .well {
+	background-color: @grayLighter;
+}
+
+.hero-unit {
+    .box-shadow(inset 0 1px 1px rgba(0,0,0,.05));
+	border: 1px solid rgba(0,0,0,.05);
+	.border-radius(0);
+}
+
+.table-bordered, .well, .prettyprint {
+	.border-radius(0);
+}

pathod/.sources/make

@@ -0,0 +1,5 @@
+#!/bin/sh
+pygmentize -f html ../examples/test_context.py > ../libpathod/templates/examples_context.html
+pygmentize -f html ../examples/test_setup.py > ../libpathod/templates/examples_setup.html
+pygmentize -f html ../examples/test_setupall.py > ../libpathod/templates/examples_setupall.html
+pygmentize -f html ../examples/libpathod_pathoc.py > ../libpathod/templates/libpathod_pathoc.html

pathod/.sources/variables.less

@@ -0,0 +1,208 @@
+// Variables.less
+// Variables to customize the look and feel of Bootstrap
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// GLOBAL VALUES
+// --------------------------------------------------
+
+
+// Grays
+// -------------------------
+@black:                 #000;
+@grayDarker:            #222;
+@grayDark:              #333;
+@gray:                  #888;
+@grayLight:             #999;
+@grayLighter:           #eee;
+@white:                 #fff;
+
+
+// Accent colors
+// -------------------------
+@blue:                  #4380D3;
+@blueDark:              darken(@blue, 15%);
+@green:                 #22B24C;
+@red:                   #C00;
+@yellow:                #FCFADB;
+@orange:                #FF7F00;
+@pink:                  #CC99CC;
+@purple:                #7a43b6;
+@tan:                   #FFCA73;
+
+
+
+// Scaffolding
+// -------------------------
+@bodyBackground:        #FCFBFD;
+@textColor:             @grayDarker;
+
+
+// Links
+// -------------------------
+@linkColor:             @blue;
+@linkColorHover:        @red;
+
+
+// Typography
+// -------------------------
+@sansFontFamily:        'Open Sans', "Helvetica Neue", Helvetica, Arial, sans-serif;
+@serifFontFamily:       Georgia, "Times New Roman", Times, serif;
+@monoFontFamily:        Menlo, Monaco, Consolas, "Courier New", monospace;
+
+@baseFontSize:          14px;
+@baseFontFamily:        @sansFontFamily;
+@baseLineHeight:        18px;
+@altFontFamily:         @serifFontFamily;
+
+@headingsFontFamily:    inherit; // empty to use BS default, @baseFontFamily
+@headingsFontWeight:    bold;    // instead of browser default, bold
+@headingsColor:         inherit; // empty to use BS default, @textColor
+
+
+// Tables
+// -------------------------
+@tableBackground:                   transparent; // overall background-color
+@tableBackgroundAccent:             @grayLighter; // for striping
+@tableBackgroundHover:              #f5f5f5; // for hover
+@tableBorder:                       #ddd; // table and cell border
+
+
+// Buttons
+// -------------------------
+@btnBackground:                     @white;
+@btnBackgroundHighlight:            darken(@white, 10%);
+@btnBorder:                         darken(@white, 20%);
+
+@btnPrimaryBackground:              @linkColor;
+@btnPrimaryBackgroundHighlight:     spin(@btnPrimaryBackground, 15%);
+
+@btnInfoBackground:                 #5bc0de;
+@btnInfoBackgroundHighlight:        #2f96b4;
+
+@btnSuccessBackground:              #62c462;
+@btnSuccessBackgroundHighlight:     #51a351;
+
+@btnWarningBackground:              lighten(@orange, 10%);
+@btnWarningBackgroundHighlight:     @orange;
+
+@btnDangerBackground:               #ee5f5b;
+@btnDangerBackgroundHighlight:      #bd362f;
+
+@btnInverseBackground:              @linkColor;
+@btnInverseBackgroundHighlight:     darken(@linkColor, 5%);
+
+
+// Forms
+// -------------------------
+@inputBackground:               @white;
+@inputBorder:                   #ccc;
+@inputBorderRadius:             3px;
+@inputDisabledBackground:       @grayLighter;
+@formActionsBackground:         @grayLighter;
+
+// Dropdowns
+// -------------------------
+@dropdownBackground:            @bodyBackground;
+@dropdownBorder:                rgba(0,0,0,.2);
+@dropdownLinkColor:             @textColor;
+@dropdownLinkColorHover:        @textColor;
+@dropdownLinkBackgroundHover:   #eee;
+@dropdownDividerTop:            #e5e5e5;
+@dropdownDividerBottom:         @white;
+
+
+
+// COMPONENT VARIABLES
+// --------------------------------------------------
+
+// Z-index master list
+// -------------------------
+// Used for a bird's eye view of components dependent on the z-axis
+// Try to avoid customizing these :)
+@zindexDropdown:          1000;
+@zindexPopover:           1010;
+@zindexTooltip:           1020;
+@zindexFixedNavbar:       1030;
+@zindexModalBackdrop:     1040;
+@zindexModal:             1050;
+
+
+// Sprite icons path
+// -------------------------
+@iconSpritePath:          "../img/glyphicons-halflings.png";
+@iconWhiteSpritePath:     "../img/glyphicons-halflings-white.png";
+
+
+// Input placeholder text color
+// -------------------------
+@placeholderText:         @grayLight;
+
+
+// Hr border color
+// -------------------------
+@hrBorder:                @grayLighter;
+
+
+// Navbar
+// -------------------------
+@navbarHeight:                    50px;
+@navbarBackground:                @bodyBackground;
+@navbarBackgroundHighlight:       @bodyBackground;
+
+@navbarText:                      @textColor;
+@navbarLinkColor:                 @linkColor;
+@navbarLinkColorHover:            @linkColor;
+@navbarLinkColorActive:           @navbarLinkColorHover;
+@navbarLinkBackgroundHover:       @grayLighter;
+@navbarLinkBackgroundActive:      @grayLighter;
+
+@navbarSearchBackground:          lighten(@navbarBackground, 25%);
+@navbarSearchBackgroundFocus:     @white;
+@navbarSearchBorder:              darken(@navbarSearchBackground, 30%);
+@navbarSearchPlaceholderColor:    #ccc;
+@navbarBrandColor:                @blue;
+
+
+// Hero unit
+// -------------------------
+@heroUnitBackground:              @grayLighter;
+@heroUnitHeadingColor:            inherit;
+@heroUnitLeadColor:               inherit;
+
+
+// Form states and alerts
+// -------------------------
+@warningText:             #c09853;
+@warningBackground:       #fcf8e3;
+@warningBorder:           darken(spin(@warningBackground, -10), 3%);
+
+@errorText:               #b94a48;
+@errorBackground:         #f2dede;
+@errorBorder:             darken(spin(@errorBackground, -10), 3%);
+
+@successText:             #468847;
+@successBackground:       #dff0d8;
+@successBorder:           darken(spin(@successBackground, -10), 5%);
+
+@infoText:                #3a87ad;
+@infoBackground:          #d9edf7;
+@infoBorder:              darken(spin(@infoBackground, -10), 7%);
+
+
+
+// GRID
+// --------------------------------------------------
+
+// Default 940px grid
+// -------------------------
+@gridColumns:             12;
+@gridColumnWidth:         60px;
+@gridGutterWidth:         20px;
+@gridRowWidth:            (@gridColumns * @gridColumnWidth) + (@gridGutterWidth * (@gridColumns - 1));
+
+// Fluid grid
+// -------------------------
+@fluidGridColumnWidth:    6.382978723%;
+@fluidGridGutterWidth:    2.127659574%;

pathod/.travis.yml

@@ -0,0 +1,70 @@
+sudo: false
+language: python
+
+matrix:
+  fast_finish: true
+  include:
+    - python: 2.7
+    - python: 2.7
+      env: OPENSSL=1.0.2
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.0.2
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: pypy
+    - python: pypy
+      env: OPENSSL=1.0.2
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.0.2
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+  allow_failures:
+    # We allow pypy to fail until Travis fixes their infrastructure to a pypy
+    # with a recent enought CFFI library to run cryptography 1.0+.
+    - python: pypy
+
+install:
+  - "pip install --src . -r requirements.txt"
+
+before_script:
+  - "openssl version -a"
+
+script:
+  - "py.test --cov libpathod -v"
+
+after_success:
+  - coveralls
+
+notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#mitmproxy"
+    on_success: change
+    on_failure: always
+  slack:
+    rooms:
+      - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu#ci
+    on_success: always
+    on_failure: always
+
+# exclude cryptography from cache
+# it depends on libssl-dev version
+# which needs to be compiled specifically to each version
+before_cache:
+  - pip uninstall -y cryptography
+
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
+    - /home/travis/virtualenv/python2.7.9/bin
+    - /home/travis/virtualenv/pypy-2.5.0/site-packages
+    - /home/travis/virtualenv/pypy-2.5.0/bin

pathod/CHANGELOG

@@ -0,0 +1,83 @@
+7 November 2014: pathod 0.11:
+
+  * Hugely improved SSL support, including dynamic generation of certificates
+    using the mitproxy cacert
+  * pathoc -S dumps information on the remote SSL certificate chain
+  * Big improvements to fuzzing, including random spec selection and memoization to avoid repeating randomly generated patterns 
+  * Reflected patterns, allowing you to embed a pathod server response specification in a pathoc request, resolving both on client side. This makes fuzzing proxies and other intermediate systems much better.
+
+
+25 August 2013: pathod 0.9.2:
+
+    * Adapt to interface changes in netlib
+    
+
+15 May 2013: pathod 0.9 (version synced with mitmproxy):
+
+    * Pathod proxy mode. You can now configure clients to use pathod as an
+      HTTP/S proxy.
+
+    * Pathoc proxy support, including using CONNECT to tunnel directly to
+      targets. 
+
+    * Pathoc client certificate support.
+    
+    * API improvements, bugfixes.
+
+
+16 November 2012: pathod 0.3:
+
+    A release focusing on shoring up our fuzzing capabilities, especially with
+    pathoc.
+
+    * pathoc -q and -r options, output full request and response text.
+
+    * pathod -q and -r options, add full request and response text to pathod's
+      log buffer.
+    
+    * pathoc and pathod -x option, makes -q and -r options log in hex dump
+      format.
+
+    * pathoc -C option, specify response codes to ignore.
+    
+    * pathoc -T option, instructs pathoc to ignore timeouts.
+    
+    * pathoc -o option, a one-shot mode that exits after the first non-ignored
+      response. 
+    
+    * pathoc and pathod -e option, which explains the resulting message by
+      expanding random and generated portions, and logging a reproducible
+      specification.
+
+    * Streamline the specification langauge. HTTP response message is now
+      specified using the "r" mnemonic.
+
+    * Add a "u" mnemonic for specifying User-Agent strings. Add a set of
+      standard user-agent strings accessible through shortcuts.
+
+    * Major internal refactoring and cleanup.
+    
+    * Many bugfixes.
+    
+
+22 August 2012: pathod 0.2:
+
+    * Add pathoc, a pathological HTTP client.
+    
+    * Add libpathod.test, a truss for using pathod in unit tests.
+    
+    * Add an injection operator to the specification language.
+    
+    * Allow Python escape sequences in value literals.
+    
+    * Allow execution of requests and responses from file, using the new + operator.
+    
+    * Add daemonization to Pathod, and make it more robust for public-facing use. 
+   
+    * Let pathod pick an arbitrary open port if -p 0 is specified.
+    
+    * Move from Tornado to netlib, the network library written for mitmproxy.
+    
+    * Move the web application to Flask.
+    
+    * Massively expand the documentation.

pathod/CONTRIBUTORS

@@ -0,0 +1,6 @@
+   426	Aldo Cortesi
+    74	Maximilian Hils
+    50	Thomas Kriechbaumer
+     1	Felix Yan
+     1	requires.io
+     1	starenka

pathod/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2008, Aldo Cortesi. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pathod/MANIFEST.in

@@ -0,0 +1,6 @@
+include LICENSE CHANGELOG README.txt
+exclude README.mkd
+recursive-include test *
+recursive-include libpathod *
+recursive-include examples *
+recursive-exclude * *.pyc *.pyo *.swo *.swp

pathod/README.mkd

@@ -0,0 +1,44 @@
+[![Build Status](https://img.shields.io/travis/mitmproxy/pathod/master.svg)](https://travis-ci.org/mitmproxy/pathod)
+[![Code Health](https://landscape.io/github/mitmproxy/pathod/master/landscape.svg?style=flat)](https://landscape.io/github/mitmproxy/pathod/master)
+[![Coverage Status](https://img.shields.io/coveralls/mitmproxy/pathod/master.svg)](https://coveralls.io/r/mitmproxy/pathod)
+[![Downloads](https://img.shields.io/pypi/dm/pathod.svg?color=orange)](https://pypi.python.org/pypi/pathod)
+[![Latest Version](https://img.shields.io/pypi/v/pathod.svg)](https://pypi.python.org/pypi/pathod)
+[![Supported Python versions](https://img.shields.io/pypi/pyversions/pathod.svg)](https://pypi.python.org/pypi/pathod)
+
+__pathod__ is a collection of pathological tools for testing and torturing HTTP
+clients and servers. The project has three components:
+
+- __pathod__, an pathological HTTP daemon.
+- __pathoc__, a perverse HTTP client.
+- __libpathod.test__, an API for easily using __pathod__ and __pathoc__ in unit tests.
+
+
+# Documentation
+
+The pathod documentation is self-hosted. Just fire up pathod, like so:
+
+    ./pathod
+
+And then browse to:
+
+    http://localhost:9999
+
+You can always view the documentation for the latest release at the pathod
+website:
+
+    http://pathod.net
+
+# Installing
+
+If you already have __pip__ on your system, installing __pathod__ and its
+dependencies is dead simple:
+
+    pip install pathod
+
+The project has the following dependencies:
+
+* [netlib](https://github.com/mitmproxy/netlib)
+* [requests](http://docs.python-requests.org/en/latest/index.html)
+
+The project's test suite uses the
+[nose](http://nose.readthedocs.org/en/latest/) unit testing framework.

pathod/README.txt

@@ -0,0 +1,43 @@
+**pathod** is a collection of pathological tools for testing and torturing HTTP
+clients and servers. The project has three components:
+
+- **pathod**, an pathological HTTP daemon.
+- **pathoc**, a perverse HTTP client. 
+- **libpathod.test**, an API for easily using pathod and pathoc in unit tests.
+
+
+Documentation
+-------------
+
+The pathod documentation is self-hosted. Just fire up pathod, like so:
+    
+    ./pathod 
+
+And then browse to:
+
+    http://localhost:9999
+
+You can always view the documentation for the latest release at the pathod
+website:
+    
+    http://pathod.net
+
+
+Installing
+----------
+
+If you already have **pip** on your system, installing **pathod** and its
+dependencies is dead simple:
+    
+    pip install pathod
+
+The project has the following dependencies:
+
+* netlib_
+* requests_
+
+The project's test suite uses the nose_ unit testing framework.
+
+.. _netlib: https://github.com/mitmproxy/netlib
+.. _requests: http://docs.python-requests.org/en/latest/index.html 
+.. _nose: http://nose.readthedocs.org/en/latest/ 

pathod/examples/libpathod_pathoc.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+from libpathod import pathoc
+
+p = pathoc.Pathoc(("google.com", 80))
+p.connect()
+print p.request("get:/")
+print p.request("get:/foo")

pathod/examples/test_context.py

@@ -0,0 +1,23 @@
+import requests
+from libpathod import test
+
+
+def test_simple():
+    """
+        Testing the requests module with
+        a pathod context manager.
+    """
+    # Start pathod in a separate thread
+    with test.Daemon() as d:
+        # Get a URL for a pathod spec
+        url = d.p("200:b@100")
+        # ... and request it
+        r = requests.put(url)
+
+        # Check the returned data
+        assert r.status_code == 200
+        assert len(r.content) == 100
+
+        # Check pathod's internal log
+        log = d.last_log()["request"]
+        assert log["method"] == "PUT"

pathod/examples/test_setup.py

@@ -0,0 +1,31 @@
+import requests
+from libpathod import test
+
+
+class Test:
+
+    """
+        Testing the requests module with
+        a pathod instance started for
+        each test.
+    """
+
+    def setup(self):
+        self.d = test.Daemon()
+
+    def teardown(self):
+        self.d.shutdown()
+
+    def test_simple(self):
+        # Get a URL for a pathod spec
+        url = self.d.p("200:b@100")
+        # ... and request it
+        r = requests.put(url)
+
+        # Check the returned data
+        assert r.status_code == 200
+        assert len(r.content) == 100
+
+        # Check pathod's internal log
+        log = self.d.last_log()["request"]
+        assert log["method"] == "PUT"

pathod/examples/test_setupall.py

@@ -0,0 +1,39 @@
+import requests
+from libpathod import test
+
+
+class Test:
+
+    """
+        Testing the requests module with
+        a single pathod instance started
+        for the test suite.
+    """
+    @classmethod
+    def setup_class(cls):
+        cls.d = test.Daemon()
+
+    @classmethod
+    def teardown_class(cls):
+        cls.d.shutdown()
+
+    def setup(self):
+        # Clear the pathod logs between tests
+        self.d.clear_log()
+
+    def test_simple(self):
+        # Get a URL for a pathod spec
+        url = self.d.p("200:b@100")
+        # ... and request it
+        r = requests.put(url)
+
+        # Check the returned data
+        assert r.status_code == 200
+        assert len(r.content) == 100
+
+        # Check pathod's internal log
+        log = self.d.last_log()["request"]
+        assert log["method"] == "PUT"
+
+    def test_two(self):
+        assert not self.d.log()

pathod/libpathod/app.py

@@ -0,0 +1,179 @@
+import logging
+import pprint
+import cStringIO
+import copy
+from flask import Flask, jsonify, render_template, request, abort, make_response
+from . import version, language, utils
+from netlib.http import user_agents
+
+logging.basicConfig(level="DEBUG")
+EXAMPLE_HOST = "example.com"
+EXAMPLE_WEBSOCKET_KEY = "examplekey"
+
+# pylint: disable=unused-variable
+
+
+def make_app(noapi, debug):
+    app = Flask(__name__)
+    app.debug = debug
+
+    if not noapi:
+        @app.route('/api/info')
+        def api_info():
+            return jsonify(
+                version=version.IVERSION
+            )
+
+    @app.route('/api/log')
+    def api_log():
+        return jsonify(
+            log=app.config["pathod"].get_log()
+        )
+
+    @app.route('/api/clear_log')
+    def api_clear_log():
+        app.config["pathod"].clear_log()
+        return "OK"
+
+    def render(s, cacheable, **kwargs):
+        kwargs["noapi"] = app.config["pathod"].noapi
+        kwargs["nocraft"] = app.config["pathod"].nocraft
+        kwargs["craftanchor"] = app.config["pathod"].craftanchor
+        resp = make_response(render_template(s, **kwargs), 200)
+        if cacheable:
+            resp.headers["Cache-control"] = "public, max-age=4320"
+        return resp
+
+    @app.route('/')
+    @app.route('/index.html')
+    def index():
+        return render(
+            "index.html",
+            True,
+            section="main",
+            version=version.VERSION
+        )
+
+    @app.route('/download')
+    @app.route('/download.html')
+    def download():
+        return render(
+            "download.html", True, section="download", version=version.VERSION
+        )
+
+    @app.route('/about')
+    @app.route('/about.html')
+    def about():
+        return render("about.html", True, section="about")
+
+    @app.route('/docs/pathod')
+    def docs_pathod():
+        return render(
+            "docs_pathod.html", True, section="docs", subsection="pathod"
+        )
+
+    @app.route('/docs/language')
+    def docs_language():
+        return render(
+            "docs_lang.html", True,
+            section="docs", uastrings=user_agents.UASTRINGS,
+            subsection="lang"
+        )
+
+    @app.route('/docs/pathoc')
+    def docs_pathoc():
+        return render(
+            "docs_pathoc.html", True, section="docs", subsection="pathoc"
+        )
+
+    @app.route('/docs/libpathod')
+    def docs_libpathod():
+        return render(
+            "docs_libpathod.html", True, section="docs", subsection="libpathod"
+        )
+
+    @app.route('/docs/test')
+    def docs_test():
+        return render(
+            "docs_test.html", True, section="docs", subsection="test"
+        )
+
+    @app.route('/log')
+    def log():
+        if app.config["pathod"].noapi:
+            abort(404)
+        return render(
+            "log.html",
+            False,
+            section="log",
+            log=app.config["pathod"].get_log()
+        )
+
+    @app.route('/log/<int:lid>')
+    def onelog(lid):
+        item = app.config["pathod"].log_by_id(int(lid))
+        if not item:
+            abort(404)
+        l = pprint.pformat(item)
+        return render("onelog.html", False, section="log", alog=l, lid=lid)
+
+    def _preview(is_request):
+        if is_request:
+            template = "request_preview.html"
+        else:
+            template = "response_preview.html"
+
+        spec = request.args["spec"]
+
+        args = dict(
+            spec=spec,
+            section="main",
+            syntaxerror=None,
+            error=None,
+        )
+        if not spec.strip():
+            args["error"] = "Can't parse an empty spec."
+            return render(template, False, **args)
+
+        try:
+            if is_request:
+                r = language.parse_pathoc(spec).next()
+            else:
+                r = language.parse_pathod(spec).next()
+        except language.ParseException as v:
+            args["syntaxerror"] = str(v)
+            args["marked"] = v.marked()
+            return render(template, False, **args)
+
+        s = cStringIO.StringIO()
+
+        settings = copy.copy(app.config["pathod"].settings)
+        settings.request_host = EXAMPLE_HOST
+        settings.websocket_key = EXAMPLE_WEBSOCKET_KEY
+
+        safe = r.preview_safe()
+        err, safe = app.config["pathod"].check_policy(
+            safe,
+            settings
+        )
+        if err:
+            args["error"] = err
+            return render(template, False, **args)
+        if is_request:
+            settings.request_host = EXAMPLE_HOST
+            language.serve(safe, s, settings)
+        else:
+            settings.websocket_key = EXAMPLE_WEBSOCKET_KEY
+            language.serve(safe, s, settings)
+
+        args["output"] = utils.escape_unprintables(s.getvalue())
+        return render(template, False, **args)
+
+    @app.route('/response_preview')
+    def response_preview():
+        return _preview(False)
+
+    @app.route('/request_preview')
+    def request_preview():
+        return _preview(True)
+    return app

pathod/libpathod/language/__init__.py

@@ -0,0 +1,113 @@
+import itertools
+import time
+
+import pyparsing as pp
+
+from . import http, http2, websockets, writer, exceptions
+
+from exceptions import *
+from base import Settings
+assert Settings  # prevent pyflakes from messing with this
+
+
+def expand(msg):
+    times = getattr(msg, "times", None)
+    if times:
+        for j_ in xrange(int(times.value)):
+            yield msg.strike_token("times")
+    else:
+        yield msg
+
+
+def parse_pathod(s, use_http2=False):
+    """
+        May raise ParseException
+    """
+    try:
+        s = s.decode("ascii")
+    except UnicodeError:
+        raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
+    try:
+        if use_http2:
+            expressions = [
+                # http2.Frame.expr(),
+                http2.Response.expr(),
+            ]
+        else:
+            expressions = [
+                websockets.WebsocketFrame.expr(),
+                http.Response.expr(),
+            ]
+        reqs = pp.Or(expressions).parseString(s, parseAll=True)
+    except pp.ParseException as v:
+        raise exceptions.ParseException(v.msg, v.line, v.col)
+    return itertools.chain(*[expand(i) for i in reqs])
+
+
+def parse_pathoc(s, use_http2=False):
+    try:
+        s = s.decode("ascii")
+    except UnicodeError:
+        raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
+    try:
+        if use_http2:
+            expressions = [
+                # http2.Frame.expr(),
+                http2.Request.expr(),
+            ]
+        else:
+            expressions = [
+                websockets.WebsocketClientFrame.expr(),
+                http.Request.expr(),
+            ]
+        reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
+    except pp.ParseException as v:
+        raise exceptions.ParseException(v.msg, v.line, v.col)
+    return itertools.chain(*[expand(i) for i in reqs])
+
+
+def parse_websocket_frame(s):
+    """
+        May raise ParseException
+    """
+    try:
+        reqs = pp.OneOrMore(
+            websockets.WebsocketFrame.expr()
+        ).parseString(
+            s,
+            parseAll=True
+        )
+    except pp.ParseException as v:
+        raise exceptions.ParseException(v.msg, v.line, v.col)
+    return itertools.chain(*[expand(i) for i in reqs])
+
+
+def serve(msg, fp, settings):
+    """
+        fp: The file pointer to write to.
+
+        request_host: If this a request, this is the connecting host. If
+        None, we assume it's a response. Used to decide what standard
+        modifications to make if raw is not set.
+
+        Calling this function may modify the object.
+    """
+    msg = msg.resolve(settings)
+    started = time.time()
+
+    vals = msg.values(settings)
+    vals.reverse()
+
+    actions = sorted(msg.actions[:])
+    actions.reverse()
+    actions = [i.intermediate(settings) for i in actions]
+
+    disconnect = writer.write_values(fp, vals, actions[:])
+    duration = time.time() - started
+    ret = dict(
+        disconnect=disconnect,
+        started=started,
+        duration=duration,
+    )
+    ret.update(msg.log(settings))
+    return ret

pathod/libpathod/language/actions.py

@@ -0,0 +1,126 @@
+import abc
+import copy
+import random
+
+import pyparsing as pp
+
+from . import base
+
+
+class _Action(base.Token):
+
+    """
+        An action that operates on the raw data stream of the message. All
+        actions have one thing in common: an offset that specifies where the
+        action should take place.
+    """
+
+    def __init__(self, offset):
+        self.offset = offset
+
+    def resolve(self, settings, msg):
+        """
+            Resolves offset specifications to a numeric offset. Returns a copy
+            of the action object.
+        """
+        c = copy.copy(self)
+        l = msg.length(settings)
+        if c.offset == "r":
+            c.offset = random.randrange(l)
+        elif c.offset == "a":
+            c.offset = l + 1
+        return c
+
+    def __cmp__(self, other):
+        return cmp(self.offset, other.offset)
+
+    def __repr__(self):
+        return self.spec()
+
+    @abc.abstractmethod
+    def spec(self):  # pragma: no cover
+        pass
+
+    @abc.abstractmethod
+    def intermediate(self, settings):  # pragma: no cover
+        pass
+
+
+class PauseAt(_Action):
+    unique_name = None
+
+    def __init__(self, offset, seconds):
+        _Action.__init__(self, offset)
+        self.seconds = seconds
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal("p").suppress()
+        e += base.TokOffset
+        e += pp.Literal(",").suppress()
+        e += pp.MatchFirst(
+            [
+                base.v_integer,
+                pp.Literal("f")
+            ]
+        )
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        return "p%s,%s" % (self.offset, self.seconds)
+
+    def intermediate(self, settings):
+        return (self.offset, "pause", self.seconds)
+
+    def freeze(self, settings_):
+        return self
+
+
+class DisconnectAt(_Action):
+
+    def __init__(self, offset):
+        _Action.__init__(self, offset)
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal("d").suppress()
+        e += base.TokOffset
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        return "d%s" % self.offset
+
+    def intermediate(self, settings):
+        return (self.offset, "disconnect")
+
+    def freeze(self, settings_):
+        return self
+
+
+class InjectAt(_Action):
+    unique_name = None
+
+    def __init__(self, offset, value):
+        _Action.__init__(self, offset)
+        self.value = value
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal("i").suppress()
+        e += base.TokOffset
+        e += pp.Literal(",").suppress()
+        e += base.TokValue
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        return "i%s,%s" % (self.offset, self.value.spec())
+
+    def intermediate(self, settings):
+        return (
+            self.offset,
+            "inject",
+            self.value.get_generator(settings)
+        )
+
+    def freeze(self, settings):
+        return InjectAt(self.offset, self.value.freeze(settings))

pathod/libpathod/language/base.py

@@ -0,0 +1,576 @@
+import operator
+import os
+import abc
+import pyparsing as pp
+
+from .. import utils
+from . import generators, exceptions
+
+class Settings(object):
+
+    def __init__(
+        self,
+        is_client=False,
+        staticdir=None,
+        unconstrained_file_access=False,
+        request_host=None,
+        websocket_key=None,
+        protocol=None,
+    ):
+        self.is_client = is_client
+        self.staticdir = staticdir
+        self.unconstrained_file_access = unconstrained_file_access
+        self.request_host = request_host
+        self.websocket_key = websocket_key  # TODO: refactor this into the protocol
+        self.protocol = protocol
+
+
+Sep = pp.Optional(pp.Literal(":")).suppress()
+
+
+v_integer = pp.Word(pp.nums)\
+    .setName("integer")\
+    .setParseAction(lambda toks: int(toks[0]))
+
+
+v_literal = pp.MatchFirst(
+    [
+        pp.QuotedString(
+            "\"",
+            unquoteResults=True,
+            multiline=True
+        ),
+        pp.QuotedString(
+            "'",
+            unquoteResults=True,
+            multiline=True
+        ),
+    ]
+)
+
+v_naked_literal = pp.MatchFirst(
+    [
+        v_literal,
+        pp.Word("".join(i for i in pp.printables if i not in ",:\n@\'\""))
+    ]
+)
+
+
+class Token(object):
+
+    """
+        A token in the specification language. Tokens are immutable. The token
+        classes have no meaning in and of themselves, and are combined into
+        Components and Actions to build the language.
+    """
+    __metaclass__ = abc.ABCMeta
+
+    @classmethod
+    def expr(cls):  # pragma: no cover
+        """
+            A parse expression.
+        """
+        return None
+
+    @abc.abstractmethod
+    def spec(self):  # pragma: no cover
+        """
+            A parseable specification for this token.
+        """
+        return None
+
+    @property
+    def unique_name(self):
+        """
+            Controls uniqueness constraints for tokens. No two tokens with the
+            same name will be allowed. If no uniquness should be applied, this
+            should be None.
+        """
+        return self.__class__.__name__.lower()
+
+    def resolve(self, settings_, msg_):
+        """
+            Resolves this token to ready it for transmission. This means that
+            the calculated offsets of actions are fixed.
+
+            settings: a language.Settings instance
+            msg: The containing message
+        """
+        return self
+
+    def __repr__(self):
+        return self.spec()
+
+
+class _TokValueLiteral(Token):
+
+    def __init__(self, val):
+        self.val = val.decode("string_escape")
+
+    def get_generator(self, settings_):
+        return self.val
+
+    def freeze(self, settings_):
+        return self
+
+
+class TokValueLiteral(_TokValueLiteral):
+
+    """
+        A literal with Python-style string escaping
+    """
+    @classmethod
+    def expr(cls):
+        e = v_literal.copy()
+        return e.setParseAction(cls.parseAction)
+
+    @classmethod
+    def parseAction(cls, x):
+        v = cls(*x)
+        return v
+
+    def spec(self):
+        inner = self.val.encode("string_escape")
+        inner = inner.replace(r"\'", r"\x27")
+        return "'" + inner + "'"
+
+
+class TokValueNakedLiteral(_TokValueLiteral):
+
+    @classmethod
+    def expr(cls):
+        e = v_naked_literal.copy()
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        return self.val.encode("string_escape")
+
+
+class TokValueGenerate(Token):
+
+    def __init__(self, usize, unit, datatype):
+        if not unit:
+            unit = "b"
+        self.usize, self.unit, self.datatype = usize, unit, datatype
+
+    def bytes(self):
+        return self.usize * utils.SIZE_UNITS[self.unit]
+
+    def get_generator(self, settings_):
+        return generators.RandomGenerator(self.datatype, self.bytes())
+
+    def freeze(self, settings):
+        g = self.get_generator(settings)
+        return TokValueLiteral(g[:].encode("string_escape"))
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal("@").suppress() + v_integer
+
+        u = reduce(
+            operator.or_,
+            [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]
+        ).leaveWhitespace()
+        e = e + pp.Optional(u, default=None)
+
+        s = pp.Literal(",").suppress()
+        s += reduce(
+            operator.or_,
+            [pp.Literal(i) for i in generators.DATATYPES.keys()]
+        )
+        e += pp.Optional(s, default="bytes")
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        s = "@%s" % self.usize
+        if self.unit != "b":
+            s += self.unit
+        if self.datatype != "bytes":
+            s += ",%s" % self.datatype
+        return s
+
+
+class TokValueFile(Token):
+
+    def __init__(self, path):
+        self.path = str(path)
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal("<").suppress()
+        e = e + v_naked_literal
+        return e.setParseAction(lambda x: cls(*x))
+
+    def freeze(self, settings_):
+        return self
+
+    def get_generator(self, settings):
+        if not settings.staticdir:
+            raise exceptions.FileAccessDenied("File access disabled.")
+        s = os.path.expanduser(self.path)
+        s = os.path.normpath(
+            os.path.abspath(os.path.join(settings.staticdir, s))
+        )
+        uf = settings.unconstrained_file_access
+        if not uf and not s.startswith(settings.staticdir):
+            raise exceptions.FileAccessDenied(
+                "File access outside of configured directory"
+            )
+        if not os.path.isfile(s):
+            raise exceptions.FileAccessDenied("File not readable")
+        return generators.FileGenerator(s)
+
+    def spec(self):
+        return "<'%s'" % self.path.encode("string_escape")
+
+
+TokValue = pp.MatchFirst(
+    [
+        TokValueGenerate.expr(),
+        TokValueFile.expr(),
+        TokValueLiteral.expr()
+    ]
+)
+
+
+TokNakedValue = pp.MatchFirst(
+    [
+        TokValueGenerate.expr(),
+        TokValueFile.expr(),
+        TokValueLiteral.expr(),
+        TokValueNakedLiteral.expr(),
+    ]
+)
+
+
+TokOffset = pp.MatchFirst(
+    [
+        v_integer,
+        pp.Literal("r"),
+        pp.Literal("a")
+    ]
+)
+
+
+class _Component(Token):
+
+    """
+        A value component of the primary specification of an message.
+        Components produce byte values desribe the bytes of the message.
+    """
+
+    def values(self, settings):  # pragma: no cover
+        """
+           A sequence of values, which can either be strings or generators.
+        """
+        pass
+
+    def string(self, settings=None):
+        """
+            A string representation of the object.
+        """
+        return "".join(i[:] for i in self.values(settings or {}))
+
+
+class KeyValue(_Component):
+
+    """
+        A key/value pair.
+        cls.preamble: leader
+    """
+
+    def __init__(self, key, value):
+        self.key, self.value = key, value
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal(cls.preamble).suppress()
+        e += TokValue
+        e += pp.Literal("=").suppress()
+        e += TokValue
+        return e.setParseAction(lambda x: cls(*x))
+
+    def spec(self):
+        return "%s%s=%s" % (self.preamble, self.key.spec(), self.value.spec())
+
+    def freeze(self, settings):
+        return self.__class__(
+            self.key.freeze(settings), self.value.freeze(settings)
+        )
+
+
+class CaselessLiteral(_Component):
+
+    """
+        A caseless token that can take only one value.
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+    @classmethod
+    def expr(cls):
+        spec = pp.CaselessLiteral(cls.TOK)
+        spec = spec.setParseAction(lambda x: cls(*x))
+        return spec
+
+    def values(self, settings):
+        return self.TOK
+
+    def spec(self):
+        return self.TOK
+
+    def freeze(self, settings_):
+        return self
+
+
+class OptionsOrValue(_Component):
+
+    """
+        Can be any of a specified set of options, or a value specifier.
+    """
+    preamble = ""
+    options = []
+
+    def __init__(self, value):
+        # If it's a string, we were passed one of the options, so we lower-case
+        # it to be canonical. The user can specify a different case by using a
+        # string value literal.
+        self.option_used = False
+        if isinstance(value, basestring):
+            for i in self.options:
+                # Find the exact option value in a case-insensitive way
+                if i.lower() == value.lower():
+                    self.option_used = True
+                    value = TokValueLiteral(i)
+                    break
+        self.value = value
+
+    @classmethod
+    def expr(cls):
+        parts = [pp.CaselessLiteral(i) for i in cls.options]
+        m = pp.MatchFirst(parts)
+        spec = m | TokValue.copy()
+        spec = spec.setParseAction(lambda x: cls(*x))
+        if cls.preamble:
+            spec = pp.Literal(cls.preamble).suppress() + spec
+        return spec
+
+    def values(self, settings):
+        return [
+            self.value.get_generator(settings)
+        ]
+
+    def spec(self):
+        s = self.value.spec()
+        if s[1:-1].lower() in self.options:
+            s = s[1:-1].lower()
+        return "%s%s" % (self.preamble, s)
+
+    def freeze(self, settings):
+        return self.__class__(self.value.freeze(settings))
+
+
+class Integer(_Component):
+    bounds = (None, None)
+    preamble = ""
+
+    def __init__(self, value):
+        v = int(value)
+        outofbounds = any([
+            self.bounds[0] is not None and v < self.bounds[0],
+            self.bounds[1] is not None and v > self.bounds[1]
+        ])
+        if outofbounds:
+            raise exceptions.ParseException(
+                "Integer value must be between %s and %s." % self.bounds,
+                0, 0
+            )
+        self.value = str(value)
+
+    @classmethod
+    def expr(cls):
+        e = v_integer.copy()
+        if cls.preamble:
+            e = pp.Literal(cls.preamble).suppress() + e
+        return e.setParseAction(lambda x: cls(*x))
+
+    def values(self, settings):
+        return self.value
+
+    def spec(self):
+        return "%s%s" % (self.preamble, self.value)
+
+    def freeze(self, settings_):
+        return self
+
+
+class Value(_Component):
+
+    """
+        A value component lead by an optional preamble.
+    """
+    preamble = ""
+
+    def __init__(self, value):
+        self.value = value
+
+    @classmethod
+    def expr(cls):
+        e = (TokValue | TokNakedValue)
+        if cls.preamble:
+            e = pp.Literal(cls.preamble).suppress() + e
+        return e.setParseAction(lambda x: cls(*x))
+
+    def values(self, settings):
+        return [self.value.get_generator(settings)]
+
+    def spec(self):
+        return "%s%s" % (self.preamble, self.value.spec())
+
+    def freeze(self, settings):
+        return self.__class__(self.value.freeze(settings))
+
+
+class FixedLengthValue(Value):
+
+    """
+        A value component lead by an optional preamble.
+    """
+    preamble = ""
+    length = None
+
+    def __init__(self, value):
+        Value.__init__(self, value)
+        lenguess = None
+        try:
+            lenguess = len(value.get_generator(Settings()))
+        except exceptions.RenderError:
+            pass
+        # This check will fail if we know the length upfront
+        if lenguess is not None and lenguess != self.length:
+            raise exceptions.RenderError(
+                "Invalid value length: '%s' is %s bytes, should be %s." % (
+                    self.spec(),
+                    lenguess,
+                    self.length
+                )
+            )
+
+    def values(self, settings):
+        ret = Value.values(self, settings)
+        l = sum(len(i) for i in ret)
+        # This check will fail if we don't know the length upfront - i.e. for
+        # file inputs
+        if l != self.length:
+            raise exceptions.RenderError(
+                "Invalid value length: '%s' is %s bytes, should be %s." % (
+                    self.spec(),
+                    l,
+                    self.length
+                )
+            )
+        return ret
+
+
+class Boolean(_Component):
+
+    """
+        A boolean flag.
+            name  = true
+            -name = false
+    """
+    name = ""
+
+    def __init__(self, value):
+        self.value = value
+
+    @classmethod
+    def expr(cls):
+        e = pp.Optional(pp.Literal("-"), default=True)
+        e += pp.Literal(cls.name).suppress()
+
+        def parse(s_, loc_, toks):
+            val = True
+            if toks[0] == "-":
+                val = False
+            return cls(val)
+
+        return e.setParseAction(parse)
+
+    def spec(self):
+        return "%s%s" % ("-" if not self.value else "", self.name)
+
+
+class IntField(_Component):
+
+    """
+        An integer field, where values can optionally specified by name.
+    """
+    names = {}
+    max = 16
+    preamble = ""
+
+    def __init__(self, value):
+        self.origvalue = value
+        self.value = self.names.get(value, value)
+        if self.value > self.max:
+            raise exceptions.ParseException(
+                "Value can't exceed %s" % self.max, 0, 0
+            )
+
+    @classmethod
+    def expr(cls):
+        parts = [pp.CaselessLiteral(i) for i in cls.names.keys()]
+        m = pp.MatchFirst(parts)
+        spec = m | v_integer.copy()
+        spec = spec.setParseAction(lambda x: cls(*x))
+        if cls.preamble:
+            spec = pp.Literal(cls.preamble).suppress() + spec
+        return spec
+
+    def values(self, settings):
+        return [str(self.value)]
+
+    def spec(self):
+        return "%s%s" % (self.preamble, self.origvalue)
+
+
+class NestedMessage(Token):
+
+    """
+        A nested message, as an escaped string with a preamble.
+    """
+    preamble = ""
+    nest_type = None
+
+    def __init__(self, value):
+        Token.__init__(self)
+        self.value = value
+        try:
+            self.parsed = self.nest_type(
+                self.nest_type.expr().parseString(
+                    value.val,
+                    parseAll=True
+                )
+            )
+        except pp.ParseException as v:
+            raise exceptions.ParseException(v.msg, v.line, v.col)
+
+    @classmethod
+    def expr(cls):
+        e = pp.Literal(cls.preamble).suppress()
+        e = e + TokValueLiteral.expr()
+        return e.setParseAction(lambda x: cls(*x))
+
+    def values(self, settings):
+        return [
+            self.value.get_generator(settings),
+        ]
+
+    def spec(self):
+        return "%s%s" % (self.preamble, self.value.spec())
+
+    def freeze(self, settings):
+        f = self.parsed.freeze(settings).spec()
+        return self.__class__(TokValueLiteral(f.encode("string_escape")))

pathod/libpathod/language/exceptions.py

@@ -0,0 +1,22 @@
+
+class RenderError(Exception):
+    pass
+
+
+class FileAccessDenied(RenderError):
+    pass
+
+
+class ParseException(Exception):
+
+    def __init__(self, msg, s, col):
+        Exception.__init__(self)
+        self.msg = msg
+        self.s = s
+        self.col = col
+
+    def marked(self):
+        return "%s\n%s" % (self.s, " " * (self.col - 1) + "^")
+
+    def __str__(self):
+        return "%s at char %s" % (self.msg, self.col)

pathod/libpathod/language/generators.py

@@ -0,0 +1,86 @@
+import string
+import random
+import mmap
+
+DATATYPES = dict(
+    ascii_letters=string.ascii_letters,
+    ascii_lowercase=string.ascii_lowercase,
+    ascii_uppercase=string.ascii_uppercase,
+    digits=string.digits,
+    hexdigits=string.hexdigits,
+    octdigits=string.octdigits,
+    punctuation=string.punctuation,
+    whitespace=string.whitespace,
+    ascii=string.printable,
+    bytes="".join(chr(i) for i in range(256))
+)
+
+
+class TransformGenerator(object):
+
+    """
+        Perform a byte-by-byte transform another generator - that is, for each
+        input byte, the transformation must produce one output byte.
+
+        gen: A generator to wrap
+        transform: A function (offset, data) -> transformed
+    """
+
+    def __init__(self, gen, transform):
+        self.gen = gen
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.gen)
+
+    def __getitem__(self, x):
+        d = self.gen.__getitem__(x)
+        return self.transform(x, d)
+
+    def __getslice__(self, a, b):
+        d = self.gen.__getslice__(a, b)
+        return self.transform(a, d)
+
+    def __repr__(self):
+        return "'transform(%s)'" % self.gen
+
+
+class RandomGenerator(object):
+
+    def __init__(self, dtype, length):
+        self.dtype = dtype
+        self.length = length
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, x):
+        return random.choice(DATATYPES[self.dtype])
+
+    def __getslice__(self, a, b):
+        b = min(b, self.length)
+        chars = DATATYPES[self.dtype]
+        return "".join(random.choice(chars) for x in range(a, b))
+
+    def __repr__(self):
+        return "%s random from %s" % (self.length, self.dtype)
+
+
+class FileGenerator(object):
+
+    def __init__(self, path):
+        self.path = path
+        self.fp = file(path, "rb")
+        self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)
+
+    def __len__(self):
+        return len(self.map)
+
+    def __getitem__(self, x):
+        return self.map.__getitem__(x)
+
+    def __getslice__(self, a, b):
+        return self.map.__getslice__(a, b)
+
+    def __repr__(self):
+        return "<%s" % self.path

pathod/libpathod/language/http.py

@@ -0,0 +1,381 @@
+
+import abc
+
+import pyparsing as pp
+
+import netlib.websockets
+from netlib.http import status_codes, user_agents
+from . import base, exceptions, actions, message
+
+# TODO: use netlib.semantics.protocol assemble method,
+# instead of duplicating the HTTP on-the-wire representation here.
+# see http2 language for an example
+
+class WS(base.CaselessLiteral):
+    TOK = "ws"
+
+
+class Raw(base.CaselessLiteral):
+    TOK = "r"
+
+
+class Path(base.Value):
+    pass
+
+
+class StatusCode(base.Integer):
+    pass
+
+
+class Reason(base.Value):
+    preamble = "m"
+
+
+class Body(base.Value):
+    preamble = "b"
+
+
+class Times(base.Integer):
+    preamble = "x"
+
+
+class Method(base.OptionsOrValue):
+    options = [
+        "GET",
+        "HEAD",
+        "POST",
+        "PUT",
+        "DELETE",
+        "OPTIONS",
+        "TRACE",
+        "CONNECT",
+    ]
+
+
+class _HeaderMixin(object):
+    unique_name = None
+
+    def format_header(self, key, value):
+        return [key, ": ", value, "\r\n"]
+
+    def values(self, settings):
+        return self.format_header(
+            self.key.get_generator(settings),
+            self.value.get_generator(settings),
+        )
+
+
+class Header(_HeaderMixin, base.KeyValue):
+    preamble = "h"
+
+
+class ShortcutContentType(_HeaderMixin, base.Value):
+    preamble = "c"
+    key = base.TokValueLiteral("Content-Type")
+
+
+class ShortcutLocation(_HeaderMixin, base.Value):
+    preamble = "l"
+    key = base.TokValueLiteral("Location")
+
+
+class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
+    preamble = "u"
+    options = [i[1] for i in user_agents.UASTRINGS]
+    key = base.TokValueLiteral("User-Agent")
+
+    def values(self, settings):
+        value = self.value.val
+        if self.option_used:
+            value = user_agents.get_by_shortcut(value.lower())[2]
+
+        return self.format_header(
+            self.key.get_generator(settings),
+            value
+        )
+
+
+def get_header(val, headers):
+    """
+        Header keys may be Values, so we have to "generate" them as we try the
+        match.
+    """
+    for h in headers:
+        k = h.key.get_generator({})
+        if len(k) == len(val) and k[:].lower() == val.lower():
+            return h
+    return None
+
+
+class _HTTPMessage(message.Message):
+    version = "HTTP/1.1"
+
+    @property
+    def actions(self):
+        return self.toks(actions._Action)
+
+    @property
+    def raw(self):
+        return bool(self.tok(Raw))
+
+    @property
+    def body(self):
+        return self.tok(Body)
+
+    @abc.abstractmethod
+    def preamble(self, settings):  # pragma: no cover
+        pass
+
+    @property
+    def headers(self):
+        return self.toks(_HeaderMixin)
+
+    def values(self, settings):
+        vals = self.preamble(settings)
+        vals.append("\r\n")
+        for h in self.headers:
+            vals.extend(h.values(settings))
+        vals.append("\r\n")
+        if self.body:
+            vals.extend(self.body.values(settings))
+        return vals
+
+
+class Response(_HTTPMessage):
+    unique_name = None
+    comps = (
+        Header,
+        ShortcutContentType,
+        ShortcutLocation,
+        Raw,
+        Reason,
+        Body,
+
+        actions.PauseAt,
+        actions.DisconnectAt,
+        actions.InjectAt,
+    )
+    logattrs = ["status_code", "reason", "version", "body"]
+
+    @property
+    def ws(self):
+        return self.tok(WS)
+
+    @property
+    def status_code(self):
+        return self.tok(StatusCode)
+
+    @property
+    def reason(self):
+        return self.tok(Reason)
+
+    def preamble(self, settings):
+        l = [self.version, " "]
+        l.extend(self.status_code.values(settings))
+        status_code = int(self.status_code.value)
+        l.append(" ")
+        if self.reason:
+            l.extend(self.reason.values(settings))
+        else:
+            l.append(
+                status_codes.RESPONSES.get(
+                    status_code,
+                    "Unknown code"
+                )
+            )
+        return l
+
+    def resolve(self, settings, msg=None):
+        tokens = self.tokens[:]
+        if self.ws:
+            if not settings.websocket_key:
+                raise exceptions.RenderError(
+                    "No websocket key - have we seen a client handshake?"
+                )
+            if not self.status_code:
+                tokens.insert(
+                    1,
+                    StatusCode(101)
+                )
+            headers = netlib.websockets.WebsocketsProtocol.server_handshake_headers(
+                settings.websocket_key
+            )
+            for i in headers.fields:
+                if not get_header(i[0], self.headers):
+                    tokens.append(
+                        Header(
+                            base.TokValueLiteral(i[0]),
+                            base.TokValueLiteral(i[1]))
+                    )
+        if not self.raw:
+            if not get_header("Content-Length", self.headers):
+                if not self.body:
+                    length = 0
+                else:
+                    length = sum(
+                        len(i) for i in self.body.values(settings)
+                    )
+                tokens.append(
+                    Header(
+                        base.TokValueLiteral("Content-Length"),
+                        base.TokValueLiteral(str(length)),
+                    )
+                )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(settings, intermediate) for i in tokens]
+        )
+
+    @classmethod
+    def expr(cls):
+        parts = [i.expr() for i in cls.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                pp.MatchFirst(
+                    [
+                        WS.expr() + pp.Optional(
+                            base.Sep + StatusCode.expr()
+                        ),
+                        StatusCode.expr(),
+                    ]
+                ),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(cls)
+        return resp
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+class NestedResponse(base.NestedMessage):
+    preamble = "s"
+    nest_type = Response
+
+
+class Request(_HTTPMessage):
+    comps = (
+        Header,
+        ShortcutContentType,
+        ShortcutUserAgent,
+        Raw,
+        NestedResponse,
+        Body,
+        Times,
+
+        actions.PauseAt,
+        actions.DisconnectAt,
+        actions.InjectAt,
+    )
+    logattrs = ["method", "path", "body"]
+
+    @property
+    def ws(self):
+        return self.tok(WS)
+
+    @property
+    def method(self):
+        return self.tok(Method)
+
+    @property
+    def path(self):
+        return self.tok(Path)
+
+    @property
+    def times(self):
+        return self.tok(Times)
+
+    @property
+    def nested_response(self):
+        return self.tok(NestedResponse)
+
+    def preamble(self, settings):
+        v = self.method.values(settings)
+        v.append(" ")
+        v.extend(self.path.values(settings))
+        if self.nested_response:
+            v.append(self.nested_response.parsed.spec())
+        v.append(" ")
+        v.append(self.version)
+        return v
+
+    def resolve(self, settings, msg=None):
+        tokens = self.tokens[:]
+        if self.ws:
+            if not self.method:
+                tokens.insert(
+                    1,
+                    Method("get")
+                )
+            for i in netlib.websockets.WebsocketsProtocol.client_handshake_headers().fields:
+                if not get_header(i[0], self.headers):
+                    tokens.append(
+                        Header(
+                            base.TokValueLiteral(i[0]),
+                            base.TokValueLiteral(i[1])
+                        )
+                    )
+        if not self.raw:
+            if not get_header("Content-Length", self.headers):
+                if self.body:
+                    length = sum(
+                        len(i) for i in self.body.values(settings)
+                    )
+                    tokens.append(
+                        Header(
+                            base.TokValueLiteral("Content-Length"),
+                            base.TokValueLiteral(str(length)),
+                        )
+                    )
+            if settings.request_host:
+                if not get_header("Host", self.headers):
+                    tokens.append(
+                        Header(
+                            base.TokValueLiteral("Host"),
+                            base.TokValueLiteral(settings.request_host)
+                        )
+                    )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(settings, intermediate) for i in tokens]
+        )
+
+    @classmethod
+    def expr(cls):
+        parts = [i.expr() for i in cls.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                pp.MatchFirst(
+                    [
+                        WS.expr() + pp.Optional(
+                            base.Sep + Method.expr()
+                        ),
+                        Method.expr(),
+                    ]
+                ),
+                base.Sep,
+                Path.expr(),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(cls)
+        return resp
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+def make_error_response(reason, body=None):
+    tokens = [
+        StatusCode("800"),
+        Header(
+            base.TokValueLiteral("Content-Type"),
+            base.TokValueLiteral("text/plain")
+        ),
+        Reason(base.TokValueLiteral(reason)),
+        Body(base.TokValueLiteral("pathod error: " + (body or reason))),
+    ]
+    return Response(tokens)

pathod/libpathod/language/http2.py

@@ -0,0 +1,299 @@
+import pyparsing as pp
+
+from netlib import http
+from netlib.http import user_agents, Headers
+from . import base, message
+
+"""
+    Normal HTTP requests:
+        <method>:<path>:<header>:<body>
+    e.g.:
+        GET:/
+        GET:/:h"foo"="bar"
+        POST:/:h"foo"="bar":b'content body payload'
+
+    Normal HTTP responses:
+        <code>:<header>:<body>
+    e.g.:
+        200
+        302:h"foo"="bar"
+        404:h"foo"="bar":b'content body payload'
+
+    Individual HTTP/2 frames:
+        h2f:<payload_length>:<type>:<flags>:<stream_id>:<payload>
+    e.g.:
+        h2f:0:PING
+        h2f:42:HEADERS:END_HEADERS:0x1234567:foo=bar,host=example.com
+        h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'
+"""
+
+def get_header(val, headers):
+    """
+        Header keys may be Values, so we have to "generate" them as we try the
+        match.
+    """
+    for h in headers:
+        k = h.key.get_generator({})
+        if len(k) == len(val) and k[:].lower() == val.lower():
+            return h
+    return None
+
+
+class _HeaderMixin(object):
+    unique_name = None
+
+    def values(self, settings):
+        return (
+            self.key.get_generator(settings),
+            self.value.get_generator(settings),
+        )
+
+class _HTTP2Message(message.Message):
+    @property
+    def actions(self):
+        return []  # self.toks(actions._Action)
+
+    @property
+    def headers(self):
+        headers = self.toks(_HeaderMixin)
+
+        if not self.raw:
+            if not get_header("content-length", headers):
+                if not self.body:
+                    length = 0
+                else:
+                    length = len(self.body.string())
+                headers.append(
+                    Header(
+                        base.TokValueLiteral("content-length"),
+                        base.TokValueLiteral(str(length)),
+                    )
+                )
+        return headers
+
+    @property
+    def raw(self):
+        return bool(self.tok(Raw))
+
+    @property
+    def body(self):
+        return self.tok(Body)
+
+    def resolve(self, settings):
+        return self
+
+
+class StatusCode(base.Integer):
+    pass
+
+
+class Method(base.OptionsOrValue):
+    options = [
+        "GET",
+        "HEAD",
+        "POST",
+        "PUT",
+        "DELETE",
+    ]
+
+
+class Path(base.Value):
+    pass
+
+
+class Header(_HeaderMixin, base.KeyValue):
+    preamble = "h"
+
+
+class ShortcutContentType(_HeaderMixin, base.Value):
+    preamble = "c"
+    key = base.TokValueLiteral("content-type")
+
+
+class ShortcutLocation(_HeaderMixin, base.Value):
+    preamble = "l"
+    key = base.TokValueLiteral("location")
+
+
+class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
+    preamble = "u"
+    options = [i[1] for i in user_agents.UASTRINGS]
+    key = base.TokValueLiteral("user-agent")
+
+    def values(self, settings):
+        value = self.value.val
+        if self.option_used:
+            value = user_agents.get_by_shortcut(value.lower())[2]
+
+        return (
+            self.key.get_generator(settings),
+            value
+        )
+
+
+class Raw(base.CaselessLiteral):
+    TOK = "r"
+
+
+class Body(base.Value):
+    preamble = "b"
+
+
+class Times(base.Integer):
+    preamble = "x"
+
+
+class Response(_HTTP2Message):
+    unique_name = None
+    comps = (
+        Header,
+        Body,
+        ShortcutContentType,
+        ShortcutLocation,
+        Raw,
+    )
+
+    def __init__(self, tokens):
+        super(Response, self).__init__(tokens)
+        self.rendered_values = None
+        self.stream_id = 2
+
+    @property
+    def status_code(self):
+        return self.tok(StatusCode)
+
+    @classmethod
+    def expr(cls):
+        parts = [i.expr() for i in cls.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                StatusCode.expr(),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(cls)
+        return resp
+
+    def values(self, settings):
+        if self.rendered_values:
+            return self.rendered_values
+        else:
+            headers = Headers([header.values(settings) for header in self.headers])
+
+            body = self.body
+            if body:
+                body = body.string()
+
+            resp = http.Response(
+                (2, 0),
+                self.status_code.string(),
+                '',
+                headers,
+                body,
+            )
+            resp.stream_id = self.stream_id
+
+            self.rendered_values = settings.protocol.assemble(resp)
+            return self.rendered_values
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+class NestedResponse(base.NestedMessage):
+    preamble = "s"
+    nest_type = Response
+
+
+class Request(_HTTP2Message):
+    comps = (
+        Header,
+        ShortcutContentType,
+        ShortcutUserAgent,
+        Raw,
+        NestedResponse,
+        Body,
+        Times,
+    )
+    logattrs = ["method", "path"]
+
+    def __init__(self, tokens):
+        super(Request, self).__init__(tokens)
+        self.rendered_values = None
+        self.stream_id = 1
+
+    @property
+    def method(self):
+        return self.tok(Method)
+
+    @property
+    def path(self):
+        return self.tok(Path)
+
+    @property
+    def nested_response(self):
+        return self.tok(NestedResponse)
+
+    @property
+    def times(self):
+        return self.tok(Times)
+
+    @classmethod
+    def expr(cls):
+        parts = [i.expr() for i in cls.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                Method.expr(),
+                base.Sep,
+                Path.expr(),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(cls)
+        return resp
+
+    def values(self, settings):
+        if self.rendered_values:
+            return self.rendered_values
+        else:
+            path = self.path.string()
+            if self.nested_response:
+                path += self.nested_response.parsed.spec()
+
+            headers = Headers([header.values(settings) for header in self.headers])
+
+            body = self.body
+            if body:
+                body = body.string()
+
+            req = http.Request(
+                '',
+                self.method.string(),
+                '',
+                '',
+                '',
+                path,
+                (2, 0),
+                headers,
+                body,
+            )
+            req.stream_id = self.stream_id
+
+            self.rendered_values = settings.protocol.assemble(req)
+            return self.rendered_values
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+def make_error_response(reason, body=None):
+    tokens = [
+        StatusCode("800"),
+        Body(base.TokValueLiteral("pathod error: " + (body or reason))),
+    ]
+    return Response(tokens)
+
+
+# class Frame(message.Message):
+#     pass

pathod/libpathod/language/message.py

@@ -0,0 +1,96 @@
+import abc
+from . import actions, exceptions
+
+LOG_TRUNCATE = 1024
+
+
+class Message(object):
+    __metaclass__ = abc.ABCMeta
+    logattrs = []
+
+    def __init__(self, tokens):
+        track = set([])
+        for i in tokens:
+            if i.unique_name:
+                if i.unique_name in track:
+                    raise exceptions.ParseException(
+                        "Message has multiple %s clauses, "
+                        "but should only have one." % i.unique_name,
+                        0, 0
+                    )
+                else:
+                    track.add(i.unique_name)
+        self.tokens = tokens
+
+    def strike_token(self, name):
+        toks = [i for i in self.tokens if i.unique_name != name]
+        return self.__class__(toks)
+
+    def toks(self, klass):
+        """
+            Fetch all tokens that are instances of klass
+        """
+        return [i for i in self.tokens if isinstance(i, klass)]
+
+    def tok(self, klass):
+        """
+            Fetch first token that is an instance of klass
+        """
+        l = self.toks(klass)
+        if l:
+            return l[0]
+
+    def length(self, settings):
+        """
+            Calculate the length of the base message without any applied
+            actions.
+        """
+        return sum(len(x) for x in self.values(settings))
+
+    def preview_safe(self):
+        """
+            Return a copy of this message that issafe for previews.
+        """
+        tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)]
+        return self.__class__(tokens)
+
+    def maximum_length(self, settings):
+        """
+            Calculate the maximum length of the base message with all applied
+            actions.
+        """
+        l = self.length(settings)
+        for i in self.actions:
+            if isinstance(i, actions.InjectAt):
+                l += len(i.value.get_generator(settings))
+        return l
+
+    @classmethod
+    def expr(cls):  # pragma: no cover
+        pass
+
+    def log(self, settings):
+        """
+            A dictionary that should be logged if this message is served.
+        """
+        ret = {}
+        for i in self.logattrs:
+            v = getattr(self, i)
+            # Careful not to log any VALUE specs without sanitizing them first.
+            # We truncate at 1k.
+            if hasattr(v, "values"):
+                v = [x[:LOG_TRUNCATE] for x in v.values(settings)]
+                v = "".join(v).encode("string_escape")
+            elif hasattr(v, "__len__"):
+                v = v[:LOG_TRUNCATE]
+                v = v.encode("string_escape")
+            ret[i] = v
+        ret["spec"] = self.spec()
+        return ret
+
+    def freeze(self, settings):
+        r = self.resolve(settings)
+        return self.__class__([i.freeze(settings) for i in r.tokens])
+
+    def __repr__(self):
+        return self.spec()

pathod/libpathod/language/websockets.py

@@ -0,0 +1,241 @@
+import os
+import netlib.websockets
+import pyparsing as pp
+from . import base, generators, actions, message
+
+NESTED_LEADER = "pathod!"
+
+
+class WF(base.CaselessLiteral):
+    TOK = "wf"
+
+
+class OpCode(base.IntField):
+    names = {
+        "continue": netlib.websockets.OPCODE.CONTINUE,
+        "text": netlib.websockets.OPCODE.TEXT,
+        "binary": netlib.websockets.OPCODE.BINARY,
+        "close": netlib.websockets.OPCODE.CLOSE,
+        "ping": netlib.websockets.OPCODE.PING,
+        "pong": netlib.websockets.OPCODE.PONG,
+    }
+    max = 15
+    preamble = "c"
+
+
+class Body(base.Value):
+    preamble = "b"
+
+
+class RawBody(base.Value):
+    unique_name = "body"
+    preamble = "r"
+
+
+class Fin(base.Boolean):
+    name = "fin"
+
+
+class RSV1(base.Boolean):
+    name = "rsv1"
+
+
+class RSV2(base.Boolean):
+    name = "rsv2"
+
+
+class RSV3(base.Boolean):
+    name = "rsv3"
+
+
+class Mask(base.Boolean):
+    name = "mask"
+
+
+class Key(base.FixedLengthValue):
+    preamble = "k"
+    length = 4
+
+
+class KeyNone(base.CaselessLiteral):
+    unique_name = "key"
+    TOK = "knone"
+
+
+class Length(base.Integer):
+    bounds = (0, 1 << 64)
+    preamble = "l"
+
+
+class Times(base.Integer):
+    preamble = "x"
+
+
+COMPONENTS = (
+    OpCode,
+    Length,
+    # Bit flags
+    Fin,
+    RSV1,
+    RSV2,
+    RSV3,
+    Mask,
+    actions.PauseAt,
+    actions.DisconnectAt,
+    actions.InjectAt,
+    KeyNone,
+    Key,
+    Times,
+
+    Body,
+    RawBody,
+)
+
+
+class WebsocketFrame(message.Message):
+    components = COMPONENTS
+    logattrs = ["body"]
+    # Used for nested frames
+    unique_name = "body"
+
+    @property
+    def actions(self):
+        return self.toks(actions._Action)
+
+    @property
+    def body(self):
+        return self.tok(Body)
+
+    @property
+    def rawbody(self):
+        return self.tok(RawBody)
+
+    @property
+    def opcode(self):
+        return self.tok(OpCode)
+
+    @property
+    def fin(self):
+        return self.tok(Fin)
+
+    @property
+    def rsv1(self):
+        return self.tok(RSV1)
+
+    @property
+    def rsv2(self):
+        return self.tok(RSV2)
+
+    @property
+    def rsv3(self):
+        return self.tok(RSV3)
+
+    @property
+    def mask(self):
+        return self.tok(Mask)
+
+    @property
+    def key(self):
+        return self.tok(Key)
+
+    @property
+    def knone(self):
+        return self.tok(KeyNone)
+
+    @property
+    def times(self):
+        return self.tok(Times)
+
+    @property
+    def toklength(self):
+        return self.tok(Length)
+
+    @classmethod
+    def expr(cls):
+        parts = [i.expr() for i in cls.components]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                WF.expr(),
+                base.Sep,
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(cls)
+        return resp
+
+    @property
+    def nested_frame(self):
+        return self.tok(NestedFrame)
+
+    def resolve(self, settings, msg=None):
+        tokens = self.tokens[:]
+        if not self.mask and settings.is_client:
+            tokens.append(
+                Mask(True)
+            )
+        if not self.knone and self.mask and self.mask.value and not self.key:
+            tokens.append(
+                Key(base.TokValueLiteral(os.urandom(4)))
+            )
+        return self.__class__(
+            [i.resolve(settings, self) for i in tokens]
+        )
+
+    def values(self, settings):
+        if self.body:
+            bodygen = self.body.value.get_generator(settings)
+            length = len(self.body.value.get_generator(settings))
+        elif self.rawbody:
+            bodygen = self.rawbody.value.get_generator(settings)
+            length = len(self.rawbody.value.get_generator(settings))
+        elif self.nested_frame:
+            bodygen = NESTED_LEADER + self.nested_frame.parsed.spec()
+            length = len(bodygen)
+        else:
+            bodygen = None
+            length = 0
+        if self.toklength:
+            length = int(self.toklength.value)
+        frameparts = dict(
+            payload_length=length
+        )
+        if self.mask and self.mask.value:
+            frameparts["mask"] = True
+        if self.knone:
+            frameparts["masking_key"] = None
+        elif self.key:
+            key = self.key.values(settings)[0][:]
+            frameparts["masking_key"] = key
+        for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]:
+            v = getattr(self, i, None)
+            if v is not None:
+                frameparts[i] = v.value
+        frame = netlib.websockets.FrameHeader(**frameparts)
+        vals = [bytes(frame)]
+        if bodygen:
+            if frame.masking_key and not self.rawbody:
+                masker = netlib.websockets.Masker(frame.masking_key)
+                vals.append(
+                    generators.TransformGenerator(
+                        bodygen,
+                        masker.mask
+                    )
+                )
+            else:
+                vals.append(bodygen)
+        return vals
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+class NestedFrame(base.NestedMessage):
+    preamble = "f"
+    nest_type = WebsocketFrame
+
+
+class WebsocketClientFrame(WebsocketFrame):
+    components = COMPONENTS + (
+        NestedFrame,
+    )

pathod/libpathod/language/writer.py

@@ -0,0 +1,67 @@
+import time
+from netlib.exceptions import TcpDisconnect
+import netlib.tcp
+
+BLOCKSIZE = 1024
+# It's not clear what the upper limit for time.sleep is. It's lower than the
+# maximum int or float. 1 year should do.
+FOREVER = 60 * 60 * 24 * 365
+
+
+def send_chunk(fp, val, blocksize, start, end):
+    """
+        (start, end): Inclusive lower bound, exclusive upper bound.
+    """
+    for i in range(start, end, blocksize):
+        fp.write(
+            val[i:min(i + blocksize, end)]
+        )
+    return end - start
+
+
+def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
+    """
+        vals: A list of values, which may be strings or Value objects.
+
+        actions: A list of (offset, action, arg) tuples. Action may be "pause"
+        or "disconnect".
+
+        Both vals and actions are in reverse order, with the first items last.
+
+        Return True if connection should disconnect.
+    """
+    sofar = 0
+    try:
+        while vals:
+            v = vals.pop()
+            offset = 0
+            while actions and actions[-1][0] < (sofar + len(v)):
+                a = actions.pop()
+                offset += send_chunk(
+                    fp,
+                    v,
+                    blocksize,
+                    offset,
+                    a[0] - sofar - offset
+                )
+                if a[1] == "pause":
+                    time.sleep(
+                        FOREVER if a[2] == "f" else a[2]
+                    )
+                elif a[1] == "disconnect":
+                    return True
+                elif a[1] == "inject":
+                    send_chunk(fp, a[2], blocksize, 0, len(a[2]))
+            send_chunk(fp, v, blocksize, offset, len(v))
+            sofar += len(v)
+        # Remainders
+        while actions:
+            a = actions.pop()
+            if a[1] == "pause":
+                time.sleep(a[2])
+            elif a[1] == "disconnect":
+                return True
+            elif a[1] == "inject":
+                send_chunk(fp, a[2], blocksize, 0, len(a[2]))
+    except TcpDisconnect:  # pragma: no cover
+        return True

pathod/libpathod/log.py

@@ -0,0 +1,83 @@
+import datetime
+
+import netlib.utils
+import netlib.tcp
+import netlib.http
+
+TIMEFMT = '%d-%m-%y %H:%M:%S'
+
+
+def write_raw(fp, lines):
+    if fp:
+        fp.write(
+            "%s: " % datetime.datetime.now().strftime(TIMEFMT)
+        )
+        for i in lines:
+            fp.write(i)
+        fp.write("\n")
+        fp.flush()
+
+
+class LogCtx(object):
+
+    def __init__(self, fp, hex, rfile, wfile):
+        self.lines = []
+        self.fp = fp
+        self.suppressed = False
+        self.hex = hex
+        self.rfile, self.wfile = rfile, wfile
+
+    def __enter__(self):
+        if self.wfile:
+            self.wfile.start_log()
+        if self.rfile:
+            self.rfile.start_log()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        wlog = self.wfile.get_log() if self.wfile else None
+        rlog = self.rfile.get_log() if self.rfile else None
+        if self.suppressed or not self.fp:
+            return
+        if wlog:
+            self("Bytes written:")
+            self.dump(wlog, self.hex)
+        if rlog:
+            self("Bytes read:")
+            self.dump(rlog, self.hex)
+        if self.lines:
+            write_raw(
+                self.fp,
+                [
+                    "\n".join(self.lines),
+                ]
+            )
+        if exc_value:
+            raise exc_type, exc_value, traceback
+
+    def suppress(self):
+        self.suppressed = True
+
+    def dump(self, data, hexdump):
+        if hexdump:
+            for line in netlib.utils.hexdump(data):
+                self("\t%s %s %s" % line)
+        else:
+            for i in netlib.utils.clean_bin(data).split("\n"):
+                self("\t%s" % i)
+
+    def __call__(self, line):
+        self.lines.append(line)
+
+
+class ConnectionLogger:
+    def __init__(self, fp, hex, rfile, wfile):
+        self.fp = fp
+        self.hex = hex
+        self.rfile, self.wfile = rfile, wfile
+
+    def ctx(self):
+        return LogCtx(self.fp, self.hex, self.rfile, self.wfile)
+
+    def write(self, lines):
+        write_raw(self.fp, lines)

pathod/libpathod/pathoc.py

@@ -0,0 +1,534 @@
+import contextlib
+import sys
+import os
+import itertools
+import hashlib
+import Queue
+import random
+import select
+import time
+import threading
+
+import OpenSSL.crypto
+import six
+
+from netlib import tcp, http, certutils, websockets, socks
+from netlib.exceptions import HttpException, TcpDisconnect, TcpTimeout, TlsException, TcpException, \
+    NetlibException
+from netlib.http import http1, http2
+
+import language.http
+import language.websockets
+from . import utils, log
+
+import logging
+from netlib.tutils import treq
+
+logging.getLogger("hpack").setLevel(logging.WARNING)
+
+
+class PathocError(Exception):
+    pass
+
+
+class SSLInfo(object):
+
+    def __init__(self, certchain, cipher, alp):
+        self.certchain, self.cipher, self.alp = certchain, cipher, alp
+
+    def __str__(self):
+        parts = [
+            "Application Layer Protocol: %s" % self.alp,
+            "Cipher: %s, %s bit, %s" % self.cipher,
+            "SSL certificate chain:"
+        ]
+        for i in self.certchain:
+            parts.append("\tSubject: ")
+            for cn in i.get_subject().get_components():
+                parts.append("\t\t%s=%s" % cn)
+            parts.append("\tIssuer: ")
+            for cn in i.get_issuer().get_components():
+                parts.append("\t\t%s=%s" % cn)
+            parts.extend(
+                [
+                    "\tVersion: %s" % i.get_version(),
+                    "\tValidity: %s - %s" % (
+                        i.get_notBefore(), i.get_notAfter()
+                    ),
+                    "\tSerial: %s" % i.get_serial_number(),
+                    "\tAlgorithm: %s" % i.get_signature_algorithm()
+                ]
+            )
+            pk = i.get_pubkey()
+            types = {
+                OpenSSL.crypto.TYPE_RSA: "RSA",
+                OpenSSL.crypto.TYPE_DSA: "DSA"
+            }
+            t = types.get(pk.type(), "Uknown")
+            parts.append("\tPubkey: %s bit %s" % (pk.bits(), t))
+            s = certutils.SSLCert(i)
+            if s.altnames:
+                parts.append("\tSANs: %s" % " ".join(s.altnames))
+            return "\n".join(parts)
+
+
+
+class WebsocketFrameReader(threading.Thread):
+
+    def __init__(
+            self,
+            rfile,
+            logfp,
+            showresp,
+            hexdump,
+            ws_read_limit,
+            timeout
+    ):
+        threading.Thread.__init__(self)
+        self.timeout = timeout
+        self.ws_read_limit = ws_read_limit
+        self.logfp = logfp
+        self.showresp = showresp
+        self.hexdump = hexdump
+        self.rfile = rfile
+        self.terminate = Queue.Queue()
+        self.frames_queue = Queue.Queue()
+        self.logger = log.ConnectionLogger(
+            self.logfp,
+            self.hexdump,
+            rfile if showresp else None,
+            None
+        )
+
+    @contextlib.contextmanager
+    def terminator(self):
+        yield
+        self.frames_queue.put(None)
+
+    def run(self):
+        starttime = time.time()
+        with self.terminator():
+            while True:
+                if self.ws_read_limit == 0:
+                    return
+                r, _, _ = select.select([self.rfile], [], [], 0.05)
+                delta = time.time() - starttime
+                if not r and self.timeout and delta > self.timeout:
+                    return
+                try:
+                    self.terminate.get_nowait()
+                    return
+                except Queue.Empty:
+                    pass
+                for rfile in r:
+                    with self.logger.ctx() as log:
+                        try:
+                            frm = websockets.Frame.from_file(self.rfile)
+                        except TcpDisconnect:
+                            return
+                        self.frames_queue.put(frm)
+                        log("<< %s" % frm.header.human_readable())
+                        if self.ws_read_limit is not None:
+                            self.ws_read_limit -= 1
+                        starttime = time.time()
+
+
+class Pathoc(tcp.TCPClient):
+
+    def __init__(
+            self,
+            address,
+
+            # SSL
+            ssl=None,
+            sni=None,
+            ssl_version=tcp.SSL_DEFAULT_METHOD,
+            ssl_options=tcp.SSL_DEFAULT_OPTIONS,
+            clientcert=None,
+            ciphers=None,
+
+            # HTTP/2
+            use_http2=False,
+            http2_skip_connection_preface=False,
+            http2_framedump=False,
+
+            # Websockets
+            ws_read_limit=None,
+
+            # Network
+            timeout=None,
+
+            # Output control
+            showreq=False,
+            showresp=False,
+            explain=False,
+            hexdump=False,
+            ignorecodes=(),
+            ignoretimeout=False,
+            showsummary=False,
+            fp=sys.stdout
+    ):
+        """
+            spec: A request specification
+            showreq: Print requests
+            showresp: Print responses
+            explain: Print request explanation
+            showssl: Print info on SSL connection
+            hexdump: When printing requests or responses, use hex dump output
+            showsummary: Show a summary of requests
+            ignorecodes: Sequence of return codes to ignore
+        """
+        tcp.TCPClient.__init__(self, address)
+
+        self.ssl, self.sni = ssl, sni
+        self.clientcert = clientcert
+        self.ssl_version = ssl_version
+        self.ssl_options = ssl_options
+        self.ciphers = ciphers
+        self.sslinfo = None
+
+        self.use_http2 = use_http2
+        self.http2_skip_connection_preface = http2_skip_connection_preface
+        self.http2_framedump = http2_framedump
+
+        self.ws_read_limit = ws_read_limit
+
+        self.timeout = timeout
+
+        self.showreq = showreq
+        self.showresp = showresp
+        self.explain = explain
+        self.hexdump = hexdump
+        self.ignorecodes = ignorecodes
+        self.ignoretimeout = ignoretimeout
+        self.showsummary = showsummary
+        self.fp = fp
+
+        self.ws_framereader = None
+
+        if self.use_http2:
+            if not OpenSSL._util.lib.Cryptography_HAS_ALPN:  # pragma: nocover
+                log.write_raw(
+                    self.fp,
+                    "HTTP/2 requires ALPN support. "
+                    "Please use OpenSSL >= 1.0.2. "
+                    "Pathoc might not be working as expected without ALPN."
+                )
+            self.protocol = http2.HTTP2Protocol(self, dump_frames=self.http2_framedump)
+        else:
+            self.protocol = http1
+
+        self.settings = language.Settings(
+            is_client=True,
+            staticdir=os.getcwd(),
+            unconstrained_file_access=True,
+            request_host=self.address.host,
+            protocol=self.protocol,
+        )
+
+    def http_connect(self, connect_to):
+        self.wfile.write(
+            'CONNECT %s:%s HTTP/1.1\r\n' % tuple(connect_to) +
+            '\r\n'
+        )
+        self.wfile.flush()
+        try:
+            resp = self.protocol.read_response(self.rfile, treq(method="CONNECT"))
+            if resp.status_code != 200:
+                raise HttpException("Unexpected status code: %s" % resp.status_code)
+        except HttpException as e:
+            six.reraise(PathocError, PathocError(
+                "Proxy CONNECT failed: %s" % repr(e)
+            ))
+
+    def socks_connect(self, connect_to):
+        try:
+            client_greet = socks.ClientGreeting(socks.VERSION.SOCKS5, [socks.METHOD.NO_AUTHENTICATION_REQUIRED])
+            client_greet.to_file(self.wfile)
+            self.wfile.flush()
+
+            server_greet = socks.ServerGreeting.from_file(self.rfile)
+            server_greet.assert_socks5()
+            if server_greet.method != socks.METHOD.NO_AUTHENTICATION_REQUIRED:
+                raise socks.SocksError(
+                    socks.METHOD.NO_ACCEPTABLE_METHODS,
+                    "pathoc only supports SOCKS without authentication"
+                )
+
+            connect_request = socks.Message(
+                socks.VERSION.SOCKS5,
+                socks.CMD.CONNECT,
+                socks.ATYP.DOMAINNAME,
+                tcp.Address.wrap(connect_to)
+            )
+            connect_request.to_file(self.wfile)
+            self.wfile.flush()
+
+            connect_reply = socks.Message.from_file(self.rfile)
+            connect_reply.assert_socks5()
+            if connect_reply.msg != socks.REP.SUCCEEDED:
+                raise socks.SocksError(
+                    connect_reply.msg,
+                    "SOCKS server error"
+                )
+        except (socks.SocksError, TcpDisconnect) as e:
+            raise PathocError(str(e))
+
+    def connect(self, connect_to=None, showssl=False, fp=sys.stdout):
+        """
+            connect_to: A (host, port) tuple, which will be connected to with
+            an HTTP CONNECT request.
+        """
+        if self.use_http2 and not self.ssl:
+            raise NotImplementedError("HTTP2 without SSL is not supported.")
+
+        tcp.TCPClient.connect(self)
+
+        if connect_to:
+            self.http_connect(connect_to)
+
+        self.sslinfo = None
+        if self.ssl:
+            try:
+                alpn_protos = [b'http/1.1']
+                if self.use_http2:
+                    alpn_protos.append(b'h2')
+
+                self.convert_to_ssl(
+                    sni=self.sni,
+                    cert=self.clientcert,
+                    method=self.ssl_version,
+                    options=self.ssl_options,
+                    cipher_list=self.ciphers,
+                    alpn_protos=alpn_protos
+                )
+            except TlsException as v:
+                raise PathocError(str(v))
+
+            self.sslinfo = SSLInfo(
+                self.connection.get_peer_cert_chain(),
+                self.get_current_cipher(),
+                self.get_alpn_proto_negotiated()
+            )
+            if showssl:
+                print >> fp, str(self.sslinfo)
+
+            if self.use_http2:
+                self.protocol.check_alpn()
+                if not self.http2_skip_connection_preface:
+                    self.protocol.perform_client_connection_preface()
+
+        if self.timeout:
+            self.settimeout(self.timeout)
+
+    def stop(self):
+        if self.ws_framereader:
+            self.ws_framereader.terminate.put(None)
+
+    def wait(self, timeout=0.01, finish=True):
+        """
+            A generator that yields frames until Pathoc terminates.
+
+            timeout: If specified None may be yielded instead if timeout is
+            reached. If timeout is None, wait forever. If timeout is 0, return
+            immedately if nothing is on the queue.
+
+            finish: If true, consume messages until the reader shuts down.
+            Otherwise, return None on timeout.
+        """
+        if self.ws_framereader:
+            while True:
+                try:
+                    frm = self.ws_framereader.frames_queue.get(
+                        timeout=timeout,
+                        block=True if timeout != 0 else False
+                    )
+                except Queue.Empty:
+                    if finish:
+                        continue
+                    else:
+                        return
+                if frm is None:
+                    self.ws_framereader.join()
+                    return
+                yield frm
+
+    def websocket_send_frame(self, r):
+        """
+            Sends a single websocket frame.
+        """
+        logger = log.ConnectionLogger(
+            self.fp,
+            self.hexdump,
+            None,
+            self.wfile if self.showreq else None,
+        )
+        with logger.ctx() as lg:
+            lg(">> %s" % r)
+            language.serve(r, self.wfile, self.settings)
+            self.wfile.flush()
+
+    def websocket_start(self, r):
+        """
+            Performs an HTTP request, and attempts to drop into websocket
+            connection.
+        """
+        resp = self.http(r)
+        if resp.status_code == 101:
+            self.ws_framereader = WebsocketFrameReader(
+                self.rfile,
+                self.fp,
+                self.showresp,
+                self.hexdump,
+                self.ws_read_limit,
+                self.timeout
+            )
+            self.ws_framereader.start()
+        return resp
+
+    def http(self, r):
+        """
+            Performs a single request.
+
+            r: A language.http.Request object, or a string representing one
+            request.
+
+            Returns Response if we have a non-ignored response.
+
+            May raise a NetlibException
+        """
+        logger = log.ConnectionLogger(
+            self.fp,
+            self.hexdump,
+            self.rfile if self.showresp else None,
+            self.wfile if self.showreq else None,
+        )
+        with logger.ctx() as lg:
+            lg(">> %s" % r)
+            resp, req = None, None
+            try:
+                req = language.serve(r, self.wfile, self.settings)
+                self.wfile.flush()
+
+                resp = self.protocol.read_response(self.rfile, treq(method=req["method"].encode()))
+                resp.sslinfo = self.sslinfo
+            except HttpException as v:
+                lg("Invalid server response: %s" % v)
+                raise
+            except TcpTimeout:
+                if self.ignoretimeout:
+                    lg("Timeout (ignored)")
+                    return None
+                lg("Timeout")
+                raise
+            finally:
+                if resp:
+                    lg("<< %s %s: %s bytes" % (
+                        resp.status_code, utils.xrepr(resp.msg), len(resp.content)
+                    ))
+                    if resp.status_code in self.ignorecodes:
+                        lg.suppress()
+            return resp
+
+    def request(self, r):
+        """
+            Performs a single request.
+
+            r: A language.message.Messsage object, or a string representing
+            one.
+
+            Returns Response if we have a non-ignored response.
+
+            May raise a NetlibException
+        """
+        if isinstance(r, basestring):
+            r = language.parse_pathoc(r, self.use_http2).next()
+
+        if isinstance(r, language.http.Request):
+            if r.ws:
+                return self.websocket_start(r)
+            else:
+                return self.http(r)
+        elif isinstance(r, language.websockets.WebsocketFrame):
+            self.websocket_send_frame(r)
+        elif isinstance(r, language.http2.Request):
+            return self.http(r)
+        # elif isinstance(r, language.http2.Frame):
+            # TODO: do something
+
+
+def main(args):  # pragma: nocover
+    memo = set([])
+    trycount = 0
+    p = None
+    try:
+        cnt = 0
+        while True:
+            if cnt == args.repeat and args.repeat != 0:
+                break
+            if args.wait and cnt != 0:
+                time.sleep(args.wait)
+
+            cnt += 1
+            playlist = itertools.chain(*args.requests)
+            if args.random:
+                playlist = random.choice(args.requests)
+            p = Pathoc(
+                (args.host, args.port),
+                ssl=args.ssl,
+                sni=args.sni,
+                ssl_version=args.ssl_version,
+                ssl_options=args.ssl_options,
+                clientcert=args.clientcert,
+                ciphers=args.ciphers,
+                use_http2=args.use_http2,
+                http2_skip_connection_preface=args.http2_skip_connection_preface,
+                http2_framedump=args.http2_framedump,
+                showreq=args.showreq,
+                showresp=args.showresp,
+                explain=args.explain,
+                hexdump=args.hexdump,
+                ignorecodes=args.ignorecodes,
+                timeout=args.timeout,
+                ignoretimeout=args.ignoretimeout,
+                showsummary=True
+            )
+            trycount = 0
+            try:
+                p.connect(args.connect_to, args.showssl)
+            except TcpException as v:
+                print >> sys.stderr, str(v)
+                continue
+            except PathocError as v:
+                print >> sys.stderr, str(v)
+                sys.exit(1)
+            for spec in playlist:
+                if args.explain or args.memo:
+                    spec = spec.freeze(p.settings)
+                if args.memo:
+                    h = hashlib.sha256(spec.spec()).digest()
+                    if h not in memo:
+                        trycount = 0
+                        memo.add(h)
+                    else:
+                        trycount += 1
+                        if trycount > args.memolimit:
+                            print >> sys.stderr, "Memo limit exceeded..."
+                            return
+                        else:
+                            continue
+                try:
+                    ret = p.request(spec)
+                    if ret and args.oneshot:
+                        return
+                    # We consume the queue when we can, so it doesn't build up.
+                    for i_ in p.wait(timeout=0, finish=False):
+                        pass
+                except NetlibException:
+                    break
+            for i_ in p.wait(timeout=0.01, finish=True):
+                pass
+    except KeyboardInterrupt:
+        pass
+    if p:
+        p.stop()

pathod/libpathod/pathoc_cmdline.py

@@ -0,0 +1,226 @@
+import sys
+import argparse
+import os
+import os.path
+
+from netlib import tcp
+from netlib.http import user_agents
+from . import pathoc, version, language
+
+
+def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
+    preparser = argparse.ArgumentParser(add_help=False)
+    preparser.add_argument(
+        "--show-uas", dest="showua", action="store_true", default=False,
+        help="Print user agent shortcuts and exit."
+    )
+    pa = preparser.parse_known_args(argv)[0]
+    if pa.showua:
+        print >> stdout, "User agent strings:"
+        for i in user_agents.UASTRINGS:
+            print >> stdout, "  ", i[1], i[0]
+        sys.exit(0)
+
+    parser = argparse.ArgumentParser(
+        description='A perverse HTTP client.', parents=[preparser]
+    )
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="pathoc " + version.VERSION
+    )
+    parser.add_argument(
+        "-c", dest="connect_to", type=str, default=False,
+        metavar="HOST:PORT",
+        help="Issue an HTTP CONNECT to connect to the specified host."
+    )
+    parser.add_argument(
+        "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
+        help='Stop if we do not find a valid request after N attempts.'
+    )
+    parser.add_argument(
+        "-m", dest='memo', action="store_true", default=False,
+        help="""
+            Remember specs, and never play the same one twice. Note that this
+            means requests have to be rendered in memory, which means that
+            large generated data can cause issues.
+        """
+    )
+    parser.add_argument(
+        "-n", dest='repeat', default=1, type=int, metavar="N",
+        help='Repeat N times. If 0 repeat for ever.'
+    )
+    parser.add_argument(
+        "-w", dest='wait', default=0, type=float, metavar="N",
+        help='Wait N seconds between each request.'
+    )
+    parser.add_argument(
+        "-r", dest="random", action="store_true", default=False,
+        help="""
+        Select a random request from those specified. If this is not specified,
+        requests are all played in sequence.
+        """
+    )
+    parser.add_argument(
+        "-t", dest="timeout", type=int, default=None,
+        help="Connection timeout"
+    )
+    parser.add_argument(
+        "--http2", dest="use_http2", action="store_true", default=False,
+        help='Perform all requests over a single HTTP/2 connection.'
+    )
+    parser.add_argument(
+        "--http2-skip-connection-preface",
+        dest="http2_skip_connection_preface",
+        action="store_true",
+        default=False,
+        help='Skips the HTTP/2 connection preface before sending requests.')
+
+    parser.add_argument(
+        'host', type=str,
+        metavar="host[:port]",
+        help='Host and port to connect to'
+    )
+    parser.add_argument(
+        'requests', type=str, nargs="+",
+        help="""
+        Request specification, or path to a file containing request
+        specifcations
+        """
+    )
+
+    group = parser.add_argument_group(
+        'SSL',
+    )
+    group.add_argument(
+        "-s", dest="ssl", action="store_true", default=False,
+        help="Connect with SSL"
+    )
+    group.add_argument(
+        "-C", dest="clientcert", type=str, default=False,
+        help="Path to a file containing client certificate and private key"
+    )
+    group.add_argument(
+        "-i", dest="sni", type=str, default=False,
+        help="SSL Server Name Indication"
+    )
+    group.add_argument(
+        "--ciphers", dest="ciphers", type=str, default=False,
+        help="SSL cipher specification"
+    )
+    group.add_argument(
+        "--ssl-version", dest="ssl_version", type=str, default="secure",
+        choices=tcp.sslversion_choices.keys(),
+        help="Set supported SSL/TLS versions. "
+             "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
+    )
+
+    group = parser.add_argument_group(
+        'Controlling Output',
+        """
+            Some of these options expand generated values for logging - if
+            you're generating large data, use them with caution.
+        """
+    )
+    group.add_argument(
+        "-I", dest="ignorecodes", type=str, default="",
+        help="Comma-separated list of response codes to ignore"
+    )
+    group.add_argument(
+        "-S", dest="showssl", action="store_true", default=False,
+        help="Show info on SSL connection"
+    )
+    group.add_argument(
+        "-e", dest="explain", action="store_true", default=False,
+        help="Explain requests"
+    )
+    group.add_argument(
+        "-o", dest="oneshot", action="store_true", default=False,
+        help="Oneshot - exit after first non-ignored response"
+    )
+    group.add_argument(
+        "-q", dest="showreq", action="store_true", default=False,
+        help="Print full request"
+    )
+    group.add_argument(
+        "-p", dest="showresp", action="store_true", default=False,
+        help="Print full response"
+    )
+    group.add_argument(
+        "-T", dest="ignoretimeout", action="store_true", default=False,
+        help="Ignore timeouts"
+    )
+    group.add_argument(
+        "-x", dest="hexdump", action="store_true", default=False,
+        help="Output in hexdump format"
+    )
+    group.add_argument(
+        "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+        help="Output all received & sent HTTP/2 frames"
+    )
+
+    args = parser.parse_args(argv[1:])
+
+    args.ssl_version, args.ssl_options = tcp.sslversion_choices[args.ssl_version]
+
+    args.port = None
+    if ":" in args.host:
+        h, p = args.host.rsplit(":", 1)
+        try:
+            p = int(p)
+        except ValueError:
+            return parser.error("Invalid port in host spec: %s" % args.host)
+        args.host = h
+        args.port = p
+
+    if args.port is None:
+        args.port = 443 if args.ssl else 80
+
+    try:
+        args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
+    except ValueError:
+        return parser.error(
+            "Invalid return code specification: %s" %
+            args.ignorecodes)
+
+    if args.connect_to:
+        parts = args.connect_to.split(":")
+        if len(parts) != 2:
+            return parser.error(
+                "Invalid CONNECT specification: %s" %
+                args.connect_to)
+        try:
+            parts[1] = int(parts[1])
+        except ValueError:
+            return parser.error(
+                "Invalid CONNECT specification: %s" %
+                args.connect_to)
+        args.connect_to = parts
+    else:
+        args.connect_to = None
+
+    if args.http2_skip_connection_preface:
+        args.use_http2 = True
+
+    if args.use_http2:
+        args.ssl = True
+
+    reqs = []
+    for r in args.requests:
+        if os.path.isfile(r):
+            data = open(r).read()
+            r = data
+        try:
+            reqs.append(language.parse_pathoc(r, args.use_http2))
+        except language.ParseException as v:
+            print >> stderr, "Error parsing request spec: %s" % v.msg
+            print >> stderr, v.marked()
+            sys.exit(1)
+    args.requests = reqs
+
+    return args
+
+
+def go_pathoc():  # pragma: nocover
+    args = args_pathoc(sys.argv)
+    pathoc.main(args)

pathod/libpathod/pathod.py

@@ -0,0 +1,503 @@
+import copy
+import logging
+import os
+import sys
+import threading
+import urllib
+
+from netlib import tcp, http, certutils, websockets
+from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \
+    TlsException
+
+from . import version, app, language, utils, log, protocols
+import language.http
+import language.actions
+import language.exceptions
+import language.websockets
+
+
+DEFAULT_CERT_DOMAIN = "pathod.net"
+CONFDIR = "~/.mitmproxy"
+CERTSTORE_BASENAME = "mitmproxy"
+CA_CERT_NAME = "mitmproxy-ca.pem"
+DEFAULT_CRAFT_ANCHOR = "/p/"
+
+logger = logging.getLogger('pathod')
+
+
+class PathodError(Exception):
+    pass
+
+
+class SSLOptions(object):
+    def __init__(
+        self,
+        confdir=CONFDIR,
+        cn=None,
+        sans=(),
+        not_after_connect=None,
+        request_client_cert=False,
+        ssl_version=tcp.SSL_DEFAULT_METHOD,
+        ssl_options=tcp.SSL_DEFAULT_OPTIONS,
+        ciphers=None,
+        certs=None,
+        alpn_select=b'h2',
+    ):
+        self.confdir = confdir
+        self.cn = cn
+        self.sans = sans
+        self.not_after_connect = not_after_connect
+        self.request_client_cert = request_client_cert
+        self.ssl_version = ssl_version
+        self.ssl_options = ssl_options
+        self.ciphers = ciphers
+        self.alpn_select = alpn_select
+        self.certstore = certutils.CertStore.from_store(
+            os.path.expanduser(confdir),
+            CERTSTORE_BASENAME
+        )
+        for i in certs or []:
+            self.certstore.add_cert_file(*i)
+
+    def get_cert(self, name):
+        if self.cn:
+            name = self.cn
+        elif not name:
+            name = DEFAULT_CERT_DOMAIN
+        return self.certstore.get_cert(name, self.sans)
+
+
+class PathodHandler(tcp.BaseHandler):
+    wbufsize = 0
+    sni = None
+
+    def __init__(
+        self,
+        connection,
+        address,
+        server,
+        logfp,
+        settings,
+        http2_framedump=False
+    ):
+        tcp.BaseHandler.__init__(self, connection, address, server)
+        self.logfp = logfp
+        self.settings = copy.copy(settings)
+        self.protocol = None
+        self.use_http2 = False
+        self.http2_framedump = http2_framedump
+
+    def handle_sni(self, connection):
+        self.sni = connection.get_servername()
+
+    def http_serve_crafted(self, crafted, logctx):
+        error, crafted = self.server.check_policy(
+            crafted, self.settings
+        )
+        if error:
+            err = self.make_http_error_response(error)
+            language.serve(err, self.wfile, self.settings)
+            return None, dict(
+                type="error",
+                msg=error
+            )
+
+        if self.server.explain and not hasattr(crafted, 'is_error_response'):
+            crafted = crafted.freeze(self.settings)
+            logctx(">> Spec: %s" % crafted.spec())
+
+        response_log = language.serve(
+            crafted,
+            self.wfile,
+            self.settings
+        )
+        if response_log["disconnect"]:
+            return None, response_log
+        return self.handle_http_request, response_log
+
+
+    def handle_http_request(self, logger):
+        """
+            Returns a (handler, log) tuple.
+
+            handler: Handler for the next request, or None to disconnect
+            log: A dictionary, or None
+        """
+        with logger.ctx() as lg:
+            try:
+                req = self.protocol.read_request(self.rfile)
+            except HttpReadDisconnect:
+                return None, None
+            except HttpException as s:
+                s = str(s)
+                lg(s)
+                return None, dict(type="error", msg=s)
+
+            if req.method == 'CONNECT':
+                return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg)
+
+            method = req.method
+            path = req.path
+            http_version = req.http_version
+            headers = req.headers
+            body = req.content
+
+            clientcert = None
+            if self.clientcert:
+                clientcert = dict(
+                    cn=self.clientcert.cn,
+                    subject=self.clientcert.subject,
+                    serial=self.clientcert.serial,
+                    notbefore=self.clientcert.notbefore.isoformat(),
+                    notafter=self.clientcert.notafter.isoformat(),
+                    keyinfo=self.clientcert.keyinfo,
+                )
+
+            retlog = dict(
+                type="crafted",
+                protocol="http",
+                request=dict(
+                    path=path,
+                    method=method,
+                    headers=headers.fields,
+                    http_version=http_version,
+                    sni=self.sni,
+                    remote_address=self.address(),
+                    clientcert=clientcert,
+                ),
+                cipher=None,
+            )
+            if self.ssl_established:
+                retlog["cipher"] = self.get_current_cipher()
+
+            m = utils.MemBool()
+            websocket_key = websockets.WebsocketsProtocol.check_client_handshake(headers)
+            self.settings.websocket_key = websocket_key
+
+            # If this is a websocket initiation, we respond with a proper
+            # server response, unless over-ridden.
+            if websocket_key:
+                anchor_gen = language.parse_pathod("ws")
+            else:
+                anchor_gen = None
+
+            for regex, spec in self.server.anchors:
+                if regex.match(path):
+                    anchor_gen = language.parse_pathod(spec, self.use_http2)
+                    break
+            else:
+                if m(path.startswith(self.server.craftanchor)):
+                    spec = urllib.unquote(path)[len(self.server.craftanchor):]
+                    if spec:
+                        try:
+                            anchor_gen = language.parse_pathod(spec, self.use_http2)
+                        except language.ParseException as v:
+                            lg("Parse error: %s" % v.msg)
+                            anchor_gen = iter([self.make_http_error_response(
+                                "Parse Error",
+                                "Error parsing response spec: %s\n" % (
+                                    v.msg + v.marked()
+                                )
+                            )])
+                else:
+                    if self.use_http2:
+                        anchor_gen = iter([self.make_http_error_response(
+                            "Spec Error",
+                            "HTTP/2 only supports request/response with the craft anchor point: %s" %
+                            self.server.craftanchor
+                        )])
+
+            if anchor_gen:
+                spec = anchor_gen.next()
+
+                if self.use_http2 and isinstance(spec, language.http2.Response):
+                    spec.stream_id = req.stream_id
+
+                lg("crafting spec: %s" % spec)
+                nexthandler, retlog["response"] = self.http_serve_crafted(
+                    spec,
+                    lg
+                )
+                if nexthandler and websocket_key:
+                    self.protocol = protocols.websockets.WebsocketsProtocol(self)
+                    return self.protocol.handle_websocket, retlog
+                else:
+                    return nexthandler, retlog
+            else:
+                return self.protocol.handle_http_app(method, path, headers, body, lg)
+
+    def make_http_error_response(self, reason, body=None):
+        resp = self.protocol.make_error_response(reason, body)
+        resp.is_error_response = True
+        return resp
+
+    def handle(self):
+        self.settimeout(self.server.timeout)
+
+        if self.server.ssl:
+            try:
+                cert, key, _ = self.server.ssloptions.get_cert(None)
+                self.convert_to_ssl(
+                    cert,
+                    key,
+                    handle_sni=self.handle_sni,
+                    request_client_cert=self.server.ssloptions.request_client_cert,
+                    cipher_list=self.server.ssloptions.ciphers,
+                    method=self.server.ssloptions.ssl_version,
+                    options=self.server.ssloptions.ssl_options,
+                    alpn_select=self.server.ssloptions.alpn_select,
+                )
+            except TlsException as v:
+                s = str(v)
+                self.server.add_log(
+                    dict(
+                        type="error",
+                        msg=s
+                    )
+                )
+                log.write_raw(self.logfp, s)
+                return
+
+            alp = self.get_alpn_proto_negotiated()
+            if alp == b'h2':
+                self.protocol = protocols.http2.HTTP2Protocol(self)
+                self.use_http2 = True
+
+        if not self.protocol:
+            self.protocol = protocols.http.HTTPProtocol(self)
+
+        lr = self.rfile if self.server.logreq else None
+        lw = self.wfile if self.server.logresp else None
+        logger = log.ConnectionLogger(self.logfp, self.server.hexdump, lr, lw)
+
+        self.settings.protocol = self.protocol
+
+        handler = self.handle_http_request
+
+        while not self.finished:
+            handler, l = handler(logger)
+            if l:
+                self.addlog(l)
+            if not handler:
+                return
+
+    def addlog(self, log):
+        # FIXME: The bytes in the log should not be escaped. We do this at the
+        # moment because JSON encoding can't handle binary data, and I don't
+        # want to base64 everything.
+        if self.server.logreq:
+            encoded_bytes = self.rfile.get_log().encode("string_escape")
+            log["request_bytes"] = encoded_bytes
+        if self.server.logresp:
+            encoded_bytes = self.wfile.get_log().encode("string_escape")
+            log["response_bytes"] = encoded_bytes
+        self.server.add_log(log)
+
+
+class Pathod(tcp.TCPServer):
+    LOGBUF = 500
+
+    def __init__(
+        self,
+        addr,
+        ssl=False,
+        ssloptions=None,
+        craftanchor=DEFAULT_CRAFT_ANCHOR,
+        staticdir=None,
+        anchors=(),
+        sizelimit=None,
+        noweb=False,
+        nocraft=False,
+        noapi=False,
+        nohang=False,
+        timeout=None,
+        logreq=False,
+        logresp=False,
+        explain=False,
+        hexdump=False,
+        http2_framedump=False,
+        webdebug=False,
+        logfp=sys.stdout,
+    ):
+        """
+            addr: (address, port) tuple. If port is 0, a free port will be
+            automatically chosen.
+            ssloptions: an SSLOptions object.
+            craftanchor: URL prefix specifying the path under which to anchor
+            response generation.
+            staticdir: path to a directory of static resources, or None.
+            anchors: List of (regex object, language.Request object) tuples, or
+            None.
+            sizelimit: Limit size of served data.
+            nocraft: Disable response crafting.
+            noapi: Disable the API.
+            nohang: Disable pauses.
+        """
+        tcp.TCPServer.__init__(self, addr)
+        self.ssl = ssl
+        self.ssloptions = ssloptions or SSLOptions()
+        self.staticdir = staticdir
+        self.craftanchor = craftanchor
+        self.sizelimit = sizelimit
+        self.noweb, self.nocraft = noweb, nocraft
+        self.noapi, self.nohang = noapi, nohang
+        self.timeout, self.logreq = timeout, logreq
+        self.logresp, self.hexdump = logresp, hexdump
+        self.http2_framedump = http2_framedump
+        self.explain = explain
+        self.logfp = logfp
+
+        self.app = app.make_app(noapi, webdebug)
+        self.app.config["pathod"] = self
+        self.log = []
+        self.logid = 0
+        self.anchors = anchors
+
+        self.settings = language.Settings(
+            staticdir=self.staticdir
+        )
+
+    def check_policy(self, req, settings):
+        """
+            A policy check that verifies the request size is within limits.
+        """
+        if self.nocraft:
+            return "Crafting disabled.", None
+        try:
+            req = req.resolve(settings)
+            l = req.maximum_length(settings)
+        except language.FileAccessDenied:
+            return "File access denied.", None
+        if self.sizelimit and l > self.sizelimit:
+            return "Response too large.", None
+        pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions]
+        if self.nohang and any(pauses):
+            return "Pauses have been disabled.", None
+        return None, req
+
+    def handle_client_connection(self, request, client_address):
+        h = PathodHandler(
+            request,
+            client_address,
+            self,
+            self.logfp,
+            self.settings,
+            self.http2_framedump,
+        )
+        try:
+            h.handle()
+            h.finish()
+        except TcpDisconnect:  # pragma: no cover
+            log.write_raw(self.logfp, "Disconnect")
+            self.add_log(
+                dict(
+                    type="error",
+                    msg="Disconnect"
+                )
+            )
+            return
+        except TcpTimeout:
+            log.write_raw(self.logfp, "Timeout")
+            self.add_log(
+                dict(
+                    type="timeout",
+                )
+            )
+            return
+
+    def add_log(self, d):
+        if not self.noapi:
+            lock = threading.Lock()
+            with lock:
+                d["id"] = self.logid
+                self.log.insert(0, d)
+                if len(self.log) > self.LOGBUF:
+                    self.log.pop()
+                self.logid += 1
+            return d["id"]
+
+    def clear_log(self):
+        lock = threading.Lock()
+        with lock:
+            self.log = []
+
+    def log_by_id(self, identifier):
+        for i in self.log:
+            if i["id"] == identifier:
+                return i
+
+    def get_log(self):
+        return self.log
+
+
+def main(args):  # pragma: nocover
+    ssloptions = SSLOptions(
+        cn=args.cn,
+        confdir=args.confdir,
+        not_after_connect=args.ssl_not_after_connect,
+        ciphers=args.ciphers,
+        ssl_version=args.ssl_version,
+        ssl_options=args.ssl_options,
+        certs=args.ssl_certs,
+        sans=args.sans,
+    )
+
+    root = logging.getLogger()
+    if root.handlers:
+        for handler in root.handlers:
+            root.removeHandler(handler)
+
+    log = logging.getLogger('pathod')
+    log.setLevel(logging.DEBUG)
+    fmt = logging.Formatter(
+        '%(asctime)s: %(message)s',
+        datefmt='%d-%m-%y %H:%M:%S',
+    )
+    if args.logfile:
+        fh = logging.handlers.WatchedFileHandler(args.logfile)
+        fh.setFormatter(fmt)
+        log.addHandler(fh)
+    if not args.daemonize:
+        sh = logging.StreamHandler()
+        sh.setFormatter(fmt)
+        log.addHandler(sh)
+
+    try:
+        pd = Pathod(
+            (args.address, args.port),
+            craftanchor=args.craftanchor,
+            ssl=args.ssl,
+            ssloptions=ssloptions,
+            staticdir=args.staticdir,
+            anchors=args.anchors,
+            sizelimit=args.sizelimit,
+            noweb=args.noweb,
+            nocraft=args.nocraft,
+            noapi=args.noapi,
+            nohang=args.nohang,
+            timeout=args.timeout,
+            logreq=args.logreq,
+            logresp=args.logresp,
+            hexdump=args.hexdump,
+            http2_framedump=args.http2_framedump,
+            explain=args.explain,
+            webdebug=args.webdebug
+        )
+    except PathodError as v:
+        print >> sys.stderr, "Error: %s" % v
+        sys.exit(1)
+    except language.FileAccessDenied as v:
+        print >> sys.stderr, "Error: %s" % v
+
+    if args.daemonize:
+        utils.daemonize()
+
+    try:
+        print "%s listening on %s:%s" % (
+            version.NAMEVERSION,
+            pd.address.host,
+            pd.address.port
+        )
+        pd.serve_forever()
+    except KeyboardInterrupt:
+        pass

pathod/libpathod/pathod_cmdline.py

@@ -0,0 +1,231 @@
+import sys
+import argparse
+import os
+import os.path
+import re
+
+from netlib import tcp
+from . import pathod, version, utils
+
+
+def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
+    parser = argparse.ArgumentParser(
+        description='A pathological HTTP/S daemon.'
+    )
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="pathod " + version.VERSION
+    )
+    parser.add_argument(
+        "-p",
+        dest='port',
+        default=9999,
+        type=int,
+        help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
+    )
+    parser.add_argument(
+        "-l",
+        dest='address',
+        default="127.0.0.1",
+        type=str,
+        help='Listening address. (127.0.0.1)'
+    )
+    parser.add_argument(
+        "-a",
+        dest='anchors',
+        default=[],
+        type=str,
+        action="append",
+        metavar="ANCHOR",
+        help="""
+            Add an anchor. Specified as a string with the form
+            pattern=spec or pattern=filepath, where pattern is a regular
+            expression.
+        """
+    )
+    parser.add_argument(
+        "-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str,
+        help="""
+            URL path specifying prefix for URL crafting
+            commands. (%s)
+        """%pathod.DEFAULT_CRAFT_ANCHOR
+    )
+    parser.add_argument(
+        "--confdir",
+        action="store", type = str, dest="confdir", default='~/.mitmproxy',
+        help = "Configuration directory. (~/.mitmproxy)"
+    )
+    parser.add_argument(
+        "-d", dest='staticdir', default=None, type=str,
+        help='Directory for static files.'
+    )
+    parser.add_argument(
+        "-D", dest='daemonize', default=False, action="store_true",
+        help='Daemonize.'
+    )
+    parser.add_argument(
+        "-t", dest="timeout", type=int, default=None,
+        help="Connection timeout"
+    )
+    parser.add_argument(
+        "--limit-size",
+        dest='sizelimit',
+        default=None,
+        type=str,
+        help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
+    parser.add_argument(
+        "--noapi", dest='noapi', default=False, action="store_true",
+        help='Disable API.'
+    )
+    parser.add_argument(
+        "--nohang", dest='nohang', default=False, action="store_true",
+        help='Disable pauses during crafted response generation.'
+    )
+    parser.add_argument(
+        "--noweb", dest='noweb', default=False, action="store_true",
+        help='Disable both web interface and API.'
+    )
+    parser.add_argument(
+        "--nocraft",
+        dest='nocraft',
+        default=False,
+        action="store_true",
+        help='Disable response crafting. If anchors are specified, they still work.')
+    parser.add_argument(
+        "--webdebug", dest='webdebug', default=False, action="store_true",
+        help='Debugging mode for the web app (dev only).'
+    )
+
+    group = parser.add_argument_group(
+        'SSL',
+    )
+    group.add_argument(
+        "-s", dest='ssl', default=False, action="store_true",
+        help='Run in HTTPS mode.'
+    )
+    group.add_argument(
+        "--cn",
+        dest="cn",
+        type=str,
+        default=None,
+        help="CN for generated SSL certs. Default: %s" %
+        pathod.DEFAULT_CERT_DOMAIN)
+    group.add_argument(
+        "-C", dest='ssl_not_after_connect', default=False, action="store_true",
+        help="Don't expect SSL after a CONNECT request."
+    )
+    group.add_argument(
+        "--cert", dest='ssl_certs', default=[], type=str,
+        metavar = "SPEC", action="append",
+        help = """
+        Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
+        may include a wildcard, and is equal to "*" if not specified. The file
+        at path is a certificate in PEM format. If a private key is included in
+        the PEM, it is used, else the default key in the conf dir is used. Can
+        be passed multiple times.
+        """
+    )
+    group.add_argument(
+        "--ciphers", dest="ciphers", type=str, default=False,
+        help="SSL cipher specification"
+    )
+    group.add_argument(
+        "--san", dest="sans", type=str, default=[], action="append",
+        metavar="SAN",
+        help="""
+            Subject Altnernate Name to add to the server certificate.
+            May be passed multiple times.
+        """
+    )
+    group.add_argument(
+        "--ssl-version", dest="ssl_version", type=str, default="secure",
+        choices=tcp.sslversion_choices.keys(),
+        help="Set supported SSL/TLS versions. "
+             "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
+    )
+
+    group = parser.add_argument_group(
+        'Controlling Logging',
+        """
+            Some of these options expand generated values for logging - if
+            you're generating large data, use them with caution.
+        """
+    )
+    group.add_argument(
+        "-e", dest="explain", action="store_true", default=False,
+        help="Explain responses"
+    )
+    group.add_argument(
+        "-f", dest='logfile', default=None, type=str,
+        help='Log to file.'
+    )
+    group.add_argument(
+        "-q", dest="logreq", action="store_true", default=False,
+        help="Log full request"
+    )
+    group.add_argument(
+        "-r", dest="logresp", action="store_true", default=False,
+        help="Log full response"
+    )
+    group.add_argument(
+        "-x", dest="hexdump", action="store_true", default=False,
+        help="Log request/response in hexdump format"
+    )
+    group.add_argument(
+        "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+        help="Output all received & sent HTTP/2 frames"
+    )
+
+
+    args = parser.parse_args(argv[1:])
+
+    args.ssl_version, args.ssl_options = tcp.sslversion_choices[args.ssl_version]
+
+    certs = []
+    for i in args.ssl_certs:
+        parts = i.split("=", 1)
+        if len(parts) == 1:
+            parts = ["*", parts[0]]
+        parts[1] = os.path.expanduser(parts[1])
+        if not os.path.isfile(parts[1]):
+            return parser.error(
+                "Certificate file does not exist: %s" %
+                parts[1])
+        certs.append(parts)
+    args.ssl_certs = certs
+
+    alst = []
+    for i in args.anchors:
+        parts = utils.parse_anchor_spec(i)
+        if not parts:
+            return parser.error("Invalid anchor specification: %s" % i)
+        alst.append(parts)
+    args.anchors = alst
+
+    sizelimit = None
+    if args.sizelimit:
+        try:
+            sizelimit = utils.parse_size(args.sizelimit)
+        except ValueError as v:
+            return parser.error(v)
+    args.sizelimit = sizelimit
+
+    anchors = []
+    for patt, spec in args.anchors:
+        if os.path.isfile(spec):
+            data = open(spec).read()
+            spec = data
+        try:
+            arex = re.compile(patt)
+        except re.error:
+            return parser.error("Invalid regex in anchor: %s" % patt)
+        anchors.append((arex, spec))
+    args.anchors = anchors
+
+    return args
+
+
+def go_pathod():  # pragma: nocover
+    args = args_pathod(sys.argv)
+    pathod.main(args)

pathod/libpathod/protocols/__init__.py

@@ -0,0 +1 @@
+from . import http, http2, websockets

pathod/libpathod/protocols/http.py

@@ -0,0 +1,71 @@
+from netlib import tcp, wsgi
+from netlib.exceptions import HttpReadDisconnect, TlsException
+from netlib.http import http1, Request
+from .. import version, language
+
+
+class HTTPProtocol(object):
+    def __init__(self, pathod_handler):
+        self.pathod_handler = pathod_handler
+
+    def make_error_response(self, reason, body):
+        return language.http.make_error_response(reason, body)
+
+    def handle_http_app(self, method, path, headers, body, lg):
+        """
+            Handle a request to the built-in app.
+        """
+        if self.pathod_handler.server.noweb:
+            crafted = self.pathod_handler.make_http_error_response("Access Denied")
+            language.serve(crafted, self.pathod_handler.wfile, self.pathod_handler.settings)
+            return None, dict(
+                type="error",
+                msg="Access denied: web interface disabled"
+            )
+        lg("app: %s %s" % (method, path))
+        req = wsgi.Request("http", method, path, b"HTTP/1.1", headers, body)
+        flow = wsgi.Flow(self.pathod_handler.address, req)
+        sn = self.pathod_handler.connection.getsockname()
+        a = wsgi.WSGIAdaptor(
+            self.pathod_handler.server.app,
+            sn[0],
+            self.pathod_handler.server.address.port,
+            version.NAMEVERSION
+        )
+        a.serve(flow, self.pathod_handler.wfile)
+        return self.pathod_handler.handle_http_request, None
+
+    def handle_http_connect(self, connect, lg):
+        """
+            Handle a CONNECT request.
+        """
+
+        self.pathod_handler.wfile.write(
+            'HTTP/1.1 200 Connection established\r\n' +
+            ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
+            '\r\n'
+        )
+        self.pathod_handler.wfile.flush()
+        if not self.pathod_handler.server.ssloptions.not_after_connect:
+            try:
+                cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert(
+                    connect[0]
+                )
+                self.pathod_handler.convert_to_ssl(
+                    cert,
+                    key,
+                    handle_sni=self.pathod_handler.handle_sni,
+                    request_client_cert=self.pathod_handler.server.ssloptions.request_client_cert,
+                    cipher_list=self.pathod_handler.server.ssloptions.ciphers,
+                    method=self.pathod_handler.server.ssloptions.ssl_version,
+                    options=self.pathod_handler.server.ssloptions.ssl_options,
+                    alpn_select=self.pathod_handler.server.ssloptions.alpn_select,
+                )
+            except TlsException as v:
+                s = str(v)
+                lg(s)
+                return None, dict(type="error", msg=s)
+        return self.pathod_handler.handle_http_request, None
+
+    def read_request(self, lg=None):
+        return http1.read_request(self.pathod_handler.rfile)

pathod/libpathod/protocols/http2.py

@@ -0,0 +1,20 @@
+from netlib.http import http2
+from .. import version, app, language, utils, log
+
+class HTTP2Protocol:
+
+    def __init__(self, pathod_handler):
+        self.pathod_handler = pathod_handler
+        self.wire_protocol = http2.HTTP2Protocol(
+            self.pathod_handler, is_server=True, dump_frames=self.pathod_handler.http2_framedump
+        )
+
+    def make_error_response(self, reason, body):
+        return language.http2.make_error_response(reason, body)
+
+    def read_request(self, lg=None):
+        self.wire_protocol.perform_server_connection_preface()
+        return self.wire_protocol.read_request(self.pathod_handler.rfile)
+
+    def assemble(self, message):
+        return self.wire_protocol.assemble(message)

pathod/libpathod/protocols/websockets.py

@@ -0,0 +1,56 @@
+import time
+
+from netlib import websockets
+from .. import language
+from netlib.exceptions import NetlibException
+
+
+class WebsocketsProtocol:
+
+    def __init__(self, pathod_handler):
+        self.pathod_handler = pathod_handler
+
+    def handle_websocket(self, logger):
+        while True:
+            with logger.ctx() as lg:
+                started = time.time()
+                try:
+                    frm = websockets.Frame.from_file(self.pathod_handler.rfile)
+                except NetlibException as e:
+                    lg("Error reading websocket frame: %s" % e)
+                    break
+                ended = time.time()
+                lg(frm.human_readable())
+            retlog = dict(
+                type="inbound",
+                protocol="websockets",
+                started=started,
+                duration=ended - started,
+                frame=dict(
+                ),
+                cipher=None,
+            )
+            if self.pathod_handler.ssl_established:
+                retlog["cipher"] = self.pathod_handler.get_current_cipher()
+            self.pathod_handler.addlog(retlog)
+            ld = language.websockets.NESTED_LEADER
+            if frm.payload.startswith(ld):
+                nest = frm.payload[len(ld):]
+                try:
+                    wf_gen = language.parse_websocket_frame(nest)
+                except language.exceptions.ParseException as v:
+                    logger.write(
+                        "Parse error in reflected frame specifcation:"
+                        " %s" % v.msg
+                    )
+                    return None, None
+                for frm in wf_gen:
+                    with logger.ctx() as lg:
+                        frame_log = language.serve(
+                            frm,
+                            self.pathod_handler.wfile,
+                            self.pathod_handler.settings
+                        )
+                        lg("crafting websocket spec: %s" % frame_log["spec"])
+                        self.pathod_handler.addlog(frame_log)
+        return self.handle_websocket, None

pathod/libpathod/static/jquery.localscroll-min.js

@@ -0,0 +1,9 @@
+/**
+ * jQuery.LocalScroll - Animated scrolling navigation, using anchors.
+ * Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
+ * Dual licensed under MIT and GPL.
+ * Date: 3/11/2009
+ * @author Ariel Flesler
+ * @version 1.2.7
+ **/
+;(function($){var l=location.href.replace(/#.*/,'');var g=$.localScroll=function(a){$('body').localScroll(a)};g.defaults={duration:1e3,axis:'y',event:'click',stop:true,target:window,reset:true};g.hash=function(a){if(location.hash){a=$.extend({},g.defaults,a);a.hash=false;if(a.reset){var e=a.duration;delete a.duration;$(a.target).scrollTo(0,a);a.duration=e}i(0,location,a)}};$.fn.localScroll=function(b){b=$.extend({},g.defaults,b);return b.lazy?this.bind(b.event,function(a){var e=$([a.target,a.target.parentNode]).filter(d)[0];if(e)i(a,e,b)}):this.find('a,area').filter(d).bind(b.event,function(a){i(a,this,b)}).end().end();function d(){return!!this.href&&!!this.hash&&this.href.replace(this.hash,'')==l&&(!b.filter||$(this).is(b.filter))}};function i(a,e,b){var d=e.hash.slice(1),f=document.getElementById(d)||document.getElementsByName(d)[0];if(!f)return;if(a)a.preventDefault();var h=$(b.target);if(b.lock&&h.is(':animated')||b.onBefore&&b.onBefore.call(b,a,f,h)===false)return;if(b.stop)h.stop(true);if(b.hash){var j=f.id==d?'id':'name',k=$('<a> </a>').attr(j,d).css({position:'absolute',top:$(window).scrollTop(),left:$(window).scrollLeft()});f[j]='';$('body').prepend(k);location=e.hash;k.remove();f[j]=d}h.scrollTo(f,b).trigger('notify.serialScroll',[f])}})(jQuery);

pathod/libpathod/static/jquery.scrollTo-min.js

@@ -0,0 +1,11 @@
+/**
+ * jQuery.ScrollTo - Easy element scrolling using jQuery.
+ * Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
+ * Dual licensed under MIT and GPL.
+ * Date: 3/9/2009
+ * @author Ariel Flesler
+ * @version 1.4.1
+ *
+ * http://flesler.blogspot.com/2007/10/jqueryscrollto.html
+ */
+;(function($){var m=$.scrollTo=function(b,h,f){$(window).scrollTo(b,h,f)};m.defaults={axis:'xy',duration:parseFloat($.fn.jquery)>=1.3?0:1};m.window=function(b){return $(window).scrollable()};$.fn.scrollable=function(){return this.map(function(){var b=this,h=!b.nodeName||$.inArray(b.nodeName.toLowerCase(),['iframe','#document','html','body'])!=-1;if(!h)return b;var f=(b.contentWindow||b).document||b.ownerDocument||b;return $.browser.safari||f.compatMode=='BackCompat'?f.body:f.documentElement})};$.fn.scrollTo=function(l,j,a){if(typeof j=='object'){a=j;j=0}if(typeof a=='function')a={onAfter:a};if(l=='max')l=9e9;a=$.extend({},m.defaults,a);j=j||a.speed||a.duration;a.queue=a.queue&&a.axis.length>1;if(a.queue)j/=2;a.offset=n(a.offset);a.over=n(a.over);return this.scrollable().each(function(){var k=this,o=$(k),d=l,p,g={},q=o.is('html,body');switch(typeof d){case'number':case'string':if(/^([+-]=)?\d+(\.\d+)?(px)?$/.test(d)){d=n(d);break}d=$(d,this);case'object':if(d.is||d.style)p=(d=$(d)).offset()}$.each(a.axis.split(''),function(b,h){var f=h=='x'?'Left':'Top',i=f.toLowerCase(),c='scroll'+f,r=k[c],s=h=='x'?'Width':'Height';if(p){g[c]=p[i]+(q?0:r-o.offset()[i]);if(a.margin){g[c]-=parseInt(d.css('margin'+f))||0;g[c]-=parseInt(d.css('border'+f+'Width'))||0}g[c]+=a.offset[i]||0;if(a.over[i])g[c]+=d[s.toLowerCase()]()*a.over[i]}else g[c]=d[i];if(/^\d+$/.test(g[c]))g[c]=g[c]<=0?0:Math.min(g[c],u(s));if(!b&&a.queue){if(r!=g[c])t(a.onAfterFirst);delete g[c]}});t(a.onAfter);function t(b){o.animate(g,j,a.easing,b&&function(){b.call(this,l,a)})};function u(b){var h='scroll'+b;if(!q)return k[h];var f='client'+b,i=k.ownerDocument.documentElement,c=k.ownerDocument.body;return Math.max(i[h],c[h])-Math.min(i[f],c[f])}}).end()};function n(b){return typeof b=='object'?b:{top:b,left:b}}})(jQuery);

pathod/libpathod/static/pathod.css

@@ -0,0 +1,56 @@
+
+
+.fronttable {
+}
+
+.bigtitle {
+    font-weight: bold;
+    font-size: 50px;
+    line-height: 55px;
+    text-align: center;
+    display: table;
+    height: 300px;
+}
+
+.bigtitle>div {
+    display: table-cell;
+    vertical-align: middle;
+}
+
+section {
+    margin-top: 50px;
+}
+
+.example {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+
+.terminal {
+    margin-top: 10px;
+    margin-bottom: 10px;
+    background: #000;
+    color: #fff;
+}
+
+.innerlink {
+    text-decoration: none;
+    border-bottom:1px dotted;
+    margin-bottom: 15px;
+}
+
+.masthead {
+    padding: 50px 0 60px;
+    text-align: center;
+
+}
+
+.header {
+    font-size: 1.5em;
+}
+
+.page-header {
+    margin-top: 10px;
+
+
+}

pathod/libpathod/static/syntax.css

@@ -0,0 +1,120 @@
+.highlight  { background: #f8f8f8; }
+.highlight .c { color: #408080; font-style: italic } /* Comment */
+.highlight .err { border: 1px solid #FF0000 } /* Error */
+.highlight .k { color: #008000; font-weight: bold } /* Keyword */
+.highlight .o { color: #666666 } /* Operator */
+.highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */
+.highlight .cp { color: #BC7A00 } /* Comment.Preproc */
+.highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */
+.highlight .cs { color: #408080; font-style: italic } /* Comment.Special */
+.highlight .gd { color: #A00000 } /* Generic.Deleted */
+.highlight .ge { font-style: italic } /* Generic.Emph */
+.highlight .gr { color: #FF0000 } /* Generic.Error */
+.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.highlight .gi { color: #00A000 } /* Generic.Inserted */
+.highlight .go { color: #808080 } /* Generic.Output */
+.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
+.highlight .gs { font-weight: bold } /* Generic.Strong */
+.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.highlight .gt { color: #0040D0 } /* Generic.Traceback */
+.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
+.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
+.highlight .kp { color: #008000 } /* Keyword.Pseudo */
+.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
+.highlight .kt { color: #B00040 } /* Keyword.Type */
+.highlight .m { color: #666666 } /* Literal.Number */
+.highlight .s { color: #BA2121 } /* Literal.String */
+.highlight .na { color: #7D9029 } /* Name.Attribute */
+.highlight .nb { color: #008000 } /* Name.Builtin */
+.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
+.highlight .no { color: #880000 } /* Name.Constant */
+.highlight .nd { color: #AA22FF } /* Name.Decorator */
+.highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */
+.highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
+.highlight .nf { color: #0000FF } /* Name.Function */
+.highlight .nl { color: #A0A000 } /* Name.Label */
+.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
+.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
+.highlight .nv { color: #19177C } /* Name.Variable */
+.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
+.highlight .w { color: #bbbbbb } /* Text.Whitespace */
+.highlight .mf { color: #666666 } /* Literal.Number.Float */
+.highlight .mh { color: #666666 } /* Literal.Number.Hex */
+.highlight .mi { color: #666666 } /* Literal.Number.Integer */
+.highlight .mo { color: #666666 } /* Literal.Number.Oct */
+.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
+.highlight .sc { color: #BA2121 } /* Literal.String.Char */
+.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
+.highlight .s2 { color: #BA2121 } /* Literal.String.Double */
+.highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
+.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */
+.highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
+.highlight .sx { color: #008000 } /* Literal.String.Other */
+.highlight .sr { color: #BB6688 } /* Literal.String.Regex */
+.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
+.highlight .ss { color: #19177C } /* Literal.String.Symbol */
+.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
+.highlight .vc { color: #19177C } /* Name.Variable.Class */
+.highlight .vg { color: #19177C } /* Name.Variable.Global */
+.highlight .vi { color: #19177C } /* Name.Variable.Instance */
+.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
+.grokdoc  { background: #f8f8f8; }
+.grokdoc .c { color: #408080; font-style: italic } /* Comment */
+.grokdoc .err { border: 1px solid #FF0000 } /* Error */
+.grokdoc .k { color: #008000; font-weight: bold } /* Keyword */
+.grokdoc .o { color: #666666 } /* Operator */
+.grokdoc .cm { color: #408080; font-style: italic } /* Comment.Multiline */
+.grokdoc .cp { color: #BC7A00 } /* Comment.Preproc */
+.grokdoc .c1 { color: #408080; font-style: italic } /* Comment.Single */
+.grokdoc .cs { color: #408080; font-style: italic } /* Comment.Special */
+.grokdoc .gd { color: #A00000 } /* Generic.Deleted */
+.grokdoc .ge { font-style: italic } /* Generic.Emph */
+.grokdoc .gr { color: #FF0000 } /* Generic.Error */
+.grokdoc .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.grokdoc .gi { color: #00A000 } /* Generic.Inserted */
+.grokdoc .go { color: #808080 } /* Generic.Output */
+.grokdoc .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
+.grokdoc .gs { font-weight: bold } /* Generic.Strong */
+.grokdoc .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.grokdoc .gt { color: #0040D0 } /* Generic.Traceback */
+.grokdoc .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
+.grokdoc .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
+.grokdoc .kp { color: #008000 } /* Keyword.Pseudo */
+.grokdoc .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
+.grokdoc .kt { color: #B00040 } /* Keyword.Type */
+.grokdoc .m { color: #666666 } /* Literal.Number */
+.grokdoc .s { color: #BA2121 } /* Literal.String */
+.grokdoc .na { color: #7D9029 } /* Name.Attribute */
+.grokdoc .nb { color: #008000 } /* Name.Builtin */
+.grokdoc .nc { color: #0000FF; font-weight: bold } /* Name.Class */
+.grokdoc .no { color: #880000 } /* Name.Constant */
+.grokdoc .nd { color: #AA22FF } /* Name.Decorator */
+.grokdoc .ni { color: #999999; font-weight: bold } /* Name.Entity */
+.grokdoc .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
+.grokdoc .nf { color: #0000FF } /* Name.Function */
+.grokdoc .nl { color: #A0A000 } /* Name.Label */
+.grokdoc .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
+.grokdoc .nt { color: #008000; font-weight: bold } /* Name.Tag */
+.grokdoc .nv { color: #19177C } /* Name.Variable */
+.grokdoc .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
+.grokdoc .w { color: #bbbbbb } /* Text.Whitespace */
+.grokdoc .mf { color: #666666 } /* Literal.Number.Float */
+.grokdoc .mh { color: #666666 } /* Literal.Number.Hex */
+.grokdoc .mi { color: #666666 } /* Literal.Number.Integer */
+.grokdoc .mo { color: #666666 } /* Literal.Number.Oct */
+.grokdoc .sb { color: #BA2121 } /* Literal.String.Backtick */
+.grokdoc .sc { color: #BA2121 } /* Literal.String.Char */
+.grokdoc .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
+.grokdoc .s2 { color: #BA2121 } /* Literal.String.Double */
+.grokdoc .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
+.grokdoc .sh { color: #BA2121 } /* Literal.String.Heredoc */
+.grokdoc .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
+.grokdoc .sx { color: #008000 } /* Literal.String.Other */
+.grokdoc .sr { color: #BB6688 } /* Literal.String.Regex */
+.grokdoc .s1 { color: #BA2121 } /* Literal.String.Single */
+.grokdoc .ss { color: #19177C } /* Literal.String.Symbol */
+.grokdoc .bp { color: #008000 } /* Name.Builtin.Pseudo */
+.grokdoc .vc { color: #19177C } /* Name.Variable.Class */
+.grokdoc .vg { color: #19177C } /* Name.Variable.Global */
+.grokdoc .vi { color: #19177C } /* Name.Variable.Instance */
+.grokdoc .il { color: #666666 } /* Literal.Number.Integer.Long */

pathod/libpathod/templates/about.html

@@ -0,0 +1,22 @@
+{% extends "frame.html" %} {% block body %}
+<section>
+    <div class="page-header">
+        <h1>About</h1>
+    </div>
+    <div class="row">
+        <div class="span6">
+            <div>
+                <p>pathod is developed by <a href="http://corte.si">Aldo Cortesi</a>.</p>
+            </div>
+
+            <div>
+                <ul>
+                    <li>email: <a href="mailto:aldo@corte.si">aldo@corte.si</a></li>
+                    <li>twitter: <a href="http://twitter.com/cortesi">@cortesi</a></li>
+                    <li>github: <a href="https://github.com/cortesi">github.com/cortesi</a></li>
+                </ul>
+            </div>
+        </div>
+    </div>
+</section>
+{% endblock %}

pathod/libpathod/templates/docframe.html

@@ -0,0 +1,26 @@
+{% extends "layout.html" %}
+
+{% macro subs(s) %}
+    {% if subsection == s %}
+        class="active"
+    {% endif %}
+{% endmacro %}
+
+{% block content %}
+<div class="row">
+    <div class="span3">
+        <div class="well sidebar-nav">
+            <ul class="nav nav-list">
+                <li {{subs( "pathod")}}><a href="/docs/pathod">pathod</a></li>
+                <li {{subs( "pathoc")}}><a href="/docs/pathoc">pathoc</a></li>
+                <li {{subs( "lang")}}><a href="/docs/language">language</a></li>
+                <li {{subs( "libpathod")}}><a href="/docs/libpathod">libpathod</a></li>
+                <li {{subs( "test")}}><a href="/docs/test">libpathod.test</a></li>
+            </ul>
+        </div>
+    </div>
+    <div class="span9">
+        {% block body %} {% endblock %}
+    </div>
+</div>
+{% endblock %}

pathod/libpathod/templates/docs_lang.html

@@ -0,0 +1,196 @@
+{% extends "docframe.html" %} {% block body %}
+<div class="page-header">
+    <h1>
+        Language Spec
+        <small>The mini-language at the heart of pathoc and pathod.</small>
+    </h1>
+</div>
+
+<ul class="nav nav-tabs">
+    <li class="active"><a href="#specifying_requests" data-toggle="tab">HTTP Requests</a></li>
+    <li><a href="#specifying_responses" data-toggle="tab">HTTP Responses</a></li>
+    <li><a href="#websockets" data-toggle="tab">Websocket Frames</a></li>
+</ul>
+
+<div class="tab-content">
+    <div class="tab-pane" id="specifying_responses">
+        {% include "docs_lang_responses.html" %}
+    </div>
+    <div class="tab-pane active" id="specifying_requests">
+        {% include "docs_lang_requests.html" %}
+    </div>
+    <div class="tab-pane" id="websockets">
+        {% include "docs_lang_websockets.html" %}
+    </div>
+</div>
+
+<section id="features">
+    <div class="page-header">
+        <h1>Features</h1>
+    </div>
+
+    <a id="offsetspec"></a>
+    <h2>OFFSET</h2>
+
+    <p>
+        Offsets are calculated relative to the base message, before any injections or other
+        transforms are applied. They have 3 flavors:
+    </p>
+
+    <ul>
+        <li>An integer byte offset </li>
+        <li><b>r</b> for a random location</li>
+        <li><b>a</b> for the end of the message</li>
+    </ul>
+
+    <a id="valuespec"></a>
+    <h2>VALUE</h2>
+
+    <h3>Literals</h3>
+
+    <p>Literal values are specified as a quoted strings: </p>
+
+    <pre class="example">"foo"</pre>
+
+    <p>
+        Either single or double quotes are accepted, and quotes can be escaped with backslashes
+        within the string:
+    </p>
+
+    <pre class="example">'fo\'o'</pre>
+
+    <p>Literal values can contain Python-style backslash escape sequences:</p>
+
+    <pre class="example">'foo\r\nbar'</pre>
+
+    <h3>Files</h3>
+
+    <p>
+        You can load a value from a specified file path. To do so, you have to specify a
+        _staticdir_ option to pathod on the command-line, like so:
+    </p>
+
+    <pre class="example">pathod -d ~/myassets</pre>
+
+    <p>
+        All paths are relative paths under this directory. File loads are indicated by starting
+        the value specifier with the left angle bracket:
+    </p>
+
+    <pre class="example">&lt;my/path</pre>
+
+    <p>The path value can also be a quoted string, with the same syntax as literals:</p>
+
+    <pre class="example">&lt;"my/path"</pre>
+
+
+    <h3>Generated values</h3>
+
+    <p>
+        An @-symbol lead-in specifies that generated data should be used. There are two components
+        to a generator specification - a size, and a data type. By default pathod
+        assumes a data type of "bytes".
+    </p>
+
+    <p>Here's a value specifier for generating 100 bytes:
+
+        <pre class="example">@100</pre>
+    </p>
+
+    <p>
+        You can use standard suffixes to indicate larger values. Here, for instance, is a
+        specifier for generating 100 megabytes:
+    </p>
+
+    <pre class="example">@100m</pre>
+
+    <p>
+        Data is generated and served efficiently - if you really want to send a terabyte
+        of data to a client, pathod can do it. The supported suffixes are:
+    </p>
+
+    <table class="table table-bordered">
+        <tbody>
+            <tr>
+                <td>b</td>
+                <td>1024**0 (bytes)</td>
+            </tr>
+            <tr>
+                <td>k</td>
+                <td>1024**1 (kilobytes)</td>
+            </tr>
+            <tr>
+                <td>m</td>
+                <td>1024**2 (megabytes)</td>
+            </tr>
+            <tr>
+                <td>g</td>
+                <td>1024**3 (gigabytes)</td>
+            </tr>
+            <tr>
+                <td>t</td>
+                <td>1024**4 (terabytes)</td>
+            </tr>
+        </tbody>
+    </table>
+
+    <p>
+        Data types are separated from the size specification by a comma. This specification
+        generates 100mb of ASCII:
+    </p>
+
+    <pre class="example">@100m,ascii</pre>
+
+    <p>Supported data types are:</p>
+
+    <table class="table table-bordered">
+        <tbody>
+            <tr>
+                <td>ascii</td>
+                <td>All ASCII characters</td>
+            </tr>
+            <tr>
+                <td>ascii_letters</td>
+                <td>A-Za-z</td>
+            </tr>
+            <tr>
+                <td>ascii_lowercase</td>
+                <td>a-z</td>
+            </tr>
+            <tr>
+                <td>ascii_uppercase</td>
+                <td>A-Z</td>
+            </tr>
+            <tr>
+                <td>bytes</td>
+                <td>All 256 byte values</td>
+            </tr>
+            <tr>
+                <td>digits</td>
+                <td>0-9</td>
+            </tr>
+            <tr>
+                <td>hexdigits</td>
+                <td>0-f</td>
+            </tr>
+            <tr>
+                <td>octdigits</td>
+                <td>0-7</td>
+            </tr>
+            <tr>
+                <td>punctuation</td>
+                <td>
+                    <pre>!"#$%&\'()*+,-./:;
+                        <=>?@[\\]^_`{|}~</pre>
+                </td>
+            </tr>
+            <tr>
+                <td>whitespace</td>
+                <td>
+                    <pre>\t\n\x0b\x0c\r and space</pre>
+                </td>
+            </tr>
+        </tbody>
+    </table>
+</section>
+{% endblock %}

pathod/libpathod/templates/docs_lang_requests.html

@@ -0,0 +1,114 @@
+<pre class="example">method:path:[colon-separated list of features]</pre>
+</p>
+
+<table class="table table-bordered">
+    <tbody>
+        <tr>
+            <td>method</td>
+            <td>
+                <p>
+                    A <a href="#valuespec">VALUE</a> specifying the HTTP method to
+                    use. Standard methods do not need to be enclosed in quotes, while
+                    non-standard methods can be specified as quoted strings.
+                </p>
+
+                <p>
+                    The special method <b>ws</b> creates a valid websocket upgrade
+                    GET request, and signals to pathoc to switch to websocket recieve
+                    mode if the server responds correctly. Apart from that, websocket
+                    requests are just like any other, and all aspects of the request
+                    can be over-ridden.
+                </p>
+            </td>
+        </tr>
+
+        <tr>
+            <td>h<a href="#valuespec">VALUE</a>=<a href="#valuespec">VALUE</a></td>
+            <td>
+                Set a header.
+            </td>
+        </tr>
+
+        <tr>
+            <td>r</td>
+            <td>
+                Set the "raw" flag on this response. Pathod will not calculate a Content-Length header
+                if a body is set.
+            </td>
+        </tr>
+
+        <tr>
+            <td>c<a href="#valuespec">VALUE</a></td>
+            <td>
+                A shortcut for setting the Content-Type header. Equivalent to h"Content-Type"=VALUE
+            </td>
+        </tr>
+
+        <tr>
+            <td>u<a href="#valuespec">VALUE</a>
+                <br> uSHORTCUT
+            </td>
+
+            <td>
+                Set a User-Agent header on this request. You can specify either a complete
+                <a href="#valuespec">VALUE</a>, or a User-Agent shortcut:
+
+                <table class="table table-condensed">
+                    {% for i in uastrings %}
+                    <tr>
+                        <td><b>{{ i[1] }}</b></td>
+                        <td>{{ i[0] }}</td>
+                    </tr>
+                    {% endfor %}
+                </table>
+            </td>
+        </tr>
+
+        <tr>
+            <td>b<a href="#valuespec">VALUE</a></td>
+            <td>
+                Set the body. The appropriate Content-Length header is added automatically unless
+                the "r" flag is set.
+            </td>
+        </tr>
+
+        <tr>
+            <td>s<a href="#valuespec">VALUE</a></td>
+            <td>
+                An embedded Response specification, appended to the path of the request.
+            </td>
+        </tr>
+
+        <tr>
+            <td>x<a href="#valuespec">INTEGER</a></td>
+            <td>
+                Repeat this message N times.
+            </td>
+        </tr>
+
+        <tr>
+            <td>d<a href="#offsetspec">OFFSET</a></td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Disconnect after
+                OFFSET bytes.
+            </td>
+        </tr>
+
+        <tr>
+            <td>i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a></td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Inject the specified
+                value at the offset.
+            </td>
+        </tr>
+
+        <tr>
+            <td>p<a href="#offsetspec">OFFSET</a>,SECONDS</td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Pause for SECONDS
+                seconds after OFFSET bytes. SECONDS can be an integer or "f" to pause
+                forever.
+            </td>
+        </tr>
+    </tbody>
+</table>

pathod/libpathod/templates/docs_lang_responses.html

@@ -0,0 +1,88 @@
+<pre class="example">code:[colon-separated list of features]</pre>
+
+<table class="table table-bordered">
+    <tbody>
+        <tr>
+            <td>code</td>
+            <td>
+                <p>An integer specifying the HTTP response code.</p>
+                <p>
+                    The special method <b>ws</b> creates a valid websocket upgrade
+                    response (code 101), and moves pathod to websocket mode. Apart
+                    from that, websocket responses are just like any other, and all
+                    aspects of the response can be over-ridden.
+                </p>
+            </td>
+        </tr>
+
+        <tr>
+            <td>m<a href="#valuespec">VALUE</a></td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> HTTP Reason message.
+                Automatically chosen according to the response code if not specified.
+            </td>
+        </tr>
+
+        <tr>
+            <td>h<a href="#valuespec">VALUE</a>=<a href="#valuespec">VALUE</a></td>
+            <td>
+                Set a header.
+            </td>
+        </tr>
+
+        <tr>
+            <td>r</td>
+            <td>
+                Set the "raw" flag on this response. Pathod will not calculate a Content-Length header
+                if a body is set, or add a Date header to the response.
+            </td>
+        </tr>
+
+        <tr>
+            <td>l<a href="#valuespec">VALUE</a></td>
+            <td>
+                A shortcut for setting the Location header. Equivalent to h"Location"=VALUE
+            </td>
+        </tr>
+
+        <tr>
+            <td>c<a href="#valuespec">VALUE</a></td>
+            <td>
+                A shortcut for setting the Content-Type header. Equivalent to h"Content-Type"=VALUE
+            </td>
+        </tr>
+
+        <tr>
+            <td>b<a href="#valuespec">VALUE</a></td>
+            <td>
+                Set the body. The appropriate Content-Length header is added automatically unless
+                the "r" flag is set.
+            </td>
+        </tr>
+
+        <tr>
+            <td>d<a href="#offsetspec">OFFSET</a></td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Disconnect after
+                OFFSET bytes.
+            </td>
+        </tr>
+
+        <tr>
+            <td>i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a></td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Inject the specified
+                value at the offset.
+            </td>
+        </tr>
+
+        <tr>
+            <td>p<a href="#offsetspec">OFFSET</a>,SECONDS</td>
+            <td>
+                <span class="badge badge-info">HTTP/1 only</span> Pause for SECONDS
+                seconds after OFFSET bytes. SECONDS can be an integer or "f" to pause
+                forever.
+            </td>
+        </tr>
+    </tbody>
+</table>

pathod/libpathod/templates/docs_lang_websockets.html

@@ -0,0 +1,115 @@
+<pre class="example">wf:[colon-separated list of features]</pre>
+</p>
+
+<table class="table table-bordered">
+    <tbody>
+
+        <tr>
+            <td> b<a href="#valuespec">VALUE</a> </td>
+            <td>
+                Set the frame payload. If a masking key is present, the value is encoded automatically.
+            </td>
+        </tr>
+
+        <tr>
+            <td> c<a href="#valuespec">INTEGER</a> </td>
+            <td>
+
+                Set the op code. This can either be an integer from 0-15, or be one of the following
+                opcode names: <b>text</b> (the default),
+                <b>continue</b>, <b>binary</b>, <b>close</b>, <b>ping</b>,
+                <b>pong</b>.
+
+            </td>
+        </tr>
+
+        <tr>
+            <td> d<a href="#offsetspec">OFFSET</a> </td>
+            <td>
+                Disconnect after OFFSET bytes.
+            </td>
+        </tr>
+
+        <tr>
+            <td> [-]fin </td>
+            <td>
+                Set or un-set the <b>fin</b> bit.
+            </td>
+        </tr>
+
+        <tr>
+            <td> i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a> </td>
+            <td>
+                Inject the specified value at the offset.
+            </td>
+        </tr>
+
+        <tr>
+            <td> k<a href="#valuespec">VALUE</a> </td>
+            <td>
+                Set the masking key. The resulting value must be exactly 4 bytes long. The special
+                form
+                <b>knone</b> specifies that no key should be set, even if the mask
+                bit is on.
+            </td>
+        </tr>
+
+        <tr>
+            <td> l<a href="#valuespec">INTEGER</a> </td>
+            <td>
+                Set the payload length in the frame header, regardless of the actual body length.
+            </td>
+        </tr>
+
+        <tr>
+            <td> [-]mask </td>
+            <td>
+                Set or un-set the <b>mask</b> bit.
+            </td>
+        </tr>
+
+        <tr>
+            <td> p<a href="#offsetspec">OFFSET</a>,SECONDS </td>
+            <td>
+                Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer or "f" to
+                pause forever.
+            </td>
+        </tr>
+
+        <tr>
+            <td> r<a href="#valuespec">VALUE</a> </td>
+            <td>
+                Set the raw frame payload. This disables masking, even if the key is present.
+            </td>
+        </tr>
+
+        <tr>
+            <td> [-]rsv1 </td>
+            <td>
+                Set or un-set the <b>rsv1</b> bit.
+            </td>
+        </tr>
+
+        <tr>
+            <td> [-]rsv2 </td>
+            <td>
+                Set or un-set the <b>rsv2</b> bit.
+            </td>
+        </tr>
+
+        <tr>
+            <td> [-]rsv3 </td>
+            <td>
+                Set or un-set the <b>rsv3</b> bit.
+            </td>
+        </tr>
+
+        <tr>
+            <td> x<a href="#valuespec">INTEGER</a> </td>
+            <td>
+                Repeat this message N times.
+            </td>
+        </tr>
+
+    </tbody>
+</table>

pathod/libpathod/templates/docs_libpathod.html

@@ -0,0 +1,23 @@
+{% extends "docframe.html" %} {% block body %}
+<div class="page-header">
+    <h1>
+        libpathod
+        <small>Using pathod and pathoc in code.</small>
+    </h1>
+</div>
+
+<div class="row">
+    <div class="span6">
+        <p>
+            Behind the pathod and pathoc command-line tools lurks <b>libpathod</b>,
+            a powerful library for manipulating and serving HTTP requests and responses.
+            The canonical documentation for the library is in the code, and can be
+            accessed using pydoc.
+        </p>
+    </div>
+    <div class="span6">
+        <h1>pathoc</h1>
+        {% include "libpathod_pathoc.html" %}
+    </div>
+</div>
+{% endblock %}

pathod/libpathod/templates/docs_pathoc.html

@@ -0,0 +1,211 @@
+{% extends "docframe.html" %} {% block body %}
+<div class="page-header">
+    <h1>
+        pathoc
+        <small>A perverse HTTP client.</small>
+    </h1>
+</div>
+
+<p>
+    Pathoc is a perverse HTTP daemon designed to let you craft almost any conceivable
+    HTTP request, including ones that creatively violate the standards. HTTP requests
+    are specified using a
+    <a href="/docs/language">small, terse language</a>, which pathod shares with
+    its server-side twin <a href="/docs/pathod">pathod</a>. To view pathoc's complete
+    range of options, use the command-line help:
+</p>
+
+<pre class="terminal">pathoc --help</pre>
+
+<section>
+    <div class="page-header">
+        <h1>Getting Started</h1>
+    </div>
+
+    <p>The basic pattern for pathoc commands is as follows: </p>
+
+    <pre class="terminal">pathoc hostname request [request ...]</pre>
+
+    <p>
+        That is, we specify the hostname to connect to, followed by one or more requests.
+        Lets start with a simple example:
+    </p>
+
+    <pre class="terminal">
+        &gt; pathoc google.com get:/ &lt;&lt; 301 Moved Permanently: 219 bytes
+    </pre>
+
+    <p>
+        Here, we make a GET request to the path / on port 80 of google.com. Pathoc's output
+        tells us that the server responded with a 301. We can tell pathoc to connect
+        using SSL, in which case the default port is changed to 443 (you can over-ride
+        the default port with the <b>-p</b> command-line option):
+    </p>
+
+    <pre class="terminal">
+        &gt; pathoc -s google.com get:/ &lt;&lt; 301 Moved Permanently: 219 bytes
+    </pre>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Multiple Requests</h1>
+    </div>
+
+    <p>
+        There are two ways to tell pathoc to issue multiple requests. The first is to specify
+        them on the command-line, like so:
+    </p>
+
+    <pre class="terminal">
+        &gt; pathoc google.com get:/ get:/ &lt;&lt; 301 Moved Permanently: 219 bytes &lt;&lt;
+        301 Moved Permanently: 219 bytes
+    </pre>
+
+    <p>
+        In this case, pathoc issues the specified requests over the same TCP connection -
+        so in the above example only one connection is made to google.com
+    </p>
+
+    <p>The other way to issue multiple requets is to use the <b>-n</b> flag:</p>
+
+    <pre class="terminal">
+        &gt; pathoc -n 2 google.com get:/ &lt;&lt; 301 Moved Permanently: 219 bytes &lt;&lt; 301
+        Moved Permanently: 219 bytes
+    </pre>
+
+    <p>
+        The output is identical, but two separate TCP connections are made to the upstream
+        server. These two specification styles can be combined:
+    </p>
+
+    <pre class="terminal">
+        &gt; pathoc -n 2 google.com get:/ get:/ &lt;&lt; 301 Moved Permanently: 219 bytes &lt;&lt;
+        301 Moved Permanently: 219 bytes &lt;&lt; 301 Moved Permanently: 219 bytes &lt;&lt;
+        301 Moved Permanently: 219 bytes
+    </pre>
+
+    <p>Here, two distinct TCP connections are made, with two requests issued over each.</p>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Basic Fuzzing</h1>
+    </div>
+
+    <p>
+        The combination of pathoc's powerful request specification language and a few of
+        its command-line options makes for quite a powerful basic fuzzer. Here's
+        an example:
+    </p>
+
+    <pre class="terminal">
+        &gt; pathoc -e -I 200 -t 2 -n 1000 localhost get:/:b@10:ir,@1
+    </pre>
+
+    <p>
+        The request specified here is a valid GET with a body consisting of 10 random bytes,
+        but with 1 random byte inserted in a random place. This could be in the headers,
+        in the initial request line, or in the body itself. There are a few things
+        to note here:
+    </p>
+
+    <ul>
+        <li>
+            Corrupting the request in this way will often make the server enter a state where
+            it's awaiting more input from the client. This is where the
+            <b>-t</b> option comes in, which sets a timeout that causes pathoc to
+            disconnect after two seconds.
+        </li>
+
+        <li>
+            The <b>-n</b> option tells pathoc to repeat the request 1000 times.
+        </li>
+
+        <li>
+            The <b>-I</b> option tells pathoc to ignore HTTP 200 response codes.
+            You can use this to fine-tune what pathoc considers to be an exceptional
+            condition, and therefore log-worthy.
+        </li>
+
+        <li>
+            The <b>-e</b> option tells pathoc to print an explanation of each logged
+            request, in the form of an expanded pathoc specification with all random
+            portions and automatic header additions resolved. This lets you precisely
+            replay a request that triggered an error.
+        </li>
+    </ul>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Interacting with Proxies</h1>
+    </div>
+
+    <p>
+        Pathoc has a reasonably sophisticated suite of features for interacting with proxies.
+        The proxy request syntax very closely mirrors that of straight HTTP, which
+        means that it is possible to make proxy-style requests using pathoc without
+        any additional syntax, by simply specifying a full URL instead of a simple
+        path:
+    </p>
+
+    <pre class="terminal">&gt; pathoc -p 8080 localhost "get:'http://google.com'"</pre>
+
+    <p>
+        Another common use case is to use an HTTP CONNECT request to probe remote servers
+        via a proxy. This is done with the <b>-c</b> command-line option,
+        which allows you to specify a remote host and port pair:
+    </p>
+
+    <pre class="terminal">&gt; pathoc -c google.com:80 -p 8080 localhost get:/</pre>
+
+    <p>
+        Note that pathoc does <b>not</b> negotiate SSL without being explictly instructed
+        to do so. If you're making a CONNECT request to an SSL-protected resource,
+        you must also pass the <b>-s</b> flag:
+    </p>
+
+    <pre class="terminal">&gt; pathoc -sc google.com:443 -p 8080 localhost get:/</pre>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Embedded response specification</h1>
+    </div>
+
+    <p>
+        One interesting feature of the Request sppecification language is that you can embed
+        a response specifcation in it, which is then added to the request path. Here's
+        an example:
+    </p>
+
+    <pre class="terminal">&gt; pathoc localhost:9999 "get:/p/:s'401:ir,@1'"</pre>
+
+    <p>
+        This crafts a request that connects to the pathod server, and which then crafts a
+        response that generates a 401, with one random byte embedded at a random
+        point. The response specification is parsed and expanded by pathoc, so you
+        see syntax errors immediately. This really becomes handy when combined with
+        the <b>-e</b> flag to show the expanded request:
+    </p>
+
+    <pre class="terminal">
+        &gt; > pathoc -e localhost:9999 "get:/p/:s'401:ir,@1'" >> Spec: get:/p/:s'401:i15,\'o\':h\'Content-Length\'=\'0\'':h'Content-Length'='0'
+        << 401 Unoauthorized: 0 bytes </pre>
+
+    <p>
+        Note that the embedded response has been resolved <i>before</i> being sent
+        to the server, so that "ir,@1" (embed a random byte at a random location)
+        has become "i15,\'o\'" (embed the character "o" at offset 15). You now have
+        a pathoc request specification that is precisely reproducable, even with
+        random components. This feature comes in terribly handy when testing a proxy,
+        since you can now drive the server repsonse completely from the client, and
+        have a complete log of reproducible requests to analyse afterwards.
+    </p>
+</section>
+{% endblock %}

pathod/libpathod/templates/docs_pathod.html

@@ -0,0 +1,172 @@
+{% extends "docframe.html" %} {% block body %}
+<div class="page-header">
+    <h1>
+        pathod
+        <small>A pathological web daemon.</small>
+    </h1>
+</div>
+
+<p>
+    Pathod is a pathological HTTP daemon designed to let you craft almost any conceivable
+    HTTP response, including ones that creatively violate the standards. HTTP responses
+    are specified using a
+    <a href="/docs/language">small, terse language</a>, which pathod shares with
+    its evil twin <a href="/docs/pathoc">pathoc</a>.
+</p>
+
+<section>
+    <div class="page-header">
+        <h1>Getting started</h1>
+    </div>
+
+    <p>To start playing with pathod, simply fire up the daemon:</p>
+
+    <pre class="terminal">./pathod</pre>
+
+    <p>
+        By default, the service listens on port 9999 of localhost. Pathod's documentation
+        is self-hosting, and the pathod daemon exposes an interface that lets you
+        play with the specifciation language, preview what responses and requests
+        would look like on the wire, and view internal logs. To access all of this,
+        just fire up your browser, and point it to the following URL:
+    </p>
+
+    <pre class="example">http://localhost:9999</pre>
+
+    <p>
+        The default crafting anchor point is the path <b>/p/</b>. Anything after
+        this URL prefix is treated as a response specifier. So, hitting the following
+        URL will generate an HTTP 200 response with 100 bytes of random data:
+    </p>
+
+    <pre class="example">http://localhost:9999/p/200:b@100</pre>
+
+    <p>
+        See the <a href="/docs/language">language documentation</a> to get (much)
+        fancier. The pathod daemon also takes a range of configuration options. To
+        view those, use the command-line help:
+    </p>
+
+    <pre class="terminal">./pathod --help</pre>
+
+</section>
+
+<section>
+    <div class="page-header">
+        <h1>Acting as a proxy</h1>
+    </div>
+
+    <p>
+        Pathod automatically responds to both straight HTTP and proxy requests. For proxy
+        requests, the upstream host is ignored, and the path portion of the URL is
+        used to match anchors. This lets you test software that supports a proxy
+        configuration by spoofing responses from upstream servers.
+    </p>
+
+    <p>
+        By default, we treat all proxy CONNECT requests as HTTPS traffic, serving the response
+        using either pathod's built-in certificates, or the cert/key pair specified
+        by the user. You can over-ride this behaviour if you're testing a client
+        that makes a non-SSL CONNECT request using the -C command-line option.
+    </p>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Anchors</h1>
+    </div>
+
+    <p>
+        Anchors provide an alternative to specifying the response in the URL. Instead, you
+        attach a response to a pre-configured anchor point, specified with a regex.
+        When a URL matching the regex is requested, the specified response is served.
+    </p>
+
+    <pre class="terminal">./pathod -a "/foo=200"</pre>
+
+    <p>
+        Here, "/foo" is the regex specifying the anchor path, and the part after the "="
+        is a response specifier.
+    </p>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>File Access</h1>
+    </div>
+
+    <p>
+        There are two operators in the <a href="/docs/language">language</a> that
+        load contents from file - the <b>+</b> operator to load an entire request
+        specification from file, and the <b>&gt;</b> value specifier. In pathod,
+        both of these operators are restricted to a directory specified at startup,
+        or disabled if no directory is specified:</p>
+    <pre class="terminal">./pathod -d ~/staticdir"</pre>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>Internal Error Responses</h1>
+    </div>
+
+    <p>
+        Pathod uses the non-standard 800 response code to indicate internal errors, to distinguish
+        them from crafted responses. For example, a request to:
+    </p>
+
+    <pre class="example">http://localhost:9999/p/foo</pre>
+
+    <p>
+        ... will return an 800 response because "foo" is not a valid page specifier.
+    </p>
+</section>
+
+
+<section>
+    <div class="page-header">
+        <h1>API</h1>
+    </div>
+
+    <p>
+        pathod exposes a simple API, intended to make it possible to drive and inspect the
+        daemon remotely for use in unit testing and the like.
+    </p>
+
+    <table class="table table-bordered">
+        <tbody>
+            <tr>
+                <td>
+                    /api/clear_log
+                </td>
+                <td>
+                    A POST to this URL clears the log buffer.
+                </td>
+            </tr>
+            <tr>
+                <td>
+                    /api/info
+                </td>
+                <td>
+                    Basic version and configuration info.
+                </td>
+            </tr>
+            <tr>
+                <td>
+                    /api/log
+                </td>
+                <td>
+                    Returns the current log buffer. At the moment the buffer size is 500 entries - when
+                    the log grows larger than this, older entries are discarded.
+                    The returned data is a JSON dictionary, with the form:
+
+                    <pre>{ 'log': [ ENTRIES ] } </pre> You can preview the JSON data
+                    returned for a log entry through the built-in web interface.
+                </td>
+            </tr>
+        </tbody>
+    </table>
+</section>
+{% endblock %}

pathod/libpathod/templates/docs_test.html

@@ -0,0 +1,50 @@
+{% extends "docframe.html" %} {% block body %}
+<div class="page-header">
+    <h1>
+        libpathod.test
+        <small>Using libpathod in unit tests.</small>
+    </h1>
+</div>
+
+<p>The <b>libpathod.test</b> module is a light, flexible testing layer for HTTP clients.
+    It works by firing up a Pathod instance in a separate thread, letting you use
+    Pathod's full abilities to generate responses, and then query Pathod's internal
+    logs to establish what happened. All the mechanics of startup, shutdown, finding
+    free ports and so forth are taken care of for you.
+</p>
+
+<p>The canonical docs can be accessed using pydoc: </p>
+
+<pre class="terminal">pydoc libpathod.test</pre>
+
+<p>
+    The remainder of this page demonstrates some common interaction patterns using
+    <a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
+    also applicable with only minor modification to most commonly used Python testing
+    engines.
+</p>
+
+<section>
+    <div class="page-header">
+        <h1>Context Manager</h1>
+    </div>
+
+    {% include "examples_context.html" %}
+</section>
+
+<section>
+    <div class="page-header">
+        <h1>One instance per test</h1>
+    </div>
+
+    {% include "examples_setup.html" %}
+</section>
+
+<section>
+    <div class="page-header">
+        <h1>One instance per suite</h1>
+    </div>
+
+    {% include "examples_setupall.html" %}
+</section>
+{% endblock %}

pathod/libpathod/templates/download.html

@@ -0,0 +1,39 @@
+{% extends "frame.html" %} {% block body %}
+<section>
+    <div class="page-header">
+        <h1>pip</h1>
+    </div>
+
+    <p>The easiest way to install pathod is to use pip:</p>
+
+    <pre>pip install pathod</pre>
+
+    <p>
+        This will automatically pull in all the dependencies, and you should be good to go.
+    </p>
+</section>
+
+<section>
+    <div class="page-header">
+        <h1>github</h1>
+    </div>
+
+    <p>You can find the project source on GitHub:</p>
+
+    <div style="margin-top: 20px; margin-bottom: 20px">
+        <a class="btn btn-primary btn-large" href="https://github.com/mitmproxy/pathod">github.com/mitmproxy/pathod</a>
+    </div>
+
+    <p>Please also use the <a href="https://github.com/mitmproxy/pathod/issues">github issue tracker</a>        to report bugs.</p>
+</section>
+
+<section>
+    <div class="page-header">
+        <h1>tarball</h1>
+    </div>
+
+    <div style="margin-top: 20px; margin-bottom: 20px">
+        <a class="btn btn-primary btn-large" href="https://github.com/downloads/mitmproxy/pathod/pathod-{{version}}.tar.gz">pathod-{{version}}.tar.gz</a>
+    </div>
+</section>
+{% endblock %}

pathod/libpathod/templates/examples_context.html

@@ -0,0 +1,24 @@
+<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span>
+<span class="kn">from</span> <span class="nn">libpathod</span> <span class="kn">import</span> <span class="n">test</span>
+
+
+<span class="k">def</span> <span class="nf">test_simple</span><span class="p">():</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Testing the requests module with</span>
+<span class="sd">        a pathod context manager.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c"># Start pathod in a separate thread</span>
+    <span class="k">with</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span> <span class="k">as</span> <span class="n">d</span><span class="p">:</span>
+        <span class="c"># Get a URL for a pathod spec</span>
+        <span class="n">url</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">&quot;200:b@100&quot;</span><span class="p">)</span>
+        <span class="c"># ... and request it</span>
+        <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
+
+        <span class="c"># Check the returned data</span>
+        <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span>
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span>
+
+        <span class="c"># Check pathod&#39;s internal log</span>
+        <span class="n">log</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">&quot;request&quot;</span><span class="p">]</span>
+        <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">&quot;method&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&quot;PUT&quot;</span>
+</pre></div>

pathod/libpathod/templates/examples_setup.html

@@ -0,0 +1,32 @@
+<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span>
+<span class="kn">from</span> <span class="nn">libpathod</span> <span class="kn">import</span> <span class="n">test</span>
+
+
+<span class="k">class</span> <span class="nc">Test</span><span class="p">:</span>
+
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Testing the requests module with</span>
+<span class="sd">        a pathod instance started for</span>
+<span class="sd">        each test.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">d</span> <span class="o">=</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">tearDown</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">test_simple</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="c"># Get a URL for a pathod spec</span>
+        <span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">&quot;200:b@100&quot;</span><span class="p">)</span>
+        <span class="c"># ... and request it</span>
+        <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
+
+        <span class="c"># Check the returned data</span>
+        <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span>
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span>
+
+        <span class="c"># Check pathod&#39;s internal log</span>
+        <span class="n">log</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">&quot;request&quot;</span><span class="p">]</span>
+        <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">&quot;method&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&quot;PUT&quot;</span>
+</pre></div>

pathod/libpathod/templates/examples_setupall.html

@@ -0,0 +1,40 @@
+<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span>
+<span class="kn">from</span> <span class="nn">libpathod</span> <span class="kn">import</span> <span class="n">test</span>
+
+
+<span class="k">class</span> <span class="nc">Test</span><span class="p">:</span>
+
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Testing the requests module with</span>
+<span class="sd">        a single pathod instance started</span>
+<span class="sd">        for the test suite.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">setUpAll</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+        <span class="n">cls</span><span class="o">.</span><span class="n">d</span> <span class="o">=</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span>
+
+    <span class="nd">@classmethod</span>
+    <span class="k">def</span> <span class="nf">tearDownAll</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
+        <span class="n">cls</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="c"># Clear the pathod logs between tests</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">clear_log</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">test_simple</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="c"># Get a URL for a pathod spec</span>
+        <span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">&quot;200:b@100&quot;</span><span class="p">)</span>
+        <span class="c"># ... and request it</span>
+        <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
+
+        <span class="c"># Check the returned data</span>
+        <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span>
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span>
+
+        <span class="c"># Check pathod&#39;s internal log</span>
+        <span class="n">log</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">&quot;request&quot;</span><span class="p">]</span>
+        <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">&quot;method&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&quot;PUT&quot;</span>
+
+    <span class="k">def</span> <span class="nf">test_two</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">assert</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">log</span><span class="p">()</span>
+</pre></div>

pathod/libpathod/templates/frame.html

@@ -0,0 +1,7 @@
+{% extends "layout.html" %} {% block content %}
+<div class="row">
+    <div class="span12">
+        {% block body %} {% endblock %}
+    </div>
+</div>
+{% endblock %}

pathod/libpathod/templates/index.html

@@ -0,0 +1,60 @@
+{% extends "frame.html" %} {% block body %}
+<div class="masthead">
+    <div class="container">
+        <h1>pathod: pathological HTTP</h1>
+
+        <p>Crafted malice for tormenting HTTP clients and servers</p>
+
+        <img src="/static/torture.png">
+    </div>
+</div>
+
+<div class="row">
+    <div class="span6">
+        <div>
+            <h2><a href="/docs/pathod">pathod</a></h2>
+
+            <p>A pathological web daemon.</p>
+
+            {% include "response_previewform.html" %}
+            <br>
+        </div>
+    </div>
+
+    <div class="span6">
+        <div>
+            <h2><a href="/docs/pathoc">pathoc</a></h2>
+
+            <p>A perverse HTTP client.</p>
+
+            {% include "request_previewform.html" %}
+        </div>
+    </div>
+</div>
+
+<section>
+    <div class="page-header">
+        <h1>Install</h1>
+    </div>
+    <div class="row">
+        <div class="span6">
+            <div>
+                <h2>pip</h2>
+
+                <pre>pip install pathod</pre>
+            </div>
+        </div>
+
+        <div class="span6">
+            <div>
+                <h2>source</h2>
+
+                <ul>
+                    <li>Current release: <a href="http://mitmproxy.org/download/pathod-{{version}}.tar.gz">pathod {{version}}</a></li>
+                    <li>GitHub: <a href="https://github.com/mitmproxy/pathod">github.com/mitmproxy/pathod</a></li>
+                </ul>
+            </div>
+        </div>
+    </div>
+</section>
+{% endblock %}

pathod/libpathod/templates/layout.html

@@ -0,0 +1,75 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>pathod</title>
+    <link href="/static/bootstrap.min.css" rel="stylesheet">
+    <link href="/static/pathod.css" rel="stylesheet">
+    <link href="/static/syntax.css" rel="stylesheet">
+    <script src="/static/jquery-1.7.2.min.js"></script>
+    <script src="/static/jquery.scrollTo-min.js"></script>
+    <script src="/static/jquery.localscroll-min.js"></script>
+    <script src="/static/bootstrap.min.js"></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="">
+    <style type="text/css">
+        body {
+            padding-top: 60px;
+            padding-bottom: 40px;
+        }
+
+    </style>
+    <!-- Le HTML5 shim, for IE6-8 support of HTML5 elements -->
+    <!--[if lt IE 9]>
+      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+    <![endif]-->
+</head>
+
+<body>
+    <div class="navbar navbar-fixed-top">
+        <div class="navbar-inner">
+            <div class="container">
+                <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                </a>
+                <a class="brand" href="/index.html">pathod</a>
+                <div class="nav-collapse">
+                    <ul class="nav">
+                        <li {% if section=="main" %} class="active" {% endif %}><a href="/">home</a></li>
+                        {% if not noapi %}
+                        <li {% if section=="log" %} class="active" {% endif %}><a href="/log">log</a></li>
+                        {% endif %}
+                        <li {% if section=="docs" %} class="active" {% endif %}><a href="/docs/pathod">docs</a></li>
+                        <li {% if section=="about" %} class="active" {% endif %}><a href="/about">about</a></li>
+                    </ul>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div class="container">
+        {% block content %} {% endblock %}
+        <hr>
+        <footer>
+            <span>&copy; Aldo Cortesi 2015</span>
+            <span class="pull-right">[served with pathod]</span>
+        </footer>
+    </div>
+</body>
+<script>
+    $(function() {
+        $.localScroll({
+            duration: 300,
+            offset: {
+                top: -45
+            }
+        });
+    });
+
+</script>
+
+</html>

pathod/libpathod/templates/libpathod_pathoc.html

@@ -0,0 +1,8 @@
+<div class="highlight"><pre><span class="c">#!/usr/bin/env python</span>
+<span class="kn">from</span> <span class="nn">libpathod</span> <span class="kn">import</span> <span class="n">pathoc</span>
+
+<span class="n">p</span> <span class="o">=</span> <span class="n">pathoc</span><span class="o">.</span><span class="n">Pathoc</span><span class="p">((</span><span class="s">&quot;google.com&quot;</span><span class="p">,</span> <span class="mi">80</span><span class="p">))</span>
+<span class="n">p</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span>
+<span class="k">print</span> <span class="n">p</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="s">&quot;get:/&quot;</span><span class="p">)</span>
+<span class="k">print</span> <span class="n">p</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="s">&quot;get:/foo&quot;</span><span class="p">)</span>
+</pre></div>

pathod/libpathod/templates/log.html

@@ -0,0 +1,31 @@
+{% extends "frame.html" %} {% block body %}
+<form style="float: right" method="POST" action="/log/clear">
+    <button type="submit" class="btn">clear</button>
+</form>
+
+<h1>Logs</h1>
+<hr>
+
+<table class="table table-striped table-condensed">
+    <thead>
+        <tr>
+            <th>id</th>
+            <th>method</th>
+            <th>path</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for i in log %}
+        <tr>
+            {% if i["type"] == 'error' %}
+            <td colspan="3">ERROR: {{ i["msg"] }}</td>
+            {% else %}
+            <td>{{ i["id"] }}</td>
+            <td>{{ i["request"]["method"] }}</td>
+            <td><a href="/log/{{ i[" id "] }}">{{ i["request"]["path"] }}</a></td>
+            {% endif %}
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% endblock %}

pathod/libpathod/templates/onelog.html

@@ -0,0 +1,8 @@
+{% extends "frame.html" %} {% block body %}
+<h2>Log entry {{ lid }}</h2>
+<hr>
+
+<pre>
+    {{ alog }}
+</pre>
+{% endblock %}

pathod/libpathod/templates/request_preview.html

@@ -0,0 +1,44 @@
+{% extends "frame.html" %} {% block body %}
+<div class="page-header">
+    <h1>pathoc preview</h1>
+</div>
+
+<div style="margin-bottom: 20px" class="row">
+    <div class="span2 header">
+        Specification:
+    </div>
+    <div class="span10">
+        {% include "request_previewform.html" %}
+    </div>
+</div>
+
+{% if syntaxerror %}
+<div class="row">
+    <div class="span2 header">
+        Error:
+    </div>
+    <div class="span10">
+        <p style="color: #ff0000">{{ syntaxerror }}</p>
+        <pre>{{ marked }}</pre>
+    </div>
+</div>
+{% elif error %}
+<div class="row">
+    <div class="span2 header">
+        Error:
+    </div>
+    <div class="span10">
+        <p style="color: #ff0000">{{ error }}</p>
+    </div>
+</div>
+{% else %}
+<div class="row">
+    <div class="span2 header">
+        Request:
+    </div>
+    <div class="span10">
+        <pre>{{ output }}</pre>
+        <p>Note: pauses are skipped when generating previews!</p>
+    </div>
+</div>
+{% endif %} {% endblock %}

pathod/libpathod/templates/request_previewform.html

@@ -0,0 +1,53 @@
+<form style="margin-bottom: 0" class="form-inline" method="GET" action="/request_preview">
+    <input style="width: 18em" id="spec" name="spec" class="input-medium" value="{{spec}}"
+    placeholder="method:path:[features]">
+    <input type="submit" class="btn" value="preview">
+</form>
+
+<a class="innerlink" data-toggle="collapse" data-target="#requestexamples">examples</a>
+
+<div id="requestexamples" class="collapse">
+    <p>
+        Check out the <a href="/docs/language">complete language docs</a>. Here are
+        some examples to get you started:
+    </p>
+
+    <table class="table table-bordered">
+        <tbody>
+            <tr>
+                <td><a href="/request_preview?spec=get:/">get:/</a></td>
+                <td>Get path /</td>
+            </tr>
+            <tr>
+                <td><a href="/request_preview?spec=get:/:b@100">get:/:b@100</a></td>
+                <td>100 random bytes as the body</td>
+            </tr>
+            <tr>
+                <td><a href='/request_preview?spec=get:/:h"Etag"="&apos;;drop table browsers;"'>get:/:h"Etag"="';drop table browsers;"</a></td>
+                <td>Add a header</td>
+            </tr>
+            <tr>
+                <td><a href='/request_preview?spec=get:/:u"&apos;;drop table browsers;"'>get:/:u"';drop table browsers;"</a></td>
+                <td>Add a User-Agent header</td>
+            </tr>
+            <tr>
+                <td><a href="/request_preview?spec=get:/:b@100:dr">get:/:b@100:dr</a></td>
+                <td>Drop the connection randomly</td>
+            </tr>
+            <tr>
+                <td>
+                    <a href="/request_preview?spec="></a>
+                </td>
+                <td></td>
+            </tr>
+            <tr>
+                <td><a href="/request_preview?spec=get:/:b@100,ascii:ir,@1">get:/:b@100,ascii:ir,@1</a></td>
+                <td>100 ASCII bytes as the body, and randomly inject a random byte</td>
+            </tr>
+            <tr>
+                <td><a href="/request_preview?spec=ws:/">ws:/</a></td>
+                <td>Initiate a websocket handshake.</td>
+            </tr>
+        </tbody>
+    </table>
+</div>

pathod/libpathod/templates/response_preview.html

@@ -0,0 +1,44 @@
+{% extends "frame.html" %} {% block body %}
+<div class="page-header">
+    <h1>pathod preview</h1>
+</div>
+
+<div style="margin-bottom: 20px" class="row">
+    <div class="span2 header">
+        Specification:
+    </div>
+    <div class="span10">
+        {% include "response_previewform.html" %}
+    </div>
+</div>
+
+{% if syntaxerror %}
+<div class="row">
+    <div class="span2 header">
+        Error:
+    </div>
+    <div class="span10">
+        <p style="color: #ff0000">{{ syntaxerror }}</p>
+        <pre>{{ marked }}</pre>
+    </div>
+</div>
+{% elif error %}
+<div class="row">
+    <div class="span2 header">
+        Error:
+    </div>
+    <div class="span10">
+        <p style="color: #ff0000">{{ error }}</p>
+    </div>
+</div>
+{% else %}
+<div class="row">
+    <div class="span2 header">
+        Response:
+    </div>
+    <div class="span10">
+        <pre>{{ output }}</pre>
+        <p>Note: pauses are skipped when generating previews!</p>
+    </div>
+</div>
+{% endif %} {% endblock %}

pathod/libpathod/templates/response_previewform.html

@@ -0,0 +1,87 @@
+<form style="margin-bottom: 0" class="form-inline" method="GET" action="/response_preview">
+    <input style="width: 18em" id="spec" name="spec" class="input-medium" value="{{spec}}"
+    placeholder="code:[features]">
+    <input type="submit" class="btn" value="preview">
+    {% if not nocraft %}
+    <a href="#" id="submitspec" class="btn">go</a>
+    {% endif %}
+</form>
+
+<a class="innerlink" data-toggle="collapse" data-target="#responseexamples">examples</a>
+
+<div id="responseexamples" class="collapse">
+    <p>
+        Check out the <a href="/docs/language">complete language docs</a>. Here are
+        some examples to get you started:
+    </p>
+
+    <table class="table table-bordered">
+        <tbody>
+            <tr>
+                <td><a href="/response_preview?spec=200">200</a></td>
+                <td>A basic HTTP 200 response.</td>
+            </tr>
+            <tr>
+                <td><a href="/response_preview?spec=200:r">200:r</a></td>
+                <td>A basic HTTP 200 response with no Content-Length header. This will
+                    hang.
+                </td>
+            </tr>
+            <tr>
+                <td><a href="/response_preview?spec=200:da">200:da</a></td>
+                <td>Server-side disconnect after all content has been sent.</td>
+            </tr>
+            <tr>
+                <td><a href="/response_preview?spec=200:b@100">200:b@100</a></td>
+                <td>
+                    100 random bytes as the body. A Content-Lenght header is added, so the disconnect
+                    is no longer needed.
+                </td>
+            </tr>
+            <tr>
+                <td><a href='/response_preview?spec=200:b@100:h"Server"="&apos;;drop table servers;"'>200:b@100:h"Etag"="';drop table servers;"</a></td>
+                <td>Add a Server header</td>
+            </tr>
+            <tr>
+                <td><a href="/response_preview?spec=200:b@100:dr">200:b@100:dr</a></td>
+                <td>Drop the connection randomly</td>
+            </tr>
+            <tr>
+                <td><a href="/response_preview?spec=200:b@100,ascii:ir,@1">200:b@100,ascii:ir,@1</a></td>
+                <td>100 ASCII bytes as the body, and randomly inject a random byte</td>
+            </tr>
+            <tr>
+                <td><a href='/response_preview?spec=200:b@1k:c"text/json"'>200:b@1k:c"text/json"</a></td>
+                <td>1k of random bytes, with a text/json content type</td>
+            </tr>
+            <tr>
+                <td><a href='/response_preview?spec=200:b@1k:p50,120'>200:b@1k:p50,120</a></td>
+                <td>1k of random bytes, pause for 120 seconds after 50 bytes</td>
+            </tr>
+            <tr>
+                <td><a href='/response_preview?spec=200:b@1k:pr,f'>200:b@1k:pr,f</a></td>
+                <td>1k of random bytes, but hang forever at a random location</td>
+            </tr>
+            <tr>
+                <td>
+                    <a href="/response_preview?spec=200:b@100:h@1k,ascii_letters='foo'">200:b@100:h@1k,ascii_letters='foo'</a>
+                </td>
+                <td>
+                    100 ASCII bytes as the body, randomly generated 100k header name, with the value
+                    'foo'.
+                </td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+
+{% if not nocraft %}
+<script>
+    $(function() {
+        $("#submitspec").click(function() {
+            document.location = "{{craftanchor}}" + $("#spec").val()
+        });
+    });
+
+</script>
+{% endif %}

pathod/libpathod/test.py

@@ -0,0 +1,103 @@
+import cStringIO
+import threading
+import Queue
+
+import requests
+import requests.packages.urllib3
+from . import pathod
+
+requests.packages.urllib3.disable_warnings()
+
+
+class Daemon:
+    IFACE = "127.0.0.1"
+
+    def __init__(self, ssl=None, **daemonargs):
+        self.q = Queue.Queue()
+        self.logfp = cStringIO.StringIO()
+        daemonargs["logfp"] = self.logfp
+        self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
+        self.thread.start()
+        self.port = self.q.get(True, 5)
+        self.urlbase = "%s://%s:%s" % (
+            "https" if ssl else "http",
+            self.IFACE,
+            self.port
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.logfp.truncate(0)
+        self.shutdown()
+        return False
+
+    def p(self, spec):
+        """
+            Return a URL that will render the response in spec.
+        """
+        return "%s/p/%s" % (self.urlbase, spec)
+
+    def info(self):
+        """
+            Return some basic info about the remote daemon.
+        """
+        resp = requests.get("%s/api/info" % self.urlbase, verify=False)
+        return resp.json()
+
+    def text_log(self):
+        return self.logfp.getvalue()
+
+    def last_log(self):
+        """
+            Returns the last logged request, or None.
+        """
+        l = self.log()
+        if not l:
+            return None
+        return l[0]
+
+    def log(self):
+        """
+            Return the log buffer as a list of dictionaries.
+        """
+        resp = requests.get("%s/api/log" % self.urlbase, verify=False)
+        return resp.json()["log"]
+
+    def clear_log(self):
+        """
+            Clear the log.
+        """
+        self.logfp.truncate(0)
+        resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False)
+        return resp.ok
+
+    def shutdown(self):
+        """
+            Shut the daemon down, return after the thread has exited.
+        """
+        self.thread.server.shutdown()
+        self.thread.join()
+
+
+class _PaThread(threading.Thread):
+
+    def __init__(self, iface, q, ssl, daemonargs):
+        threading.Thread.__init__(self)
+        self.name = "PathodThread"
+        self.iface, self.q, self.ssl = iface, q, ssl
+        self.daemonargs = daemonargs
+
+    def run(self):
+        self.server = pathod.Pathod(
+            (self.iface, 0),
+            ssl=self.ssl,
+            **self.daemonargs
+        )
+        self.name = "PathodThread (%s:%s)" % (
+            self.server.address.host,
+            self.server.address.port
+        )
+        self.q.put(self.server.address.port)
+        self.server.serve_forever()

pathod/libpathod/utils.py

@@ -0,0 +1,124 @@
+import os
+import sys
+
+
+SIZE_UNITS = dict(
+    b=1024 ** 0,
+    k=1024 ** 1,
+    m=1024 ** 2,
+    g=1024 ** 3,
+    t=1024 ** 4,
+)
+
+
+class MemBool(object):
+
+    """
+        Truth-checking with a memory, for use in chained if statements.
+    """
+
+    def __init__(self):
+        self.v = None
+
+    def __call__(self, v):
+        self.v = v
+        return bool(v)
+
+
+def parse_size(s):
+    try:
+        return int(s)
+    except ValueError:
+        pass
+    for i in SIZE_UNITS.keys():
+        if s.endswith(i):
+            try:
+                return int(s[:-1]) * SIZE_UNITS[i]
+            except ValueError:
+                break
+    raise ValueError("Invalid size specification.")
+
+
+def parse_anchor_spec(s):
+    """
+        Return a tuple, or None on error.
+    """
+    if "=" not in s:
+        return None
+    return tuple(s.split("=", 1))
+
+
+def xrepr(s):
+    return repr(s)[1:-1]
+
+
+def inner_repr(s):
+    """
+        Returns the inner portion of a string or unicode repr (i.e. without the
+        quotes)
+    """
+    if isinstance(s, unicode):
+        return repr(s)[2:-1]
+    else:
+        return repr(s)[1:-1]
+
+
+def escape_unprintables(s):
+    """
+        Like inner_repr, but preserves line breaks.
+    """
+    s = s.replace("\r\n", "PATHOD_MARKER_RN")
+    s = s.replace("\n", "PATHOD_MARKER_N")
+    s = inner_repr(s)
+    s = s.replace("PATHOD_MARKER_RN", "\n")
+    s = s.replace("PATHOD_MARKER_N", "\n")
+    return s
+
+
+class Data(object):
+
+    def __init__(self, name):
+        m = __import__(name)
+        dirname, _ = os.path.split(m.__file__)
+        self.dirname = os.path.abspath(dirname)
+
+    def path(self, path):
+        """
+            Returns a path to the package data housed at 'path' under this
+            module.Path can be a path to a file, or to a directory.
+
+            This function will raise ValueError if the path does not exist.
+        """
+        fullpath = os.path.join(self.dirname, path)
+        if not os.path.exists(fullpath):
+            raise ValueError("dataPath: %s does not exist." % fullpath)
+        return fullpath
+
+
+data = Data(__name__)
+
+
+def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):  # pragma: nocover
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError as e:
+        sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    os.chdir("/")
+    os.umask(0)
+    os.setsid()
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError as e:
+        sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    si = open(stdin, 'rb')
+    so = open(stdout, 'a+b')
+    se = open(stderr, 'a+b', 0)
+    os.dup2(si.fileno(), sys.stdin.fileno())
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(se.fileno(), sys.stderr.fileno())

pathod/libpathod/version.py

@@ -0,0 +1,11 @@
+from __future__ import (absolute_import, print_function, division)
+
+IVERSION = (0, 17)
+VERSION = ".".join(str(i) for i in IVERSION)
+MINORVERSION = ".".join(str(i) for i in IVERSION[:2])
+NAME = "pathod"
+NAMEVERSION = NAME + " " + VERSION
+
+NEXT_MINORVERSION = list(IVERSION)
+NEXT_MINORVERSION[1] += 1
+NEXT_MINORVERSION = ".".join(str(i) for i in NEXT_MINORVERSION[:2])

pathod/pathoc

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from libpathod import pathoc_cmdline as cmdline
+
+if __name__ == "__main__":
+    cmdline.go_pathoc()

pathod/pathod

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+from libpathod import pathod_cmdline as cmdline
+
+if __name__ == "__main__":
+    cmdline.go_pathod()

pathod/release/pathoc.spec

@@ -0,0 +1,22 @@
+# -*- mode: python -*-
+
+from PyInstaller.utils.hooks import collect_data_files
+
+a = Analysis(['../pathoc'],
+             binaries=None,
+             datas=None,
+             hiddenimports=['_cffi_backend'],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None)
+pyz = PYZ(a.pure, a.zipped_data)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='pathoc',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )

pathod/release/pathod.spec

@@ -0,0 +1,22 @@
+# -*- mode: python -*-
+
+from PyInstaller.utils.hooks import collect_data_files
+
+a = Analysis(['../pathod'],
+             binaries=None,
+             datas=collect_data_files("libpathod"),
+             hiddenimports=['_cffi_backend'],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None)
+pyz = PYZ(a.pure, a.zipped_data)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='pathod',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )

pathod/requirements.txt

@@ -0,0 +1,2 @@
+-e git+https://github.com/mitmproxy/netlib.git#egg=netlib
+-e .[dev]

pathod/setup.py

@@ -0,0 +1,63 @@
+from setuptools import setup, find_packages
+from codecs import open
+import os
+from libpathod import version
+
+# Based on https://github.com/pypa/sampleproject/blob/master/setup.py
+# and https://python-packaging-user-guide.readthedocs.org/
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(here, 'README.txt'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name="pathod",
+    version=version.VERSION,
+    description="A pathological HTTP/S daemon for testing and stressing clients.",
+    long_description=long_description,
+    url="http://pathod.net",
+    author="Aldo Cortesi",
+    author_email="aldo@corte.si",
+    license="MIT",
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Development Status :: 5 - Production/Stable",
+        "Operating System :: POSIX",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
+        "Topic :: Internet",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
+        "Topic :: Software Development :: Testing",
+        "Topic :: Software Development :: Testing :: Traffic Generation",
+    ],
+
+    packages=find_packages(exclude=["test", "test.*"]),
+    include_package_data=True,
+    entry_points={
+        'console_scripts': [
+            "pathod = libpathod.pathod_cmdline:go_pathod",
+            "pathoc = libpathod.pathoc_cmdline:go_pathoc"
+        ]
+    },
+    install_requires=[
+        "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
+        "requests>=2.9.1, <2.10",
+        "Flask>=0.10.1, <0.11",
+        "pyparsing>=2.1,<2.2"
+    ],
+    extras_require={
+        'dev': [
+            "mock>=1.3.0, <1.4",
+            "pytest>=2.8.0",
+            "pytest-xdist>=1.14, <1.15",
+            "pytest-cov>=2.2.1, <2.3",
+            "pytest-timeout>=1.0.0, <1.1",
+            "coveralls>=1.1, <1.2"
+        ]
+    }
+)

pathod/test/data/clientcert/.gitignore

@@ -0,0 +1,3 @@
+client.crt
+client.key
+client.req

pathod/test/data/clientcert/client.cnf

@@ -0,0 +1,5 @@
+[ ssl_client ]
+basicConstraints = CA:FALSE
+nsCertType = client
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth

pathod/test/data/clientcert/client.pem

@@ -0,0 +1,42 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0
+EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+
+ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G
+3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/
+SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP
+G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABAoIBAFE3FV/IDltbmHEP
+iky93hbJm+6QgKepFReKpRVTyqb7LaygUvueQyPWQMIriKTsy675nxo8DQr7tQsO
+y3YlSZgra/xNMikIB6e82c7K8DgyrDQw/rCqjZB3Xt4VCqsWJDLXnQMSn98lx0g7
+d7Lbf8soUpKWXqfdVpSDTi4fibSX6kshXyfSTpcz4AdoncEpViUfU1xkEEmZrjT8
+1GcCsDC41xdNmzCpqRuZX7DKSFRoB+0hUzsC1oiqM7FD5kixonRd4F5PbRXImIzt
+6YCsT2okxTA04jX7yByis7LlOLTlkmLtKQYuc3erOFvwx89s4vW+AeFei+GGNitn
+tHfSwbECgYEA7SzV+nN62hAERHlg8cEQT4TxnsWvbronYWcc/ev44eHSPDWL5tPi
+GHfSbW6YAq5Wa0I9jMWfXyhOYEC3MZTC5EEeLOB71qVrTwcy/sY66rOrcgjFI76Q
+5JFHQ4wy3SWU50KxE0oWJO9LIowprG+pW1vzqC3VF0T7q0FqESrY4LUCgYEA3F7Z
+80ndnCUlooJAb+Hfotv7peFf1o6+m1PTRcz1lLnVt5R5lXj86kn+tXEpYZo1RiGR
+2rE2N0seeznWCooakHcsBN7/qmFIhhooJNF7yW+JP2I4P2UV5+tJ+8bcs/voUkQD
+1x+rGOuMn8nvHBd2+Vharft8eGL2mgooPVI2XusCgYEAlMZpO3+w8pTVeHaDP2MR
+7i/AuQ3cbCLNjSX3Y7jgGCFllWspZRRIYXzYPNkA9b2SbBnTLjjRLgnEkFBIGgvs
+7O2EFjaCuDRvydUEQhjq4ErwIsopj7B8h0QyZcbOKTbn3uFQ3n68wVJx2Sv/ADHT
+FIHrp/WIE96r19Niy34LKXkCgYB2W59VsuOKnMz01l5DeR5C+0HSWxS9SReIl2IO
+yEFSKullWyJeLIgyUaGy0990430feKI8whcrZXYumuah7IDN/KOwzhCk8vEfzWao
+N7bzfqtJVrh9HA7C7DVlO+6H4JFrtcoWPZUIomJ549w/yz6EN3ckoMC+a/Ck1TW9
+ka1QFwKBgQCywG6TrZz0UmOjyLQZ+8Q4uvZklSW5NAKBkNnyuQ2kd5rzyYgMPE8C
+Er8T88fdVIKvkhDyHhwcI7n58xE5Gr7wkwsrk/Hbd9/ZB2GgAPY3cATskK1v1McU
+YeX38CU0fUS4aoy26hWQXkViB47IGQ3jWo3ZCtzIJl8DI9/RsBWTnw==
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICYDCCAckCAQEwDQYJKoZIhvcNAQEFBQAwKDESMBAGA1UEAxMJbWl0bXByb3h5
+MRIwEAYDVQQKEwltaXRtcHJveHkwHhcNMTMwMTIwMDEwODEzWhcNMTUxMDE3MDEw
+ODEzWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UE
+ChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0
+EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+
+ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G
+3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/
+SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP
+G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABMA0GCSqGSIb3DQEBBQUA
+A4GBAFvI+cd47B85PQ970n2dU/PlA2/Hb1ldrrXh2guR4hX6vYx/uuk5yRI/n0Rd
+KOXJ3czO0bd2Fpe3ZoNpkW0pOSDej/Q+58ScuJd0gWCT/Sh1eRk6ZdC0kusOuWoY
+bPOPMkG45LPgUMFOnZEsfJP6P5mZIxlbCvSMFC25nPHWlct7
+-----END CERTIFICATE-----

pathod/test/data/clientcert/make

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+openssl genrsa -out client.key 2048
+openssl req -key client.key -new -out client.req
+openssl x509 -req -days 365 -in client.req -signkey client.key -out client.crt -extfile client.cnf -extensions ssl_client
+openssl x509 -req -days 1000 -in client.req -CA ~/.mitmproxy/mitmproxy-ca.pem -CAkey ~/.mitmproxy/mitmproxy-ca.pem -set_serial 00001 -out client.crt -extensions ssl_client
+cat client.key client.crt > client.pem
+openssl x509 -text -noout -in client.pem

pathod/test/data/file

@@ -0,0 +1 @@
+testfile

pathod/test/data/request

@@ -0,0 +1 @@
+get:/foo

pathod/test/data/response

@@ -0,0 +1 @@
+202

pathod/test/data/testkey.pem

@@ -0,0 +1,68 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5QIBAAKCAYEAwvtKxoZvBV2AxPAkCx8PXbuE7KeqK9bBvk8x+JchPMdf/KZj
+sdu2v6Gm8Hi053i7ZGxouFvonJxHAiK6cwk9OYQwa9fbOFf2mgWKEBO4fbCH93tW
+DCTdWVxFyNViAvxGHlJs3/IU03pIG29AgUnhRW8pGbabAfx8emcOZJZ3ykEuimaC
+4s7mRwdc63GXnbcjTtRkrJsBATI+xvPwuR2+4daX7sPCf0kel3bN2jMpwXfvk/Ww
+kJ2BIEeZCg0qIvyMjH9qrUirUnsmQnpPln0CGBbQEBsW9yMfGoFdREiMYls5jZeq
+NxjWNv1RTRIm/4RjMwyxnoTA9eDS9wwO2NnJS4vfXAnUTP4BYx8Pe4ZMA2Gm6YrC
+ysT6YA1xdHNpcuHXClxwmPj/cm8Z5kIg5clbNIK60ts9yFr/Ao3KPPYJ2GBv8/Oe
+ApPBJuubews+/9/13Ew/SJ1t2u28+sPbgXUG8dC2n4vWTvJwKf6Duqxgnm82zdzj
+SZoXRQsP984qiN7NAgMBAAECggGBALB6rqWdzCL5DLI0AQun40qdjaR95UKksNvF
+5p7we379nl2ZZKb5DSHJ+MWzG1pfJo2wqeAkIBiQQp0mPcgdVrMWeJVD3QHUbDng
+RaRjlRr+izJvCeUYANj+8ZLjwECfgf+z7yOLg1oeVeGvAp2C90jXYkYJx6c2lpxb
+ZuWYY3hHIw7V1iXfywIDIhFg0TBJMMYK68xmx7QDfFqrNPj4eWsDxqSvvv1iezPw
+rkWPBX49RjWPrW5XgSZsZ5J3c+oS1rZmIY7EAgopTWB/3wJjZR1Idz/9l9LIWlBP
+6zVC27CIZzSEeGguqNVeyzJ0TPWh5idYNRmSZr6eTUF0245LNO/gqvWKgRSNIZko
+HoBa2F1AvCiB67S1kxjwS5y3VkudZE4jkgGKcC2Ws/9QmOZ0HAsjI8VAMp2hj6iN
+0HdPMTNtsLgbhKyXsoZuW4YmwfSTPxGi2gvcI7GUozpTz84n1cOneJnz1ygx6Uru
+v8DpQg+VX6xTy4X6AK1F8OYNMZ/jaQKBwQDv30NevQStnGbTmcSr+0fd4rmWFklK
+V6B2X7zWynVpSGvCWkqVSp3mG6aiZItAltVMRL/9LT6zIheyClyd+vXIjR2+W210
+XMxrvz7A8qCXkvB2dyEhrMdCfZ7p8+kf+eD2c/Mnxb7VpmDfHYLx30JeQoBwjrwU
+Eul+dE1P+r8bWBaLTjlsipTya74yItWWAToXAo+s1BXBtXhEsLoe4FghlC0u724d
+ucjDaeICdLcerApdvg6Q6p4kVHaoF6ka6I8CgcEA0Bdc05ery9gLC6CclV+BhA5Q
+dfDq2P7qhc7e1ipwNRrQo2gy5HhgOkTL3dJWc+8rV6CBP/JfchnsW40tDOnPCTLT
+gg3n7vv3RHrtncApXuhIFR+B5xjohTPBzxRUMiAOre2d0F5b6eBXFjptf/1i2tQ+
+qdqJoyOGOZP0hKVslGIfz+CKc6WEkIqX7c91Msdr5myeaWDI5TsurfuKRBH395T3
+BMAi6oinAAEb1rdySenLO2A/0kVmBVlTpaN3TNjjAoHBAMvS4uQ1qSv8okNbfgrF
+UqPwa9JkzZImM2tinovFLU9xAl/7aTTCWrmU9Vs4JDuV71kHcjwnngeJCKl4tIpp
+HUB06Lk/5xnhYLKNpz087cjeSwXe5IBA2HBfXhFd+NH6+nVwwUUieq4A2n+8C/CK
+zVJbH9iE8Lv99fpFyQwU/R63EzD8Hz9j4ny7oLnpb6QvFrVGr98jt/kJwlBb+0sR
+RtIBnwMq4F7R5w5lgm6jzpZ5ibVuMeJh+k7Ulp7uu/rpcQKBwQDE3sWIvf7f7PaO
+OpbJz0CmYjCHVLWrNIlGrPAv6Jid9U+cuXEkrCpGFl5V77CxIH59+bEuga0BMztl
+ZkxP4khoqHhom6VpeWJ3nGGAFJRPYS0JJvTsYalilBPxSYdaoO+iZ6MdxpfozcE2
+m3KLW3uSEqlyYvpCqNJNWQhGEoeGXstADWyPevHPGgAhElwL/ZW8u9inU9Tc4sAI
+BGnMer+BsaJ+ERU3lK+Clony+z2aZiFLfIUE93lM6DT2CZBN2QcCgcAVk4L0bfA6
+HFnP/ZWNlnYWpOVFKcq57PX+J5/k7Tf34e2cYM2P0eqYggWZbzVd8qoCOQCHrAx0
+aZSSvEyKAVvzRNeqbm1oXaMojksMnrSX5henHjPbZlr1EmM7+zMnSTMkfVOx/6g1
+97sASej31XdOAgKCBJGymrwvYrCLW+P5cHqd+D8v/PvfpRIQM54p5ixRt3EYZvtR
+zGrzsr0OGyOLZtj1DB0a3kvajAAOCl3TawJSzviKo2mwc+/xj28MCQM=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIE4TCCA0mgAwIBAgIJALONCAWZxPhUMA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV
+BAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYDVQQD
+DAh0ZXN0LmNvbTAeFw0xNTA0MTgyMjA0NTNaFw00MjA5MDIyMjA0NTNaMEExCzAJ
+BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD
+VQQDDAh0ZXN0LmNvbTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAML7
+SsaGbwVdgMTwJAsfD127hOynqivWwb5PMfiXITzHX/ymY7Hbtr+hpvB4tOd4u2Rs
+aLhb6JycRwIiunMJPTmEMGvX2zhX9poFihATuH2wh/d7Vgwk3VlcRcjVYgL8Rh5S
+bN/yFNN6SBtvQIFJ4UVvKRm2mwH8fHpnDmSWd8pBLopmguLO5kcHXOtxl523I07U
+ZKybAQEyPsbz8LkdvuHWl+7Dwn9JHpd2zdozKcF375P1sJCdgSBHmQoNKiL8jIx/
+aq1Iq1J7JkJ6T5Z9AhgW0BAbFvcjHxqBXURIjGJbOY2XqjcY1jb9UU0SJv+EYzMM
+sZ6EwPXg0vcMDtjZyUuL31wJ1Ez+AWMfD3uGTANhpumKwsrE+mANcXRzaXLh1wpc
+cJj4/3JvGeZCIOXJWzSCutLbPcha/wKNyjz2Cdhgb/PzngKTwSbrm3sLPv/f9dxM
+P0idbdrtvPrD24F1BvHQtp+L1k7ycCn+g7qsYJ5vNs3c40maF0ULD/fOKojezQID
+AQABo4HbMIHYMAsGA1UdDwQEAwIFoDAdBgNVHQ4EFgQUbEgfTauEqEP/bnBtby1K
+bihJvcswcQYDVR0jBGowaIAUbEgfTauEqEP/bnBtby1KbihJvcuhRaRDMEExCzAJ
+BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD
+VQQDDAh0ZXN0LmNvbYIJALONCAWZxPhUMAwGA1UdEwQFMAMBAf8wKQYDVR0RBCIw
+IIIIdGVzdC5jb22CCXRlc3QyLmNvbYIJdGVzdDMuY29tMA0GCSqGSIb3DQEBCwUA
+A4IBgQBcTedXtUb91DxQRtg73iomz7cQ4niZntUBW8iE5rpoA7prtQNGHMCbHwaX
+tbWFkzBmL5JTBWvd/6AQ2LtiB3rYB3W/iRhbpsNJ501xaoOguPEQ9720Ph8TEveM
+208gNzGsEOcNALwyXj2y9M19NGu9zMa8eu1Tc3IsQaVaGKHx8XZn5HTNUx8EdcwI
+Z/Ji9ETDCL7+e5INv0tqfFSazWaQUwxM4IzPMkKTYRcMuN/6eog609k9r9pp32Ut
+rKlzc6GIkAlgJJ0Wkoz1V46DmJNJdJG7eLu/mtsB85j6hytIQeWTf1fll5YnMZLF
+HgNZtfYn8Q0oTdBQ0ZOaZeQCfZ8emYBdLJf2YB83uGRMjQ1FoeIxzQqiRq8WHRdb
+9Q45i0DINMnNp0DbLMA4numZ7wT9SQb6sql9eUyuCNDw7nGIWTHUNfLtU1Er3h1d
+icJuApx9+//UN/pGh0yTXb3fZbiI4IehRmkpnIWonIAwaVGm6JZU04wiIn8CuBho
+/qQdlS8=
+-----END CERTIFICATE-----

pathod/test/scripts/generate.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ! -f ./private.key ]
+then
+    openssl genrsa -out private.key 3072
+fi
+openssl req \
+    -batch \
+    -new -x509 \
+    -key private.key \
+    -sha256 \
+    -out cert.pem \
+    -days 9999 \
+    -config ./openssl.cnf
+openssl x509 -in cert.pem -text -noout
+cat ./private.key ./cert.pem > testcert.pem
+rm ./private.key ./cert.pem

pathod/test/scripts/openssl.cnf

@@ -0,0 +1,39 @@
+[ req ]
+default_bits		= 1024
+default_keyfile 	= privkey.pem
+distinguished_name	= req_distinguished_name
+x509_extensions	= v3_ca
+
+[ req_distinguished_name ]
+countryName			= Country Name (2 letter code)
+countryName_default		= NZ
+countryName_min			= 2
+countryName_max			= 2
+stateOrProvinceName		= State or Province Name (full name)
+stateOrProvinceName_default	= Otago
+localityName			= Locality Name (eg, city)
+0.organizationName		= Organization Name (eg, company)
+0.organizationName_default	= Pathod
+commonName			= Common Name (e.g. server FQDN or YOUR name)
+commonName_default = test.com
+commonName_max			= 64
+
+[ v3_req ]
+
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+
+[ v3_ca ]
+
+keyUsage = digitalSignature, keyEncipherment
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always,issuer:always
+basicConstraints = CA:true
+subjectAltName      = @alternate_names
+
+
+[ alternate_names ]
+
+DNS.1        = test.com
+DNS.2        = test2.com
+DNS.3        = test3.com

pathod/test/test_app.py

@@ -0,0 +1,85 @@
+import tutils
+
+
+class TestApp(tutils.DaemonTests):
+    SSL = False
+
+    def test_index(self):
+        r = self.getpath("/")
+        assert r.status_code == 200
+        assert r.content
+
+    def test_about(self):
+        r = self.getpath("/about")
+        assert r.ok
+
+    def test_download(self):
+        r = self.getpath("/download")
+        assert r.ok
+
+    def test_docs(self):
+        assert self.getpath("/docs/pathod").status_code == 200
+        assert self.getpath("/docs/pathoc").status_code == 200
+        assert self.getpath("/docs/language").status_code == 200
+        assert self.getpath("/docs/libpathod").status_code == 200
+        assert self.getpath("/docs/test").status_code == 200
+
+    def test_log(self):
+        assert self.getpath("/log").status_code == 200
+        assert self.get("200:da").status_code == 200
+        id = self.d.log()[0]["id"]
+        assert self.getpath("/log").status_code == 200
+        assert self.getpath("/log/%s" % id).status_code == 200
+        assert self.getpath("/log/9999999").status_code == 404
+
+    def test_log_binary(self):
+        assert self.get("200:h@10b=@10b:da")
+
+    def test_response_preview(self):
+        r = self.getpath("/response_preview", params=dict(spec="200"))
+        assert r.status_code == 200
+        assert 'Response' in r.content
+
+        r = self.getpath("/response_preview", params=dict(spec="foo"))
+        assert r.status_code == 200
+        assert 'Error' in r.content
+
+        r = self.getpath("/response_preview", params=dict(spec="200:b@100m"))
+        assert r.status_code == 200
+        assert "too large" in r.content
+
+        r = self.getpath("/response_preview", params=dict(spec="200:b@5k"))
+        assert r.status_code == 200
+        assert 'Response' in r.content
+
+        r = self.getpath(
+            "/response_preview",
+            params=dict(
+                spec="200:b<nonexistent"))
+        assert r.status_code == 200
+        assert 'File access denied' in r.content
+
+        r = self.getpath("/response_preview", params=dict(spec="200:b<file"))
+        assert r.status_code == 200
+        assert 'testfile' in r.content
+
+    def test_request_preview(self):
+        r = self.getpath("/request_preview", params=dict(spec="get:/"))
+        assert r.status_code == 200
+        assert 'Request' in r.content
+
+        r = self.getpath("/request_preview", params=dict(spec="foo"))
+        assert r.status_code == 200
+        assert 'Error' in r.content
+
+        r = self.getpath("/request_preview", params=dict(spec="get:/:b@100m"))
+        assert r.status_code == 200
+        assert "too large" in r.content
+
+        r = self.getpath("/request_preview", params=dict(spec="get:/:b@5k"))
+        assert r.status_code == 200
+        assert 'Request' in r.content
+
+        r = self.getpath("/request_preview", params=dict(spec=""))
+        assert r.status_code == 200
+        assert 'empty spec' in r.content

pathod/test/test_language_actions.py

@@ -0,0 +1,135 @@
+import cStringIO
+
+from libpathod.language import actions
+from libpathod import language
+
+
+def parse_request(s):
+    return language.parse_pathoc(s).next()
+
+
+def test_unique_name():
+    assert not actions.PauseAt(0, "f").unique_name
+    assert actions.DisconnectAt(0).unique_name
+
+
+class TestDisconnects:
+
+    def test_parse_pathod(self):
+        a = language.parse_pathod("400:d0").next().actions[0]
+        assert a.spec() == "d0"
+        a = language.parse_pathod("400:dr").next().actions[0]
+        assert a.spec() == "dr"
+
+    def test_at(self):
+        e = actions.DisconnectAt.expr()
+        v = e.parseString("d0")[0]
+        assert isinstance(v, actions.DisconnectAt)
+        assert v.offset == 0
+
+        v = e.parseString("d100")[0]
+        assert v.offset == 100
+
+        e = actions.DisconnectAt.expr()
+        v = e.parseString("dr")[0]
+        assert v.offset == "r"
+
+    def test_spec(self):
+        assert actions.DisconnectAt("r").spec() == "dr"
+        assert actions.DisconnectAt(10).spec() == "d10"
+
+
+class TestInject:
+
+    def test_parse_pathod(self):
+        a = language.parse_pathod("400:ir,@100").next().actions[0]
+        assert a.offset == "r"
+        assert a.value.datatype == "bytes"
+        assert a.value.usize == 100
+
+        a = language.parse_pathod("400:ia,@100").next().actions[0]
+        assert a.offset == "a"
+
+    def test_at(self):
+        e = actions.InjectAt.expr()
+        v = e.parseString("i0,'foo'")[0]
+        assert v.value.val == "foo"
+        assert v.offset == 0
+        assert isinstance(v, actions.InjectAt)
+
+        v = e.parseString("ir,'foo'")[0]
+        assert v.offset == "r"
+
+    def test_serve(self):
+        s = cStringIO.StringIO()
+        r = language.parse_pathod("400:i0,'foo'").next()
+        assert language.serve(r, s, {})
+
+    def test_spec(self):
+        e = actions.InjectAt.expr()
+        v = e.parseString("i0,'foo'")[0]
+        assert v.spec() == 'i0,"foo"'
+
+    def test_spec(self):
+        e = actions.InjectAt.expr()
+        v = e.parseString("i0,@100")[0]
+        v2 = v.freeze({})
+        v3 = v2.freeze({})
+        assert v2.value.val == v3.value.val
+
+
+class TestPauses:
+
+    def test_parse_pathod(self):
+        e = actions.PauseAt.expr()
+        v = e.parseString("p10,10")[0]
+        assert v.seconds == 10
+        assert v.offset == 10
+
+        v = e.parseString("p10,f")[0]
+        assert v.seconds == "f"
+
+        v = e.parseString("pr,f")[0]
+        assert v.offset == "r"
+
+        v = e.parseString("pa,f")[0]
+        assert v.offset == "a"
+
+    def test_request(self):
+        r = language.parse_pathod('400:p10,10').next()
+        assert r.actions[0].spec() == "p10,10"
+
+    def test_spec(self):
+        assert actions.PauseAt("r", 5).spec() == "pr,5"
+        assert actions.PauseAt(0, 5).spec() == "p0,5"
+        assert actions.PauseAt(0, "f").spec() == "p0,f"
+
+    def test_freeze(self):
+        l = actions.PauseAt("r", 5)
+        assert l.freeze({}).spec() == l.spec()
+
+
+class Test_Action:
+
+    def test_cmp(self):
+        a = actions.DisconnectAt(0)
+        b = actions.DisconnectAt(1)
+        c = actions.DisconnectAt(0)
+        assert a < b
+        assert a == c
+        l = sorted([b, a])
+        assert l[0].offset == 0
+
+    def test_resolve(self):
+        r = parse_request('GET:"/foo"')
+        e = actions.DisconnectAt("r")
+        ret = e.resolve({}, r)
+        assert isinstance(ret.offset, int)
+
+    def test_repr(self):
+        e = actions.DisconnectAt("r")
+        assert repr(e)
+
+    def test_freeze(self):
+        l = actions.DisconnectAt(5)
+        assert l.freeze({}).spec() == l.spec()

pathod/test/test_language_base.py

@@ -0,0 +1,352 @@
+import os
+from libpathod import language
+from libpathod.language import base, exceptions
+import tutils
+
+
+def parse_request(s):
+    return language.parse_pathoc(s).next()
+
+
+def test_times():
+    reqs = list(language.parse_pathoc("get:/:x5"))
+    assert len(reqs) == 5
+    assert not reqs[0].times
+
+
+def test_caseless_literal():
+    class CL(base.CaselessLiteral):
+        TOK = "foo"
+    v = CL("foo")
+    assert v.expr()
+    assert v.values(language.Settings())
+
+
+class TestTokValueNakedLiteral:
+
+    def test_expr(self):
+        v = base.TokValueNakedLiteral("foo")
+        assert v.expr()
+
+    def test_spec(self):
+        v = base.TokValueNakedLiteral("foo")
+        assert v.spec() == repr(v) == "foo"
+
+        v = base.TokValueNakedLiteral("f\x00oo")
+        assert v.spec() == repr(v) == r"f\x00oo"
+
+
+class TestTokValueLiteral:
+
+    def test_espr(self):
+        v = base.TokValueLiteral("foo")
+        assert v.expr()
+        assert v.val == "foo"
+
+        v = base.TokValueLiteral("foo\n")
+        assert v.expr()
+        assert v.val == "foo\n"
+        assert repr(v)
+
+    def test_spec(self):
+        v = base.TokValueLiteral("foo")
+        assert v.spec() == r"'foo'"
+
+        v = base.TokValueLiteral("f\x00oo")
+        assert v.spec() == repr(v) == r"'f\x00oo'"
+
+        v = base.TokValueLiteral("\"")
+        assert v.spec() == repr(v) == '\'"\''
+
+    def roundtrip(self, spec):
+        e = base.TokValueLiteral.expr()
+        v = base.TokValueLiteral(spec)
+        v2 = e.parseString(v.spec())
+        assert v.val == v2[0].val
+        assert v.spec() == v2[0].spec()
+
+    def test_roundtrip(self):
+        self.roundtrip("'")
+        self.roundtrip('\'')
+        self.roundtrip("a")
+        self.roundtrip("\"")
+        # self.roundtrip("\\")
+        self.roundtrip("200:b'foo':i23,'\\''")
+        self.roundtrip("\a")
+
+
+class TestTokValueGenerate:
+
+    def test_basic(self):
+        v = base.TokValue.parseString("@10b")[0]
+        assert v.usize == 10
+        assert v.unit == "b"
+        assert v.bytes() == 10
+        v = base.TokValue.parseString("@10")[0]
+        assert v.unit == "b"
+        v = base.TokValue.parseString("@10k")[0]
+        assert v.bytes() == 10240
+        v = base.TokValue.parseString("@10g")[0]
+        assert v.bytes() == 1024 ** 3 * 10
+
+        v = base.TokValue.parseString("@10g,digits")[0]
+        assert v.datatype == "digits"
+        g = v.get_generator({})
+        assert g[:100]
+
+        v = base.TokValue.parseString("@10,digits")[0]
+        assert v.unit == "b"
+        assert v.datatype == "digits"
+
+    def test_spec(self):
+        v = base.TokValueGenerate(1, "b", "bytes")
+        assert v.spec() == repr(v) == "@1"
+
+        v = base.TokValueGenerate(1, "k", "bytes")
+        assert v.spec() == repr(v) == "@1k"
+
+        v = base.TokValueGenerate(1, "k", "ascii")
+        assert v.spec() == repr(v) == "@1k,ascii"
+
+        v = base.TokValueGenerate(1, "b", "ascii")
+        assert v.spec() == repr(v) == "@1,ascii"
+
+    def test_freeze(self):
+        v = base.TokValueGenerate(100, "b", "ascii")
+        f = v.freeze(language.Settings())
+        assert len(f.val) == 100
+
+
+class TestTokValueFile:
+
+    def test_file_value(self):
+        v = base.TokValue.parseString("<'one two'")[0]
+        assert str(v)
+        assert v.path == "one two"
+
+        v = base.TokValue.parseString("<path")[0]
+        assert v.path == "path"
+
+    def test_access_control(self):
+        v = base.TokValue.parseString("<path")[0]
+        with tutils.tmpdir() as t:
+            p = os.path.join(t, "path")
+            with open(p, "wb") as f:
+                f.write("x" * 10000)
+
+            assert v.get_generator(language.Settings(staticdir=t))
+
+            v = base.TokValue.parseString("<path2")[0]
+            tutils.raises(
+                exceptions.FileAccessDenied,
+                v.get_generator,
+                language.Settings(staticdir=t)
+            )
+            tutils.raises(
+                "access disabled",
+                v.get_generator,
+                language.Settings()
+            )
+
+            v = base.TokValue.parseString("</outside")[0]
+            tutils.raises(
+                "outside",
+                v.get_generator,
+                language.Settings(staticdir=t)
+            )
+
+    def test_spec(self):
+        v = base.TokValue.parseString("<'one two'")[0]
+        v2 = base.TokValue.parseString(v.spec())[0]
+        assert v2.path == "one two"
+
+    def test_freeze(self):
+        v = base.TokValue.parseString("<'one two'")[0]
+        v2 = v.freeze({})
+        assert v2.path == v.path
+
+
+class TestMisc:
+
+    def test_generators(self):
+        v = base.TokValue.parseString("'val'")[0]
+        g = v.get_generator({})
+        assert g[:] == "val"
+
+    def test_value(self):
+        assert base.TokValue.parseString("'val'")[0].val == "val"
+        assert base.TokValue.parseString('"val"')[0].val == "val"
+        assert base.TokValue.parseString('"\'val\'"')[0].val == "'val'"
+
+    def test_value(self):
+        class TT(base.Value):
+            preamble = "m"
+        e = TT.expr()
+        v = e.parseString("m'msg'")[0]
+        assert v.value.val == "msg"
+
+        s = v.spec()
+        assert s == e.parseString(s)[0].spec()
+
+        v = e.parseString("m@100")[0]
+        v2 = v.freeze({})
+        v3 = v2.freeze({})
+        assert v2.value.val == v3.value.val
+
+    def test_fixedlengthvalue(self):
+        class TT(base.FixedLengthValue):
+            preamble = "m"
+            length = 4
+
+        e = TT.expr()
+        assert e.parseString("m@4")
+        tutils.raises("invalid value length", e.parseString, "m@100")
+        tutils.raises("invalid value length", e.parseString, "m@1")
+
+        with tutils.tmpdir() as t:
+            p = os.path.join(t, "path")
+            s = base.Settings(staticdir=t)
+            with open(p, "wb") as f:
+                f.write("a" * 20)
+            v = e.parseString("m<path")[0]
+            tutils.raises("invalid value length", v.values, s)
+
+            p = os.path.join(t, "path")
+            with open(p, "wb") as f:
+                f.write("a" * 4)
+            v = e.parseString("m<path")[0]
+            assert v.values(s)
+
+
+class TKeyValue(base.KeyValue):
+    preamble = "h"
+
+    def values(self, settings):
+        return [
+            self.key.get_generator(settings),
+            ": ",
+            self.value.get_generator(settings),
+            "\r\n",
+        ]
+
+
+class TestKeyValue:
+
+    def test_simple(self):
+        e = TKeyValue.expr()
+        v = e.parseString("h'foo'='bar'")[0]
+        assert v.key.val == "foo"
+        assert v.value.val == "bar"
+
+        v2 = e.parseString(v.spec())[0]
+        assert v2.key.val == v.key.val
+        assert v2.value.val == v.value.val
+
+        s = v.spec()
+        assert s == e.parseString(s)[0].spec()
+
+    def test_freeze(self):
+        e = TKeyValue.expr()
+        v = e.parseString("h@10=@10'")[0]
+        v2 = v.freeze({})
+        v3 = v2.freeze({})
+        assert v2.key.val == v3.key.val
+        assert v2.value.val == v3.value.val
+
+
+def test_intfield():
+    class TT(base.IntField):
+        preamble = "t"
+        names = {
+            "one": 1,
+            "two": 2,
+            "three": 3
+        }
+        max = 4
+    e = TT.expr()
+
+    v = e.parseString("tone")[0]
+    assert v.value == 1
+    assert v.spec() == "tone"
+    assert v.values(language.Settings())
+
+    v = e.parseString("t1")[0]
+    assert v.value == 1
+    assert v.spec() == "t1"
+
+    v = e.parseString("t4")[0]
+    assert v.value == 4
+    assert v.spec() == "t4"
+
+    tutils.raises("can't exceed", e.parseString, "t5")
+
+
+def test_options_or_value():
+    class TT(base.OptionsOrValue):
+        options = [
+            "one",
+            "two",
+            "three"
+        ]
+    e = TT.expr()
+    assert e.parseString("one")[0].value.val == "one"
+    assert e.parseString("'foo'")[0].value.val == "foo"
+    assert e.parseString("'get'")[0].value.val == "get"
+
+    assert e.parseString("one")[0].spec() == "one"
+    assert e.parseString("'foo'")[0].spec() == "'foo'"
+
+    s = e.parseString("one")[0].spec()
+    assert s == e.parseString(s)[0].spec()
+
+    s = e.parseString("'foo'")[0].spec()
+    assert s == e.parseString(s)[0].spec()
+
+    v = e.parseString("@100")[0]
+    v2 = v.freeze({})
+    v3 = v2.freeze({})
+    assert v2.value.val == v3.value.val
+
+
+def test_integer():
+    e = base.Integer.expr()
+    v = e.parseString("200")[0]
+    assert v.string() == "200"
+    assert v.spec() == "200"
+
+    assert v.freeze({}).value == v.value
+
+    class BInt(base.Integer):
+        bounds = (1, 5)
+
+    tutils.raises("must be between", BInt, 0)
+    tutils.raises("must be between", BInt, 6)
+    assert BInt(5)
+    assert BInt(1)
+    assert BInt(3)
+
+
+class TBoolean(base.Boolean):
+    name = "test"
+
+
+def test_unique_name():
+    b = TBoolean(True)
+    assert b.unique_name
+
+
+class test_boolean():
+    e = TBoolean.expr()
+    assert e.parseString("test")[0].value
+    assert not e.parseString("-test")[0].value
+
+    def roundtrip(s):
+        e = TBoolean.expr()
+        s2 = e.parseString(s)[0].spec()
+        v1 = e.parseString(s)[0].value
+        v2 = e.parseString(s2)[0].value
+        assert s == s2
+        assert v1 == v2
+
+    roundtrip("test")
+    roundtrip("-test")

pathod/test/test_language_generators.py

@@ -0,0 +1,42 @@
+import os
+
+from libpathod.language import generators
+import tutils
+
+
+def test_randomgenerator():
+    g = generators.RandomGenerator("bytes", 100)
+    assert repr(g)
+    assert len(g[:10]) == 10
+    assert len(g[1:10]) == 9
+    assert len(g[:1000]) == 100
+    assert len(g[1000:1001]) == 0
+    assert g[0]
+
+
+def test_filegenerator():
+    with tutils.tmpdir() as t:
+        path = os.path.join(t, "foo")
+        f = open(path, "wb")
+        f.write("x" * 10000)
+        f.close()
+        g = generators.FileGenerator(path)
+        assert len(g) == 10000
+        assert g[0] == "x"
+        assert g[-1] == "x"
+        assert g[0:5] == "xxxxx"
+        assert repr(g)
+        # remove all references to FileGenerator instance to close the file
+        # handle.
+        del g
+
+
+def test_transform_generator():
+    def trans(offset, data):
+        return "a" * len(data)
+    g = "one"
+    t = generators.TransformGenerator(g, trans)
+    assert len(t) == len(g)
+    assert t[0] == "a"
+    assert t[:] == "a" * len(g)
+    assert repr(t)

pathod/test/test_language_http.py

@@ -0,0 +1,358 @@
+import cStringIO
+
+from libpathod import language
+from libpathod.language import http, base
+import tutils
+
+
+def parse_request(s):
+    return language.parse_pathoc(s).next()
+
+
+def test_make_error_response():
+    d = cStringIO.StringIO()
+    s = http.make_error_response("foo")
+    language.serve(s, d, {})
+
+
+class TestRequest:
+
+    def test_nonascii(self):
+        tutils.raises("ascii", parse_request, "get:\xf0")
+
+    def test_err(self):
+        tutils.raises(language.ParseException, parse_request, 'GET')
+
+    def test_simple(self):
+        r = parse_request('GET:"/foo"')
+        assert r.method.string() == "GET"
+        assert r.path.string() == "/foo"
+        r = parse_request('GET:/foo')
+        assert r.path.string() == "/foo"
+        r = parse_request('GET:@1k')
+        assert len(r.path.string()) == 1024
+
+    def test_multiple(self):
+        r = list(language.parse_pathoc("GET:/ PUT:/"))
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "PUT"
+        assert len(r) == 2
+
+        l = """
+            GET
+            "/foo"
+            ir,@1
+
+            PUT
+
+            "/foo
+
+
+
+            bar"
+
+            ir,@1
+        """
+        r = list(language.parse_pathoc(l))
+        assert len(r) == 2
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "PUT"
+
+        l = """
+            get:"http://localhost:9999/p/200":ir,@1
+            get:"http://localhost:9999/p/200":ir,@2
+        """
+        r = list(language.parse_pathoc(l))
+        assert len(r) == 2
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "GET"
+
+    def test_nested_response(self):
+        l = "get:/p:s'200'"
+        r = list(language.parse_pathoc(l))
+        assert len(r) == 1
+        assert len(r[0].tokens) == 3
+        assert isinstance(r[0].tokens[2], http.NestedResponse)
+        assert r[0].values({})
+
+    def test_render(self):
+        s = cStringIO.StringIO()
+        r = parse_request("GET:'/foo'")
+        assert language.serve(
+            r,
+            s,
+            language.Settings(request_host="foo.com")
+        )
+
+    def test_multiline(self):
+        l = """
+            GET
+            "/foo"
+            ir,@1
+        """
+        r = parse_request(l)
+        assert r.method.string() == "GET"
+        assert r.path.string() == "/foo"
+        assert r.actions
+
+        l = """
+            GET
+
+            "/foo
+
+
+
+            bar"
+
+            ir,@1
+        """
+        r = parse_request(l)
+        assert r.method.string() == "GET"
+        assert r.path.string().endswith("bar")
+        assert r.actions
+
+    def test_spec(self):
+        def rt(s):
+            s = parse_request(s).spec()
+            assert parse_request(s).spec() == s
+        rt("get:/foo")
+        rt("get:/foo:da")
+
+    def test_freeze(self):
+        r = parse_request("GET:/:b@100").freeze(language.Settings())
+        assert len(r.spec()) > 100
+
+    def test_path_generator(self):
+        r = parse_request("GET:@100").freeze(language.Settings())
+        assert len(r.spec()) > 100
+
+    def test_websocket(self):
+        r = parse_request('ws:/path/')
+        res = r.resolve(language.Settings())
+        assert res.method.string().lower() == "get"
+        assert res.tok(http.Path).value.val == "/path/"
+        assert res.tok(http.Method).value.val.lower() == "get"
+        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+
+        r = parse_request('ws:put:/path/')
+        res = r.resolve(language.Settings())
+        assert r.method.string().lower() == "put"
+        assert res.tok(http.Path).value.val == "/path/"
+        assert res.tok(http.Method).value.val.lower() == "put"
+        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+
+
+class TestResponse:
+
+    def dummy_response(self):
+        return language.parse_pathod("400'msg'").next()
+
+    def test_response(self):
+        r = language.parse_pathod("400:m'msg'").next()
+        assert r.status_code.string() == "400"
+        assert r.reason.string() == "msg"
+
+        r = language.parse_pathod("400:m'msg':b@100b").next()
+        assert r.reason.string() == "msg"
+        assert r.body.values({})
+        assert str(r)
+
+        r = language.parse_pathod("200").next()
+        assert r.status_code.string() == "200"
+        assert not r.reason
+        assert "OK" in [i[:] for i in r.preamble({})]
+
+    def test_render(self):
+        s = cStringIO.StringIO()
+        r = language.parse_pathod("400:m'msg'").next()
+        assert language.serve(r, s, {})
+
+        r = language.parse_pathod("400:p0,100:dr").next()
+        assert "p0" in r.spec()
+        s = r.preview_safe()
+        assert "p0" not in s.spec()
+
+    def test_raw(self):
+        s = cStringIO.StringIO()
+        r = language.parse_pathod("400:b'foo'").next()
+        language.serve(r, s, {})
+        v = s.getvalue()
+        assert "Content-Length" in v
+
+        s = cStringIO.StringIO()
+        r = language.parse_pathod("400:b'foo':r").next()
+        language.serve(r, s, {})
+        v = s.getvalue()
+        assert "Content-Length" not in v
+
+    def test_length(self):
+        def testlen(x):
+            s = cStringIO.StringIO()
+            x = x.next()
+            language.serve(x, s, language.Settings())
+            assert x.length(language.Settings()) == len(s.getvalue())
+        testlen(language.parse_pathod("400:m'msg':r"))
+        testlen(language.parse_pathod("400:m'msg':h'foo'='bar':r"))
+        testlen(language.parse_pathod("400:m'msg':h'foo'='bar':b@100b:r"))
+
+    def test_maximum_length(self):
+        def testlen(x):
+            x = x.next()
+            s = cStringIO.StringIO()
+            m = x.maximum_length({})
+            language.serve(x, s, {})
+            assert m >= len(s.getvalue())
+
+        r = language.parse_pathod("400:m'msg':b@100:d0")
+        testlen(r)
+
+        r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'")
+        testlen(r)
+
+        r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'")
+        testlen(r)
+
+    def test_parse_err(self):
+        tutils.raises(
+            language.ParseException, language.parse_pathod, "400:msg,b:"
+        )
+        try:
+            language.parse_pathod("400'msg':b:")
+        except language.ParseException as v:
+            assert v.marked()
+            assert str(v)
+
+    def test_nonascii(self):
+        tutils.raises("ascii", language.parse_pathod, "foo:b\xf0")
+
+    def test_parse_header(self):
+        r = language.parse_pathod('400:h"foo"="bar"').next()
+        assert http.get_header("foo", r.headers)
+
+    def test_parse_pause_before(self):
+        r = language.parse_pathod("400:p0,10").next()
+        assert r.actions[0].spec() == "p0,10"
+
+    def test_parse_pause_after(self):
+        r = language.parse_pathod("400:pa,10").next()
+        assert r.actions[0].spec() == "pa,10"
+
+    def test_parse_pause_random(self):
+        r = language.parse_pathod("400:pr,10").next()
+        assert r.actions[0].spec() == "pr,10"
+
+    def test_parse_stress(self):
+        # While larger values are known to work on linux, len() technically
+        # returns an int and a python 2.7 int on windows has 32bit precision.
+        # Therefore, we should keep the body length < 2147483647 bytes in our
+        # tests.
+        r = language.parse_pathod("400:b@1g").next()
+        assert r.length({})
+
+    def test_spec(self):
+        def rt(s):
+            s = language.parse_pathod(s).next().spec()
+            assert language.parse_pathod(s).next().spec() == s
+        rt("400:b@100g")
+        rt("400")
+        rt("400:da")
+
+    def test_websockets(self):
+        r = language.parse_pathod("ws").next()
+        tutils.raises("no websocket key", r.resolve, language.Settings())
+        res = r.resolve(language.Settings(websocket_key="foo"))
+        assert res.status_code.string() == "101"
+
+
+def test_ctype_shortcut():
+    e = http.ShortcutContentType.expr()
+    v = e.parseString("c'foo'")[0]
+    assert v.key.val == "Content-Type"
+    assert v.value.val == "foo"
+
+    s = v.spec()
+    assert s == e.parseString(s)[0].spec()
+
+    e = http.ShortcutContentType.expr()
+    v = e.parseString("c@100")[0]
+    v2 = v.freeze({})
+    v3 = v2.freeze({})
+    assert v2.value.val == v3.value.val
+
+
+def test_location_shortcut():
+    e = http.ShortcutLocation.expr()
+    v = e.parseString("l'foo'")[0]
+    assert v.key.val == "Location"
+    assert v.value.val == "foo"
+
+    s = v.spec()
+    assert s == e.parseString(s)[0].spec()
+
+    e = http.ShortcutLocation.expr()
+    v = e.parseString("l@100")[0]
+    v2 = v.freeze({})
+    v3 = v2.freeze({})
+    assert v2.value.val == v3.value.val
+
+
+def test_shortcuts():
+    assert language.parse_pathod(
+        "400:c'foo'").next().headers[0].key.val == "Content-Type"
+    assert language.parse_pathod(
+        "400:l'foo'").next().headers[0].key.val == "Location"
+
+    assert "Android" in tutils.render(parse_request("get:/:ua"))
+    assert "User-Agent" in tutils.render(parse_request("get:/:ua"))
+
+
+def test_user_agent():
+    e = http.ShortcutUserAgent.expr()
+    v = e.parseString("ua")[0]
+    assert "Android" in v.string()
+
+    e = http.ShortcutUserAgent.expr()
+    v = e.parseString("u'a'")[0]
+    assert "Android" not in v.string()
+
+    v = e.parseString("u@100'")[0]
+    assert len(str(v.freeze({}).value)) > 100
+    v2 = v.freeze({})
+    v3 = v2.freeze({})
+    assert v2.value.val == v3.value.val
+
+
+def test_nested_response():
+    e = http.NestedResponse.expr()
+    v = e.parseString("s'200'")[0]
+    assert v.value.val == "200"
+    tutils.raises(
+        language.ParseException,
+        e.parseString,
+        "s'foo'"
+    )
+
+    v = e.parseString('s"200:b@1"')[0]
+    assert "@1" in v.spec()
+    f = v.freeze({})
+    assert "@1" not in f.spec()
+
+
+def test_nested_response_freeze():
+    e = http.NestedResponse(
+        base.TokValueLiteral(
+            "200:b'foo':i10,'\\x27'".encode(
+                "string_escape"
+            )
+        )
+    )
+    assert e.freeze({})
+    assert e.values({})
+
+
+def test_unique_components():
+    tutils.raises(
+        "multiple body clauses",
+        language.parse_pathod,
+        "400:b@1:b@1"
+    )

pathod/test/test_language_http2.py

@@ -0,0 +1,233 @@
+import cStringIO
+
+import netlib
+from netlib import tcp
+from netlib.http import user_agents
+
+from libpathod import language
+from libpathod.language import http2, base
+import tutils
+
+
+def parse_request(s):
+    return language.parse_pathoc(s, True).next()
+
+
+def parse_response(s):
+    return language.parse_pathod(s, True).next()
+
+
+def default_settings():
+    return language.Settings(
+        request_host="foo.com",
+        protocol=netlib.http.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234)))
+    )
+
+
+def test_make_error_response():
+    d = cStringIO.StringIO()
+    s = http2.make_error_response("foo", "bar")
+    language.serve(s, d, default_settings())
+
+
+class TestRequest:
+
+    def test_cached_values(self):
+        req = parse_request("get:/")
+        req_id = id(req)
+        assert req_id == id(req.resolve(default_settings()))
+        assert req.values(default_settings()) == req.values(default_settings())
+
+    def test_nonascii(self):
+        tutils.raises("ascii", parse_request, "get:\xf0")
+
+    def test_err(self):
+        tutils.raises(language.ParseException, parse_request, 'GET')
+
+    def test_simple(self):
+        r = parse_request('GET:"/foo"')
+        assert r.method.string() == "GET"
+        assert r.path.string() == "/foo"
+        r = parse_request('GET:/foo')
+        assert r.path.string() == "/foo"
+
+    def test_multiple(self):
+        r = list(language.parse_pathoc("GET:/ PUT:/"))
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "PUT"
+        assert len(r) == 2
+
+        l = """
+            GET
+            "/foo"
+
+            PUT
+
+            "/foo
+
+
+
+            bar"
+        """
+        r = list(language.parse_pathoc(l, True))
+        assert len(r) == 2
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "PUT"
+
+        l = """
+            get:"http://localhost:9999/p/200"
+            get:"http://localhost:9999/p/200"
+        """
+        r = list(language.parse_pathoc(l, True))
+        assert len(r) == 2
+        assert r[0].method.string() == "GET"
+        assert r[1].method.string() == "GET"
+
+    def test_render_simple(self):
+        s = cStringIO.StringIO()
+        r = parse_request("GET:'/foo'")
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_raw_content_length(self):
+        r = parse_request('GET:/:r')
+        assert len(r.headers) == 0
+
+        r = parse_request('GET:/:r:b"foobar"')
+        assert len(r.headers) == 0
+
+        r = parse_request('GET:/')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-length", "0")
+
+        r = parse_request('GET:/:b"foobar"')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-length", "6")
+
+        r = parse_request('GET:/:b"foobar":h"content-length"="42"')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-length", "42")
+
+        r = parse_request('GET:/:r:b"foobar":h"content-length"="42"')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-length", "42")
+
+    def test_content_type(self):
+        r = parse_request('GET:/:r:c"foobar"')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-type", "foobar")
+
+    def test_user_agent(self):
+        r = parse_request('GET:/:r:ua')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("user-agent", user_agents.get_by_shortcut('a')[2])
+
+    def test_render_with_headers(self):
+        s = cStringIO.StringIO()
+        r = parse_request('GET:/foo:h"foo"="bar"')
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_nested_response(self):
+        l = "get:/p/:s'200'"
+        r = parse_request(l)
+        assert len(r.tokens) == 3
+        assert isinstance(r.tokens[2], http2.NestedResponse)
+        assert r.values(default_settings())
+
+
+    def test_render_with_body(self):
+        s = cStringIO.StringIO()
+        r = parse_request("GET:'/foo':bfoobar")
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_spec(self):
+        def rt(s):
+            s = parse_request(s).spec()
+            assert parse_request(s).spec() == s
+        rt("get:/foo")
+
+
+class TestResponse:
+
+    def test_cached_values(self):
+        res = parse_response("200")
+        res_id = id(res)
+        assert res_id == id(res.resolve(default_settings()))
+        assert res.values(default_settings()) == res.values(default_settings())
+
+    def test_nonascii(self):
+        tutils.raises("ascii", parse_response, "200:\xf0")
+
+    def test_err(self):
+        tutils.raises(language.ParseException, parse_response, 'GET:/')
+
+    def test_raw_content_length(self):
+        r = parse_response('200:r')
+        assert len(r.headers) == 0
+
+        r = parse_response('200')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-length", "0")
+
+    def test_content_type(self):
+        r = parse_response('200:r:c"foobar"')
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("content-type", "foobar")
+
+    def test_simple(self):
+        r = parse_response('200:r:h"foo"="bar"')
+        assert r.status_code.string() == "200"
+        assert len(r.headers) == 1
+        assert r.headers[0].values(default_settings()) == ("foo", "bar")
+        assert r.body is None
+
+        r = parse_response('200:r:h"foo"="bar":bfoobar:h"bla"="fasel"')
+        assert r.status_code.string() == "200"
+        assert len(r.headers) == 2
+        assert r.headers[0].values(default_settings()) == ("foo", "bar")
+        assert r.headers[1].values(default_settings()) == ("bla", "fasel")
+        assert r.body.string() == "foobar"
+
+    def test_render_simple(self):
+        s = cStringIO.StringIO()
+        r = parse_response('200')
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_render_with_headers(self):
+        s = cStringIO.StringIO()
+        r = parse_response('200:h"foo"="bar"')
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_render_with_body(self):
+        s = cStringIO.StringIO()
+        r = parse_response('200:bfoobar')
+        assert language.serve(
+            r,
+            s,
+            default_settings(),
+        )
+
+    def test_spec(self):
+        def rt(s):
+            s = parse_response(s).spec()
+            assert parse_response(s).spec() == s
+        rt("200:bfoobar")

pathod/test/test_language_websocket.py

@@ -0,0 +1,142 @@
+
+from libpathod import language
+from libpathod.language import websockets
+import netlib.websockets
+import tutils
+
+
+def parse_request(s):
+    return language.parse_pathoc(s).next()
+
+
+class TestWebsocketFrame:
+
+    def _test_messages(self, specs, message_klass):
+        for i in specs:
+            wf = parse_request(i)
+            assert isinstance(wf, message_klass)
+            assert wf
+            assert wf.values(language.Settings())
+            assert wf.resolve(language.Settings())
+
+            spec = wf.spec()
+            wf2 = parse_request(spec)
+            assert wf2.spec() == spec
+
+    def test_server_values(self):
+        specs = [
+            "wf",
+            "wf:dr",
+            "wf:b'foo'",
+            "wf:mask:r'foo'",
+            "wf:l1024:b'foo'",
+            "wf:cbinary",
+            "wf:c1",
+            "wf:mask:knone",
+            "wf:fin",
+            "wf:fin:rsv1:rsv2:rsv3:mask",
+            "wf:-fin:-rsv1:-rsv2:-rsv3:-mask",
+            "wf:k@4",
+            "wf:x10",
+        ]
+        self._test_messages(specs, websockets.WebsocketFrame)
+
+    def test_parse_websocket_frames(self):
+        wf = language.parse_websocket_frame("wf:x10")
+        assert len(list(wf)) == 10
+        tutils.raises(
+            language.ParseException,
+            language.parse_websocket_frame,
+            "wf:x"
+        )
+
+    def test_client_values(self):
+        specs = [
+            "wf:f'wf'",
+        ]
+        self._test_messages(specs, websockets.WebsocketClientFrame)
+
+    def test_nested_frame(self):
+        wf = parse_request("wf:f'wf'")
+        assert wf.nested_frame
+
+    def test_flags(self):
+        wf = parse_request("wf:fin:mask:rsv1:rsv2:rsv3")
+        frm = netlib.websockets.Frame.from_bytes(tutils.render(wf))
+        assert frm.header.fin
+        assert frm.header.mask
+        assert frm.header.rsv1
+        assert frm.header.rsv2
+        assert frm.header.rsv3
+
+        wf = parse_request("wf:-fin:-mask:-rsv1:-rsv2:-rsv3")
+        frm = netlib.websockets.Frame.from_bytes(tutils.render(wf))
+        assert not frm.header.fin
+        assert not frm.header.mask
+        assert not frm.header.rsv1
+        assert not frm.header.rsv2
+        assert not frm.header.rsv3
+
+    def fr(self, spec, **kwargs):
+        settings = language.base.Settings(**kwargs)
+        wf = parse_request(spec)
+        return netlib.websockets.Frame.from_bytes(tutils.render(wf, settings))
+
+    def test_construction(self):
+        assert self.fr("wf:c1").header.opcode == 1
+        assert self.fr("wf:c0").header.opcode == 0
+        assert self.fr("wf:cbinary").header.opcode ==\
+            netlib.websockets.OPCODE.BINARY
+        assert self.fr("wf:ctext").header.opcode ==\
+            netlib.websockets.OPCODE.TEXT
+
+    def test_rawbody(self):
+        frm = self.fr("wf:mask:r'foo'")
+        assert len(frm.payload) == 3
+        assert frm.payload != "foo"
+
+        assert self.fr("wf:r'foo'").payload == "foo"
+
+    def test_construction(self):
+        # Simple server frame
+        frm = self.fr("wf:b'foo'")
+        assert not frm.header.mask
+        assert not frm.header.masking_key
+
+        # Simple client frame
+        frm = self.fr("wf:b'foo'", is_client=True)
+        assert frm.header.mask
+        assert frm.header.masking_key
+        frm = self.fr("wf:b'foo':k'abcd'", is_client=True)
+        assert frm.header.mask
+        assert frm.header.masking_key == 'abcd'
+
+        # Server frame, mask explicitly set
+        frm = self.fr("wf:b'foo':mask")
+        assert frm.header.mask
+        assert frm.header.masking_key
+        frm = self.fr("wf:b'foo':k'abcd'")
+        assert frm.header.mask
+        assert frm.header.masking_key == 'abcd'
+
+        # Client frame, mask explicitly unset
+        frm = self.fr("wf:b'foo':-mask", is_client=True)
+        assert not frm.header.mask
+        assert not frm.header.masking_key
+
+        frm = self.fr("wf:b'foo':-mask:k'abcd'", is_client=True)
+        assert not frm.header.mask
+        # We're reading back a corrupted frame - the first 3 characters of the
+        # mask is mis-interpreted as the payload
+        assert frm.payload == "abc"
+
+    def test_knone(self):
+        with tutils.raises("expected 4 bytes"):
+            self.fr("wf:b'foo':mask:knone")
+
+    def test_length(self):
+        assert self.fr("wf:l3:b'foo'").header.payload_length == 3
+        frm = self.fr("wf:l2:b'foo'")
+        assert frm.header.payload_length == 2
+        assert frm.payload == "fo"
+        tutils.raises("expected 1024 bytes", self.fr, "wf:l1024:b'foo'")