Setting up a Python environment with ECH

setup.sh

# Download and install the OpenSSL fork and CPython fork

PROJECT_DIR=$HOME/code
mkdir -p $PROJECT_DIR
cd $PROJECT_DIR
git clone https://github.com/defo-project/openssl.git openssl-defo-src
pushd openssl-defo-src
./config --libdir=lib --prefix=$PROJECT_DIR/openssl-defo
make -j8 && make install_sw
popd
git clone https://github.com/irl/cpython.git cpython-defo
pushd cpython-defo
git checkout ech
./configure --with-openssl=$PROJECT_DIR/openssl-defo
make -j8
popd

# Create a new folder and create a virtual environment within that folder using the CPython fork

mkdir test-code
cd test-code
../cpython-defo/python.exe -m venv env

# Run pip and python with the virtual environment activated

. env/bin/activate
pip install ...
python3 ...

test.py

import json
import logging
import ssl
import socket
import urllib.parse

import dns.resolver
import httptools

class HTTPResponseParser:
    def __init__(self):
        self.headers = {}
        self.body = bytearray()
        self.status_code = None
        self.reason = None
        self.http_version = None
        self.parser = httptools.HttpResponseParser(self)

    def on_message_begin(self):
        pass

    def on_status(self, status):
        self.reason = status.decode('utf-8', errors='replace')

    def on_header(self, name, value):
        self.headers[name.decode('utf-8')] = value.decode('utf-8')

    def on_headers_complete(self):
        pass

    def on_body(self, body):
        self.body.extend(body)

    def on_message_complete(self):
        pass

    def feed_data(self, data):
        self.parser.feed_data(data)

def parse_http_response(response_bytes):
    parser = HTTPResponseParser()
    parser.feed_data(response_bytes)
    return {
        'status_code': parser.parser.get_status_code(),
        'reason': parser.reason,
        'headers': parser.headers,
        'body': bytes(parser.body),
    }

def get_echconfigs(domain):
    try:
        answers = dns.resolver.resolve(domain, 'HTTPS')
    except dns.resolver.NoAnswer:
        logging.warning(f"No HTTPS record found for {domain}")
        return None
    except Exception as e:
        logging.error(f"DNS query failed: {e}")
        return None

    configs = []

    for rdata in answers:
        if hasattr(rdata, 'params'):
            params = rdata.params
            echconfig = params.get(5)
            if echconfig:
                configs.append(echconfig.ech)

    if len(configs) == 0:
        logging.warning(f"No echconfig found in HTTPS record for {domain}")

    return configs

def get_http(hostname, port, path, echconfigs) -> bytes:
    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    context.load_verify_locations(cafile='/etc/ssl/cert.pem')
    context.options |= ssl.OP_ECH_GREASE
    for echconfig in echconfigs:
        try:
            context.set_ech_config(echconfig)
            context.check_hostname = False
        except ssl.SSLError as e:
            pass
    with socket.create_connection((hostname, port)) as sock:
        with context.wrap_socket(sock, server_hostname=hostname, do_handshake_on_connect=False) as ssock:
            ssock.do_handshake()
            print(ssock.get_ech_status().name, ssock.server_hostname, ssock.outer_server_hostname)
            if ssock.get_ech_status().name == ssl.ECHStatus.ECH_STATUS_GREASE_ECH:
                echconfigs = [ssock._sslobj.get_ech_retry_config()]
                return get_http(hostname, port, path, echconfigs)
            request = f'GET {path} HTTP/1.1\r\nHost: {hostname}\r\nConnection: close\r\n\r\n'
            ssock.sendall(request.encode('utf-8'))
            response = b''
            while True:
                data = ssock.recv(4096)
                if not data:
                    break
                response += data
            return response

def get(url):
    parsed = urllib.parse.urlparse(url)
    domain = parsed.hostname
    echconfigs = get_echconfigs(domain)
    request_path = (parsed.path or '/') + ('?' + parsed.query if parsed.query else '')
    raw = get_http(domain, parsed.port or 443, request_path, echconfigs)
    return parse_http_response(raw)

if __name__ == '__main__':
    with open("tests.json") as tests_file:
        tests = json.load(tests_file)
        for test in tests:
            print('-----')
            print(f"{test['description']}: {test['url']}")
            response = get(test['url'])
            result = json.loads(response['body'])
            print(result)

tests.json

[
  {
    "description": "nginx server/minimal HTTPS RR",
    "expected": "success",
    "url": "https://min-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/nominal, HTTPS RR",
    "expected": "success",
    "url": "https://v1-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/nominal, HTTPS RR",
    "expected": "success",
    "url": "https://v2-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/two RRvals for nominal, minimal, HTTPS RR",
    "expected": "success",
    "url": "https://v3-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/three RRvals, 1st bad, 2nd good, 3rd bad, HTTPS RR",
    "expected": "error",
    "url": "https://v4-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/ECHConfigList with bad alg type (0xcccc) for ech kem",
    "expected": "error",
    "url": "https://bk1-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/zero-length ECHConfig within ECHConfigList",
    "expected": "error",
    "url": "https://bk2-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/ECHConfigList with bad ECH version (0xcccc)",
    "expected": "error",
    "url": "https://bv-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/nominal, HTTPS RR, bad alpn",
    "expected": "client-dependent",
    "url": "https://badalpn-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/20 values in HTTPS RR",
    "expected": "success",
    "url": "https://many-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/AliasMode (0) and ServiceMode (!=0) are not allowed together",
    "expected": "error",
    "url": "https://mixedmode-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/uses p256, hkdf-385 and chacha",
    "expected": "success",
    "url": "https://p256-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/two RRVALs one using x25519 and one with p256, same priority",
    "expected": "success",
    "url": "https://curves1-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/two RRVALs one using x25519 (priority=1) and one with p256 (priority=2)",
    "expected": "success",
    "url": "https://curves2-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/two RRVALs one using x25519 (priority=2) and one with p256 (priority=1)",
    "expected": "success",
    "url": "https://curves3-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/alpn is only h2",
    "expected": "success",
    "url": "https://h2alpn-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/alpn is only http/1.1",
    "expected": "success",
    "url": "https://h1alpn-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/alpn is http/1.1,foo,bar,bar,bom,h2",
    "expected": "success",
    "url": "https://mixedalpn-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/alpn is very long ending with http/1.1,h2",
    "expected": "success",
    "url": "https://longalpn-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/ECHConfiglist with 2 entries a 25519 one then a p256 one (both good keys)",
    "expected": "success",
    "url": "https://2thenp-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/ECHConfiglist with 2 entries a p256 one then a 25519 one (both good keys)",
    "expected": "success",
    "url": "https://pthen2-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server/minimal HTTPS RR but with 2 ECHConfig extensions",
    "expected": "success",
    "url": "https://withext-ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "nginx server",
    "expected": "success",
    "url": "https://ng.test.defo.ie/echstat.php?format=json"
  },
  {
    "description": "apache server",
    "expected": "success",
    "url": "https://ap.test.defo.ie/echstat.php?format=json"
  }
]