sample for using a yubihsm2 with python requests

engine.py

try:
  from OpenSSL._util import (
      ffi as _ffi,
      lib as O,
      lib as S
      )
  pyopenssl = True

  import sys

  null = _ffi.NULL
  # the engine loaded here has to have an ABI matching that of the openssl that
  # is used by cryptography, that means it's probably openssl 1.1
  p11 = b"/tmp/openssl-p11/prefix/lib/engines-1.1/pkcs11.so"

except ImportError:
  pyopenssl = False
  import ctypes

  O = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libcrypto.so")
  S = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libssl.so")

  null = None
  p11 = b"/usr/lib/ssl/engines/libpkcs11.so"

O.ENGINE_load_dynamic()
e = O.ENGINE_by_id(b"dynamic")
print(e)
print(O.ENGINE_ctrl_cmd_string(e, b"SO_PATH", p11, 0))
print(O.ENGINE_ctrl_cmd_string(e, b"LOAD", null, 0))
# where to find the yubihsm_pkcs11.so
print(O.ENGINE_ctrl_cmd_string(e, b"MODULE_PATH", b"yubihsm_pkcs11.so", 0))
# authenticate to the YubiHSM (id+password)
print(O.ENGINE_ctrl_cmd_string(e, b"PIN", b"0001password", 0))
print(O.ENGINE_init(e))
# load a key with id 0x0001
key = O.ENGINE_load_private_key(e, b"0:0001", null, null)

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.ssl_ import create_urllib3_context
from requests.packages.urllib3.poolmanager import PoolManager

class YubihsmAdapter(HTTPAdapter):
  def init_poolmanager(self, connections, maxsize, block=False, *args, **kwargs):
    context = create_urllib3_context()
    if(pyopenssl):
      # if pyopenssl is in use we can find the SSL_CTX in a semi-clean way
      ctx = context._ctx._context
    else:
      # if the builtin ssl module is in use we have to jump blindly to the
      # third element in the struct and rely on cpython returning the adress
      # with id() on an object.
      ctx = ctypes.c_void_p.from_address(id(context) + ctypes.sizeof(ctypes.c_void_p) * 2)

    # point the SSL_CTX to a certificate matching our key
    print(S.SSL_CTX_use_certificate_file(ctx, b"cert.pem", 1))
    # and load the key into the SSL_CTX
    print(S.SSL_CTX_use_PrivateKey(ctx, key))
    kwargs['ssl_context'] = context
    self.poolmanager = PoolManager(
        num_pools=connections, maxsize=maxsize,
        block=block, *args, **kwargs)

s = requests.Session()
# the URL "mounted" has to be longer than https:// and match whatever it is
# we're going to use for our requests
s.mount("https://127.0.0.1", YubihsmAdapter())
print(s.get("https://127.0.0.1:8443", verify=False))

Twisted looks like it's using pyopenssl, looking at what seems to be stored in twisted that should maybe be self._context._context.

Thank you so much @klali. I tried that as your suggested. But I still get the same error:

ctx = self._context._context
print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))
print(_lib2.SSL_CTX_use_certificate_file(ctx, b"/home/dwang3/Desktop/tls-tpm/tpm-gen-cert/tpm-client-cert.pem", 1))

print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))
ctypes.ArgumentError: argument 1: <class 'TypeError'>: Don't know how to convert parameter 1

I'm actually using ctypes not the OpenSSL._util. Here is my setup:

import ctypes
_lib = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libcrypto.so")
_lib2 = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libssl.so")

I assume it should not matter but I do not know if it makes a difference?

Second question, When I import OpenSSL._util as your code. Python reports error as following do you know why? Did I do something wrong?

_lib.ENGINE_load_builtin_engines()
AttributeError: module 'lib' has no attribute 'ENGINE_load_builtin_engines'
or 
AttributeError: module 'lib' has no attribute 'ENGINE_load_dynamic'

Thank you so much for your help!

This was useful to me, wanted to share a cleaner (IMO) way of accessing the context object with ssl:

You have this:

ctx = ctypes.cast(id(context) + ctypes.sizeof(ctypes.c_void_p) * 2, ctypes.POINTER(ctypes.c_void_p)).contents

This also works:

from ctypes import c_void_p, sizeof
ctx = c_void_p.from_address(id(context) + sizeof(c_void_p) * 2)

from_address uses the value at the given address to create the c_void_p.

Second question, When I import OpenSSL._util as your code. Python reports error as following do you know why? Did I do something wrong?

_lib.ENGINE_load_builtin_engines()
AttributeError: module 'lib' has no attribute 'ENGINE_load_builtin_engines'
or 
AttributeError: module 'lib' has no attribute 'ENGINE_load_dynamic'

You probably need to use cryptography==2.5 as cryptography==2.6.1 remove a lot of "unused" function.
Commit: