Skip to content

Instantly share code, notes, and snippets.

@ageis

ageis/certdata2bundle.py

Last active April 28, 2019 12:57
Download ZIP

Revisions

  1. ageis revised this gist Apr 28, 2019. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -78,7 +78,8 @@ def parse_cmd_args():
    # Remotely retrieves CA certificates from the Mozilla/NSS root trust store, using TLS
    # in base64/DER format and re-encodes them as a PEM sans metadata
    # Default mode (--bundle) is to write all CA certificates to /etc/ssl/cacerts.pem
    # However (--individual) will write CA certificates to separate files in /etc/ssl/certs/.
    # However (--individual) will write CA certificates to separate files in /etc/ssl/certs/ but only
    # if the content has changed, in which case you'll be prompted, or if the CA isn't already found in the system trust store.
    """
    parser = argparse.ArgumentParser(
    description=description, formatter_class=argparse.RawTextHelpFormatter
  2. ageis revised this gist Apr 28, 2019. 1 changed file with 5 additions and 1 deletion.
    6 changes: 5 additions & 1 deletion certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -151,6 +151,10 @@ def main():
    f = open(ca_filename, "wb")
    f.write(ca_dict["pubkey"])
    f.close()
    else:
    f = open(ca_filename, "wb")
    f.write(ca_dict["pubkey"])
    f.close()
    print(
    "Processed "
    + ca_dict["name"]
    @@ -188,4 +192,4 @@ def main():


    if __name__ == "__main__":
    main()
    main()
  3. ageis revised this gist Apr 28, 2019. 1 changed file with 79 additions and 20 deletions.
    99 changes: 79 additions & 20 deletions certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -23,13 +23,17 @@
    # along with this program. If not, see <https://www.gnu.org/licenses/>.

    # import base64
    # import os.path
    import os.path
    import argparse
    import sys
    import hashlib
    import certifi
    import filecmp
    import tempfile

    # import textwrap
    # import io
    from pick import pick
    import re
    import urllib3
    from cryptography import x509
    @@ -69,8 +73,36 @@ def extract_certs(data):
    return list(filter(None, certs))


    def main():
    def parse_cmd_args():
    description = """
    # Remotely retrieves CA certificates from the Mozilla/NSS root trust store, using TLS
    # in base64/DER format and re-encodes them as a PEM sans metadata
    # Default mode (--bundle) is to write all CA certificates to /etc/ssl/cacerts.pem
    # However (--individual) will write CA certificates to separate files in /etc/ssl/certs/.
    """
    parser = argparse.ArgumentParser(
    description=description, formatter_class=argparse.RawTextHelpFormatter
    )
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
    "--bundle",
    required=False,
    action="store_true",
    default=True,
    help="Write a single concatenated PEM bundle of all CA certs",
    )
    group.add_argument(
    "--individual",
    required=False,
    action="store_true",
    default=False,
    help="Write CA certificates to individual *.PEM files in /etc/ssl/certs",
    )
    return parser.parse_args()


    def main():
    args = parse_cmd_args()
    base64_certs = extract_certs(certdata())
    print("Found " + str(len(base64_certs)) + " trusted certificates.")

    @@ -98,6 +130,27 @@ def main():
    )
    ca_dict["serial"] = str(x509_cert.serial_number)
    x509_certs.append(ca_dict)
    if args.individual:
    ca_filename = "/etc/ssl/certs/" + ca_dict["name"].replace(" ", "_") + ".pem"
    if os.path.isfile(ca_filename):
    with tempfile.NamedTemporaryFile(delete=False) as ca_tmpfile:
    ca_tmpfile.write(ca_dict["pubkey"])
    ca_tmpfile_name = ca_tmpfile.name
    ca_tmpfile.flush()
    ca_tmpfile.close()
    if not filecmp.cmp(ca_filename, ca_tmpfile_name):
    prompt = (
    ca_filename
    + " already exists, but the content has changed. Do you want to overwrite it?"
    )
    options = ["Yes", "No"]
    selected = pick(
    options, prompt, multi_select=False, min_selection_count=1
    )
    if selected[1] == 0:
    f = open(ca_filename, "wb")
    f.write(ca_dict["pubkey"])
    f.close()
    print(
    "Processed "
    + ca_dict["name"]
    @@ -108,24 +161,30 @@ def main():
    + "."
    )

    f = open(ca_bundle_output, "wb")

    for cert in x509_certs:
    f.write(cert["pubkey"])
    f.close()
    print(
    "Wrote "
    + str(len(x509_certs))
    + " PEM CA certificates to "
    + ca_bundle_output
    + "."
    )
    f = open(ca_bundle_output, "rb")
    f_bytes = f.read()
    ca_bundle_hash = hashlib.sha256(f_bytes).hexdigest()
    f.close()
    print("For integrity, " + ca_bundle_output + " has SHA256 hash of " + ca_bundle_hash)
    sys.exit(0)
    if args.bundle:
    f = open(ca_bundle_output, "wb")

    for cert in x509_certs:
    f.write(cert["pubkey"])
    f.close()
    print(
    "Wrote "
    + str(len(x509_certs))
    + " PEM CA certificates to "
    + ca_bundle_output
    + "."
    )
    f = open(ca_bundle_output, "rb")
    f_bytes = f.read()
    ca_bundle_hash = hashlib.sha256(f_bytes).hexdigest()
    f.close()
    print(
    "For integrity, "
    + ca_bundle_output
    + " has SHA256 hash of "
    + ca_bundle_hash
    )
    sys.exit(0)


    if __name__ == "__main__":
  4. ageis revised this gist Apr 28, 2019. 1 changed file with 12 additions and 3 deletions.
    15 changes: 12 additions & 3 deletions certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -24,8 +24,9 @@

    # import base64
    # import os.path
    import re
    import sys
    import hashlib
    import certifi

    # import textwrap
    # import io
    @@ -44,10 +45,13 @@

    def certdata():
    certdata_url = "https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt"
    http = urllib3.PoolManager()
    http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=certifi.where())
    try:
    r = http.request(
    "GET", certdata_url, retries=urllib3.Retry(5, raise_on_redirect=False)
    "GET",
    certdata_url,
    timeout=30,
    retries=urllib3.Retry(5, raise_on_redirect=False),
    )
    except Exception as e:
    print(e)
    @@ -116,6 +120,11 @@ def main():
    + ca_bundle_output
    + "."
    )
    f = open(ca_bundle_output, "rb")
    f_bytes = f.read()
    ca_bundle_hash = hashlib.sha256(f_bytes).hexdigest()
    f.close()
    print("For integrity, " + ca_bundle_output + " has SHA256 hash of " + ca_bundle_hash)
    sys.exit(0)


  5. ageis revised this gist Apr 28, 2019. 1 changed file with 63 additions and 46 deletions.
    109 changes: 63 additions & 46 deletions certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -22,12 +22,13 @@
    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <https://www.gnu.org/licenses/>.

    #import base64
    #import os.path
    # import base64
    # import os.path
    import re
    import sys
    #import textwrap
    #import io

    # import textwrap
    # import io
    import re
    import urllib3
    from cryptography import x509
    @@ -58,49 +59,65 @@ def certdata():


    def extract_certs(data):
    certs = re.findall(r"CKA_VALUE MULTILINE_OCTAL.*?END", data, flags=re.DOTALL | re.MULTILINE)
    certs = re.findall(
    r"CKA_VALUE MULTILINE_OCTAL.*?END", data, flags=re.DOTALL | re.MULTILINE
    )
    return list(filter(None, certs))


    base64_certs = extract_certs(certdata())
    print("Found " + str(len(base64_certs)) + " trusted certificates.")
    def main():

    for cert in base64_certs:
    ca_dict = {"name": None, "pubkey": None, "fingerprint": None, "serial": None}
    cert = cert.lstrip("CKA_VALUE MULTILINE_OCTAL")
    cert = cert.rstrip("END")
    cert = cert.replace("\n", "")
    cert = cert.encode("utf-8").decode("unicode_escape").encode("latin-1")
    try:
    x509_cert = x509.load_der_x509_certificate(cert, default_backend())
    except Exception as e:
    print(e)
    continue
    try:
    ca_dict["name"] = str(x509_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value)
    except IndexError as e:
    ca_dict["name"] = ""
    pass
    ca_dict["pubkey"] = x509_cert.public_bytes(serialization.Encoding.PEM)
    ca_dict["fingerprint"] = "".join("{:02x}".format(x) for x in x509_cert.fingerprint(hashes.SHA256()))
    ca_dict["serial"] = str(x509_cert.serial_number)
    x509_certs.append(ca_dict)
    print("Processed "
    + ca_dict["name"]
    + ", SHA256 fingerprint: "
    + ca_dict["fingerprint"]
    + ", serial: "
    + ca_dict["serial"]
    + ".")

    f = open(ca_bundle_output, "wb")

    for cert in x509_certs:
    f.write(cert["pubkey"])
    f.close()
    print("Wrote "
    + str(len(x509_certs))
    + " PEM CA certificates to "
    + ca_bundle_output
    + ".")
    sys.exit(0)
    base64_certs = extract_certs(certdata())
    print("Found " + str(len(base64_certs)) + " trusted certificates.")

    for cert in base64_certs:
    ca_dict = {"name": None, "pubkey": None, "fingerprint": None, "serial": None}
    cert = cert.lstrip("CKA_VALUE MULTILINE_OCTAL")
    cert = cert.rstrip("END")
    cert = cert.replace("\n", "")
    cert = cert.encode("utf-8").decode("unicode_escape").encode("latin-1")
    try:
    x509_cert = x509.load_der_x509_certificate(cert, default_backend())
    except Exception as e:
    print(e)
    continue
    try:
    ca_dict["name"] = str(
    x509_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
    )
    except IndexError as e:
    ca_dict["name"] = ""
    pass
    ca_dict["pubkey"] = x509_cert.public_bytes(serialization.Encoding.PEM)
    ca_dict["fingerprint"] = "".join(
    "{:02x}".format(x) for x in x509_cert.fingerprint(hashes.SHA256())
    )
    ca_dict["serial"] = str(x509_cert.serial_number)
    x509_certs.append(ca_dict)
    print(
    "Processed "
    + ca_dict["name"]
    + ", SHA256 fingerprint: "
    + ca_dict["fingerprint"]
    + ", serial: "
    + ca_dict["serial"]
    + "."
    )

    f = open(ca_bundle_output, "wb")

    for cert in x509_certs:
    f.write(cert["pubkey"])
    f.close()
    print(
    "Wrote "
    + str(len(x509_certs))
    + " PEM CA certificates to "
    + ca_bundle_output
    + "."
    )
    sys.exit(0)


    if __name__ == "__main__":
    main()
  6. ageis revised this gist Apr 28, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -80,7 +80,7 @@ def extract_certs(data):
    ca_dict["name"] = str(x509_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value)
    except IndexError as e:
    ca_dict["name"] = ""
    continue
    pass
    ca_dict["pubkey"] = x509_cert.public_bytes(serialization.Encoding.PEM)
    ca_dict["fingerprint"] = "".join("{:02x}".format(x) for x in x509_cert.fingerprint(hashes.SHA256()))
    ca_dict["serial"] = str(x509_cert.serial_number)
  7. ageis created this gist Apr 28, 2019.
    106 changes: 106 additions & 0 deletions certdata2bundle.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,106 @@
    #!/usr/bin/env python3
    # -*- coding: utf-8 -*-
    # vim:set et sw=4:
    #
    # certdata2bundle.py
    # retrieves CA certificates from the Mozilla/NSS root trust store in base64/DER
    # format and re-encodes them as a concatenated PEM bundle sans metadata
    # writes all CA certificates to /etc/ssl/cacerts.pem
    #
    # Copyright (C) 2019 Kevin M. Gallagher <kevingallagher@gmail.com>
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, either version 3 of the License, or
    # (at your option) any later version.

    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.

    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <https://www.gnu.org/licenses/>.

    #import base64
    #import os.path
    import re
    import sys
    #import textwrap
    #import io
    import re
    import urllib3
    from cryptography import x509
    from cryptography.hazmat.backends import default_backend
    from cryptography.hazmat.primitives import serialization
    from cryptography.hazmat.primitives import hashes
    from cryptography.x509.oid import NameOID

    ca_bundle_output = "/etc/ssl/cacerts.crt"
    base64_certs = []
    x509_certs = []


    def certdata():
    certdata_url = "https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt"
    http = urllib3.PoolManager()
    try:
    r = http.request(
    "GET", certdata_url, retries=urllib3.Retry(5, raise_on_redirect=False)
    )
    except Exception as e:
    print(e)
    sys.exit(1)
    if r.status == 200:
    return r.data.decode("utf-8")
    else:
    return None


    def extract_certs(data):
    certs = re.findall(r"CKA_VALUE MULTILINE_OCTAL.*?END", data, flags=re.DOTALL | re.MULTILINE)
    return list(filter(None, certs))


    base64_certs = extract_certs(certdata())
    print("Found " + str(len(base64_certs)) + " trusted certificates.")

    for cert in base64_certs:
    ca_dict = {"name": None, "pubkey": None, "fingerprint": None, "serial": None}
    cert = cert.lstrip("CKA_VALUE MULTILINE_OCTAL")
    cert = cert.rstrip("END")
    cert = cert.replace("\n", "")
    cert = cert.encode("utf-8").decode("unicode_escape").encode("latin-1")
    try:
    x509_cert = x509.load_der_x509_certificate(cert, default_backend())
    except Exception as e:
    print(e)
    continue
    try:
    ca_dict["name"] = str(x509_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value)
    except IndexError as e:
    ca_dict["name"] = ""
    continue
    ca_dict["pubkey"] = x509_cert.public_bytes(serialization.Encoding.PEM)
    ca_dict["fingerprint"] = "".join("{:02x}".format(x) for x in x509_cert.fingerprint(hashes.SHA256()))
    ca_dict["serial"] = str(x509_cert.serial_number)
    x509_certs.append(ca_dict)
    print("Processed "
    + ca_dict["name"]
    + ", SHA256 fingerprint: "
    + ca_dict["fingerprint"]
    + ", serial: "
    + ca_dict["serial"]
    + ".")

    f = open(ca_bundle_output, "wb")

    for cert in x509_certs:
    f.write(cert["pubkey"])
    f.close()
    print("Wrote "
    + str(len(x509_certs))
    + " PEM CA certificates to "
    + ca_bundle_output
    + ".")
    sys.exit(0)