bad.rs

// This does not


    // 1. Start by reading the encrypted session key and displaying it as an ascii string
    let session_key = parsed_packet.get_encrypted_session_key();
    debug!("Encrypted session key: {}", session_key);

    // 2. Convert the ascii string to bytes
    let session_key_bytes = hex::decode(session_key).unwrap();

    //3. Verify the length of the session key bytes is 128
    if session_key_bytes.len() != 128 {
        error!("Invalid session key length: {}", session_key_bytes.len());
        return Err(());
    }

    // Print the session key bytes as values
    debug!(
        "Session key bytes: {}",
        String::from_utf8_lossy(&session_key_bytes)
    );

    // 2. Load the RSA private key from the file and read it into a vector
    let mut file = File::open("data/private_key.pem").await.unwrap();
    let mut private_key_bytes = Vec::new();
    file.read_to_end(&mut private_key_bytes).await.unwrap();

    // 3. Create a new RSA private key from the bytes
    let private_key = openssl::rsa::Rsa::private_key_from_pem(&private_key_bytes).unwrap();

    // 4. Decrypt the session key bytes using the RSA private key. Do not use padding.
    let mut decrypted_session_key_bytes = vec![0; private_key.size() as usize];
    let decrypted_session_key_length = private_key
        .private_decrypt(
            &session_key_bytes,
            &mut decrypted_session_key_bytes,
            openssl::rsa::Padding::NONE,
        )
        .unwrap();

    // Let's print the decrypted session key bytes as a hex string
    debug!(
        "Decrypted session key bytes: {}",
        hex::encode(&decrypted_session_key_bytes[0..decrypted_session_key_length])
    );

good.ts

// This works

extractSessionKeyFromPacket(rawPacket: Buffer): void {
        this.log.debug("Extracting key");

        // Extract the session key which is 128 acsii characters (256 bytes)
        const sessionKeyAsAscii = rawPacket.subarray(52, 308).toString("utf8");
        this.log.trace(`Session key: ${sessionKeyAsAscii}`);

        // length of the session key should be 128 bytes
        const sessionkeyString = Buffer.from(sessionKeyAsAscii, "hex");
        // Decrypt the sessionkey
        try {
            if (!this._config.privateKeyFile) {
                throw new ServerError("No private key file specified");
            }
            const privatekeyContents = readFileSync(
                this._config.privateKeyFile,
            );

            const decrypted = privateDecrypt(
                {
                    key: privatekeyContents,
                },
                sessionkeyString,
            ); // length of decrypted should be 128 bytes
            this.log.trace(`decrypted: ${decrypted.toString("hex")}`);

            this.sessionKey = decrypted.subarray(2, -4).toString("hex"); // length of session key should be 12 bytes
        } catch (error) {
            this.log.trace(`Session key: ${sessionkeyString.toString("utf8")}`); // 128 bytes
            this.log.trace(`decrypted: ${this.sessionKey}`); // 12 bytes
            this.log.error(`Error decrypting session key: ${String(error)}`);
            throw new ServerError(
                `Unable to extract session key: ${String(error)}`,
            );
        }
    }