Added support for different encryption schemes to private/public keys

src/libcharon/plugins/unit_tester/tests/test_rsa_gen.c

@@ -59,12 +59,12 @@ bool test_rsa_gen()
 			return FALSE;
 		}
 		free(sig.ptr);
-		if (!public->encrypt(public, data, &crypt))
+		if (!public->encrypt(public, ENCRYPT_RSA_PKCS1, data, &crypt))
 		{
 			DBG1(DBG_CFG, "encrypting data with RSA failed");
 			return FALSE;
 		}
-		if (!private->decrypt(private, crypt, &plain))
+		if (!private->decrypt(private, ENCRYPT_RSA_PKCS1, crypt, &plain))
 		{
 			DBG1(DBG_CFG, "decrypting data with RSA failed");
 			return FALSE;

src/libstrongswan/credentials/keys/private_key.h

@@ -51,11 +51,13 @@ struct private_key_t {
 	/**
 	 * Decrypt a chunk of data.
 	 *
+	 * @param scheme	expected encryption scheme used
 	 * @param crypto	chunk containing encrypted data
 	 * @param plain		where to allocate decrypted data
 	 * @return			TRUE if data decrypted and plaintext allocated
 	 */
-	bool (*decrypt)(private_key_t *this, chunk_t crypto, chunk_t *plain);
+	bool (*decrypt)(private_key_t *this, encryption_scheme_t scheme,
+					chunk_t crypto, chunk_t *plain);
 
 	/**
 	 * Get the strength of the key in bytes.

src/libstrongswan/credentials/keys/public_key.c

@@ -42,6 +42,16 @@ ENUM(signature_scheme_names, SIGN_UNKNOWN, SIGN_ECDSA_521,
 	"ECDSA-521",
 );
 
+ENUM(encryption_scheme_names, ENCRYPT_UNKNOWN, ENCRYPT_RSA_OAEP_SHA512,
+	"ENCRYPT_UNKNOWN",
+	"ENCRYPT_RSA_PKCS1",
+	"ENCRYPT_RSA_OAEP_SHA1",
+	"ENCRYPT_RSA_OAEP_SHA224",
+	"ENCRYPT_RSA_OAEP_SHA256",
+	"ENCRYPT_RSA_OAEP_SHA384",
+	"ENCRYPT_RSA_OAEP_SHA512",
+);
+
 /**
  * See header.
  */

src/libstrongswan/credentials/keys/public_key.h

@@ -24,6 +24,7 @@
 typedef struct public_key_t public_key_t;
 typedef enum key_type_t key_type_t;
 typedef enum signature_scheme_t signature_scheme_t;
+typedef enum encryption_scheme_t encryption_scheme_t;
 
 #include <library.h>
 #include <utils/identification.h>
@@ -96,6 +97,31 @@ enum signature_scheme_t {
  */
 extern enum_name_t *signature_scheme_names;
 
+/**
+ * Encryption scheme for public key data encryption.
+ */
+enum encryption_scheme_t {
+	/** Unknown encryption scheme                                      */
+	ENCRYPT_UNKNOWN,
+	/** RSAES-PKCS1-v1_5 as in PKCS#1                                  */
+	ENCRYPT_RSA_PKCS1,
+	/** RSAES-OAEP as in PKCS#1, using SHA1 as hash, no label          */
+	ENCRYPT_RSA_OAEP_SHA1,
+	/** RSAES-OAEP as in PKCS#1, using SHA-224 as hash, no label       */
+	ENCRYPT_RSA_OAEP_SHA224,
+	/** RSAES-OAEP as in PKCS#1, using SHA-256 as hash, no label       */
+	ENCRYPT_RSA_OAEP_SHA256,
+	/** RSAES-OAEP as in PKCS#1, using SHA-384 as hash, no label       */
+	ENCRYPT_RSA_OAEP_SHA384,
+	/** RSAES-OAEP as in PKCS#1, using SHA-512 as hash, no label       */
+	ENCRYPT_RSA_OAEP_SHA512,
+};
+
+/**
+ * Enum names for encryption_scheme_t
+ */
+extern enum_name_t *encryption_scheme_names;
+
 /**
  * Abstract interface of a public key.
  */
@@ -122,11 +148,13 @@ struct public_key_t {
 	/**
 	 * Encrypt a chunk of data.
 	 *
+	 * @param scheme	encryption scheme to use
 	 * @param plain		chunk containing plaintext data
 	 * @param crypto	where to allocate encrypted data
 	 * @return 			TRUE if data successfully encrypted
 	 */
-	bool (*encrypt)(public_key_t *this, chunk_t plain, chunk_t *crypto);
+	bool (*encrypt)(public_key_t *this, encryption_scheme_t scheme,
+					chunk_t plain, chunk_t *crypto);
 
 	/**
 	 * Check if two public keys are equal.

src/libstrongswan/plugins/agent/agent_private_key.c

@@ -299,7 +299,8 @@ METHOD(private_key_t, get_type, key_type_t,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_agent_private_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_agent_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "private key decryption not supported by ssh-agent");
 	return FALSE;

src/libstrongswan/plugins/gcrypt/gcrypt_rsa_private_key.c

@@ -226,13 +226,20 @@ METHOD(private_key_t, sign, bool,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_gcrypt_rsa_private_key_t *this, chunk_t encrypted, chunk_t *plain)
+	private_gcrypt_rsa_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t encrypted, chunk_t *plain)
 {
 	gcry_error_t err;
 	gcry_sexp_t in, out;
 	chunk_t padded;
 	u_char *pos = NULL;;
 
+	if (scheme != ENCRYPT_RSA_PKCS1)
+	{
+		DBG1(DBG_LIB, "encryption scheme %N not supported",
+			 encryption_scheme_names, scheme);
+		return FALSE;
+	}
 	err = gcry_sexp_build(&in, NULL, "(enc-val(flags)(rsa(a %b)))",
 						  encrypted.len, encrypted.ptr);
 	if (err)

src/libstrongswan/plugins/gcrypt/gcrypt_rsa_public_key.c

@@ -193,11 +193,18 @@ METHOD(public_key_t, verify, bool,
 }
 
 METHOD(public_key_t, encrypt_, bool,
-	private_gcrypt_rsa_public_key_t *this, chunk_t plain, chunk_t *encrypted)
+	private_gcrypt_rsa_public_key_t *this, encryption_scheme_t scheme,
+	chunk_t plain, chunk_t *encrypted)
 {
 	gcry_sexp_t in, out;
 	gcry_error_t err;
 
+	if (scheme != ENCRYPT_RSA_PKCS1)
+	{
+		DBG1(DBG_LIB, "encryption scheme %N not supported",
+			 encryption_scheme_names, scheme);
+		return FALSE;
+	}
 	/* "pkcs1" uses PKCS 1.5 (section 8.1) block type 2 encryption:
 	 * 00 | 02 | RANDOM | 00 | DATA */
 	err = gcry_sexp_build(&in, NULL, "(data(flags pkcs1)(value %b))",

src/libstrongswan/plugins/gmp/gmp_rsa_private_key.c

@@ -314,11 +314,18 @@ METHOD(private_key_t, sign, bool,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_gmp_rsa_private_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_gmp_rsa_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	chunk_t em, stripped;
 	bool success = FALSE;
 
+	if (scheme != ENCRYPT_RSA_PKCS1)
+	{
+		DBG1(DBG_LIB, "encryption scheme %N not supported",
+			 encryption_scheme_names, scheme);
+		return FALSE;
+	}
 	/* rsa decryption using PKCS#1 RSADP */
 	stripped = em = rsadp(this, crypto);
 

src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c

@@ -309,20 +309,20 @@ METHOD(public_key_t, verify, bool,
 #define MIN_PS_PADDING 8
 
 METHOD(public_key_t, encrypt_, bool,
-	private_gmp_rsa_public_key_t *this, chunk_t plain, chunk_t *crypto)
+	private_gmp_rsa_public_key_t *this, encryption_scheme_t scheme,
+	chunk_t plain, chunk_t *crypto)
 {
 	chunk_t em;
 	u_char *pos;
 	int padding, i;
 	rng_t *rng;
 
-	rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
+	if (scheme != ENCRYPT_RSA_PKCS1)
-	if (rng == NULL)
 	{
-		DBG1(DBG_LIB, "no random generator available");
+		DBG1(DBG_LIB, "encryption scheme %N not supported",
+			 encryption_scheme_names, scheme);
 		return FALSE;
 	}
-
 	/* number of pseudo-random padding octets */
 	padding = this->k - plain.len - 3;
 	if (padding < MIN_PS_PADDING)
@@ -331,6 +331,12 @@ METHOD(public_key_t, encrypt_, bool,
 			 MIN_PS_PADDING);
 		return FALSE;
 	}
+	rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
+	if (rng == NULL)
+	{
+		DBG1(DBG_LIB, "no random generator available");
+		return FALSE;
+	}
 
 	/* padding according to PKCS#1 7.2.1 (RSAES-PKCS1-v1.5-ENCRYPT) */
 	DBG2(DBG_LIB, "padding %u bytes of data to the rsa modulus size of"

src/libstrongswan/plugins/openssl/openssl_ec_private_key.c

@@ -171,7 +171,8 @@ METHOD(private_key_t, sign, bool,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_openssl_ec_private_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_openssl_ec_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "EC private key decryption not implemented");
 	return FALSE;

src/libstrongswan/plugins/openssl/openssl_ec_public_key.c

@@ -169,7 +169,8 @@ METHOD(public_key_t, verify, bool,
 }
 
 METHOD(public_key_t, encrypt, bool,
-	private_openssl_ec_public_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_openssl_ec_public_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "EC public key encryption not implemented");
 	return FALSE;

src/libstrongswan/plugins/openssl/openssl_rsa_private_key.c

@@ -166,7 +166,8 @@ METHOD(private_key_t, sign, bool,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_openssl_rsa_private_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_openssl_rsa_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "RSA private key decryption not implemented");
 	return FALSE;

src/libstrongswan/plugins/openssl/openssl_rsa_public_key.c

@@ -148,7 +148,8 @@ METHOD(public_key_t, verify, bool,
 }
 
 METHOD(public_key_t, encrypt, bool,
-	private_openssl_rsa_public_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_openssl_rsa_public_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "RSA public key encryption not implemented");
 	return FALSE;

src/libstrongswan/plugins/pgp/pgp_builder.c

@@ -129,7 +129,7 @@ static bool sign_not_allowed(private_key_t *this, signature_scheme_t scheme,
 /**
  * Implementation of private_key_t.decrypt for signature-only keys
  */
-static bool decrypt_not_allowed(private_key_t *this,
+static bool decrypt_not_allowed(private_key_t *this, encryption_scheme_t scheme,
 								chunk_t crypto, chunk_t *plain)
 {
 	DBG1(DBG_LIB, "decryption failed - signature only key");

src/libstrongswan/plugins/pkcs11/pkcs11_private_key.c

@@ -193,7 +193,8 @@ METHOD(private_key_t, sign, bool,
 }
 
 METHOD(private_key_t, decrypt, bool,
-	private_pkcs11_private_key_t *this, chunk_t crypto, chunk_t *plain)
+	private_pkcs11_private_key_t *this, encryption_scheme_t scheme,
+	chunk_t crypto, chunk_t *plain)
 {
 	return FALSE;
 }

src/libstrongswan/plugins/pkcs11/pkcs11_public_key.c

@@ -119,7 +119,8 @@ METHOD(public_key_t, verify, bool,
 }
 
 METHOD(public_key_t, encrypt, bool,
-	private_pkcs11_public_key_t *this, chunk_t plain, chunk_t *crypto)
+	private_pkcs11_public_key_t *this, encryption_scheme_t scheme,
+	chunk_t plain, chunk_t *crypto)
 {
 	return FALSE;
 }

src/libtls/tls_peer.c

@@ -505,7 +505,8 @@ static status_t send_key_exchange(private_tls_peer_t *this,
 		DBG1(DBG_IKE, "no TLS public key found for server '%Y'", this->server);
 		return FAILED;
 	}
-	if (!public->encrypt(public, chunk_from_thing(premaster), &encrypted))
+	if (!public->encrypt(public, ENCRYPT_RSA_PKCS1,
+						 chunk_from_thing(premaster), &encrypted))
 	{
 		public->destroy(public);
 		DBG1(DBG_IKE, "encrypting TLS premaster secret failed");

src/libtls/tls_server.c

@@ -228,7 +228,8 @@ static status_t process_key_exchange(private_tls_server_t *this,
 	}
 
 	if (!this->private ||
-		!this->private->decrypt(this->private, encrypted, &premaster))
+		!this->private->decrypt(this->private, ENCRYPT_RSA_PKCS1,
+								encrypted, &premaster))
 	{
 		DBG1(DBG_IKE, "decrypting Client Key Exchange data failed");
 		return FAILED;

src/pluto/pkcs7.c

@@ -407,7 +407,7 @@ bool pkcs7_parse_envelopedData(chunk_t blob, chunk_t *data,
 			}
 			break;
 		case PKCS7_ENCRYPTED_KEY:
-			if (!key->decrypt(key, object, &symmetric_key))
+			if (!key->decrypt(key, ENCRYPT_RSA_PKCS1, object, &symmetric_key))
 			{
 				DBG1(DBG_LIB, "symmetric key could not be decrypted with rsa");
 				goto end;
@@ -710,7 +710,7 @@ chunk_t pkcs7_build_envelopedData(chunk_t data, certificate_t *cert, int enc_alg
 			chunk_free(&out);
 			return chunk_empty;
 		}
-		key->encrypt(key, symmetricKey, &protectedKey);
+		key->encrypt(key, ENCRYPT_RSA_PKCS1, symmetricKey, &protectedKey);
 		key->destroy(key);
 	}