From 3da0ca796cae6625bd26418afe0a1dc47bf5a77f Mon Sep 17 00:00:00 2001
From: "Dr. Stephen Henson" <steve@openssl.org>
Date: Mon, 7 Feb 2011 14:36:08 +0000
Subject: [PATCH] New flags EVP_CIPH_FLAG_CUSTOM_CIPHER in cipher structures if
 an underlying cipher handles all cipher symantics itself.

---
 CHANGES              | 10 ++++++++++
 crypto/evp/evp.h     |  4 ++++
 crypto/evp/evp_enc.c | 45 +++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index c76090d41c..2b052dd850 100644
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,16 @@
 
  Changes between 1.0.1 and 1.1.0  [xx XXX xxxx]
 
+  *) New flag in ciphers: EVP_CIPH_FLAG_CUSTOM_CIPHER. This means the
+     underlying do_cipher function handles all cipher semantics itself
+     including padding and finalisation. This is useful if (for example)
+     an ENGINE cipher handles block padding itself. The behaviour of
+     do_cipher is subtly changed if this flag is set: the return value
+     is the number of characters written to the output buffer (zero is
+     no longer an error code) or a negative error code. Also if the
+     input buffer is NULL and length -1 finalisation should be performed.
+     [Steve Henson]
+
   *) If a candidate issuer certificate is already part of the constructed
      path ignore it: new debug notification X509_V_ERR_PATH_LOOP for this case.
      [Steve Henson]
diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h
index 2fa0aa916e..6604f3484b 100644
--- a/crypto/evp/evp.h
+++ b/crypto/evp/evp.h
@@ -354,6 +354,10 @@ struct evp_cipher_st
 #define		EVP_CIPH_FLAG_FIPS		0x4000
 /* Allow non FIPS cipher in FIPS mode */
 #define		EVP_CIPH_FLAG_NON_FIPS_ALLOW	0x8000
+/* Cipher handles any and all padding logic as well
+ * as finalisation.
+ */
+#define 	EVP_CIPH_FLAG_CUSTOM_CIPHER	0x10000
 
 /* ctrl() values */
 
diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index a0bdf9856c..3f8473b348 100644
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -286,6 +286,16 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
 	{
 	int i,j,bl;
 
+	if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER)
+		{
+		i = ctx->cipher->do_cipher(ctx, out, in, inl);
+		if (i < 0)
+			return 0;
+		else
+			*outl = i;
+		return 1;
+		}
+
 	if (inl <= 0)
 		{
 		*outl = 0;
@@ -356,6 +366,16 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
 	int n,ret;
 	unsigned int i, b, bl;
 
+	if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER)
+		{
+		i = ctx->cipher->do_cipher(ctx, out, NULL, -1);
+		if (i < 0)
+			return 0;
+		else
+			*outl = i;
+		return 1;
+		}
+
 	b=ctx->cipher->block_size;
 	OPENSSL_assert(b <= sizeof ctx->buf);
 	if (b == 1)
@@ -393,6 +413,19 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
 	int fix_len;
 	unsigned int b;
 
+	if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER)
+		{
+		fix_len = ctx->cipher->do_cipher(ctx, out, in, inl);
+		if (fix_len < 0)
+			{
+			*outl = 0;
+			return 0;
+			}
+		else
+			*outl = fix_len;
+		return 1;
+		}
+
 	if (inl <= 0)
 		{
 		*outl = 0;
@@ -446,8 +479,18 @@ int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
 	{
 	int i,n;
 	unsigned int b;
-
 	*outl=0;
+
+	if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER)
+		{
+		i = ctx->cipher->do_cipher(ctx, out, NULL, -1);
+		if (i < 0)
+			return 0;
+		else
+			*outl = i;
+		return 1;
+		}
+
 	b=ctx->cipher->block_size;
 	if (ctx->flags & EVP_CIPH_NO_PADDING)
 		{
-- 
2.25.1