From f89b16a831d94b3013ccdd68f5d99572a1b679d2 Mon Sep 17 00:00:00 2001
From: Nikos Mavrogiannopoulos <nmav@gnutls.org>
Date: Thu, 1 Dec 2011 09:45:33 +0100
Subject: [PATCH 151/291] cryptodev: reorganization

---
 drivers/staging/crypto/cryptodev/Makefile          |    2 +-
 drivers/staging/crypto/cryptodev/authenc.c         |  724 ++++++++++
 drivers/staging/crypto/cryptodev/cryptlib.c        |  319 +++++
 drivers/staging/crypto/cryptodev/cryptodev_auth.c  |  724 ----------
 .../staging/crypto/cryptodev/cryptodev_cipher.c    |  319 -----
 drivers/staging/crypto/cryptodev/cryptodev_int.h   |   10 +-
 drivers/staging/crypto/cryptodev/cryptodev_main.c  | 1414 --------------------
 drivers/staging/crypto/cryptodev/ioctl.c           | 1125 ++++++++++++++++
 drivers/staging/crypto/cryptodev/main.c            |  313 +++++
 9 files changed, 2483 insertions(+), 2467 deletions(-)
 create mode 100644 drivers/staging/crypto/cryptodev/authenc.c
 create mode 100644 drivers/staging/crypto/cryptodev/cryptlib.c
 delete mode 100644 drivers/staging/crypto/cryptodev/cryptodev_auth.c
 delete mode 100644 drivers/staging/crypto/cryptodev/cryptodev_cipher.c
 delete mode 100644 drivers/staging/crypto/cryptodev/cryptodev_main.c
 create mode 100644 drivers/staging/crypto/cryptodev/ioctl.c
 create mode 100644 drivers/staging/crypto/cryptodev/main.c

diff --git a/drivers/staging/crypto/cryptodev/Makefile b/drivers/staging/crypto/cryptodev/Makefile
index a90cd8bda24d..a01bab1bb66d 100644
--- a/drivers/staging/crypto/cryptodev/Makefile
+++ b/drivers/staging/crypto/cryptodev/Makefile
@@ -2,7 +2,7 @@ KBUILD_CFLAGS += -I$(src)
 KERNEL_DIR = /lib/modules/$(shell uname -r)/build
 VERSION = 1.0
 
-cryptodev-objs = cryptodev_main.o cryptodev_cipher.o cryptodev_auth.o
+cryptodev-objs = ioctl.o main.o cryptlib.o authenc.o
 
 obj-m += cryptodev.o
 
diff --git a/drivers/staging/crypto/cryptodev/authenc.c b/drivers/staging/crypto/cryptodev/authenc.c
new file mode 100644
index 000000000000..597515989daa
--- /dev/null
+++ b/drivers/staging/crypto/cryptodev/authenc.c
@@ -0,0 +1,724 @@
+/*
+ * Driver for /dev/crypto device (aka CryptoDev)
+ *
+ * Copyright (c) 2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
+ *
+ * This file is part of linux cryptodev.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+/*
+ * This file handles the AEAD part of /dev/crypto.
+ *
+ */
+
+#include <crypto/hash.h>
+#include <linux/crypto.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/ioctl.h>
+#include <linux/random.h>
+#include <linux/syscalls.h>
+#include <linux/pagemap.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <crypto/cryptodev.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include "cryptodev_int.h"
+#include "version.h"
+
+
+/* make cop->src and cop->dst available in scatterlists */
+static int get_userbuf_aead(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
+			struct scatterlist **auth_sg, struct scatterlist **dst_sg, 
+			int *tot_pages)
+{
+	int dst_pagecount = 0, pagecount;
+	int auth_pagecount = 0;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	int rc;
+
+	if (caop->dst == NULL && caop->auth_src == NULL)
+		return -EINVAL;
+
+	if (ses->alignmask) {
+		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
+			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
+				__func__, (unsigned long)caop->dst, ses->alignmask + 1);
+		if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
+			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
+				__func__, (unsigned long)caop->auth_src, ses->alignmask + 1);
+	}
+
+	if (kcaop->dst_len == 0) {
+		dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
+		return -EINVAL;
+	}
+
+	if (caop->auth_len > 0)
+		auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
+
+	dst_pagecount = PAGECOUNT(caop->dst, kcaop->dst_len);
+
+	(*tot_pages) = pagecount = auth_pagecount + dst_pagecount;
+
+	rc = adjust_sg_array(ses, pagecount);
+	if (rc)
+		return rc;
+
+	if (auth_pagecount > 0) {
+		rc = __get_userbuf(caop->auth_src, caop->auth_len, 0, auth_pagecount,
+				   ses->pages, ses->sg, kcaop->task, kcaop->mm);
+		if (unlikely(rc)) {
+			dprintk(1, KERN_ERR,
+				"failed to get user pages for data input\n");
+			return -EINVAL;
+		}
+		(*auth_sg) = ses->sg;
+		(*dst_sg) = ses->sg + auth_pagecount;
+	} else {
+		(*auth_sg) = NULL;
+		(*dst_sg) = ses->sg;
+	}
+
+	rc = __get_userbuf(caop->dst, kcaop->dst_len, 1, dst_pagecount,
+	                   ses->pages + auth_pagecount, *dst_sg, kcaop->task, kcaop->mm);
+	if (unlikely(rc)) {
+		release_user_pages(ses->pages, auth_pagecount);
+		dprintk(1, KERN_ERR,
+			"failed to get user pages for data input\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Taken from Maxim Levitsky's patch
+ */
+static struct scatterlist *sg_advance(struct scatterlist *sg, int consumed)
+{
+	while (consumed >= sg->length) {
+		consumed -= sg->length;
+
+		sg = sg_next(sg);
+		if (!sg)
+			break;
+	}
+
+	WARN_ON(!sg && consumed);
+
+	if (!sg)
+		return NULL;
+
+	sg->offset += consumed;
+	sg->length -= consumed;
+
+	if (sg->offset >= PAGE_SIZE) {
+		struct page *page =
+			nth_page(sg_page(sg), sg->offset / PAGE_SIZE);
+		sg_set_page(sg, page, sg->length, sg->offset % PAGE_SIZE);
+	}
+
+	return sg;
+}
+
+/**
+ * sg_copy - copies sg entries from sg_from to sg_to, such
+ * as sg_to covers first 'len' bytes from sg_from.
+ */
+static int sg_copy(struct scatterlist *sg_from, struct scatterlist *sg_to, int len)
+{
+	while (len > sg_from->length) {
+		len -= sg_from->length;
+
+		sg_set_page(sg_to, sg_page(sg_from),
+				sg_from->length, sg_from->offset);
+
+		sg_to = sg_next(sg_to);
+		sg_from = sg_next(sg_from);
+
+		if (len && (!sg_from || !sg_to))
+			return -ENOMEM;
+	}
+
+	if (len)
+		sg_set_page(sg_to, sg_page(sg_from),
+				len, sg_from->offset);
+	sg_mark_end(sg_to);
+	return 0;
+}
+
+static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
+			struct scatterlist **auth_sg, struct scatterlist **dst_sg, 
+			int *tot_pages)
+{
+	int pagecount, diff;
+	int auth_pagecount = 0;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	int rc;
+
+	if (caop->dst == NULL && caop->auth_src == NULL)
+		return -EINVAL;
+
+	if (ses->alignmask) {
+		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
+			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
+				__func__, (unsigned long)caop->dst, ses->alignmask + 1);
+		if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
+			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
+				__func__, (unsigned long)caop->auth_src, ses->alignmask + 1);
+	}
+
+	if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {
+		dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
+		return -EINVAL;
+	}
+
+	/* Note that in SRTP auth data overlap with data to be encrypted (dst)
+         */
+
+	auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
+	diff = (int)(caop->src - caop->auth_src);
+	if (diff > PAGE_SIZE || diff < 0) {
+		dprintk(1, KERN_WARNING, "auth_src must overlap with src (diff: %d).\n", diff);
+		return -EINVAL;
+	}
+
+	(*tot_pages) = pagecount = auth_pagecount;
+
+	rc = adjust_sg_array(ses, pagecount*2); /* double pages to have pages for dst(=auth_src) */
+	if (rc)
+		return rc;
+
+	rc = __get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,
+			   ses->pages, ses->sg, kcaop->task, kcaop->mm);
+	if (unlikely(rc)) {
+		dprintk(1, KERN_ERR,
+			"failed to get user pages for data input\n");
+		return -EINVAL;
+	}
+	(*auth_sg) = ses->sg;
+
+	(*dst_sg) = ses->sg + auth_pagecount;
+	sg_init_table(*dst_sg, auth_pagecount);
+	sg_copy(ses->sg, (*dst_sg), caop->auth_len);
+	(*dst_sg) = sg_advance(*dst_sg, diff);
+	if (*dst_sg == NULL) {
+		release_user_pages(ses->pages, pagecount);
+		dprintk(1, KERN_ERR,
+			"failed to get enough pages for auth data\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int copy_from_user_to_user( void* __user dst, void* __user src, int len)
+{
+uint8_t *buffer;
+int buffer_size = min(len, 16*1024);
+int rc;
+
+	if (len > buffer_size) {
+		dprintk(1, KERN_ERR,
+			"The provided buffer is too large\n");
+		return -EINVAL;
+	}
+
+	buffer = kmalloc(buffer_size, GFP_KERNEL);
+	if (buffer == NULL)
+		return -ENOMEM;
+
+	if (unlikely(copy_from_user(buffer, src, len))) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	if (unlikely(copy_to_user(dst, buffer, len))) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	rc = 0;
+out:
+	kfree(buffer);
+	return rc;
+}
+
+static int fill_kcaop_from_caop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
+{
+	struct crypt_auth_op *caop = &kcaop->caop;
+	struct csession *ses_ptr;
+	int rc;
+
+	/* this also enters ses_ptr->sem */
+	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", caop->ses);
+		return -EINVAL;
+	}
+
+	if (caop->src != caop->dst) {
+		dprintk(2, KERN_ERR,
+			"Non-inplace encryption and decryption is not efficient\n");
+		
+		rc = copy_from_user_to_user( caop->dst, caop->src, caop->len);
+		if (rc < 0)
+			goto out_unlock;
+
+
+	}
+
+	if (caop->tag_len == 0)
+		caop->tag_len = ses_ptr->hdata.digestsize;
+
+	kcaop->ivlen = caop->iv ? ses_ptr->cdata.ivsize : 0;
+
+	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE)
+		kcaop->dst_len = caop->len + ses_ptr->cdata.blocksize /* pad */ + caop->tag_len;
+	else
+		kcaop->dst_len = caop->len;
+
+	kcaop->task = current;
+	kcaop->mm = current->mm;
+
+	if (caop->iv) {
+		rc = copy_from_user(kcaop->iv, caop->iv, kcaop->ivlen);
+		if (unlikely(rc)) {
+			dprintk(1, KERN_ERR,
+				"error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
+				kcaop->ivlen, rc, (unsigned long)caop->iv);
+			rc = -EFAULT;
+			goto out_unlock;
+		}
+	}
+	
+	rc = 0;
+
+out_unlock:
+	crypto_put_session(ses_ptr);
+	return rc;
+
+}
+
+static int fill_caop_from_kcaop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
+{
+	int ret;
+
+	kcaop->caop.len = kcaop->dst_len;
+
+	if (kcaop->ivlen && kcaop->caop.flags & COP_FLAG_WRITE_IV) {
+		ret = copy_to_user(kcaop->caop.iv,
+				kcaop->iv, kcaop->ivlen);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_ERR, "Error in copying to userspace\n");
+			return -EFAULT;
+		}
+	}
+	return 0;
+}
+
+
+int kcaop_from_user(struct kernel_crypt_auth_op *kcaop,
+			struct fcrypt *fcr, void __user *arg)
+{
+	if (unlikely(copy_from_user(&kcaop->caop, arg, sizeof(kcaop->caop)))) {
+		dprintk(1, KERN_ERR, "Error in copying from userspace\n");
+		return -EFAULT;
+	}
+
+	return fill_kcaop_from_caop(kcaop, fcr);
+}
+
+int kcaop_to_user(struct kernel_crypt_auth_op *kcaop,
+		struct fcrypt *fcr, void __user *arg)
+{
+	int ret;
+
+	ret = fill_caop_from_kcaop(kcaop, fcr);
+	if (unlikely(ret)) {
+		dprintk(1, KERN_ERR, "fill_caop_from_kcaop\n");
+		return ret;
+	}
+
+	if (unlikely(copy_to_user(arg, &kcaop->caop, sizeof(kcaop->caop)))) {
+		dprintk(1, KERN_ERR, "Error in copying to userspace\n");
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static void copy_tls_hash( struct scatterlist *dst_sg, int len, void* hash, int hash_len)
+{
+	scatterwalk_map_and_copy(hash, dst_sg, len, hash_len, 1);
+}
+
+static void read_tls_hash( struct scatterlist *dst_sg, int len, void* hash, int hash_len)
+{
+	scatterwalk_map_and_copy(hash, dst_sg, len-hash_len, hash_len, 0);
+}
+
+static int pad_record( struct scatterlist *dst_sg, int len, int block_size)
+{
+	uint8_t pad[block_size];
+	int pad_size = block_size - (len % block_size);
+
+	memset(pad, pad_size-1, pad_size);
+
+	scatterwalk_map_and_copy(pad, dst_sg, len, pad_size, 1);
+
+	return pad_size;
+}
+
+static int verify_tls_record_pad( struct scatterlist *dst_sg, int len, int block_size)
+{
+	uint8_t pad[256]; /* the maximum allowed */
+	uint8_t pad_size;
+	int i;
+
+	scatterwalk_map_and_copy(&pad_size, dst_sg, len-1, 1, 0);
+
+	if (pad_size+1 > len) {
+		dprintk(1, KERN_ERR, "Pad size: %d\n", pad_size);
+		return -ECANCELED;
+	}
+
+	scatterwalk_map_and_copy(pad, dst_sg, len-pad_size-1, pad_size+1, 0);
+
+	for (i=0;i<pad_size;i++)
+		if (pad[i] != pad_size) {
+			dprintk(1, KERN_ERR, "Pad size: %d, pad: %d\n", pad_size, (int)pad[i]);
+			return -ECANCELED;
+		}
+
+	return pad_size+1;
+}
+
+static int
+tls_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
+		struct scatterlist *auth_sg, uint32_t auth_len,
+		struct scatterlist *dst_sg, uint32_t len)
+{
+	int ret, fail = 0;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	uint8_t vhash[AALG_MAX_RESULT_LEN];
+	uint8_t hash_output[AALG_MAX_RESULT_LEN];
+
+	/* TLS authenticates the plaintext except for the padding.
+	 */
+	if (caop->op == COP_ENCRYPT) {
+		if (ses_ptr->hdata.init != 0) {
+			if (auth_len > 0) {
+				ret = cryptodev_hash_update(&ses_ptr->hdata,
+								auth_sg, auth_len);
+				if (unlikely(ret)) {
+					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+					goto out_err;
+				}
+			}
+
+			if (len > 0) {
+				ret = cryptodev_hash_update(&ses_ptr->hdata,
+								dst_sg, len);
+				if (unlikely(ret)) {
+					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+					goto out_err;
+				}
+			}
+
+			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
+				goto out_err;
+			}
+
+			copy_tls_hash( dst_sg, len, hash_output, caop->tag_len);
+			len += caop->tag_len;
+		}
+
+		if (ses_ptr->cdata.init != 0) {
+			if (ses_ptr->cdata.blocksize > 1) {
+				ret = pad_record(dst_sg, len, ses_ptr->cdata.blocksize);
+				len += ret;
+			}
+
+			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
+							dst_sg, dst_sg, len);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_cipher_encrypt: %d\n", ret);
+				goto out_err;
+			}
+		}
+	} else {
+		if (ses_ptr->cdata.init != 0) {
+			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
+							dst_sg, dst_sg, len);
+
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_cipher_decrypt: %d\n", ret);
+				goto out_err;
+			}
+
+			if (ses_ptr->cdata.blocksize > 1) {
+				ret = verify_tls_record_pad(dst_sg, len, ses_ptr->cdata.blocksize);
+				if (unlikely(ret < 0)) {
+					dprintk(0, KERN_ERR, "verify_record_pad: %d\n", ret);
+					fail = 1;
+				} else {
+					len -= ret;
+				}
+			}
+		}
+
+		if (ses_ptr->hdata.init != 0) {
+			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
+				dprintk(1, KERN_ERR, "Illegal tag len size\n");
+				ret = -EINVAL;
+				goto out_err;
+			}
+
+			read_tls_hash( dst_sg, len, vhash, caop->tag_len);
+			len -= caop->tag_len;
+
+			if (auth_len > 0) {
+				ret = cryptodev_hash_update(&ses_ptr->hdata,
+								auth_sg, auth_len);
+				if (unlikely(ret)) {
+					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+					goto out_err;
+				}
+			}
+
+			if (len > 0) {
+				ret = cryptodev_hash_update(&ses_ptr->hdata,
+									dst_sg, len);
+				if (unlikely(ret)) {
+					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+					goto out_err;
+				}
+			}
+
+			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
+				goto out_err;
+			}
+
+			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
+				dprintk(1, KERN_ERR, "MAC verification failed (tag_len: %d)\n", caop->tag_len);
+				ret = -ECANCELED;
+				goto out_err;
+			}
+		}
+	}
+	kcaop->dst_len = len;
+	return 0;
+out_err:
+	return ret;
+}
+
+static int
+srtp_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
+		  struct scatterlist *auth_sg, uint32_t auth_len,
+		  struct scatterlist *dst_sg, uint32_t len)
+{
+	int ret, fail = 0;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	uint8_t vhash[AALG_MAX_RESULT_LEN];
+	uint8_t hash_output[AALG_MAX_RESULT_LEN];
+
+	/* SRTP authenticates the encrypted data.
+	 */
+	if (caop->op == COP_ENCRYPT) {
+		if (ses_ptr->cdata.init != 0) {
+			if (ses_ptr->cdata.stream == 0) {
+				dprintk(0, KERN_ERR, "Only stream modes are allowed in SRTP mode\n");
+				ret = -EINVAL;
+				goto out_err;
+			}
+
+			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
+							dst_sg, dst_sg, len);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_cipher_encrypt: %d\n", ret);
+				goto out_err;
+			}
+		}
+
+		if (ses_ptr->hdata.init != 0) {
+			if (auth_len > 0) {
+				ret = cryptodev_hash_update(&ses_ptr->hdata,
+								auth_sg, auth_len);
+				if (unlikely(ret)) {
+					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+					goto out_err;
+				}
+			}
+
+			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
+				goto out_err;
+			}
+
+			if (unlikely(copy_to_user(caop->tag, hash_output, caop->tag_len))) {
+				ret = -EFAULT;
+				goto out_err;
+			}
+		}
+
+	} else {
+		if (ses_ptr->hdata.init != 0) {
+			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
+				dprintk(1, KERN_ERR, "Illegal tag len size\n");
+				ret = -EINVAL;
+				goto out_err;
+			}
+
+			if (unlikely(copy_from_user(vhash, caop->tag, caop->tag_len))) {
+				ret = -EFAULT;
+				goto out_err;
+			}
+
+			ret = cryptodev_hash_update(&ses_ptr->hdata,
+							auth_sg, auth_len);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
+				goto out_err;
+			}
+
+			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
+				goto out_err;
+			}
+
+			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
+				dprintk(1, KERN_ERR, "MAC verification failed\n");
+				ret = -ECANCELED;
+				goto out_err;
+			}
+		}
+
+		if (ses_ptr->cdata.init != 0) {
+			if (ses_ptr->cdata.stream == 0) {
+				dprintk(0, KERN_ERR, "Only stream modes are allowed in SRTP mode\n");
+				ret = -EINVAL;
+				goto out_err;
+			}
+
+			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
+							dst_sg, dst_sg, len);
+
+			if (unlikely(ret)) {
+				dprintk(0, KERN_ERR, "cryptodev_cipher_decrypt: %d\n", ret);
+				goto out_err;
+			}
+		}
+
+	}
+	kcaop->dst_len = len;
+	return 0;
+out_err:
+	return ret;
+}
+
+/* This is the main crypto function - zero-copy edition */
+static int
+__crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
+{
+	struct scatterlist *dst_sg, *auth_sg;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	int ret = 0, pagecount;
+
+	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) {
+		ret = get_userbuf_aead(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_ERR, "Error getting user pages.\n");
+			return ret;
+		}
+
+		ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, 
+			   dst_sg, caop->len);
+	} else if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {
+		ret = get_userbuf_srtp(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_ERR, "Error getting user pages.\n");
+			return ret;
+		}
+
+		ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, 
+			   dst_sg, caop->len);
+	} else {
+		dprintk(1, KERN_ERR, "Unsupported flag for authenc\n");
+		return -EINVAL;
+	}
+
+	release_user_pages(ses_ptr->pages, pagecount);
+	return ret;
+}
+
+
+int crypto_auth_run(struct fcrypt *fcr, struct kernel_crypt_auth_op *kcaop)
+{
+	struct csession *ses_ptr;
+	struct crypt_auth_op *caop = &kcaop->caop;
+	int ret;
+
+	if (unlikely(caop->op != COP_ENCRYPT && caop->op != COP_DECRYPT)) {
+		dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", caop->op);
+		return -EINVAL;
+	}
+
+	/* this also enters ses_ptr->sem */
+	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", caop->ses);
+		return -EINVAL;
+	}
+
+	if (unlikely(ses_ptr->cdata.init == 0)) {
+		dprintk(1, KERN_ERR, "cipher context not initialized\n");
+		return -EINVAL;
+	}
+
+	if (ses_ptr->hdata.init != 0) {
+		ret = cryptodev_hash_reset(&ses_ptr->hdata);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_ERR,
+				"error in cryptodev_hash_reset()\n");
+			goto out_unlock;
+		}
+	}
+
+	cryptodev_cipher_set_iv(&ses_ptr->cdata, kcaop->iv,
+				min(ses_ptr->cdata.ivsize, kcaop->ivlen));
+
+	if (likely(caop->len || caop->auth_len)) {
+		ret = __crypto_auth_run_zc(ses_ptr, kcaop);
+		if (unlikely(ret))
+			goto out_unlock;
+	}
+
+	cryptodev_cipher_get_iv(&ses_ptr->cdata, kcaop->iv,
+				min(ses_ptr->cdata.ivsize, kcaop->ivlen));
+
+out_unlock:
+	mutex_unlock(&ses_ptr->sem);
+	return ret;
+}
diff --git a/drivers/staging/crypto/cryptodev/cryptlib.c b/drivers/staging/crypto/cryptodev/cryptlib.c
new file mode 100644
index 000000000000..ba6d7c0afa19
--- /dev/null
+++ b/drivers/staging/crypto/cryptodev/cryptlib.c
@@ -0,0 +1,319 @@
+/*
+ * Driver for /dev/crypto device (aka CryptoDev)
+ *
+ * Copyright (c) 2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
+ * Portions Copyright (c) 2010 Michael Weiser
+ * Portions Copyright (c) 2010 Phil Sutter
+ *
+ * This file is part of linux cryptodev.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/crypto.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/ioctl.h>
+#include <linux/random.h>
+#include <linux/scatterlist.h>
+#include <linux/uaccess.h>
+#include <crypto/algapi.h>
+#include <crypto/hash.h>
+#include <crypto/cryptodev.h>
+#include "cryptodev_int.h"
+
+
+struct cryptodev_result {
+	struct completion completion;
+	int err;
+};
+
+static void cryptodev_complete(struct crypto_async_request *req, int err)
+{
+	struct cryptodev_result *res = req->data;
+
+	if (err == -EINPROGRESS)
+		return;
+
+	res->err = err;
+	complete(&res->completion);
+}
+
+int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
+				uint8_t *keyp, size_t keylen, int stream)
+{
+	struct ablkcipher_alg *alg;
+	int ret;
+
+	memset(out, 0, sizeof(*out));
+
+	out->async.s = crypto_alloc_ablkcipher(alg_name, 0, 0);
+	if (unlikely(IS_ERR(out->async.s))) {
+		dprintk(1, KERN_DEBUG, "%s: Failed to load cipher %s\n",
+			__func__, alg_name);
+		return -EINVAL;
+	}
+
+	alg = crypto_ablkcipher_alg(out->async.s);
+
+	if (alg != NULL) {
+		/* Was correct key length supplied? */
+		if (alg->max_keysize > 0 &&
+				unlikely((keylen < alg->min_keysize) ||
+					(keylen > alg->max_keysize))) {
+			dprintk(1, KERN_DEBUG,
+				"Wrong keylen '%zu' for algorithm '%s'. \
+				Use %u to %u.\n",
+				   keylen, alg_name, alg->min_keysize,
+				   alg->max_keysize);
+			ret = -EINVAL;
+			goto error;
+		}
+	}
+
+	ret = crypto_ablkcipher_setkey(out->async.s, keyp, keylen);
+	if (unlikely(ret)) {
+		dprintk(1, KERN_DEBUG, "Setting key failed for %s-%zu.\n",
+			alg_name, keylen*8);
+		ret = -EINVAL;
+		goto error;
+	}
+
+	out->stream = stream;
+	out->blocksize = crypto_ablkcipher_blocksize(out->async.s);
+	out->ivsize = crypto_ablkcipher_ivsize(out->async.s);
+	out->alignmask = crypto_ablkcipher_alignmask(out->async.s);
+
+	out->async.result = kmalloc(sizeof(*out->async.result), GFP_KERNEL);
+	if (unlikely(!out->async.result)) {
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	memset(out->async.result, 0, sizeof(*out->async.result));
+	init_completion(&out->async.result->completion);
+
+	out->async.request = ablkcipher_request_alloc(out->async.s, GFP_KERNEL);
+	if (unlikely(!out->async.request)) {
+		dprintk(1, KERN_ERR, "error allocating async crypto request\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	ablkcipher_request_set_callback(out->async.request,
+					CRYPTO_TFM_REQ_MAY_BACKLOG,
+					cryptodev_complete, out->async.result);
+
+	out->init = 1;
+	return 0;
+error:
+	if (out->async.request)
+		ablkcipher_request_free(out->async.request);
+	kfree(out->async.result);
+	if (out->async.s)
+		crypto_free_ablkcipher(out->async.s);
+
+	return ret;
+}
+
+void cryptodev_cipher_deinit(struct cipher_data *cdata)
+{
+	if (cdata->init) {
+		if (cdata->async.request)
+			ablkcipher_request_free(cdata->async.request);
+		kfree(cdata->async.result);
+		if (cdata->async.s)
+			crypto_free_ablkcipher(cdata->async.s);
+
+		cdata->init = 0;
+	}
+}
+
+static inline int waitfor(struct cryptodev_result *cr, ssize_t ret)
+{
+	switch (ret) {
+	case 0:
+		break;
+	case -EINPROGRESS:
+	case -EBUSY:
+		wait_for_completion(&cr->completion);
+		/* At this point we known for sure the request has finished,
+		 * because wait_for_completion above was not interruptible.
+		 * This is important because otherwise hardware or driver
+		 * might try to access memory which will be freed or reused for
+		 * another request. */
+
+		if (unlikely(cr->err)) {
+			dprintk(0, KERN_ERR, "error from async request: %d\n",
+				cr->err);
+			return cr->err;
+		}
+
+		break;
+	default:
+		return ret;
+	}
+
+	return 0;
+}
+
+ssize_t cryptodev_cipher_encrypt(struct cipher_data *cdata,
+		const struct scatterlist *sg1, struct scatterlist *sg2,
+		size_t len)
+{
+	int ret;
+
+	INIT_COMPLETION(cdata->async.result->completion);
+	ablkcipher_request_set_crypt(cdata->async.request,
+			(struct scatterlist *)sg1, sg2,
+			len, cdata->async.iv);
+	ret = crypto_ablkcipher_encrypt(cdata->async.request);
+
+	return waitfor(cdata->async.result, ret);
+}
+
+ssize_t cryptodev_cipher_decrypt(struct cipher_data *cdata,
+		const struct scatterlist *sg1, struct scatterlist *sg2,
+		size_t len)
+{
+	int ret;
+
+	INIT_COMPLETION(cdata->async.result->completion);
+	ablkcipher_request_set_crypt(cdata->async.request,
+			(struct scatterlist *)sg1, sg2,
+			len, cdata->async.iv);
+	ret = crypto_ablkcipher_decrypt(cdata->async.request);
+
+	return waitfor(cdata->async.result, ret);
+}
+
+/* Hash functions */
+
+int cryptodev_hash_init(struct hash_data *hdata, const char *alg_name,
+			int hmac_mode, void *mackey, size_t mackeylen)
+{
+	int ret;
+
+	hdata->async.s = crypto_alloc_ahash(alg_name, 0, 0);
+	if (unlikely(IS_ERR(hdata->async.s))) {
+		dprintk(1, KERN_DEBUG, "%s: Failed to load transform for %s\n",
+			__func__, alg_name);
+		return -EINVAL;
+	}
+
+	/* Copy the key from user and set to TFM. */
+	if (hmac_mode != 0) {
+		ret = crypto_ahash_setkey(hdata->async.s, mackey, mackeylen);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_DEBUG,
+				"Setting hmac key failed for %s-%zu.\n",
+				alg_name, mackeylen*8);
+			ret = -EINVAL;
+			goto error;
+		}
+	}
+
+	hdata->digestsize = crypto_ahash_digestsize(hdata->async.s);
+	hdata->alignmask = crypto_ahash_alignmask(hdata->async.s);
+
+	hdata->async.result = kmalloc(sizeof(*hdata->async.result), GFP_KERNEL);
+	if (unlikely(!hdata->async.result)) {
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	memset(hdata->async.result, 0, sizeof(*hdata->async.result));
+	init_completion(&hdata->async.result->completion);
+
+	hdata->async.request = ahash_request_alloc(hdata->async.s, GFP_KERNEL);
+	if (unlikely(!hdata->async.request)) {
+		dprintk(0, KERN_ERR, "error allocating async crypto request\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	ahash_request_set_callback(hdata->async.request,
+			CRYPTO_TFM_REQ_MAY_BACKLOG,
+			cryptodev_complete, hdata->async.result);
+
+	ret = crypto_ahash_init(hdata->async.request);
+	if (unlikely(ret)) {
+		dprintk(0, KERN_ERR, "error in crypto_hash_init()\n");
+		goto error_request;
+	}
+
+	hdata->init = 1;
+	return 0;
+
+error_request:
+	ahash_request_free(hdata->async.request);
+error:
+	kfree(hdata->async.result);
+	crypto_free_ahash(hdata->async.s);
+	return ret;
+}
+
+void cryptodev_hash_deinit(struct hash_data *hdata)
+{
+	if (hdata->init) {
+		if (hdata->async.request)
+			ahash_request_free(hdata->async.request);
+		kfree(hdata->async.result);
+		if (hdata->async.s)
+			crypto_free_ahash(hdata->async.s);
+		hdata->init = 0;
+	}
+}
+
+int cryptodev_hash_reset(struct hash_data *hdata)
+{
+	int ret;
+
+	ret = crypto_ahash_init(hdata->async.request);
+	if (unlikely(ret)) {
+		dprintk(0, KERN_ERR, "error in crypto_hash_init()\n");
+		return ret;
+	}
+
+	return 0;
+
+}
+
+ssize_t cryptodev_hash_update(struct hash_data *hdata,
+				struct scatterlist *sg, size_t len)
+{
+	int ret;
+
+	INIT_COMPLETION(hdata->async.result->completion);
+	ahash_request_set_crypt(hdata->async.request, sg, NULL, len);
+
+	ret = crypto_ahash_update(hdata->async.request);
+
+	return waitfor(hdata->async.result, ret);
+}
+
+int cryptodev_hash_final(struct hash_data *hdata, void* output)
+{
+	int ret;
+
+	INIT_COMPLETION(hdata->async.result->completion);
+	ahash_request_set_crypt(hdata->async.request, NULL, output, 0);
+
+	ret = crypto_ahash_final(hdata->async.request);
+
+	return waitfor(hdata->async.result, ret);
+}
+
diff --git a/drivers/staging/crypto/cryptodev/cryptodev_auth.c b/drivers/staging/crypto/cryptodev/cryptodev_auth.c
deleted file mode 100644
index 597515989daa..000000000000
--- a/drivers/staging/crypto/cryptodev/cryptodev_auth.c
+++ /dev/null
@@ -1,724 +0,0 @@
-/*
- * Driver for /dev/crypto device (aka CryptoDev)
- *
- * Copyright (c) 2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
- *
- * This file is part of linux cryptodev.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-/*
- * This file handles the AEAD part of /dev/crypto.
- *
- */
-
-#include <crypto/hash.h>
-#include <linux/crypto.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/ioctl.h>
-#include <linux/random.h>
-#include <linux/syscalls.h>
-#include <linux/pagemap.h>
-#include <linux/poll.h>
-#include <linux/uaccess.h>
-#include <crypto/cryptodev.h>
-#include <crypto/scatterwalk.h>
-#include <linux/scatterlist.h>
-#include "cryptodev_int.h"
-#include "version.h"
-
-
-/* make cop->src and cop->dst available in scatterlists */
-static int get_userbuf_aead(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
-			struct scatterlist **auth_sg, struct scatterlist **dst_sg, 
-			int *tot_pages)
-{
-	int dst_pagecount = 0, pagecount;
-	int auth_pagecount = 0;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	int rc;
-
-	if (caop->dst == NULL && caop->auth_src == NULL)
-		return -EINVAL;
-
-	if (ses->alignmask) {
-		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
-			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
-				__func__, (unsigned long)caop->dst, ses->alignmask + 1);
-		if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
-			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
-				__func__, (unsigned long)caop->auth_src, ses->alignmask + 1);
-	}
-
-	if (kcaop->dst_len == 0) {
-		dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
-		return -EINVAL;
-	}
-
-	if (caop->auth_len > 0)
-		auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
-
-	dst_pagecount = PAGECOUNT(caop->dst, kcaop->dst_len);
-
-	(*tot_pages) = pagecount = auth_pagecount + dst_pagecount;
-
-	rc = adjust_sg_array(ses, pagecount);
-	if (rc)
-		return rc;
-
-	if (auth_pagecount > 0) {
-		rc = __get_userbuf(caop->auth_src, caop->auth_len, 0, auth_pagecount,
-				   ses->pages, ses->sg, kcaop->task, kcaop->mm);
-		if (unlikely(rc)) {
-			dprintk(1, KERN_ERR,
-				"failed to get user pages for data input\n");
-			return -EINVAL;
-		}
-		(*auth_sg) = ses->sg;
-		(*dst_sg) = ses->sg + auth_pagecount;
-	} else {
-		(*auth_sg) = NULL;
-		(*dst_sg) = ses->sg;
-	}
-
-	rc = __get_userbuf(caop->dst, kcaop->dst_len, 1, dst_pagecount,
-	                   ses->pages + auth_pagecount, *dst_sg, kcaop->task, kcaop->mm);
-	if (unlikely(rc)) {
-		release_user_pages(ses->pages, auth_pagecount);
-		dprintk(1, KERN_ERR,
-			"failed to get user pages for data input\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* Taken from Maxim Levitsky's patch
- */
-static struct scatterlist *sg_advance(struct scatterlist *sg, int consumed)
-{
-	while (consumed >= sg->length) {
-		consumed -= sg->length;
-
-		sg = sg_next(sg);
-		if (!sg)
-			break;
-	}
-
-	WARN_ON(!sg && consumed);
-
-	if (!sg)
-		return NULL;
-
-	sg->offset += consumed;
-	sg->length -= consumed;
-
-	if (sg->offset >= PAGE_SIZE) {
-		struct page *page =
-			nth_page(sg_page(sg), sg->offset / PAGE_SIZE);
-		sg_set_page(sg, page, sg->length, sg->offset % PAGE_SIZE);
-	}
-
-	return sg;
-}
-
-/**
- * sg_copy - copies sg entries from sg_from to sg_to, such
- * as sg_to covers first 'len' bytes from sg_from.
- */
-static int sg_copy(struct scatterlist *sg_from, struct scatterlist *sg_to, int len)
-{
-	while (len > sg_from->length) {
-		len -= sg_from->length;
-
-		sg_set_page(sg_to, sg_page(sg_from),
-				sg_from->length, sg_from->offset);
-
-		sg_to = sg_next(sg_to);
-		sg_from = sg_next(sg_from);
-
-		if (len && (!sg_from || !sg_to))
-			return -ENOMEM;
-	}
-
-	if (len)
-		sg_set_page(sg_to, sg_page(sg_from),
-				len, sg_from->offset);
-	sg_mark_end(sg_to);
-	return 0;
-}
-
-static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
-			struct scatterlist **auth_sg, struct scatterlist **dst_sg, 
-			int *tot_pages)
-{
-	int pagecount, diff;
-	int auth_pagecount = 0;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	int rc;
-
-	if (caop->dst == NULL && caop->auth_src == NULL)
-		return -EINVAL;
-
-	if (ses->alignmask) {
-		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask))
-			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
-				__func__, (unsigned long)caop->dst, ses->alignmask + 1);
-		if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask))
-			dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
-				__func__, (unsigned long)caop->auth_src, ses->alignmask + 1);
-	}
-
-	if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {
-		dprintk(1, KERN_WARNING, "Destination length cannot be zero\n");
-		return -EINVAL;
-	}
-
-	/* Note that in SRTP auth data overlap with data to be encrypted (dst)
-         */
-
-	auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
-	diff = (int)(caop->src - caop->auth_src);
-	if (diff > PAGE_SIZE || diff < 0) {
-		dprintk(1, KERN_WARNING, "auth_src must overlap with src (diff: %d).\n", diff);
-		return -EINVAL;
-	}
-
-	(*tot_pages) = pagecount = auth_pagecount;
-
-	rc = adjust_sg_array(ses, pagecount*2); /* double pages to have pages for dst(=auth_src) */
-	if (rc)
-		return rc;
-
-	rc = __get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,
-			   ses->pages, ses->sg, kcaop->task, kcaop->mm);
-	if (unlikely(rc)) {
-		dprintk(1, KERN_ERR,
-			"failed to get user pages for data input\n");
-		return -EINVAL;
-	}
-	(*auth_sg) = ses->sg;
-
-	(*dst_sg) = ses->sg + auth_pagecount;
-	sg_init_table(*dst_sg, auth_pagecount);
-	sg_copy(ses->sg, (*dst_sg), caop->auth_len);
-	(*dst_sg) = sg_advance(*dst_sg, diff);
-	if (*dst_sg == NULL) {
-		release_user_pages(ses->pages, pagecount);
-		dprintk(1, KERN_ERR,
-			"failed to get enough pages for auth data\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int copy_from_user_to_user( void* __user dst, void* __user src, int len)
-{
-uint8_t *buffer;
-int buffer_size = min(len, 16*1024);
-int rc;
-
-	if (len > buffer_size) {
-		dprintk(1, KERN_ERR,
-			"The provided buffer is too large\n");
-		return -EINVAL;
-	}
-
-	buffer = kmalloc(buffer_size, GFP_KERNEL);
-	if (buffer == NULL)
-		return -ENOMEM;
-
-	if (unlikely(copy_from_user(buffer, src, len))) {
-		rc = -EFAULT;
-		goto out;
-	}
-
-	if (unlikely(copy_to_user(dst, buffer, len))) {
-		rc = -EFAULT;
-		goto out;
-	}
-
-	rc = 0;
-out:
-	kfree(buffer);
-	return rc;
-}
-
-static int fill_kcaop_from_caop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
-{
-	struct crypt_auth_op *caop = &kcaop->caop;
-	struct csession *ses_ptr;
-	int rc;
-
-	/* this also enters ses_ptr->sem */
-	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", caop->ses);
-		return -EINVAL;
-	}
-
-	if (caop->src != caop->dst) {
-		dprintk(2, KERN_ERR,
-			"Non-inplace encryption and decryption is not efficient\n");
-		
-		rc = copy_from_user_to_user( caop->dst, caop->src, caop->len);
-		if (rc < 0)
-			goto out_unlock;
-
-
-	}
-
-	if (caop->tag_len == 0)
-		caop->tag_len = ses_ptr->hdata.digestsize;
-
-	kcaop->ivlen = caop->iv ? ses_ptr->cdata.ivsize : 0;
-
-	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE)
-		kcaop->dst_len = caop->len + ses_ptr->cdata.blocksize /* pad */ + caop->tag_len;
-	else
-		kcaop->dst_len = caop->len;
-
-	kcaop->task = current;
-	kcaop->mm = current->mm;
-
-	if (caop->iv) {
-		rc = copy_from_user(kcaop->iv, caop->iv, kcaop->ivlen);
-		if (unlikely(rc)) {
-			dprintk(1, KERN_ERR,
-				"error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
-				kcaop->ivlen, rc, (unsigned long)caop->iv);
-			rc = -EFAULT;
-			goto out_unlock;
-		}
-	}
-	
-	rc = 0;
-
-out_unlock:
-	crypto_put_session(ses_ptr);
-	return rc;
-
-}
-
-static int fill_caop_from_kcaop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
-{
-	int ret;
-
-	kcaop->caop.len = kcaop->dst_len;
-
-	if (kcaop->ivlen && kcaop->caop.flags & COP_FLAG_WRITE_IV) {
-		ret = copy_to_user(kcaop->caop.iv,
-				kcaop->iv, kcaop->ivlen);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_ERR, "Error in copying to userspace\n");
-			return -EFAULT;
-		}
-	}
-	return 0;
-}
-
-
-int kcaop_from_user(struct kernel_crypt_auth_op *kcaop,
-			struct fcrypt *fcr, void __user *arg)
-{
-	if (unlikely(copy_from_user(&kcaop->caop, arg, sizeof(kcaop->caop)))) {
-		dprintk(1, KERN_ERR, "Error in copying from userspace\n");
-		return -EFAULT;
-	}
-
-	return fill_kcaop_from_caop(kcaop, fcr);
-}
-
-int kcaop_to_user(struct kernel_crypt_auth_op *kcaop,
-		struct fcrypt *fcr, void __user *arg)
-{
-	int ret;
-
-	ret = fill_caop_from_kcaop(kcaop, fcr);
-	if (unlikely(ret)) {
-		dprintk(1, KERN_ERR, "fill_caop_from_kcaop\n");
-		return ret;
-	}
-
-	if (unlikely(copy_to_user(arg, &kcaop->caop, sizeof(kcaop->caop)))) {
-		dprintk(1, KERN_ERR, "Error in copying to userspace\n");
-		return -EFAULT;
-	}
-	return 0;
-}
-
-static void copy_tls_hash( struct scatterlist *dst_sg, int len, void* hash, int hash_len)
-{
-	scatterwalk_map_and_copy(hash, dst_sg, len, hash_len, 1);
-}
-
-static void read_tls_hash( struct scatterlist *dst_sg, int len, void* hash, int hash_len)
-{
-	scatterwalk_map_and_copy(hash, dst_sg, len-hash_len, hash_len, 0);
-}
-
-static int pad_record( struct scatterlist *dst_sg, int len, int block_size)
-{
-	uint8_t pad[block_size];
-	int pad_size = block_size - (len % block_size);
-
-	memset(pad, pad_size-1, pad_size);
-
-	scatterwalk_map_and_copy(pad, dst_sg, len, pad_size, 1);
-
-	return pad_size;
-}
-
-static int verify_tls_record_pad( struct scatterlist *dst_sg, int len, int block_size)
-{
-	uint8_t pad[256]; /* the maximum allowed */
-	uint8_t pad_size;
-	int i;
-
-	scatterwalk_map_and_copy(&pad_size, dst_sg, len-1, 1, 0);
-
-	if (pad_size+1 > len) {
-		dprintk(1, KERN_ERR, "Pad size: %d\n", pad_size);
-		return -ECANCELED;
-	}
-
-	scatterwalk_map_and_copy(pad, dst_sg, len-pad_size-1, pad_size+1, 0);
-
-	for (i=0;i<pad_size;i++)
-		if (pad[i] != pad_size) {
-			dprintk(1, KERN_ERR, "Pad size: %d, pad: %d\n", pad_size, (int)pad[i]);
-			return -ECANCELED;
-		}
-
-	return pad_size+1;
-}
-
-static int
-tls_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
-		struct scatterlist *auth_sg, uint32_t auth_len,
-		struct scatterlist *dst_sg, uint32_t len)
-{
-	int ret, fail = 0;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	uint8_t vhash[AALG_MAX_RESULT_LEN];
-	uint8_t hash_output[AALG_MAX_RESULT_LEN];
-
-	/* TLS authenticates the plaintext except for the padding.
-	 */
-	if (caop->op == COP_ENCRYPT) {
-		if (ses_ptr->hdata.init != 0) {
-			if (auth_len > 0) {
-				ret = cryptodev_hash_update(&ses_ptr->hdata,
-								auth_sg, auth_len);
-				if (unlikely(ret)) {
-					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-					goto out_err;
-				}
-			}
-
-			if (len > 0) {
-				ret = cryptodev_hash_update(&ses_ptr->hdata,
-								dst_sg, len);
-				if (unlikely(ret)) {
-					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-					goto out_err;
-				}
-			}
-
-			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
-				goto out_err;
-			}
-
-			copy_tls_hash( dst_sg, len, hash_output, caop->tag_len);
-			len += caop->tag_len;
-		}
-
-		if (ses_ptr->cdata.init != 0) {
-			if (ses_ptr->cdata.blocksize > 1) {
-				ret = pad_record(dst_sg, len, ses_ptr->cdata.blocksize);
-				len += ret;
-			}
-
-			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
-							dst_sg, dst_sg, len);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_cipher_encrypt: %d\n", ret);
-				goto out_err;
-			}
-		}
-	} else {
-		if (ses_ptr->cdata.init != 0) {
-			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
-							dst_sg, dst_sg, len);
-
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_cipher_decrypt: %d\n", ret);
-				goto out_err;
-			}
-
-			if (ses_ptr->cdata.blocksize > 1) {
-				ret = verify_tls_record_pad(dst_sg, len, ses_ptr->cdata.blocksize);
-				if (unlikely(ret < 0)) {
-					dprintk(0, KERN_ERR, "verify_record_pad: %d\n", ret);
-					fail = 1;
-				} else {
-					len -= ret;
-				}
-			}
-		}
-
-		if (ses_ptr->hdata.init != 0) {
-			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
-				dprintk(1, KERN_ERR, "Illegal tag len size\n");
-				ret = -EINVAL;
-				goto out_err;
-			}
-
-			read_tls_hash( dst_sg, len, vhash, caop->tag_len);
-			len -= caop->tag_len;
-
-			if (auth_len > 0) {
-				ret = cryptodev_hash_update(&ses_ptr->hdata,
-								auth_sg, auth_len);
-				if (unlikely(ret)) {
-					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-					goto out_err;
-				}
-			}
-
-			if (len > 0) {
-				ret = cryptodev_hash_update(&ses_ptr->hdata,
-									dst_sg, len);
-				if (unlikely(ret)) {
-					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-					goto out_err;
-				}
-			}
-
-			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
-				goto out_err;
-			}
-
-			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
-				dprintk(1, KERN_ERR, "MAC verification failed (tag_len: %d)\n", caop->tag_len);
-				ret = -ECANCELED;
-				goto out_err;
-			}
-		}
-	}
-	kcaop->dst_len = len;
-	return 0;
-out_err:
-	return ret;
-}
-
-static int
-srtp_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
-		  struct scatterlist *auth_sg, uint32_t auth_len,
-		  struct scatterlist *dst_sg, uint32_t len)
-{
-	int ret, fail = 0;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	uint8_t vhash[AALG_MAX_RESULT_LEN];
-	uint8_t hash_output[AALG_MAX_RESULT_LEN];
-
-	/* SRTP authenticates the encrypted data.
-	 */
-	if (caop->op == COP_ENCRYPT) {
-		if (ses_ptr->cdata.init != 0) {
-			if (ses_ptr->cdata.stream == 0) {
-				dprintk(0, KERN_ERR, "Only stream modes are allowed in SRTP mode\n");
-				ret = -EINVAL;
-				goto out_err;
-			}
-
-			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
-							dst_sg, dst_sg, len);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_cipher_encrypt: %d\n", ret);
-				goto out_err;
-			}
-		}
-
-		if (ses_ptr->hdata.init != 0) {
-			if (auth_len > 0) {
-				ret = cryptodev_hash_update(&ses_ptr->hdata,
-								auth_sg, auth_len);
-				if (unlikely(ret)) {
-					dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-					goto out_err;
-				}
-			}
-
-			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
-				goto out_err;
-			}
-
-			if (unlikely(copy_to_user(caop->tag, hash_output, caop->tag_len))) {
-				ret = -EFAULT;
-				goto out_err;
-			}
-		}
-
-	} else {
-		if (ses_ptr->hdata.init != 0) {
-			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
-				dprintk(1, KERN_ERR, "Illegal tag len size\n");
-				ret = -EINVAL;
-				goto out_err;
-			}
-
-			if (unlikely(copy_from_user(vhash, caop->tag, caop->tag_len))) {
-				ret = -EFAULT;
-				goto out_err;
-			}
-
-			ret = cryptodev_hash_update(&ses_ptr->hdata,
-							auth_sg, auth_len);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_hash_update: %d\n", ret);
-				goto out_err;
-			}
-
-			ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_hash_final: %d\n", ret);
-				goto out_err;
-			}
-
-			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
-				dprintk(1, KERN_ERR, "MAC verification failed\n");
-				ret = -ECANCELED;
-				goto out_err;
-			}
-		}
-
-		if (ses_ptr->cdata.init != 0) {
-			if (ses_ptr->cdata.stream == 0) {
-				dprintk(0, KERN_ERR, "Only stream modes are allowed in SRTP mode\n");
-				ret = -EINVAL;
-				goto out_err;
-			}
-
-			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
-							dst_sg, dst_sg, len);
-
-			if (unlikely(ret)) {
-				dprintk(0, KERN_ERR, "cryptodev_cipher_decrypt: %d\n", ret);
-				goto out_err;
-			}
-		}
-
-	}
-	kcaop->dst_len = len;
-	return 0;
-out_err:
-	return ret;
-}
-
-/* This is the main crypto function - zero-copy edition */
-static int
-__crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
-{
-	struct scatterlist *dst_sg, *auth_sg;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	int ret = 0, pagecount;
-
-	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) {
-		ret = get_userbuf_aead(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_ERR, "Error getting user pages.\n");
-			return ret;
-		}
-
-		ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, 
-			   dst_sg, caop->len);
-	} else if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {
-		ret = get_userbuf_srtp(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_ERR, "Error getting user pages.\n");
-			return ret;
-		}
-
-		ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, 
-			   dst_sg, caop->len);
-	} else {
-		dprintk(1, KERN_ERR, "Unsupported flag for authenc\n");
-		return -EINVAL;
-	}
-
-	release_user_pages(ses_ptr->pages, pagecount);
-	return ret;
-}
-
-
-int crypto_auth_run(struct fcrypt *fcr, struct kernel_crypt_auth_op *kcaop)
-{
-	struct csession *ses_ptr;
-	struct crypt_auth_op *caop = &kcaop->caop;
-	int ret;
-
-	if (unlikely(caop->op != COP_ENCRYPT && caop->op != COP_DECRYPT)) {
-		dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", caop->op);
-		return -EINVAL;
-	}
-
-	/* this also enters ses_ptr->sem */
-	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", caop->ses);
-		return -EINVAL;
-	}
-
-	if (unlikely(ses_ptr->cdata.init == 0)) {
-		dprintk(1, KERN_ERR, "cipher context not initialized\n");
-		return -EINVAL;
-	}
-
-	if (ses_ptr->hdata.init != 0) {
-		ret = cryptodev_hash_reset(&ses_ptr->hdata);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_ERR,
-				"error in cryptodev_hash_reset()\n");
-			goto out_unlock;
-		}
-	}
-
-	cryptodev_cipher_set_iv(&ses_ptr->cdata, kcaop->iv,
-				min(ses_ptr->cdata.ivsize, kcaop->ivlen));
-
-	if (likely(caop->len || caop->auth_len)) {
-		ret = __crypto_auth_run_zc(ses_ptr, kcaop);
-		if (unlikely(ret))
-			goto out_unlock;
-	}
-
-	cryptodev_cipher_get_iv(&ses_ptr->cdata, kcaop->iv,
-				min(ses_ptr->cdata.ivsize, kcaop->ivlen));
-
-out_unlock:
-	mutex_unlock(&ses_ptr->sem);
-	return ret;
-}
diff --git a/drivers/staging/crypto/cryptodev/cryptodev_cipher.c b/drivers/staging/crypto/cryptodev/cryptodev_cipher.c
deleted file mode 100644
index ba6d7c0afa19..000000000000
--- a/drivers/staging/crypto/cryptodev/cryptodev_cipher.c
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Driver for /dev/crypto device (aka CryptoDev)
- *
- * Copyright (c) 2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
- * Portions Copyright (c) 2010 Michael Weiser
- * Portions Copyright (c) 2010 Phil Sutter
- *
- * This file is part of linux cryptodev.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-#include <linux/crypto.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/ioctl.h>
-#include <linux/random.h>
-#include <linux/scatterlist.h>
-#include <linux/uaccess.h>
-#include <crypto/algapi.h>
-#include <crypto/hash.h>
-#include <crypto/cryptodev.h>
-#include "cryptodev_int.h"
-
-
-struct cryptodev_result {
-	struct completion completion;
-	int err;
-};
-
-static void cryptodev_complete(struct crypto_async_request *req, int err)
-{
-	struct cryptodev_result *res = req->data;
-
-	if (err == -EINPROGRESS)
-		return;
-
-	res->err = err;
-	complete(&res->completion);
-}
-
-int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
-				uint8_t *keyp, size_t keylen, int stream)
-{
-	struct ablkcipher_alg *alg;
-	int ret;
-
-	memset(out, 0, sizeof(*out));
-
-	out->async.s = crypto_alloc_ablkcipher(alg_name, 0, 0);
-	if (unlikely(IS_ERR(out->async.s))) {
-		dprintk(1, KERN_DEBUG, "%s: Failed to load cipher %s\n",
-			__func__, alg_name);
-		return -EINVAL;
-	}
-
-	alg = crypto_ablkcipher_alg(out->async.s);
-
-	if (alg != NULL) {
-		/* Was correct key length supplied? */
-		if (alg->max_keysize > 0 &&
-				unlikely((keylen < alg->min_keysize) ||
-					(keylen > alg->max_keysize))) {
-			dprintk(1, KERN_DEBUG,
-				"Wrong keylen '%zu' for algorithm '%s'. \
-				Use %u to %u.\n",
-				   keylen, alg_name, alg->min_keysize,
-				   alg->max_keysize);
-			ret = -EINVAL;
-			goto error;
-		}
-	}
-
-	ret = crypto_ablkcipher_setkey(out->async.s, keyp, keylen);
-	if (unlikely(ret)) {
-		dprintk(1, KERN_DEBUG, "Setting key failed for %s-%zu.\n",
-			alg_name, keylen*8);
-		ret = -EINVAL;
-		goto error;
-	}
-
-	out->stream = stream;
-	out->blocksize = crypto_ablkcipher_blocksize(out->async.s);
-	out->ivsize = crypto_ablkcipher_ivsize(out->async.s);
-	out->alignmask = crypto_ablkcipher_alignmask(out->async.s);
-
-	out->async.result = kmalloc(sizeof(*out->async.result), GFP_KERNEL);
-	if (unlikely(!out->async.result)) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	memset(out->async.result, 0, sizeof(*out->async.result));
-	init_completion(&out->async.result->completion);
-
-	out->async.request = ablkcipher_request_alloc(out->async.s, GFP_KERNEL);
-	if (unlikely(!out->async.request)) {
-		dprintk(1, KERN_ERR, "error allocating async crypto request\n");
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	ablkcipher_request_set_callback(out->async.request,
-					CRYPTO_TFM_REQ_MAY_BACKLOG,
-					cryptodev_complete, out->async.result);
-
-	out->init = 1;
-	return 0;
-error:
-	if (out->async.request)
-		ablkcipher_request_free(out->async.request);
-	kfree(out->async.result);
-	if (out->async.s)
-		crypto_free_ablkcipher(out->async.s);
-
-	return ret;
-}
-
-void cryptodev_cipher_deinit(struct cipher_data *cdata)
-{
-	if (cdata->init) {
-		if (cdata->async.request)
-			ablkcipher_request_free(cdata->async.request);
-		kfree(cdata->async.result);
-		if (cdata->async.s)
-			crypto_free_ablkcipher(cdata->async.s);
-
-		cdata->init = 0;
-	}
-}
-
-static inline int waitfor(struct cryptodev_result *cr, ssize_t ret)
-{
-	switch (ret) {
-	case 0:
-		break;
-	case -EINPROGRESS:
-	case -EBUSY:
-		wait_for_completion(&cr->completion);
-		/* At this point we known for sure the request has finished,
-		 * because wait_for_completion above was not interruptible.
-		 * This is important because otherwise hardware or driver
-		 * might try to access memory which will be freed or reused for
-		 * another request. */
-
-		if (unlikely(cr->err)) {
-			dprintk(0, KERN_ERR, "error from async request: %d\n",
-				cr->err);
-			return cr->err;
-		}
-
-		break;
-	default:
-		return ret;
-	}
-
-	return 0;
-}
-
-ssize_t cryptodev_cipher_encrypt(struct cipher_data *cdata,
-		const struct scatterlist *sg1, struct scatterlist *sg2,
-		size_t len)
-{
-	int ret;
-
-	INIT_COMPLETION(cdata->async.result->completion);
-	ablkcipher_request_set_crypt(cdata->async.request,
-			(struct scatterlist *)sg1, sg2,
-			len, cdata->async.iv);
-	ret = crypto_ablkcipher_encrypt(cdata->async.request);
-
-	return waitfor(cdata->async.result, ret);
-}
-
-ssize_t cryptodev_cipher_decrypt(struct cipher_data *cdata,
-		const struct scatterlist *sg1, struct scatterlist *sg2,
-		size_t len)
-{
-	int ret;
-
-	INIT_COMPLETION(cdata->async.result->completion);
-	ablkcipher_request_set_crypt(cdata->async.request,
-			(struct scatterlist *)sg1, sg2,
-			len, cdata->async.iv);
-	ret = crypto_ablkcipher_decrypt(cdata->async.request);
-
-	return waitfor(cdata->async.result, ret);
-}
-
-/* Hash functions */
-
-int cryptodev_hash_init(struct hash_data *hdata, const char *alg_name,
-			int hmac_mode, void *mackey, size_t mackeylen)
-{
-	int ret;
-
-	hdata->async.s = crypto_alloc_ahash(alg_name, 0, 0);
-	if (unlikely(IS_ERR(hdata->async.s))) {
-		dprintk(1, KERN_DEBUG, "%s: Failed to load transform for %s\n",
-			__func__, alg_name);
-		return -EINVAL;
-	}
-
-	/* Copy the key from user and set to TFM. */
-	if (hmac_mode != 0) {
-		ret = crypto_ahash_setkey(hdata->async.s, mackey, mackeylen);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_DEBUG,
-				"Setting hmac key failed for %s-%zu.\n",
-				alg_name, mackeylen*8);
-			ret = -EINVAL;
-			goto error;
-		}
-	}
-
-	hdata->digestsize = crypto_ahash_digestsize(hdata->async.s);
-	hdata->alignmask = crypto_ahash_alignmask(hdata->async.s);
-
-	hdata->async.result = kmalloc(sizeof(*hdata->async.result), GFP_KERNEL);
-	if (unlikely(!hdata->async.result)) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	memset(hdata->async.result, 0, sizeof(*hdata->async.result));
-	init_completion(&hdata->async.result->completion);
-
-	hdata->async.request = ahash_request_alloc(hdata->async.s, GFP_KERNEL);
-	if (unlikely(!hdata->async.request)) {
-		dprintk(0, KERN_ERR, "error allocating async crypto request\n");
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	ahash_request_set_callback(hdata->async.request,
-			CRYPTO_TFM_REQ_MAY_BACKLOG,
-			cryptodev_complete, hdata->async.result);
-
-	ret = crypto_ahash_init(hdata->async.request);
-	if (unlikely(ret)) {
-		dprintk(0, KERN_ERR, "error in crypto_hash_init()\n");
-		goto error_request;
-	}
-
-	hdata->init = 1;
-	return 0;
-
-error_request:
-	ahash_request_free(hdata->async.request);
-error:
-	kfree(hdata->async.result);
-	crypto_free_ahash(hdata->async.s);
-	return ret;
-}
-
-void cryptodev_hash_deinit(struct hash_data *hdata)
-{
-	if (hdata->init) {
-		if (hdata->async.request)
-			ahash_request_free(hdata->async.request);
-		kfree(hdata->async.result);
-		if (hdata->async.s)
-			crypto_free_ahash(hdata->async.s);
-		hdata->init = 0;
-	}
-}
-
-int cryptodev_hash_reset(struct hash_data *hdata)
-{
-	int ret;
-
-	ret = crypto_ahash_init(hdata->async.request);
-	if (unlikely(ret)) {
-		dprintk(0, KERN_ERR, "error in crypto_hash_init()\n");
-		return ret;
-	}
-
-	return 0;
-
-}
-
-ssize_t cryptodev_hash_update(struct hash_data *hdata,
-				struct scatterlist *sg, size_t len)
-{
-	int ret;
-
-	INIT_COMPLETION(hdata->async.result->completion);
-	ahash_request_set_crypt(hdata->async.request, sg, NULL, len);
-
-	ret = crypto_ahash_update(hdata->async.request);
-
-	return waitfor(hdata->async.result, ret);
-}
-
-int cryptodev_hash_final(struct hash_data *hdata, void* output)
-{
-	int ret;
-
-	INIT_COMPLETION(hdata->async.result->completion);
-	ahash_request_set_crypt(hdata->async.request, NULL, output, 0);
-
-	ret = crypto_ahash_final(hdata->async.request);
-
-	return waitfor(hdata->async.result, ret);
-}
-
diff --git a/drivers/staging/crypto/cryptodev/cryptodev_int.h b/drivers/staging/crypto/cryptodev/cryptodev_int.h
index 905e3854a9ce..d7412a69cee6 100644
--- a/drivers/staging/crypto/cryptodev/cryptodev_int.h
+++ b/drivers/staging/crypto/cryptodev/cryptodev_int.h
@@ -2,8 +2,6 @@
 #ifndef CRYPTODEV_INT_H
 # define CRYPTODEV_INT_H
 
-#define CRYPTODEV_STATS
-
 #include <linux/init.h>
 #include <linux/sched.h>
 #include <linux/fs.h>
@@ -171,6 +169,7 @@ int kcaop_from_user(struct kernel_crypt_auth_op *kcop,
 int kcaop_to_user(struct kernel_crypt_auth_op *kcaop,
 		struct fcrypt *fcr, void __user *arg);
 int crypto_auth_run(struct fcrypt *fcr, struct kernel_crypt_auth_op *kcaop);
+int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop);
 
 /* other internal structs */
 struct csession {
@@ -180,13 +179,6 @@ struct csession {
 	struct hash_data hdata;
 	uint32_t sid;
 	uint32_t alignmask;
-#ifdef CRYPTODEV_STATS
-#if !((COP_ENCRYPT < 2) && (COP_DECRYPT < 2))
-#error Struct csession.stat uses COP_{ENCRYPT,DECRYPT} as indices. Do something!
-#endif
-	unsigned long long stat[2];
-	size_t stat_max_size, stat_count;
-#endif
 	int array_size;
 	struct page **pages;
 	struct scatterlist *sg;
diff --git a/drivers/staging/crypto/cryptodev/cryptodev_main.c b/drivers/staging/crypto/cryptodev/cryptodev_main.c
deleted file mode 100644
index 0c97f0f79c15..000000000000
--- a/drivers/staging/crypto/cryptodev/cryptodev_main.c
+++ /dev/null
@@ -1,1414 +0,0 @@
-/*
- * Driver for /dev/crypto device (aka CryptoDev)
- *
- * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
- * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
- *
- * This file is part of linux cryptodev.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-/*
- * Device /dev/crypto provides an interface for
- * accessing kernel CryptoAPI algorithms (ciphers,
- * hashes) from userspace programs.
- *
- * /dev/crypto interface was originally introduced in
- * OpenBSD and this module attempts to keep the API.
- *
- */
-
-#include <crypto/hash.h>
-#include <linux/crypto.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/ioctl.h>
-#include <linux/random.h>
-#include <linux/syscalls.h>
-#include <linux/pagemap.h>
-#include <linux/poll.h>
-#include <linux/uaccess.h>
-#include <crypto/cryptodev.h>
-#include <linux/scatterlist.h>
-#include "cryptodev_int.h"
-#include "version.h"
-
-MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
-MODULE_DESCRIPTION("CryptoDev driver");
-MODULE_LICENSE("GPL");
-
-/* ====== Compile-time config ====== */
-
-/* Default (pre-allocated) and maximum size of the job queue.
- * These are free, pending and done items all together. */
-#define DEF_COP_RINGSIZE 16
-#define MAX_COP_RINGSIZE 64
-
-/* ====== Module parameters ====== */
-
-int cryptodev_verbosity;
-module_param(cryptodev_verbosity, int, 0644);
-MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
-
-#ifdef CRYPTODEV_STATS
-static int enable_stats;
-module_param(enable_stats, int, 0644);
-MODULE_PARM_DESC(enable_stats, "collect statictics about cryptodev usage");
-#endif
-
-/* ====== CryptoAPI ====== */
-struct todo_list_item {
-	struct list_head __hook;
-	struct kernel_crypt_op kcop;
-	int result;
-};
-
-struct locked_list {
-	struct list_head list;
-	struct mutex lock;
-};
-
-struct crypt_priv {
-	struct fcrypt fcrypt;
-	struct locked_list free, todo, done;
-	int itemcount;
-	struct work_struct cryptask;
-	wait_queue_head_t user_waiter;
-};
-
-#define FILL_SG(sg, ptr, len)					\
-	do {							\
-		(sg)->page = virt_to_page(ptr);			\
-		(sg)->offset = offset_in_page(ptr);		\
-		(sg)->length = len;				\
-		(sg)->dma_address = 0;				\
-	} while (0)
-
-/* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
-static struct workqueue_struct *cryptodev_wq;
-
-/* Prepare session for future use. */
-static int
-crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
-{
-	struct csession	*ses_new = NULL, *ses_ptr;
-	int ret = 0;
-	const char *alg_name = NULL;
-	const char *hash_name = NULL;
-	int hmac_mode = 1, stream = 0;
-
-	/* Does the request make sense? */
-	if (unlikely(!sop->cipher && !sop->mac)) {
-		dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
-		return -EINVAL;
-	}
-
-	switch (sop->cipher) {
-	case 0:
-		break;
-	case CRYPTO_DES_CBC:
-		alg_name = "cbc(des)";
-		break;
-	case CRYPTO_3DES_CBC:
-		alg_name = "cbc(des3_ede)";
-		break;
-	case CRYPTO_BLF_CBC:
-		alg_name = "cbc(blowfish)";
-		break;
-	case CRYPTO_AES_CBC:
-		alg_name = "cbc(aes)";
-		break;
-	case CRYPTO_AES_ECB:
-		alg_name = "ecb(aes)";
-		break;
-	case CRYPTO_CAMELLIA_CBC:
-		alg_name = "cbc(camelia)";
-		break;
-	case CRYPTO_AES_CTR:
-		alg_name = "ctr(aes)";
-		stream = 1;
-		break;
-	case CRYPTO_NULL:
-		alg_name = "ecb(cipher_null)";
-		stream = 1;
-		break;
-	default:
-		dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
-			sop->cipher);
-		return -EINVAL;
-	}
-
-	switch (sop->mac) {
-	case 0:
-		break;
-	case CRYPTO_MD5_HMAC:
-		hash_name = "hmac(md5)";
-		break;
-	case CRYPTO_RIPEMD160_HMAC:
-		hash_name = "hmac(rmd160)";
-		break;
-	case CRYPTO_SHA1_HMAC:
-		hash_name = "hmac(sha1)";
-		break;
-	case CRYPTO_SHA2_256_HMAC:
-		hash_name = "hmac(sha256)";
-		break;
-	case CRYPTO_SHA2_384_HMAC:
-		hash_name = "hmac(sha384)";
-		break;
-	case CRYPTO_SHA2_512_HMAC:
-		hash_name = "hmac(sha512)";
-		break;
-
-	/* non-hmac cases */
-	case CRYPTO_MD5:
-		hash_name = "md5";
-		hmac_mode = 0;
-		break;
-	case CRYPTO_RIPEMD160:
-		hash_name = "rmd160";
-		hmac_mode = 0;
-		break;
-	case CRYPTO_SHA1:
-		hash_name = "sha1";
-		hmac_mode = 0;
-		break;
-	case CRYPTO_SHA2_256:
-		hash_name = "sha256";
-		hmac_mode = 0;
-		break;
-	case CRYPTO_SHA2_384:
-		hash_name = "sha384";
-		hmac_mode = 0;
-		break;
-	case CRYPTO_SHA2_512:
-		hash_name = "sha512";
-		hmac_mode = 0;
-		break;
-
-	default:
-		dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
-			sop->mac);
-		return -EINVAL;
-	}
-
-	/* Create a session and put it to the list. */
-	ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
-	if (!ses_new)
-		return -ENOMEM;
-
-	/* Set-up crypto transform. */
-	if (alg_name) {
-		uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
-
-		if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
-			dprintk(1, KERN_DEBUG,
-				"Setting key failed for %s-%zu.\n",
-				alg_name, (size_t)sop->keylen*8);
-			ret = -EINVAL;
-			goto error_cipher;
-		}
-
-		if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
-			ret = -EFAULT;
-			goto error_cipher;
-		}
-
-		ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
-						sop->keylen, stream);
-		if (ret < 0) {
-			dprintk(1, KERN_DEBUG,
-				"%s: Failed to load cipher for %s\n",
-				__func__, alg_name);
-			ret = -EINVAL;
-			goto error_cipher;
-		}
-	}
-
-	if (hash_name) {
-		uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
-
-		if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
-			dprintk(1, KERN_DEBUG,
-				"Setting key failed for %s-%zu.\n",
-				alg_name, (size_t)sop->mackeylen*8);
-			ret = -EINVAL;
-			goto error_hash;
-		}
-
-		if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
-					    sop->mackeylen))) {
-			ret = -EFAULT;
-			goto error_hash;
-		}
-
-		ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
-							keyp, sop->mackeylen);
-		if (ret != 0) {
-			dprintk(1, KERN_DEBUG,
-			"%s: Failed to load hash for %s\n",
-			__func__, hash_name);
-			ret = -EINVAL;
-			goto error_hash;
-		}
-	}
-
-	ses_new->alignmask = max(ses_new->cdata.alignmask,
-	                                          ses_new->hdata.alignmask);
-	dprintk(2, KERN_DEBUG, "%s: got alignmask %d\n", __func__, ses_new->alignmask);
-
-	ses_new->array_size = DEFAULT_PREALLOC_PAGES;
-	dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
-			__func__, ses_new->array_size);
-	ses_new->pages = kzalloc(ses_new->array_size *
-			sizeof(struct page *), GFP_KERNEL);
-	ses_new->sg = kzalloc(ses_new->array_size *
-			sizeof(struct scatterlist), GFP_KERNEL);
-	if (ses_new->sg == NULL || ses_new->pages == NULL) {
-		dprintk(0, KERN_DEBUG, "Memory error\n");
-		ret = -ENOMEM;
-		goto error_hash;
-	}
-
-	/* put the new session to the list */
-	get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
-	mutex_init(&ses_new->sem);
-
-	mutex_lock(&fcr->sem);
-restart:
-	list_for_each_entry(ses_ptr, &fcr->list, entry) {
-		/* Check for duplicate SID */
-		if (unlikely(ses_new->sid == ses_ptr->sid)) {
-			get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
-			/* Unless we have a broken RNG this
-			   shouldn't loop forever... ;-) */
-			goto restart;
-		}
-	}
-
-	list_add(&ses_new->entry, &fcr->list);
-	mutex_unlock(&fcr->sem);
-
-	/* Fill in some values for the user. */
-	sop->ses = ses_new->sid;
-
-	return 0;
-
-error_hash:
-	cryptodev_cipher_deinit(&ses_new->cdata);
-	kfree(ses_new->sg);
-	kfree(ses_new->pages);
-error_cipher:
-	kfree(ses_new);
-
-	return ret;
-
-}
-
-/* Everything that needs to be done when remowing a session. */
-static inline void
-crypto_destroy_session(struct csession *ses_ptr)
-{
-	if (!mutex_trylock(&ses_ptr->sem)) {
-		dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
-			ses_ptr->sid);
-		mutex_lock(&ses_ptr->sem);
-	}
-	dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
-#if defined(CRYPTODEV_STATS)
-	if (enable_stats)
-		dprintk(2, KERN_DEBUG,
-			"Usage in Bytes: enc=%llu, dec=%llu, "
-			"max=%zu, avg=%lu, cnt=%zu\n",
-			ses_ptr->stat[COP_ENCRYPT], ses_ptr->stat[COP_DECRYPT],
-			ses_ptr->stat_max_size, ses_ptr->stat_count > 0
-				? ((unsigned long)(ses_ptr->stat[COP_ENCRYPT]+
-						   ses_ptr->stat[COP_DECRYPT]) /
-				   ses_ptr->stat_count) : 0,
-			ses_ptr->stat_count);
-#endif
-	cryptodev_cipher_deinit(&ses_ptr->cdata);
-	cryptodev_hash_deinit(&ses_ptr->hdata);
-	dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
-			__func__, ses_ptr->array_size);
-	kfree(ses_ptr->pages);
-	kfree(ses_ptr->sg);
-	mutex_unlock(&ses_ptr->sem);
-	kfree(ses_ptr);
-}
-
-/* Look up a session by ID and remove. */
-static int
-crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
-{
-	struct csession *tmp, *ses_ptr;
-	struct list_head *head;
-	int ret = 0;
-
-	mutex_lock(&fcr->sem);
-	head = &fcr->list;
-	list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
-		if (ses_ptr->sid == sid) {
-			list_del(&ses_ptr->entry);
-			crypto_destroy_session(ses_ptr);
-			break;
-		}
-	}
-
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
-			sid);
-		ret = -ENOENT;
-	}
-	mutex_unlock(&fcr->sem);
-
-	return ret;
-}
-
-/* Remove all sessions when closing the file */
-static int
-crypto_finish_all_sessions(struct fcrypt *fcr)
-{
-	struct csession *tmp, *ses_ptr;
-	struct list_head *head;
-
-	mutex_lock(&fcr->sem);
-
-	head = &fcr->list;
-	list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
-		list_del(&ses_ptr->entry);
-		crypto_destroy_session(ses_ptr);
-	}
-	mutex_unlock(&fcr->sem);
-
-	return 0;
-}
-
-/* Look up session by session ID. The returned session is locked. */
-struct csession *
-crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
-{
-	struct csession *ses_ptr, *retval = 0;
-
-	mutex_lock(&fcr->sem);
-	list_for_each_entry(ses_ptr, &fcr->list, entry) {
-		if (ses_ptr->sid == sid) {
-			mutex_lock(&ses_ptr->sem);
-			retval = ses_ptr;
-			break;
-		}
-	}
-	mutex_unlock(&fcr->sem);
-
-	return retval;
-}
-
-static int
-hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
-		struct scatterlist *src_sg, struct scatterlist *dst_sg,
-		uint32_t len)
-{
-	int ret;
-
-	/* Always hash before encryption and after decryption. Maybe
-	 * we should introduce a flag to switch... TBD later on.
-	 */
-	if (cop->op == COP_ENCRYPT) {
-		if (ses_ptr->hdata.init != 0) {
-			ret = cryptodev_hash_update(&ses_ptr->hdata,
-							src_sg, len);
-			if (unlikely(ret))
-				goto out_err;
-		}
-		if (ses_ptr->cdata.init != 0) {
-			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
-							src_sg, dst_sg, len);
-
-			if (unlikely(ret))
-				goto out_err;
-		}
-	} else {
-		if (ses_ptr->cdata.init != 0) {
-			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
-							src_sg, dst_sg, len);
-
-			if (unlikely(ret))
-				goto out_err;
-		}
-
-		if (ses_ptr->hdata.init != 0) {
-			ret = cryptodev_hash_update(&ses_ptr->hdata,
-								dst_sg, len);
-			if (unlikely(ret))
-				goto out_err;
-		}
-	}
-	return 0;
-out_err:
-	dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
-	return ret;
-}
-
-
-/* This is the main crypto function - feed it with plaintext
-   and get a ciphertext (or vice versa :-) */
-static int
-__crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
-{
-	char *data;
-	char __user *src, *dst;
-	struct scatterlist sg;
-	size_t nbytes, bufsize;
-	int ret = 0;
-
-	nbytes = cop->len;
-	data = (char *)__get_free_page(GFP_KERNEL);
-
-	if (unlikely(!data))
-		return -ENOMEM;
-
-	bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
-
-	src = cop->src;
-	dst = cop->dst;
-
-	while (nbytes > 0) {
-		size_t current_len = nbytes > bufsize ? bufsize : nbytes;
-
-		if (unlikely(copy_from_user(data, src, current_len))) {
-			ret = -EFAULT;
-			break;
-		}
-
-		sg_init_one(&sg, data, current_len);
-
-		ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
-
-		if (unlikely(ret))
-			break;
-
-		if (ses_ptr->cdata.init != 0) {
-			if (unlikely(copy_to_user(dst, data, current_len))) {
-				ret = -EFAULT;
-				break;
-			}
-		}
-
-		dst += current_len;
-		nbytes -= current_len;
-		src += current_len;
-	}
-
-	free_page((unsigned long)data);
-	return ret;
-}
-
-void release_user_pages(struct page **pg, int pagecount)
-{
-	while (pagecount--) {
-		if (!PageReserved(pg[pagecount]))
-			SetPageDirty(pg[pagecount]);
-		page_cache_release(pg[pagecount]);
-	}
-}
-
-/* offset of buf in it's first page */
-#define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
-
-/* fetch the pages addr resides in into pg and initialise sg with them */
-int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
-		int pgcount, struct page **pg, struct scatterlist *sg,
-		struct task_struct *task, struct mm_struct *mm)
-{
-	int ret, pglen, i = 0;
-	struct scatterlist *sgp;
-
-	down_write(&mm->mmap_sem);
-	ret = get_user_pages(task, mm,
-			(unsigned long)addr, pgcount, write, 0, pg, NULL);
-	up_write(&mm->mmap_sem);
-	if (ret != pgcount)
-		return -EINVAL;
-
-	sg_init_table(sg, pgcount);
-
-	pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
-	sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
-
-	len -= pglen;
-	for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
-		pglen = min((uint32_t)PAGE_SIZE, len);
-		sg_set_page(sgp, pg[i++], pglen, 0);
-		len -= pglen;
-	}
-	sg_mark_end(sg_last(sg, pgcount));
-	return 0;
-}
-
-int adjust_sg_array(struct csession * ses, int pagecount)
-{
-struct scatterlist *sg;
-struct page **pages;
-int array_size;
-
-	for (array_size = ses->array_size; array_size < pagecount;
-	     array_size *= 2)
-		;
-
-	dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
-			__func__, array_size);
-	pages = krealloc(ses->pages, array_size * sizeof(struct page *),
-			 GFP_KERNEL);
-	if (unlikely(!pages))
-		return -ENOMEM;
-	ses->pages = pages;
-	sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
-		      GFP_KERNEL);
-	if (unlikely(!sg))
-		return -ENOMEM;
-	ses->sg = sg;
-	ses->array_size = array_size;
-
-	return 0;
-}
-
-/* make cop->src and cop->dst available in scatterlists */
-static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop,
-                       struct scatterlist **src_sg, struct scatterlist **dst_sg,
-                       int *tot_pages)
-{
-	int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
-	struct crypt_op *cop = &kcop->cop;
-	int rc;
-
-	if (cop->src == NULL)
-		return -EINVAL;
-
-	if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) {
-		dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
-				__func__, (unsigned long)cop->src, ses->alignmask + 1);
-	}
-
-	src_pagecount = PAGECOUNT(cop->src, cop->len);
-	if (!ses->cdata.init) {		/* hashing only */
-		write_src = 0;
-	} else if (cop->src != cop->dst) {	/* non-in-situ transformation */
-		if (cop->dst == NULL)
-			return -EINVAL;
-
-		dst_pagecount = PAGECOUNT(cop->dst, cop->len);
-		write_src = 0;
-
-		if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) {
-			dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
-					__func__, (unsigned long)cop->dst, ses->alignmask + 1);
-		}
-
-	}
-	(*tot_pages) = pagecount = src_pagecount + dst_pagecount;
-
-	if (pagecount > ses->array_size) {
-		rc = adjust_sg_array(ses, pagecount);
-		if (rc)
-			return rc;
-	}
-
-	rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount,
-	                   ses->pages, ses->sg, kcop->task, kcop->mm);
-	if (unlikely(rc)) {
-		dprintk(1, KERN_ERR,
-			"failed to get user pages for data input\n");
-		return -EINVAL;
-	}
-	(*src_sg) = (*dst_sg) = ses->sg;
-
-	if (!dst_pagecount)
-		return 0;
-
-	(*dst_sg) = ses->sg + src_pagecount;
-
-	rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount,
-	                   ses->pages + src_pagecount, *dst_sg,
-			   kcop->task, kcop->mm);
-	if (unlikely(rc)) {
-		dprintk(1, KERN_ERR,
-		        "failed to get user pages for data output\n");
-		release_user_pages(ses->pages, src_pagecount);
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* This is the main crypto function - zero-copy edition */
-static int
-__crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
-{
-	struct scatterlist *src_sg, *dst_sg;
-	struct crypt_op *cop = &kcop->cop;
-	int ret = 0, pagecount;
-
-	ret = get_userbuf(ses_ptr, kcop, &src_sg, &dst_sg, &pagecount);
-	if (unlikely(ret)) {
-		dprintk(1, KERN_ERR, "Error getting user pages. "
-					"Falling back to non zero copy.\n");
-		return __crypto_run_std(ses_ptr, cop);
-	}
-
-	ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
-
-	release_user_pages(ses_ptr->pages, pagecount);
-	return ret;
-}
-
-static int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
-{
-	struct csession *ses_ptr;
-	struct crypt_op *cop = &kcop->cop;
-	int ret;
-
-	if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
-		dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
-		return -EINVAL;
-	}
-
-	/* this also enters ses_ptr->sem */
-	ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
-		return -EINVAL;
-	}
-
-	if (ses_ptr->hdata.init != 0 && !(cop->flags & (COP_FLAG_UPDATE | COP_FLAG_FINAL))) {
-		ret = cryptodev_hash_reset(&ses_ptr->hdata);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_ERR,
-				"error in cryptodev_hash_reset()\n");
-			goto out_unlock;
-		}
-	}
-
-	if (ses_ptr->cdata.init != 0) {
-		int blocksize = ses_ptr->cdata.blocksize;
-
-		if (unlikely(cop->len % blocksize)) {
-			dprintk(1, KERN_ERR,
-				"data size (%u) isn't a multiple "
-				"of block size (%u)\n",
-				cop->len, blocksize);
-			ret = -EINVAL;
-			goto out_unlock;
-		}
-
-		cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
-				min(ses_ptr->cdata.ivsize, kcop->ivlen));
-	}
-
-	if (likely(cop->len)) {
-		if (cop->flags & COP_FLAG_NO_ZC)
-			ret = __crypto_run_std(ses_ptr, &kcop->cop);
-		else
-			ret = __crypto_run_zc(ses_ptr, kcop);
-		if (unlikely(ret))
-			goto out_unlock;
-	}
-
-	if (ses_ptr->cdata.init != 0) {
-		cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
-				min(ses_ptr->cdata.ivsize, kcop->ivlen));
-	}
-
-	if (ses_ptr->hdata.init != 0 &&
-		((cop->flags & COP_FLAG_FINAL) ||
-		   (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
-
-		ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
-		if (unlikely(ret)) {
-			dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
-			goto out_unlock;
-		}
-		kcop->digestsize = ses_ptr->hdata.digestsize;
-	}
-
-#if defined(CRYPTODEV_STATS)
-	if (enable_stats) {
-		/* this is safe - we check cop->op at the function entry */
-		ses_ptr->stat[cop->op] += cop->len;
-		if (ses_ptr->stat_max_size < cop->len)
-			ses_ptr->stat_max_size = cop->len;
-		ses_ptr->stat_count++;
-	}
-#endif
-
-out_unlock:
-	crypto_put_session(ses_ptr);
-	return ret;
-}
-
-static void cryptask_routine(struct work_struct *work)
-{
-	struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
-	struct todo_list_item *item;
-	LIST_HEAD(tmp);
-
-	/* fetch all pending jobs into the temporary list */
-	mutex_lock(&pcr->todo.lock);
-	list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
-	mutex_unlock(&pcr->todo.lock);
-
-	/* handle each job locklessly */
-	list_for_each_entry(item, &tmp, __hook) {
-		item->result = crypto_run(&pcr->fcrypt, &item->kcop);
-		if (unlikely(item->result))
-			dprintk(0, KERN_ERR, "%s: crypto_run() failed: %d\n",
-					__func__, item->result);
-	}
-
-	/* push all handled jobs to the done list at once */
-	mutex_lock(&pcr->done.lock);
-	list_splice_tail(&tmp, &pcr->done.list);
-	mutex_unlock(&pcr->done.lock);
-
-	/* wake for POLLIN */
-	wake_up_interruptible(&pcr->user_waiter);
-}
-
-/* ====== /dev/crypto ====== */
-
-static int
-cryptodev_open(struct inode *inode, struct file *filp)
-{
-	struct todo_list_item *tmp;
-	struct crypt_priv *pcr;
-	int i;
-
-	pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
-	if (!pcr)
-		return -ENOMEM;
-
-	memset(pcr, 0, sizeof(*pcr));
-	mutex_init(&pcr->fcrypt.sem);
-	INIT_LIST_HEAD(&pcr->fcrypt.list);
-
-	INIT_LIST_HEAD(&pcr->free.list);
-	INIT_LIST_HEAD(&pcr->todo.list);
-	INIT_LIST_HEAD(&pcr->done.list);
-	INIT_WORK(&pcr->cryptask, cryptask_routine);
-	mutex_init(&pcr->free.lock);
-	mutex_init(&pcr->todo.lock);
-	mutex_init(&pcr->done.lock);
-	init_waitqueue_head(&pcr->user_waiter);
-
-	for (i = 0; i < DEF_COP_RINGSIZE; i++) {
-		tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
-		pcr->itemcount++;
-		dprintk(2, KERN_DEBUG, "%s: allocated new item at %lx\n",
-				__func__, (unsigned long)tmp);
-		list_add(&tmp->__hook, &pcr->free.list);
-	}
-
-	filp->private_data = pcr;
-	dprintk(2, KERN_DEBUG,
-	        "Cryptodev handle initialised, %d elements in queue\n",
-		DEF_COP_RINGSIZE);
-	return 0;
-}
-
-static int
-cryptodev_release(struct inode *inode, struct file *filp)
-{
-	struct crypt_priv *pcr = filp->private_data;
-	struct todo_list_item *item, *item_safe;
-	int items_freed = 0;
-
-	if (!pcr)
-		return 0;
-
-	cancel_work_sync(&pcr->cryptask);
-
-	mutex_destroy(&pcr->todo.lock);
-	mutex_destroy(&pcr->done.lock);
-	mutex_destroy(&pcr->free.lock);
-
-	list_splice_tail(&pcr->todo.list, &pcr->free.list);
-	list_splice_tail(&pcr->done.list, &pcr->free.list);
-
-	list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
-		dprintk(2, KERN_DEBUG, "%s: freeing item at %lx\n",
-				__func__, (unsigned long)item);
-		list_del(&item->__hook);
-		kfree(item);
-		items_freed++;
-
-	}
-	if (items_freed != pcr->itemcount) {
-		dprintk(0, KERN_ERR,
-		        "%s: freed %d items, but %d should exist!\n",
-		        __func__, items_freed, pcr->itemcount);
-	}
-
-	crypto_finish_all_sessions(&pcr->fcrypt);
-	kfree(pcr);
-	filp->private_data = NULL;
-
-	dprintk(2, KERN_DEBUG,
-	        "Cryptodev handle deinitialised, %d elements freed\n",
-	        items_freed);
-	return 0;
-}
-
-static int
-clonefd(struct file *filp)
-{
-	int ret;
-	ret = get_unused_fd();
-	if (ret >= 0) {
-			get_file(filp);
-			fd_install(ret, filp);
-	}
-
-	return ret;
-}
-
-/* enqueue a job for asynchronous completion
- *
- * returns:
- * -EBUSY when there are no free queue slots left
- *        (and the number of slots has reached it MAX_COP_RINGSIZE)
- * -EFAULT when there was a memory allocation error
- * 0 on success */
-static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
-{
-	struct todo_list_item *item = NULL;
-
-	mutex_lock(&pcr->free.lock);
-	if (likely(!list_empty(&pcr->free.list))) {
-		item = list_first_entry(&pcr->free.list,
-				struct todo_list_item, __hook);
-		list_del(&item->__hook);
-	} else if (pcr->itemcount < MAX_COP_RINGSIZE) {
-		pcr->itemcount++;
-	} else {
-		mutex_unlock(&pcr->free.lock);
-		return -EBUSY;
-	}
-	mutex_unlock(&pcr->free.lock);
-
-	if (unlikely(!item)) {
-		item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
-		if (unlikely(!item))
-			return -EFAULT;
-		dprintk(1, KERN_INFO, "%s: increased item count to %d\n",
-				__func__, pcr->itemcount);
-	}
-
-	memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
-
-	mutex_lock(&pcr->todo.lock);
-	list_add_tail(&item->__hook, &pcr->todo.list);
-	mutex_unlock(&pcr->todo.lock);
-
-	queue_work(cryptodev_wq, &pcr->cryptask);
-	return 0;
-}
-
-/* get the first completed job from the "done" queue
- *
- * returns:
- * -EBUSY if no completed jobs are ready (yet)
- * the return value of crypto_run() otherwise */
-static int crypto_async_fetch(struct crypt_priv *pcr,
-		struct kernel_crypt_op *kcop)
-{
-	struct todo_list_item *item;
-	int retval;
-
-	mutex_lock(&pcr->done.lock);
-	if (list_empty(&pcr->done.list)) {
-		mutex_unlock(&pcr->done.lock);
-		return -EBUSY;
-	}
-	item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
-	list_del(&item->__hook);
-	mutex_unlock(&pcr->done.lock);
-
-	memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
-	retval = item->result;
-
-	mutex_lock(&pcr->free.lock);
-	list_add_tail(&item->__hook, &pcr->free.list);
-	mutex_unlock(&pcr->free.lock);
-
-	/* wake for POLLOUT */
-	wake_up_interruptible(&pcr->user_waiter);
-
-	return retval;
-}
-
-/* this function has to be called from process context */
-static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
-{
-	struct crypt_op *cop = &kcop->cop;
-	struct csession *ses_ptr;
-	int rc;
-
-	/* this also enters ses_ptr->sem */
-	ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
-		return -EINVAL;
-	}
-	kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
-	kcop->digestsize = 0; /* will be updated during operation */
-
-	crypto_put_session(ses_ptr);
-
-	kcop->task = current;
-	kcop->mm = current->mm;
-
-	if (cop->iv) {
-		rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
-		if (unlikely(rc)) {
-			dprintk(1, KERN_ERR,
-				"error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
-				kcop->ivlen, rc, (unsigned long)cop->iv);
-			return -EFAULT;
-		}
-	}
-
-	return 0;
-}
-
-/* this function has to be called from process context */
-static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
-{
-	int ret;
-
-	if (kcop->digestsize) {
-		ret = copy_to_user(kcop->cop.mac,
-				kcop->hash_output, kcop->digestsize);
-		if (unlikely(ret))
-			return -EFAULT;
-	}
-	if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
-		ret = copy_to_user(kcop->cop.iv,
-				kcop->iv, kcop->ivlen);
-		if (unlikely(ret))
-			return -EFAULT;
-	}
-	return 0;
-}
-
-static int kcop_from_user(struct kernel_crypt_op *kcop,
-			struct fcrypt *fcr, void __user *arg)
-{
-	if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
-		return -EFAULT;
-
-	return fill_kcop_from_cop(kcop, fcr);
-}
-
-static int kcop_to_user(struct kernel_crypt_op *kcop,
-			struct fcrypt *fcr, void __user *arg)
-{
-	int ret;
-
-	ret = fill_cop_from_kcop(kcop, fcr);
-	if (unlikely(ret))
-		return ret;
-
-	if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop))))
-		return -EFAULT;
-	return 0;
-}
-
-static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
-{
-	snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
-			"%s", crypto_tfm_alg_name(tfm));
-	snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
-			"%s", crypto_tfm_alg_driver_name(tfm));
-}
-
-static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
-{
-	struct csession *ses_ptr;
-
-	/* this also enters ses_ptr->sem */
-	ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
-	if (unlikely(!ses_ptr)) {
-		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
-		return -EINVAL;
-	}
-
-	if (ses_ptr->cdata.init) {
-		tfm_info_to_alg_info(&siop->cipher_info,
-				crypto_ablkcipher_tfm(ses_ptr->cdata.async.s));
-	}
-	if (ses_ptr->hdata.init) {
-		tfm_info_to_alg_info(&siop->hash_info,
-				crypto_ahash_tfm(ses_ptr->hdata.async.s));
-	}
-
-	siop->alignmask = ses_ptr->alignmask;
-
-	crypto_put_session(ses_ptr);
-	return 0;
-}
-
-static long
-cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
-{
-	void __user *arg = (void __user *)arg_;
-	int __user *p = arg;
-	struct session_op sop;
-	struct kernel_crypt_op kcop;
-	struct kernel_crypt_auth_op kcaop;
-	struct crypt_priv *pcr = filp->private_data;
-	struct fcrypt *fcr;
-	struct session_info_op siop;
-	uint32_t ses;
-	int ret, fd;
-
-	if (unlikely(!pcr))
-		BUG();
-
-	fcr = &pcr->fcrypt;
-
-	switch (cmd) {
-	case CIOCASYMFEAT:
-		return put_user(0, p);
-	case CRIOGET:
-		fd = clonefd(filp);
-		ret = put_user(fd, p);
-		if (unlikely(ret)) {
-			sys_close(fd);
-			return ret;
-		}
-		return ret;
-	case CIOCGSESSION:
-		if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
-			return -EFAULT;
-
-		ret = crypto_create_session(fcr, &sop);
-		if (unlikely(ret))
-			return ret;
-		ret = copy_to_user(arg, &sop, sizeof(sop));
-		if (unlikely(ret)) {
-			crypto_finish_session(fcr, sop.ses);
-			return -EFAULT;
-		}
-		return ret;
-	case CIOCFSESSION:
-		ret = get_user(ses, (uint32_t __user *)arg);
-		if (unlikely(ret))
-			return ret;
-		ret = crypto_finish_session(fcr, ses);
-		return ret;
-	case CIOCGSESSINFO:
-		if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
-			return -EFAULT;
-
-		ret = get_session_info(fcr, &siop);
-		if (unlikely(ret))
-			return ret;
-		return copy_to_user(arg, &siop, sizeof(siop));
-	case CIOCCRYPT:
-		if (unlikely(ret = kcop_from_user(&kcop, fcr, arg))) {
-			dprintk(1, KERN_WARNING, "Error copying from user");
-			return ret;
-		}
-
-		ret = crypto_run(fcr, &kcop);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_WARNING, "Error in crypto_run");
-			return ret;
-		}
-
-		return kcop_to_user(&kcop, fcr, arg);
-	case CIOCAUTHCRYPT:
-		if (unlikely(ret = kcaop_from_user(&kcaop, fcr, arg))) {
-			dprintk(1, KERN_WARNING, "Error copying from user");
-			return ret;
-		}
-
-		ret = crypto_auth_run(fcr, &kcaop);
-		if (unlikely(ret)) {
-			dprintk(1, KERN_WARNING, "Error in crypto_auth_run");
-			return ret;
-		}
-		return kcaop_to_user(&kcaop, fcr, arg);
-	case CIOCASYNCCRYPT:
-		if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
-			return ret;
-
-		return crypto_async_run(pcr, &kcop);
-	case CIOCASYNCFETCH:
-		ret = crypto_async_fetch(pcr, &kcop);
-		if (unlikely(ret))
-			return ret;
-
-		return kcop_to_user(&kcop, fcr, arg);
-	default:
-		return -EINVAL;
-	}
-}
-
-/* compatibility code for 32bit userlands */
-#ifdef CONFIG_COMPAT
-
-static inline void
-compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
-{
-	sop->cipher = compat->cipher;
-	sop->mac = compat->mac;
-	sop->keylen = compat->keylen;
-
-	sop->key       = compat_ptr(compat->key);
-	sop->mackeylen = compat->mackeylen;
-	sop->mackey    = compat_ptr(compat->mackey);
-	sop->ses       = compat->ses;
-}
-
-static inline void
-session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
-{
-	compat->cipher = sop->cipher;
-	compat->mac = sop->mac;
-	compat->keylen = sop->keylen;
-
-	compat->key       = ptr_to_compat(sop->key);
-	compat->mackeylen = sop->mackeylen;
-	compat->mackey    = ptr_to_compat(sop->mackey);
-	compat->ses       = sop->ses;
-}
-
-static inline void
-compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
-{
-	cop->ses = compat->ses;
-	cop->op = compat->op;
-	cop->flags = compat->flags;
-	cop->len = compat->len;
-
-	cop->src = compat_ptr(compat->src);
-	cop->dst = compat_ptr(compat->dst);
-	cop->mac = compat_ptr(compat->mac);
-	cop->iv  = compat_ptr(compat->iv);
-}
-
-static inline void
-crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
-{
-	compat->ses = cop->ses;
-	compat->op = cop->op;
-	compat->flags = cop->flags;
-	compat->len = cop->len;
-
-	compat->src = ptr_to_compat(cop->src);
-	compat->dst = ptr_to_compat(cop->dst);
-	compat->mac = ptr_to_compat(cop->mac);
-	compat->iv  = ptr_to_compat(cop->iv);
-}
-
-static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
-                                 struct fcrypt *fcr, void __user *arg)
-{
-	struct compat_crypt_op compat_cop;
-
-	if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
-		return -EFAULT;
-	compat_to_crypt_op(&compat_cop, &kcop->cop);
-
-	return fill_kcop_from_cop(kcop, fcr);
-}
-
-static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
-                                 struct fcrypt *fcr, void __user *arg)
-{
-	int ret;
-	struct compat_crypt_op compat_cop;
-
-	ret = fill_cop_from_kcop(kcop, fcr);
-	if (unlikely(ret)) {
-		dprintk(1, KERN_WARNING, "Error in fill_cop_from_kcop");
-		return ret;
-	}
-	crypt_op_to_compat(&kcop->cop, &compat_cop);
-
-	if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop)))) {
-		dprintk(1, KERN_WARNING, "Error copying to user");
-		return -EFAULT;
-	}
-	return 0;
-}
-
-static long
-cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
-{
-	void __user *arg = (void __user *)arg_;
-	struct crypt_priv *pcr = file->private_data;
-	struct fcrypt *fcr;
-	struct session_op sop;
-	struct compat_session_op compat_sop;
-	struct kernel_crypt_op kcop;
-	int ret;
-
-	if (unlikely(!pcr))
-		BUG();
-
-	fcr = &pcr->fcrypt;
-
-	switch (cmd) {
-	case CIOCASYMFEAT:
-	case CRIOGET:
-	case CIOCFSESSION:
-	case CIOCGSESSINFO:
-		return cryptodev_ioctl(file, cmd, arg_);
-
-	case COMPAT_CIOCGSESSION:
-		if (unlikely(copy_from_user(&compat_sop, arg,
-					    sizeof(compat_sop))))
-			return -EFAULT;
-		compat_to_session_op(&compat_sop, &sop);
-
-		ret = crypto_create_session(fcr, &sop);
-		if (unlikely(ret))
-			return ret;
-
-		session_op_to_compat(&sop, &compat_sop);
-		ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
-		if (unlikely(ret)) {
-			crypto_finish_session(fcr, sop.ses);
-			return -EFAULT;
-		}
-		return ret;
-
-	case COMPAT_CIOCCRYPT:
-		ret = compat_kcop_from_user(&kcop, fcr, arg);
-		if (unlikely(ret))
-			return ret;
-
-		ret = crypto_run(fcr, &kcop);
-		if (unlikely(ret))
-			return ret;
-
-		return compat_kcop_to_user(&kcop, fcr, arg);
-	case COMPAT_CIOCASYNCCRYPT:
-		if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
-			return ret;
-
-		return crypto_async_run(pcr, &kcop);
-	case COMPAT_CIOCASYNCFETCH:
-		ret = crypto_async_fetch(pcr, &kcop);
-		if (unlikely(ret))
-			return ret;
-
-		return compat_kcop_to_user(&kcop, fcr, arg);
-
-	default:
-		return -EINVAL;
-	}
-}
-
-#endif /* CONFIG_COMPAT */
-
-static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
-{
-	struct crypt_priv *pcr = file->private_data;
-	int ret = 0;
-
-	poll_wait(file, &pcr->user_waiter, wait);
-
-	if (!list_empty_careful(&pcr->done.list))
-		ret |= POLLIN | POLLRDNORM;
-	if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
-		ret |= POLLOUT | POLLWRNORM;
-
-	return ret;
-}
-
-static const struct file_operations cryptodev_fops = {
-	.owner = THIS_MODULE,
-	.open = cryptodev_open,
-	.release = cryptodev_release,
-	.unlocked_ioctl = cryptodev_ioctl,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl = cryptodev_compat_ioctl,
-#endif /* CONFIG_COMPAT */
-	.poll = cryptodev_poll,
-};
-
-static struct miscdevice cryptodev = {
-	.minor = MISC_DYNAMIC_MINOR,
-	.name = "crypto",
-	.fops = &cryptodev_fops,
-};
-
-static int __init
-cryptodev_register(void)
-{
-	int rc;
-
-	rc = misc_register(&cryptodev);
-	if (unlikely(rc)) {
-		printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
-		return rc;
-	}
-
-	return 0;
-}
-
-static void __exit
-cryptodev_deregister(void)
-{
-	misc_deregister(&cryptodev);
-}
-
-/* ====== Module init/exit ====== */
-static int __init init_cryptodev(void)
-{
-	int rc;
-
-	cryptodev_wq = create_workqueue("cryptodev_queue");
-	if (unlikely(!cryptodev_wq)) {
-		printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
-		return -EFAULT;
-	}
-
-	rc = cryptodev_register();
-	if (unlikely(rc)) {
-		destroy_workqueue(cryptodev_wq);
-		return rc;
-	}
-
-	printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
-
-	return 0;
-}
-
-static void __exit exit_cryptodev(void)
-{
-	flush_workqueue(cryptodev_wq);
-	destroy_workqueue(cryptodev_wq);
-
-	cryptodev_deregister();
-	printk(KERN_INFO PFX "driver unloaded.\n");
-}
-
-module_init(init_cryptodev);
-module_exit(exit_cryptodev);
-
diff --git a/drivers/staging/crypto/cryptodev/ioctl.c b/drivers/staging/crypto/cryptodev/ioctl.c
new file mode 100644
index 000000000000..6177007cf489
--- /dev/null
+++ b/drivers/staging/crypto/cryptodev/ioctl.c
@@ -0,0 +1,1125 @@
+/*
+ * Driver for /dev/crypto device (aka CryptoDev)
+ *
+ * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
+ * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
+ *
+ * This file is part of linux cryptodev.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+/*
+ * Device /dev/crypto provides an interface for
+ * accessing kernel CryptoAPI algorithms (ciphers,
+ * hashes) from userspace programs.
+ *
+ * /dev/crypto interface was originally introduced in
+ * OpenBSD and this module attempts to keep the API.
+ *
+ */
+
+#include <crypto/hash.h>
+#include <linux/crypto.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/ioctl.h>
+#include <linux/random.h>
+#include <linux/syscalls.h>
+#include <linux/pagemap.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <crypto/cryptodev.h>
+#include <linux/scatterlist.h>
+#include "cryptodev_int.h"
+#include "version.h"
+
+MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
+MODULE_DESCRIPTION("CryptoDev driver");
+MODULE_LICENSE("GPL");
+
+/* ====== Compile-time config ====== */
+
+/* Default (pre-allocated) and maximum size of the job queue.
+ * These are free, pending and done items all together. */
+#define DEF_COP_RINGSIZE 16
+#define MAX_COP_RINGSIZE 64
+
+/* ====== Module parameters ====== */
+
+int cryptodev_verbosity;
+module_param(cryptodev_verbosity, int, 0644);
+MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
+
+/* ====== CryptoAPI ====== */
+struct todo_list_item {
+	struct list_head __hook;
+	struct kernel_crypt_op kcop;
+	int result;
+};
+
+struct locked_list {
+	struct list_head list;
+	struct mutex lock;
+};
+
+struct crypt_priv {
+	struct fcrypt fcrypt;
+	struct locked_list free, todo, done;
+	int itemcount;
+	struct work_struct cryptask;
+	wait_queue_head_t user_waiter;
+};
+
+#define FILL_SG(sg, ptr, len)					\
+	do {							\
+		(sg)->page = virt_to_page(ptr);			\
+		(sg)->offset = offset_in_page(ptr);		\
+		(sg)->length = len;				\
+		(sg)->dma_address = 0;				\
+	} while (0)
+
+/* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
+static struct workqueue_struct *cryptodev_wq;
+
+/* Prepare session for future use. */
+static int
+crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
+{
+	struct csession	*ses_new = NULL, *ses_ptr;
+	int ret = 0;
+	const char *alg_name = NULL;
+	const char *hash_name = NULL;
+	int hmac_mode = 1, stream = 0;
+
+	/* Does the request make sense? */
+	if (unlikely(!sop->cipher && !sop->mac)) {
+		dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
+		return -EINVAL;
+	}
+
+	switch (sop->cipher) {
+	case 0:
+		break;
+	case CRYPTO_DES_CBC:
+		alg_name = "cbc(des)";
+		break;
+	case CRYPTO_3DES_CBC:
+		alg_name = "cbc(des3_ede)";
+		break;
+	case CRYPTO_BLF_CBC:
+		alg_name = "cbc(blowfish)";
+		break;
+	case CRYPTO_AES_CBC:
+		alg_name = "cbc(aes)";
+		break;
+	case CRYPTO_AES_ECB:
+		alg_name = "ecb(aes)";
+		break;
+	case CRYPTO_CAMELLIA_CBC:
+		alg_name = "cbc(camelia)";
+		break;
+	case CRYPTO_AES_CTR:
+		alg_name = "ctr(aes)";
+		stream = 1;
+		break;
+	case CRYPTO_NULL:
+		alg_name = "ecb(cipher_null)";
+		stream = 1;
+		break;
+	default:
+		dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
+			sop->cipher);
+		return -EINVAL;
+	}
+
+	switch (sop->mac) {
+	case 0:
+		break;
+	case CRYPTO_MD5_HMAC:
+		hash_name = "hmac(md5)";
+		break;
+	case CRYPTO_RIPEMD160_HMAC:
+		hash_name = "hmac(rmd160)";
+		break;
+	case CRYPTO_SHA1_HMAC:
+		hash_name = "hmac(sha1)";
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		hash_name = "hmac(sha256)";
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		hash_name = "hmac(sha384)";
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		hash_name = "hmac(sha512)";
+		break;
+
+	/* non-hmac cases */
+	case CRYPTO_MD5:
+		hash_name = "md5";
+		hmac_mode = 0;
+		break;
+	case CRYPTO_RIPEMD160:
+		hash_name = "rmd160";
+		hmac_mode = 0;
+		break;
+	case CRYPTO_SHA1:
+		hash_name = "sha1";
+		hmac_mode = 0;
+		break;
+	case CRYPTO_SHA2_256:
+		hash_name = "sha256";
+		hmac_mode = 0;
+		break;
+	case CRYPTO_SHA2_384:
+		hash_name = "sha384";
+		hmac_mode = 0;
+		break;
+	case CRYPTO_SHA2_512:
+		hash_name = "sha512";
+		hmac_mode = 0;
+		break;
+
+	default:
+		dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
+			sop->mac);
+		return -EINVAL;
+	}
+
+	/* Create a session and put it to the list. */
+	ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
+	if (!ses_new)
+		return -ENOMEM;
+
+	/* Set-up crypto transform. */
+	if (alg_name) {
+		uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
+
+		if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
+			dprintk(1, KERN_DEBUG,
+				"Setting key failed for %s-%zu.\n",
+				alg_name, (size_t)sop->keylen*8);
+			ret = -EINVAL;
+			goto error_cipher;
+		}
+
+		if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
+			ret = -EFAULT;
+			goto error_cipher;
+		}
+
+		ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
+						sop->keylen, stream);
+		if (ret < 0) {
+			dprintk(1, KERN_DEBUG,
+				"%s: Failed to load cipher for %s\n",
+				__func__, alg_name);
+			ret = -EINVAL;
+			goto error_cipher;
+		}
+	}
+
+	if (hash_name) {
+		uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
+
+		if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
+			dprintk(1, KERN_DEBUG,
+				"Setting key failed for %s-%zu.\n",
+				alg_name, (size_t)sop->mackeylen*8);
+			ret = -EINVAL;
+			goto error_hash;
+		}
+
+		if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
+					    sop->mackeylen))) {
+			ret = -EFAULT;
+			goto error_hash;
+		}
+
+		ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
+							keyp, sop->mackeylen);
+		if (ret != 0) {
+			dprintk(1, KERN_DEBUG,
+			"%s: Failed to load hash for %s\n",
+			__func__, hash_name);
+			ret = -EINVAL;
+			goto error_hash;
+		}
+	}
+
+	ses_new->alignmask = max(ses_new->cdata.alignmask,
+	                                          ses_new->hdata.alignmask);
+	dprintk(2, KERN_DEBUG, "%s: got alignmask %d\n", __func__, ses_new->alignmask);
+
+	ses_new->array_size = DEFAULT_PREALLOC_PAGES;
+	dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
+			__func__, ses_new->array_size);
+	ses_new->pages = kzalloc(ses_new->array_size *
+			sizeof(struct page *), GFP_KERNEL);
+	ses_new->sg = kzalloc(ses_new->array_size *
+			sizeof(struct scatterlist), GFP_KERNEL);
+	if (ses_new->sg == NULL || ses_new->pages == NULL) {
+		dprintk(0, KERN_DEBUG, "Memory error\n");
+		ret = -ENOMEM;
+		goto error_hash;
+	}
+
+	/* put the new session to the list */
+	get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
+	mutex_init(&ses_new->sem);
+
+	mutex_lock(&fcr->sem);
+restart:
+	list_for_each_entry(ses_ptr, &fcr->list, entry) {
+		/* Check for duplicate SID */
+		if (unlikely(ses_new->sid == ses_ptr->sid)) {
+			get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
+			/* Unless we have a broken RNG this
+			   shouldn't loop forever... ;-) */
+			goto restart;
+		}
+	}
+
+	list_add(&ses_new->entry, &fcr->list);
+	mutex_unlock(&fcr->sem);
+
+	/* Fill in some values for the user. */
+	sop->ses = ses_new->sid;
+
+	return 0;
+
+error_hash:
+	cryptodev_cipher_deinit(&ses_new->cdata);
+	kfree(ses_new->sg);
+	kfree(ses_new->pages);
+error_cipher:
+	kfree(ses_new);
+
+	return ret;
+
+}
+
+/* Everything that needs to be done when remowing a session. */
+static inline void
+crypto_destroy_session(struct csession *ses_ptr)
+{
+	if (!mutex_trylock(&ses_ptr->sem)) {
+		dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
+			ses_ptr->sid);
+		mutex_lock(&ses_ptr->sem);
+	}
+	dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
+	cryptodev_cipher_deinit(&ses_ptr->cdata);
+	cryptodev_hash_deinit(&ses_ptr->hdata);
+	dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
+			__func__, ses_ptr->array_size);
+	kfree(ses_ptr->pages);
+	kfree(ses_ptr->sg);
+	mutex_unlock(&ses_ptr->sem);
+	kfree(ses_ptr);
+}
+
+/* Look up a session by ID and remove. */
+static int
+crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
+{
+	struct csession *tmp, *ses_ptr;
+	struct list_head *head;
+	int ret = 0;
+
+	mutex_lock(&fcr->sem);
+	head = &fcr->list;
+	list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
+		if (ses_ptr->sid == sid) {
+			list_del(&ses_ptr->entry);
+			crypto_destroy_session(ses_ptr);
+			break;
+		}
+	}
+
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
+			sid);
+		ret = -ENOENT;
+	}
+	mutex_unlock(&fcr->sem);
+
+	return ret;
+}
+
+/* Remove all sessions when closing the file */
+static int
+crypto_finish_all_sessions(struct fcrypt *fcr)
+{
+	struct csession *tmp, *ses_ptr;
+	struct list_head *head;
+
+	mutex_lock(&fcr->sem);
+
+	head = &fcr->list;
+	list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
+		list_del(&ses_ptr->entry);
+		crypto_destroy_session(ses_ptr);
+	}
+	mutex_unlock(&fcr->sem);
+
+	return 0;
+}
+
+/* Look up session by session ID. The returned session is locked. */
+struct csession *
+crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
+{
+	struct csession *ses_ptr, *retval = 0;
+
+	mutex_lock(&fcr->sem);
+	list_for_each_entry(ses_ptr, &fcr->list, entry) {
+		if (ses_ptr->sid == sid) {
+			mutex_lock(&ses_ptr->sem);
+			retval = ses_ptr;
+			break;
+		}
+	}
+	mutex_unlock(&fcr->sem);
+
+	return retval;
+}
+
+void release_user_pages(struct page **pg, int pagecount)
+{
+	while (pagecount--) {
+		if (!PageReserved(pg[pagecount]))
+			SetPageDirty(pg[pagecount]);
+		page_cache_release(pg[pagecount]);
+	}
+}
+
+/* offset of buf in it's first page */
+#define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
+
+/* fetch the pages addr resides in into pg and initialise sg with them */
+int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
+		int pgcount, struct page **pg, struct scatterlist *sg,
+		struct task_struct *task, struct mm_struct *mm)
+{
+	int ret, pglen, i = 0;
+	struct scatterlist *sgp;
+
+	down_write(&mm->mmap_sem);
+	ret = get_user_pages(task, mm,
+			(unsigned long)addr, pgcount, write, 0, pg, NULL);
+	up_write(&mm->mmap_sem);
+	if (ret != pgcount)
+		return -EINVAL;
+
+	sg_init_table(sg, pgcount);
+
+	pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
+	sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
+
+	len -= pglen;
+	for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
+		pglen = min((uint32_t)PAGE_SIZE, len);
+		sg_set_page(sgp, pg[i++], pglen, 0);
+		len -= pglen;
+	}
+	sg_mark_end(sg_last(sg, pgcount));
+	return 0;
+}
+
+int adjust_sg_array(struct csession * ses, int pagecount)
+{
+struct scatterlist *sg;
+struct page **pages;
+int array_size;
+
+	for (array_size = ses->array_size; array_size < pagecount;
+	     array_size *= 2)
+		;
+
+	dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
+			__func__, array_size);
+	pages = krealloc(ses->pages, array_size * sizeof(struct page *),
+			 GFP_KERNEL);
+	if (unlikely(!pages))
+		return -ENOMEM;
+	ses->pages = pages;
+	sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
+		      GFP_KERNEL);
+	if (unlikely(!sg))
+		return -ENOMEM;
+	ses->sg = sg;
+	ses->array_size = array_size;
+
+	return 0;
+}
+
+
+static void cryptask_routine(struct work_struct *work)
+{
+	struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
+	struct todo_list_item *item;
+	LIST_HEAD(tmp);
+
+	/* fetch all pending jobs into the temporary list */
+	mutex_lock(&pcr->todo.lock);
+	list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
+	mutex_unlock(&pcr->todo.lock);
+
+	/* handle each job locklessly */
+	list_for_each_entry(item, &tmp, __hook) {
+		item->result = crypto_run(&pcr->fcrypt, &item->kcop);
+		if (unlikely(item->result))
+			dprintk(0, KERN_ERR, "%s: crypto_run() failed: %d\n",
+					__func__, item->result);
+	}
+
+	/* push all handled jobs to the done list at once */
+	mutex_lock(&pcr->done.lock);
+	list_splice_tail(&tmp, &pcr->done.list);
+	mutex_unlock(&pcr->done.lock);
+
+	/* wake for POLLIN */
+	wake_up_interruptible(&pcr->user_waiter);
+}
+
+/* ====== /dev/crypto ====== */
+
+static int
+cryptodev_open(struct inode *inode, struct file *filp)
+{
+	struct todo_list_item *tmp;
+	struct crypt_priv *pcr;
+	int i;
+
+	pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
+	if (!pcr)
+		return -ENOMEM;
+
+	memset(pcr, 0, sizeof(*pcr));
+	mutex_init(&pcr->fcrypt.sem);
+	INIT_LIST_HEAD(&pcr->fcrypt.list);
+
+	INIT_LIST_HEAD(&pcr->free.list);
+	INIT_LIST_HEAD(&pcr->todo.list);
+	INIT_LIST_HEAD(&pcr->done.list);
+	INIT_WORK(&pcr->cryptask, cryptask_routine);
+	mutex_init(&pcr->free.lock);
+	mutex_init(&pcr->todo.lock);
+	mutex_init(&pcr->done.lock);
+	init_waitqueue_head(&pcr->user_waiter);
+
+	for (i = 0; i < DEF_COP_RINGSIZE; i++) {
+		tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
+		pcr->itemcount++;
+		dprintk(2, KERN_DEBUG, "%s: allocated new item at %lx\n",
+				__func__, (unsigned long)tmp);
+		list_add(&tmp->__hook, &pcr->free.list);
+	}
+
+	filp->private_data = pcr;
+	dprintk(2, KERN_DEBUG,
+	        "Cryptodev handle initialised, %d elements in queue\n",
+		DEF_COP_RINGSIZE);
+	return 0;
+}
+
+static int
+cryptodev_release(struct inode *inode, struct file *filp)
+{
+	struct crypt_priv *pcr = filp->private_data;
+	struct todo_list_item *item, *item_safe;
+	int items_freed = 0;
+
+	if (!pcr)
+		return 0;
+
+	cancel_work_sync(&pcr->cryptask);
+
+	mutex_destroy(&pcr->todo.lock);
+	mutex_destroy(&pcr->done.lock);
+	mutex_destroy(&pcr->free.lock);
+
+	list_splice_tail(&pcr->todo.list, &pcr->free.list);
+	list_splice_tail(&pcr->done.list, &pcr->free.list);
+
+	list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
+		dprintk(2, KERN_DEBUG, "%s: freeing item at %lx\n",
+				__func__, (unsigned long)item);
+		list_del(&item->__hook);
+		kfree(item);
+		items_freed++;
+
+	}
+	if (items_freed != pcr->itemcount) {
+		dprintk(0, KERN_ERR,
+		        "%s: freed %d items, but %d should exist!\n",
+		        __func__, items_freed, pcr->itemcount);
+	}
+
+	crypto_finish_all_sessions(&pcr->fcrypt);
+	kfree(pcr);
+	filp->private_data = NULL;
+
+	dprintk(2, KERN_DEBUG,
+	        "Cryptodev handle deinitialised, %d elements freed\n",
+	        items_freed);
+	return 0;
+}
+
+static int
+clonefd(struct file *filp)
+{
+	int ret;
+	ret = get_unused_fd();
+	if (ret >= 0) {
+			get_file(filp);
+			fd_install(ret, filp);
+	}
+
+	return ret;
+}
+
+/* enqueue a job for asynchronous completion
+ *
+ * returns:
+ * -EBUSY when there are no free queue slots left
+ *        (and the number of slots has reached it MAX_COP_RINGSIZE)
+ * -EFAULT when there was a memory allocation error
+ * 0 on success */
+static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
+{
+	struct todo_list_item *item = NULL;
+
+	mutex_lock(&pcr->free.lock);
+	if (likely(!list_empty(&pcr->free.list))) {
+		item = list_first_entry(&pcr->free.list,
+				struct todo_list_item, __hook);
+		list_del(&item->__hook);
+	} else if (pcr->itemcount < MAX_COP_RINGSIZE) {
+		pcr->itemcount++;
+	} else {
+		mutex_unlock(&pcr->free.lock);
+		return -EBUSY;
+	}
+	mutex_unlock(&pcr->free.lock);
+
+	if (unlikely(!item)) {
+		item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
+		if (unlikely(!item))
+			return -EFAULT;
+		dprintk(1, KERN_INFO, "%s: increased item count to %d\n",
+				__func__, pcr->itemcount);
+	}
+
+	memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
+
+	mutex_lock(&pcr->todo.lock);
+	list_add_tail(&item->__hook, &pcr->todo.list);
+	mutex_unlock(&pcr->todo.lock);
+
+	queue_work(cryptodev_wq, &pcr->cryptask);
+	return 0;
+}
+
+/* get the first completed job from the "done" queue
+ *
+ * returns:
+ * -EBUSY if no completed jobs are ready (yet)
+ * the return value of crypto_run() otherwise */
+static int crypto_async_fetch(struct crypt_priv *pcr,
+		struct kernel_crypt_op *kcop)
+{
+	struct todo_list_item *item;
+	int retval;
+
+	mutex_lock(&pcr->done.lock);
+	if (list_empty(&pcr->done.list)) {
+		mutex_unlock(&pcr->done.lock);
+		return -EBUSY;
+	}
+	item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
+	list_del(&item->__hook);
+	mutex_unlock(&pcr->done.lock);
+
+	memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
+	retval = item->result;
+
+	mutex_lock(&pcr->free.lock);
+	list_add_tail(&item->__hook, &pcr->free.list);
+	mutex_unlock(&pcr->free.lock);
+
+	/* wake for POLLOUT */
+	wake_up_interruptible(&pcr->user_waiter);
+
+	return retval;
+}
+
+/* this function has to be called from process context */
+static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
+{
+	struct crypt_op *cop = &kcop->cop;
+	struct csession *ses_ptr;
+	int rc;
+
+	/* this also enters ses_ptr->sem */
+	ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
+		return -EINVAL;
+	}
+	kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
+	kcop->digestsize = 0; /* will be updated during operation */
+
+	crypto_put_session(ses_ptr);
+
+	kcop->task = current;
+	kcop->mm = current->mm;
+
+	if (cop->iv) {
+		rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
+		if (unlikely(rc)) {
+			dprintk(1, KERN_ERR,
+				"error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
+				kcop->ivlen, rc, (unsigned long)cop->iv);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+/* this function has to be called from process context */
+static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
+{
+	int ret;
+
+	if (kcop->digestsize) {
+		ret = copy_to_user(kcop->cop.mac,
+				kcop->hash_output, kcop->digestsize);
+		if (unlikely(ret))
+			return -EFAULT;
+	}
+	if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
+		ret = copy_to_user(kcop->cop.iv,
+				kcop->iv, kcop->ivlen);
+		if (unlikely(ret))
+			return -EFAULT;
+	}
+	return 0;
+}
+
+static int kcop_from_user(struct kernel_crypt_op *kcop,
+			struct fcrypt *fcr, void __user *arg)
+{
+	if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
+		return -EFAULT;
+
+	return fill_kcop_from_cop(kcop, fcr);
+}
+
+static int kcop_to_user(struct kernel_crypt_op *kcop,
+			struct fcrypt *fcr, void __user *arg)
+{
+	int ret;
+
+	ret = fill_cop_from_kcop(kcop, fcr);
+	if (unlikely(ret))
+		return ret;
+
+	if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop))))
+		return -EFAULT;
+	return 0;
+}
+
+static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
+{
+	snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
+			"%s", crypto_tfm_alg_name(tfm));
+	snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
+			"%s", crypto_tfm_alg_driver_name(tfm));
+}
+
+static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
+{
+	struct csession *ses_ptr;
+
+	/* this also enters ses_ptr->sem */
+	ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
+		return -EINVAL;
+	}
+
+	if (ses_ptr->cdata.init) {
+		tfm_info_to_alg_info(&siop->cipher_info,
+				crypto_ablkcipher_tfm(ses_ptr->cdata.async.s));
+	}
+	if (ses_ptr->hdata.init) {
+		tfm_info_to_alg_info(&siop->hash_info,
+				crypto_ahash_tfm(ses_ptr->hdata.async.s));
+	}
+
+	siop->alignmask = ses_ptr->alignmask;
+
+	crypto_put_session(ses_ptr);
+	return 0;
+}
+
+static long
+cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
+{
+	void __user *arg = (void __user *)arg_;
+	int __user *p = arg;
+	struct session_op sop;
+	struct kernel_crypt_op kcop;
+	struct kernel_crypt_auth_op kcaop;
+	struct crypt_priv *pcr = filp->private_data;
+	struct fcrypt *fcr;
+	struct session_info_op siop;
+	uint32_t ses;
+	int ret, fd;
+
+	if (unlikely(!pcr))
+		BUG();
+
+	fcr = &pcr->fcrypt;
+
+	switch (cmd) {
+	case CIOCASYMFEAT:
+		return put_user(0, p);
+	case CRIOGET:
+		fd = clonefd(filp);
+		ret = put_user(fd, p);
+		if (unlikely(ret)) {
+			sys_close(fd);
+			return ret;
+		}
+		return ret;
+	case CIOCGSESSION:
+		if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
+			return -EFAULT;
+
+		ret = crypto_create_session(fcr, &sop);
+		if (unlikely(ret))
+			return ret;
+		ret = copy_to_user(arg, &sop, sizeof(sop));
+		if (unlikely(ret)) {
+			crypto_finish_session(fcr, sop.ses);
+			return -EFAULT;
+		}
+		return ret;
+	case CIOCFSESSION:
+		ret = get_user(ses, (uint32_t __user *)arg);
+		if (unlikely(ret))
+			return ret;
+		ret = crypto_finish_session(fcr, ses);
+		return ret;
+	case CIOCGSESSINFO:
+		if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
+			return -EFAULT;
+
+		ret = get_session_info(fcr, &siop);
+		if (unlikely(ret))
+			return ret;
+		return copy_to_user(arg, &siop, sizeof(siop));
+	case CIOCCRYPT:
+		if (unlikely(ret = kcop_from_user(&kcop, fcr, arg))) {
+			dprintk(1, KERN_WARNING, "Error copying from user");
+			return ret;
+		}
+
+		ret = crypto_run(fcr, &kcop);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_WARNING, "Error in crypto_run");
+			return ret;
+		}
+
+		return kcop_to_user(&kcop, fcr, arg);
+	case CIOCAUTHCRYPT:
+		if (unlikely(ret = kcaop_from_user(&kcaop, fcr, arg))) {
+			dprintk(1, KERN_WARNING, "Error copying from user");
+			return ret;
+		}
+
+		ret = crypto_auth_run(fcr, &kcaop);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_WARNING, "Error in crypto_auth_run");
+			return ret;
+		}
+		return kcaop_to_user(&kcaop, fcr, arg);
+	case CIOCASYNCCRYPT:
+		if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
+			return ret;
+
+		return crypto_async_run(pcr, &kcop);
+	case CIOCASYNCFETCH:
+		ret = crypto_async_fetch(pcr, &kcop);
+		if (unlikely(ret))
+			return ret;
+
+		return kcop_to_user(&kcop, fcr, arg);
+	default:
+		return -EINVAL;
+	}
+}
+
+/* compatibility code for 32bit userlands */
+#ifdef CONFIG_COMPAT
+
+static inline void
+compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
+{
+	sop->cipher = compat->cipher;
+	sop->mac = compat->mac;
+	sop->keylen = compat->keylen;
+
+	sop->key       = compat_ptr(compat->key);
+	sop->mackeylen = compat->mackeylen;
+	sop->mackey    = compat_ptr(compat->mackey);
+	sop->ses       = compat->ses;
+}
+
+static inline void
+session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
+{
+	compat->cipher = sop->cipher;
+	compat->mac = sop->mac;
+	compat->keylen = sop->keylen;
+
+	compat->key       = ptr_to_compat(sop->key);
+	compat->mackeylen = sop->mackeylen;
+	compat->mackey    = ptr_to_compat(sop->mackey);
+	compat->ses       = sop->ses;
+}
+
+static inline void
+compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
+{
+	cop->ses = compat->ses;
+	cop->op = compat->op;
+	cop->flags = compat->flags;
+	cop->len = compat->len;
+
+	cop->src = compat_ptr(compat->src);
+	cop->dst = compat_ptr(compat->dst);
+	cop->mac = compat_ptr(compat->mac);
+	cop->iv  = compat_ptr(compat->iv);
+}
+
+static inline void
+crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
+{
+	compat->ses = cop->ses;
+	compat->op = cop->op;
+	compat->flags = cop->flags;
+	compat->len = cop->len;
+
+	compat->src = ptr_to_compat(cop->src);
+	compat->dst = ptr_to_compat(cop->dst);
+	compat->mac = ptr_to_compat(cop->mac);
+	compat->iv  = ptr_to_compat(cop->iv);
+}
+
+static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
+                                 struct fcrypt *fcr, void __user *arg)
+{
+	struct compat_crypt_op compat_cop;
+
+	if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
+		return -EFAULT;
+	compat_to_crypt_op(&compat_cop, &kcop->cop);
+
+	return fill_kcop_from_cop(kcop, fcr);
+}
+
+static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
+                                 struct fcrypt *fcr, void __user *arg)
+{
+	int ret;
+	struct compat_crypt_op compat_cop;
+
+	ret = fill_cop_from_kcop(kcop, fcr);
+	if (unlikely(ret)) {
+		dprintk(1, KERN_WARNING, "Error in fill_cop_from_kcop");
+		return ret;
+	}
+	crypt_op_to_compat(&kcop->cop, &compat_cop);
+
+	if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop)))) {
+		dprintk(1, KERN_WARNING, "Error copying to user");
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static long
+cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
+{
+	void __user *arg = (void __user *)arg_;
+	struct crypt_priv *pcr = file->private_data;
+	struct fcrypt *fcr;
+	struct session_op sop;
+	struct compat_session_op compat_sop;
+	struct kernel_crypt_op kcop;
+	int ret;
+
+	if (unlikely(!pcr))
+		BUG();
+
+	fcr = &pcr->fcrypt;
+
+	switch (cmd) {
+	case CIOCASYMFEAT:
+	case CRIOGET:
+	case CIOCFSESSION:
+	case CIOCGSESSINFO:
+		return cryptodev_ioctl(file, cmd, arg_);
+
+	case COMPAT_CIOCGSESSION:
+		if (unlikely(copy_from_user(&compat_sop, arg,
+					    sizeof(compat_sop))))
+			return -EFAULT;
+		compat_to_session_op(&compat_sop, &sop);
+
+		ret = crypto_create_session(fcr, &sop);
+		if (unlikely(ret))
+			return ret;
+
+		session_op_to_compat(&sop, &compat_sop);
+		ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
+		if (unlikely(ret)) {
+			crypto_finish_session(fcr, sop.ses);
+			return -EFAULT;
+		}
+		return ret;
+
+	case COMPAT_CIOCCRYPT:
+		ret = compat_kcop_from_user(&kcop, fcr, arg);
+		if (unlikely(ret))
+			return ret;
+
+		ret = crypto_run(fcr, &kcop);
+		if (unlikely(ret))
+			return ret;
+
+		return compat_kcop_to_user(&kcop, fcr, arg);
+	case COMPAT_CIOCASYNCCRYPT:
+		if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
+			return ret;
+
+		return crypto_async_run(pcr, &kcop);
+	case COMPAT_CIOCASYNCFETCH:
+		ret = crypto_async_fetch(pcr, &kcop);
+		if (unlikely(ret))
+			return ret;
+
+		return compat_kcop_to_user(&kcop, fcr, arg);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+#endif /* CONFIG_COMPAT */
+
+static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
+{
+	struct crypt_priv *pcr = file->private_data;
+	int ret = 0;
+
+	poll_wait(file, &pcr->user_waiter, wait);
+
+	if (!list_empty_careful(&pcr->done.list))
+		ret |= POLLIN | POLLRDNORM;
+	if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
+		ret |= POLLOUT | POLLWRNORM;
+
+	return ret;
+}
+
+static const struct file_operations cryptodev_fops = {
+	.owner = THIS_MODULE,
+	.open = cryptodev_open,
+	.release = cryptodev_release,
+	.unlocked_ioctl = cryptodev_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = cryptodev_compat_ioctl,
+#endif /* CONFIG_COMPAT */
+	.poll = cryptodev_poll,
+};
+
+static struct miscdevice cryptodev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "crypto",
+	.fops = &cryptodev_fops,
+};
+
+static int __init
+cryptodev_register(void)
+{
+	int rc;
+
+	rc = misc_register(&cryptodev);
+	if (unlikely(rc)) {
+		printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+static void __exit
+cryptodev_deregister(void)
+{
+	misc_deregister(&cryptodev);
+}
+
+/* ====== Module init/exit ====== */
+static int __init init_cryptodev(void)
+{
+	int rc;
+
+	cryptodev_wq = create_workqueue("cryptodev_queue");
+	if (unlikely(!cryptodev_wq)) {
+		printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
+		return -EFAULT;
+	}
+
+	rc = cryptodev_register();
+	if (unlikely(rc)) {
+		destroy_workqueue(cryptodev_wq);
+		return rc;
+	}
+
+	printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
+
+	return 0;
+}
+
+static void __exit exit_cryptodev(void)
+{
+	flush_workqueue(cryptodev_wq);
+	destroy_workqueue(cryptodev_wq);
+
+	cryptodev_deregister();
+	printk(KERN_INFO PFX "driver unloaded.\n");
+}
+
+module_init(init_cryptodev);
+module_exit(exit_cryptodev);
+
diff --git a/drivers/staging/crypto/cryptodev/main.c b/drivers/staging/crypto/cryptodev/main.c
new file mode 100644
index 000000000000..b79282db1d6e
--- /dev/null
+++ b/drivers/staging/crypto/cryptodev/main.c
@@ -0,0 +1,313 @@
+/*
+ * Driver for /dev/crypto device (aka CryptoDev)
+ *
+ * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
+ * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
+ *
+ * This file is part of linux cryptodev.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+/*
+ * Device /dev/crypto provides an interface for
+ * accessing kernel CryptoAPI algorithms (ciphers,
+ * hashes) from userspace programs.
+ *
+ * /dev/crypto interface was originally introduced in
+ * OpenBSD and this module attempts to keep the API.
+ *
+ */
+#include <crypto/hash.h>
+#include <linux/crypto.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/ioctl.h>
+#include <linux/random.h>
+#include <linux/syscalls.h>
+#include <linux/pagemap.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <crypto/cryptodev.h>
+#include <crypto/scatterwalk.h>
+#include <linux/scatterlist.h>
+#include "cryptodev_int.h"
+#include "version.h"
+
+/* This file contains the traditional operations of encryption
+ * and hashing of /dev/crypto.
+ */
+
+/* make cop->src and cop->dst available in scatterlists */
+static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop,
+                       struct scatterlist **src_sg, struct scatterlist **dst_sg,
+                       int *tot_pages)
+{
+	int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
+	struct crypt_op *cop = &kcop->cop;
+	int rc;
+
+	if (cop->src == NULL)
+		return -EINVAL;
+
+	if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) {
+		dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
+				__func__, (unsigned long)cop->src, ses->alignmask + 1);
+	}
+
+	src_pagecount = PAGECOUNT(cop->src, cop->len);
+	if (!ses->cdata.init) {		/* hashing only */
+		write_src = 0;
+	} else if (cop->src != cop->dst) {	/* non-in-situ transformation */
+		if (cop->dst == NULL)
+			return -EINVAL;
+
+		dst_pagecount = PAGECOUNT(cop->dst, cop->len);
+		write_src = 0;
+
+		if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) {
+			dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
+					__func__, (unsigned long)cop->dst, ses->alignmask + 1);
+		}
+
+	}
+	(*tot_pages) = pagecount = src_pagecount + dst_pagecount;
+
+	if (pagecount > ses->array_size) {
+		rc = adjust_sg_array(ses, pagecount);
+		if (rc)
+			return rc;
+	}
+
+	rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount,
+	                   ses->pages, ses->sg, kcop->task, kcop->mm);
+	if (unlikely(rc)) {
+		dprintk(1, KERN_ERR,
+			"failed to get user pages for data input\n");
+		return -EINVAL;
+	}
+	(*src_sg) = (*dst_sg) = ses->sg;
+
+	if (!dst_pagecount)
+		return 0;
+
+	(*dst_sg) = ses->sg + src_pagecount;
+
+	rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount,
+	                   ses->pages + src_pagecount, *dst_sg,
+			   kcop->task, kcop->mm);
+	if (unlikely(rc)) {
+		dprintk(1, KERN_ERR,
+		        "failed to get user pages for data output\n");
+		release_user_pages(ses->pages, src_pagecount);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int
+hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
+		struct scatterlist *src_sg, struct scatterlist *dst_sg,
+		uint32_t len)
+{
+	int ret;
+
+	/* Always hash before encryption and after decryption. Maybe
+	 * we should introduce a flag to switch... TBD later on.
+	 */
+	if (cop->op == COP_ENCRYPT) {
+		if (ses_ptr->hdata.init != 0) {
+			ret = cryptodev_hash_update(&ses_ptr->hdata,
+							src_sg, len);
+			if (unlikely(ret))
+				goto out_err;
+		}
+		if (ses_ptr->cdata.init != 0) {
+			ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
+							src_sg, dst_sg, len);
+
+			if (unlikely(ret))
+				goto out_err;
+		}
+	} else {
+		if (ses_ptr->cdata.init != 0) {
+			ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
+							src_sg, dst_sg, len);
+
+			if (unlikely(ret))
+				goto out_err;
+		}
+
+		if (ses_ptr->hdata.init != 0) {
+			ret = cryptodev_hash_update(&ses_ptr->hdata,
+								dst_sg, len);
+			if (unlikely(ret))
+				goto out_err;
+		}
+	}
+	return 0;
+out_err:
+	dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
+	return ret;
+}
+
+/* This is the main crypto function - feed it with plaintext
+   and get a ciphertext (or vice versa :-) */
+static int
+__crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
+{
+	char *data;
+	char __user *src, *dst;
+	struct scatterlist sg;
+	size_t nbytes, bufsize;
+	int ret = 0;
+
+	nbytes = cop->len;
+	data = (char *)__get_free_page(GFP_KERNEL);
+
+	if (unlikely(!data))
+		return -ENOMEM;
+
+	bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
+
+	src = cop->src;
+	dst = cop->dst;
+
+	while (nbytes > 0) {
+		size_t current_len = nbytes > bufsize ? bufsize : nbytes;
+
+		if (unlikely(copy_from_user(data, src, current_len))) {
+			ret = -EFAULT;
+			break;
+		}
+
+		sg_init_one(&sg, data, current_len);
+
+		ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
+
+		if (unlikely(ret))
+			break;
+
+		if (ses_ptr->cdata.init != 0) {
+			if (unlikely(copy_to_user(dst, data, current_len))) {
+				ret = -EFAULT;
+				break;
+			}
+		}
+
+		dst += current_len;
+		nbytes -= current_len;
+		src += current_len;
+	}
+
+	free_page((unsigned long)data);
+	return ret;
+}
+
+/* This is the main crypto function - zero-copy edition */
+static int
+__crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
+{
+	struct scatterlist *src_sg, *dst_sg;
+	struct crypt_op *cop = &kcop->cop;
+	int ret = 0, pagecount;
+
+	ret = get_userbuf(ses_ptr, kcop, &src_sg, &dst_sg, &pagecount);
+	if (unlikely(ret)) {
+		dprintk(1, KERN_ERR, "Error getting user pages. "
+					"Falling back to non zero copy.\n");
+		return __crypto_run_std(ses_ptr, cop);
+	}
+
+	ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
+
+	release_user_pages(ses_ptr->pages, pagecount);
+	return ret;
+}
+
+int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
+{
+	struct csession *ses_ptr;
+	struct crypt_op *cop = &kcop->cop;
+	int ret;
+
+	if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
+		dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
+		return -EINVAL;
+	}
+
+	/* this also enters ses_ptr->sem */
+	ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
+	if (unlikely(!ses_ptr)) {
+		dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
+		return -EINVAL;
+	}
+
+	if (ses_ptr->hdata.init != 0 && !(cop->flags & (COP_FLAG_UPDATE | COP_FLAG_FINAL))) {
+		ret = cryptodev_hash_reset(&ses_ptr->hdata);
+		if (unlikely(ret)) {
+			dprintk(1, KERN_ERR,
+				"error in cryptodev_hash_reset()\n");
+			goto out_unlock;
+		}
+	}
+
+	if (ses_ptr->cdata.init != 0) {
+		int blocksize = ses_ptr->cdata.blocksize;
+
+		if (unlikely(cop->len % blocksize)) {
+			dprintk(1, KERN_ERR,
+				"data size (%u) isn't a multiple "
+				"of block size (%u)\n",
+				cop->len, blocksize);
+			ret = -EINVAL;
+			goto out_unlock;
+		}
+
+		cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
+				min(ses_ptr->cdata.ivsize, kcop->ivlen));
+	}
+
+	if (likely(cop->len)) {
+		if (cop->flags & COP_FLAG_NO_ZC)
+			ret = __crypto_run_std(ses_ptr, &kcop->cop);
+		else
+			ret = __crypto_run_zc(ses_ptr, kcop);
+		if (unlikely(ret))
+			goto out_unlock;
+	}
+
+	if (ses_ptr->cdata.init != 0) {
+		cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
+				min(ses_ptr->cdata.ivsize, kcop->ivlen));
+	}
+
+	if (ses_ptr->hdata.init != 0 &&
+		((cop->flags & COP_FLAG_FINAL) ||
+		   (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
+
+		ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
+		if (unlikely(ret)) {
+			dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
+			goto out_unlock;
+		}
+		kcop->digestsize = ses_ptr->hdata.digestsize;
+	}
+
+out_unlock:
+	crypto_put_session(ses_ptr);
+	return ret;
+}
-- 
2.1.0

