diff --git a/arch/s390/lib/uaccess.S b/arch/s390/lib/uaccess.S
index 5d59e262504..837275284d9 100644
--- a/arch/s390/lib/uaccess.S
+++ b/arch/s390/lib/uaccess.S
@@ -88,30 +88,31 @@ __copy_to_user_asm:
         .globl __copy_in_user_asm
 	# %r2 = from, %r3 = n, %r4 = to
 __copy_in_user_asm:
+	ahi	%r3,-1
+	jo	6f
 	sacf	256
-	bras	1,1f
-	mvc	0(1,%r4),0(%r2)
-0:	mvc	0(256,%r4),0(%r2)
-	la	%r2,256(%r2)
-	la	%r4,256(%r4)
-1:	ahi	%r3,-256
-	jnm	0b
-2:	ex	%r3,0(%r1)
-	sacf	0
-	slr	%r2,%r2
-	br	14
-3:	mvc	0(1,%r4),0(%r2)
+	bras	%r1,4f
+0:	ahi	%r3,257
+1:	mvc	0(1,%r4),0(%r2)
 	la	%r2,1(%r2)
 	la	%r4,1(%r4)
 	ahi	%r3,-1
+	jnz	1b
+2:	lr	%r2,%r3
+	br	%r14
+3:	mvc	0(256,%r4),0(%r2)
+	la	%r2,256(%r2)
+	la	%r4,256(%r4)
+4:	ahi	%r3,-256
 	jnm	3b
-4:	lr	%r2,%r3
+5:	ex	%r3,4(%r1)
 	sacf	0
+6:	slr	%r2,%r2
 	br	%r14
         .section __ex_table,"a"
-	.long	0b,3b
-	.long	2b,3b
-	.long	3b,4b
+	.long	1b,2b
+	.long	3b,0b
+	.long	5b,0b
         .previous
 
         .align 4
diff --git a/arch/s390/lib/uaccess64.S b/arch/s390/lib/uaccess64.S
index 19b41a33c23..1f755be22f9 100644
--- a/arch/s390/lib/uaccess64.S
+++ b/arch/s390/lib/uaccess64.S
@@ -88,30 +88,31 @@ __copy_to_user_asm:
         .globl __copy_in_user_asm
 	# %r2 = from, %r3 = n, %r4 = to
 __copy_in_user_asm:
+	aghi	%r3,-1
+	jo	6f
 	sacf	256
-	bras	1,1f
-	mvc	0(1,%r4),0(%r2)
-0:	mvc	0(256,%r4),0(%r2)
-	la	%r2,256(%r2)
-	la	%r4,256(%r4)
-1:	aghi	%r3,-256
-	jnm	0b
-2:	ex	%r3,0(%r1)
-	sacf	0
-	slgr	%r2,%r2
-	br	14
-3:	mvc	0(1,%r4),0(%r2)
+	bras	%r1,4f
+0:	aghi	%r3,257
+1:	mvc	0(1,%r4),0(%r2)
 	la	%r2,1(%r2)
 	la	%r4,1(%r4)
 	aghi	%r3,-1
-	jnm	3b
-4:	lgr	%r2,%r3
-	sacf	0
+	jnz	1b
+2:	lgr	%r2,%r3
 	br	%r14
+3:	mvc	0(256,%r4),0(%r2)
+	la	%r2,256(%r2)
+	la	%r4,256(%r4)
+4:	aghi	%r3,-256
+	jnm	3b
+5:	ex	%r3,4(%r1)
+	sacf	0
+6:	slgr	%r2,%r2
+	br	14
         .section __ex_table,"a"
-	.quad	0b,3b
-	.quad	2b,3b
-	.quad	3b,4b
+	.quad	1b,2b
+	.quad	3b,0b
+	.quad	5b,0b
         .previous
 
         .align 4