Skip to content

Commit 6fe0ea9

Browse files
hcahcaAlexander Gordeev
hcahca
authored and
Alexander Gordeev
committed
s390/uaccess: Make cmpxchg_user_key() library code
Move cmpxchg_user_key() handling to uaccess library code. The generated code is large in any case so that there is hardly any benefit if it is inlined. Reviewed-by: Claudio Imbrenda <[email protected]> Signed-off-by: Heiko Carstens <[email protected]> Signed-off-by: Alexander Gordeev <[email protected]>
1 parent ac5bf06 commit 6fe0ea9

2 files changed

Lines changed: 224 additions & 181 deletions

File tree

‎arch/s390/include/asm/uaccess.h‎

Lines changed: 23 additions & 181 deletions
Original file line numberDiff line numberDiff line change
@@ -473,188 +473,30 @@ do { \
473473

474474
void __cmpxchg_user_key_called_with_bad_pointer(void);
475475

476-
#define CMPXCHG_USER_KEY_MAX_LOOPS 128
477-
478-
static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
479-
__uint128_t old, __uint128_t new,
480-
unsigned long key, int size)
476+
int __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
477+
unsigned char old, unsigned char new, unsigned long key);
478+
int __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
479+
unsigned short old, unsigned short new, unsigned long key);
480+
int __cmpxchg_user_key4(unsigned long address, unsigned int *uval,
481+
unsigned int old, unsigned int new, unsigned long key);
482+
int __cmpxchg_user_key8(unsigned long address, unsigned long *uval,
483+
unsigned long old, unsigned long new, unsigned long key);
484+
int __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
485+
__uint128_t old, __uint128_t new, unsigned long key);
486+
487+
static __always_inline int _cmpxchg_user_key(unsigned long address, void *uval,
488+
__uint128_t old, __uint128_t new,
489+
unsigned long key, int size)
481490
{
482-
bool sacf_flag;
483-
int rc = 0;
484-
485491
switch (size) {
486-
case 1: {
487-
unsigned int prev, shift, mask, _old, _new;
488-
unsigned long count;
489-
490-
shift = (3 ^ (address & 3)) << 3;
491-
address ^= address & 3;
492-
_old = ((unsigned int)old & 0xff) << shift;
493-
_new = ((unsigned int)new & 0xff) << shift;
494-
mask = ~(0xff << shift);
495-
sacf_flag = enable_sacf_uaccess();
496-
asm_inline volatile(
497-
" spka 0(%[key])\n"
498-
" sacf 256\n"
499-
" llill %[count],%[max_loops]\n"
500-
"0: l %[prev],%[address]\n"
501-
"1: nr %[prev],%[mask]\n"
502-
" xilf %[mask],0xffffffff\n"
503-
" or %[new],%[prev]\n"
504-
" or %[prev],%[tmp]\n"
505-
"2: lr %[tmp],%[prev]\n"
506-
"3: cs %[prev],%[new],%[address]\n"
507-
"4: jnl 5f\n"
508-
" xr %[tmp],%[prev]\n"
509-
" xr %[new],%[tmp]\n"
510-
" nr %[tmp],%[mask]\n"
511-
" jnz 5f\n"
512-
" brct %[count],2b\n"
513-
"5: sacf 768\n"
514-
" spka %[default_key]\n"
515-
EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
516-
EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
517-
EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
518-
EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
519-
: [rc] "+&d" (rc),
520-
[prev] "=&d" (prev),
521-
[address] "+Q" (*(int *)address),
522-
[tmp] "+&d" (_old),
523-
[new] "+&d" (_new),
524-
[mask] "+&d" (mask),
525-
[count] "=a" (count)
526-
: [key] "%[count]" (key << 4),
527-
[default_key] "J" (PAGE_DEFAULT_KEY),
528-
[max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
529-
: "memory", "cc");
530-
disable_sacf_uaccess(sacf_flag);
531-
*(unsigned char *)uval = prev >> shift;
532-
if (!count)
533-
rc = -EAGAIN;
534-
return rc;
535-
}
536-
case 2: {
537-
unsigned int prev, shift, mask, _old, _new;
538-
unsigned long count;
539-
540-
shift = (2 ^ (address & 2)) << 3;
541-
address ^= address & 2;
542-
_old = ((unsigned int)old & 0xffff) << shift;
543-
_new = ((unsigned int)new & 0xffff) << shift;
544-
mask = ~(0xffff << shift);
545-
sacf_flag = enable_sacf_uaccess();
546-
asm_inline volatile(
547-
" spka 0(%[key])\n"
548-
" sacf 256\n"
549-
" llill %[count],%[max_loops]\n"
550-
"0: l %[prev],%[address]\n"
551-
"1: nr %[prev],%[mask]\n"
552-
" xilf %[mask],0xffffffff\n"
553-
" or %[new],%[prev]\n"
554-
" or %[prev],%[tmp]\n"
555-
"2: lr %[tmp],%[prev]\n"
556-
"3: cs %[prev],%[new],%[address]\n"
557-
"4: jnl 5f\n"
558-
" xr %[tmp],%[prev]\n"
559-
" xr %[new],%[tmp]\n"
560-
" nr %[tmp],%[mask]\n"
561-
" jnz 5f\n"
562-
" brct %[count],2b\n"
563-
"5: sacf 768\n"
564-
" spka %[default_key]\n"
565-
EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
566-
EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
567-
EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
568-
EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
569-
: [rc] "+&d" (rc),
570-
[prev] "=&d" (prev),
571-
[address] "+Q" (*(int *)address),
572-
[tmp] "+&d" (_old),
573-
[new] "+&d" (_new),
574-
[mask] "+&d" (mask),
575-
[count] "=a" (count)
576-
: [key] "%[count]" (key << 4),
577-
[default_key] "J" (PAGE_DEFAULT_KEY),
578-
[max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
579-
: "memory", "cc");
580-
disable_sacf_uaccess(sacf_flag);
581-
*(unsigned short *)uval = prev >> shift;
582-
if (!count)
583-
rc = -EAGAIN;
584-
return rc;
585-
}
586-
case 4: {
587-
unsigned int prev = old;
588-
589-
sacf_flag = enable_sacf_uaccess();
590-
asm_inline volatile(
591-
" spka 0(%[key])\n"
592-
" sacf 256\n"
593-
"0: cs %[prev],%[new],%[address]\n"
594-
"1: sacf 768\n"
595-
" spka %[default_key]\n"
596-
EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
597-
EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
598-
: [rc] "+&d" (rc),
599-
[prev] "+&d" (prev),
600-
[address] "+Q" (*(int *)address)
601-
: [new] "d" ((unsigned int)new),
602-
[key] "a" (key << 4),
603-
[default_key] "J" (PAGE_DEFAULT_KEY)
604-
: "memory", "cc");
605-
disable_sacf_uaccess(sacf_flag);
606-
*(unsigned int *)uval = prev;
607-
return rc;
608-
}
609-
case 8: {
610-
unsigned long prev = old;
611-
612-
sacf_flag = enable_sacf_uaccess();
613-
asm_inline volatile(
614-
" spka 0(%[key])\n"
615-
" sacf 256\n"
616-
"0: csg %[prev],%[new],%[address]\n"
617-
"1: sacf 768\n"
618-
" spka %[default_key]\n"
619-
EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
620-
EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
621-
: [rc] "+&d" (rc),
622-
[prev] "+&d" (prev),
623-
[address] "+QS" (*(long *)address)
624-
: [new] "d" ((unsigned long)new),
625-
[key] "a" (key << 4),
626-
[default_key] "J" (PAGE_DEFAULT_KEY)
627-
: "memory", "cc");
628-
disable_sacf_uaccess(sacf_flag);
629-
*(unsigned long *)uval = prev;
630-
return rc;
631-
}
632-
case 16: {
633-
__uint128_t prev = old;
634-
635-
sacf_flag = enable_sacf_uaccess();
636-
asm_inline volatile(
637-
" spka 0(%[key])\n"
638-
" sacf 256\n"
639-
"0: cdsg %[prev],%[new],%[address]\n"
640-
"1: sacf 768\n"
641-
" spka %[default_key]\n"
642-
EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
643-
EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
644-
: [rc] "+&d" (rc),
645-
[prev] "+&d" (prev),
646-
[address] "+QS" (*(__int128_t *)address)
647-
: [new] "d" (new),
648-
[key] "a" (key << 4),
649-
[default_key] "J" (PAGE_DEFAULT_KEY)
650-
: "memory", "cc");
651-
disable_sacf_uaccess(sacf_flag);
652-
*(__uint128_t *)uval = prev;
653-
return rc;
654-
}
492+
case 1: return __cmpxchg_user_key1(address, uval, old, new, key);
493+
case 2: return __cmpxchg_user_key2(address, uval, old, new, key);
494+
case 4: return __cmpxchg_user_key4(address, uval, old, new, key);
495+
case 8: return __cmpxchg_user_key8(address, uval, old, new, key);
496+
case 16: return __cmpxchg_user_key16(address, uval, old, new, key);
497+
default: __cmpxchg_user_key_called_with_bad_pointer();
655498
}
656-
__cmpxchg_user_key_called_with_bad_pointer();
657-
return rc;
499+
return 0;
658500
}
659501

660502
/**
@@ -686,8 +528,8 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
686528
BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \
687529
might_fault(); \
688530
__chk_user_ptr(__ptr); \
689-
__cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \
690-
(old), (new), (key), sizeof(*(__ptr))); \
531+
_cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \
532+
(old), (new), (key), sizeof(*(__ptr))); \
691533
})
692534

693535
#endif /* __S390_UACCESS_H */

0 commit comments

Comments
 (0)