[glibc.git] / sysdeps / powerpc / powerpc64 / sysdep.h

/* Assembly macros for 64-bit PowerPC.
   Copyright (C) 2002-2023 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdeps/powerpc/sysdep.h>
#include <tls.h>

#ifdef __ASSEMBLER__

/* Stack frame offsets.  */
#define FRAME_BACKCHAIN		0
#define FRAME_CR_SAVE		8
#define FRAME_LR_SAVE		16
#if _CALL_ELF != 2
#define FRAME_MIN_SIZE		112
#define FRAME_MIN_SIZE_PARM	112
#define FRAME_TOC_SAVE		40
#define FRAME_PARM_SAVE		48
#else
#define FRAME_MIN_SIZE		32
#define FRAME_MIN_SIZE_PARM	96
#define FRAME_TOC_SAVE		24
#define FRAME_PARM_SAVE		32
#endif

/* Support macros for CALL_MCOUNT.  */
	.macro SAVE_ARG NARG
	.if \NARG
	SAVE_ARG \NARG-1
	std	2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
	.endif
	.endm

	.macro REST_ARG NARG
	.if \NARG
	REST_ARG \NARG-1
	ld	2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
	.endif
	.endm

	.macro CFI_SAVE_ARG NARG
	.if \NARG
	CFI_SAVE_ARG \NARG-1
	cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
	.endif
	.endm

	.macro CFI_REST_ARG NARG
	.if \NARG
	CFI_REST_ARG \NARG-1
	cfi_restore(2+\NARG)
	.endif
	.endm

/* If compiled for profiling, call `_mcount' at the start of each function.
   see ppc-mcount.S for more details.  */
	.macro CALL_MCOUNT NARG
#ifdef	PROF
	mflr	r0
	SAVE_ARG \NARG
	std	r0,FRAME_LR_SAVE(r1)
	stdu	r1,-FRAME_MIN_SIZE_PARM(r1)
	cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
	cfi_offset(lr,FRAME_LR_SAVE)
	CFI_SAVE_ARG \NARG
	bl	JUMPTARGET (_mcount)
#ifndef SHARED
	nop
#endif
	ld	r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
	REST_ARG \NARG
	mtlr	r0
	addi	r1,r1,FRAME_MIN_SIZE_PARM
	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
	cfi_restore(lr)
	CFI_REST_ARG \NARG
#endif
	.endm

#if _CALL_ELF != 2

/* Macro to prepare for calling via a function pointer.  */
	.macro PPC64_LOAD_FUNCPTR PTR
	ld      r12,0(\PTR)
	ld      r2,8(\PTR)
	mtctr   r12
	ld      r11,16(\PTR)
	.endm

#ifdef USE_PPC64_OVERLAPPING_OPD
# define OPD_ENT(name)	.quad BODY_LABEL (name), .TOC.@tocbase
#else
# define OPD_ENT(name)	.quad BODY_LABEL (name), .TOC.@tocbase, 0
#endif

#define ENTRY_1(name)				\
	.type BODY_LABEL(name),@function;	\
	.globl name;				\
	.section ".opd","aw";			\
	.p2align 3;FUNC_LABEL(name):		\
	OPD_ENT (name);				\
	.previous

#define FUNC_LABEL(X) X
#define BODY_LABEL(X) .LY##X
#define ENTRY_2(name)				\
	.type name,@function;			\
	ENTRY_1(name)
#define END_2(name)				\
	.size name,.-BODY_LABEL(name);		\
	.size BODY_LABEL(name),.-BODY_LABEL(name)
#define LOCALENTRY(name)

#else /* _CALL_ELF == 2 */

/* Macro to prepare for calling via a function pointer.  */
	.macro PPC64_LOAD_FUNCPTR PTR
	mr	r12,\PTR
	mtctr   r12
	.endm

#define FUNC_LABEL(X) X
#define BODY_LABEL(X) X
#define ENTRY_2(name)				\
	.globl name;				\
	.type name,@function
#define END_2(name)				\
	.size name,.-name
#define LOCALENTRY(name)			\
1:      addis	r2,r12,.TOC.-1b@ha;		\
        addi	r2,r2,.TOC.-1b@l;		\
	.localentry name,.-name

#endif /* _CALL_ELF */

	.macro NOPS NARG
	.if \NARG
	NOPS \NARG-1
	nop
	.endif
	.endm

	.macro ENTRY_3 name, alignp2=2, nopwords=0
	.text
	ENTRY_2(\name)
	.p2align \alignp2
	NOPS \nopwords
BODY_LABEL(\name):
	.endm

/* Use ENTRY_TOCLESS for functions that make no use of r2 and
   guarantee r2 is unchanged on exit.  Any function that has @toc or
   @got relocs uses r2.  Functions that call other functions via the
   PLT use r2.  Use ENTRY for functions that may use or change r2.
   The first argument is the function name.
   The optional second argument specifies alignment of the function's
   code, as the logarithm base two of the byte alignment.  For
   example, a value of four aligns to a sixteen byte boundary.
   The optional third argument specifies the number of NOPs to emit
   before the start of the function's code.   */
#ifndef PROF
#define ENTRY_TOCLESS(name, ...)		\
	ENTRY_3 name, ## __VA_ARGS__;		\
	cfi_startproc

#define ENTRY(name, ...)			\
	ENTRY_TOCLESS(name, ## __VA_ARGS__);	\
	LOCALENTRY(name)
#else
/* The call to _mcount is potentially via the plt, so profiling code
   is never free of an r2 use.  */
#define ENTRY_TOCLESS(name, ...)		\
	ENTRY_3 name, ## __VA_ARGS__;		\
	cfi_startproc;				\
	LOCALENTRY(name)

#define ENTRY(name, ...)			\
	ENTRY_TOCLESS(name, ## __VA_ARGS__)
#endif

/* Local labels stripped out by the linker.  */
#undef L
#define L(x) .L##x

#define tostring(s) #s
#define stringify(s) tostring(s)
#define XGLUE(a,b) a##b
#define GLUE(a,b) XGLUE(a,b)
#define LT_LABEL(name) GLUE(.LT,name)
#define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)

/* Support Traceback tables */
#define TB_ASM			0x000c000000000000
#define TB_GLOBALLINK		0x0000800000000000
#define TB_IS_EPROL		0x0000400000000000
#define TB_HAS_TBOFF		0x0000200000000000
#define TB_INT_PROC		0x0000100000000000
#define TB_HAS_CTL		0x0000080000000000
#define TB_TOCLESS		0x0000040000000000
#define TB_FP_PRESENT		0x0000020000000000
#define TB_LOG_ABORT		0x0000010000000000
#define TB_INT_HANDL		0x0000008000000000
#define TB_NAME_PRESENT		0x0000004000000000
#define TB_USES_ALLOCA		0x0000002000000000
#define TB_SAVES_CR		0x0000000200000000
#define TB_SAVES_LR		0x0000000100000000
#define TB_STORES_BC		0x0000000080000000
#define TB_FIXUP		0x0000000040000000
#define TB_FP_SAVED(fprs)	(((fprs) & 0x3f) << 24)
#define TB_GPR_SAVED(gprs)	(((fprs) & 0x3f) << 16)
#define TB_FIXEDPARMS(parms)	(((parms) & 0xff) << 8)
#define TB_FLOATPARMS(parms)	(((parms) & 0x7f) << 1)
#define TB_PARMSONSTK		0x0000000000000001

#define PPC_HIGHER(v) 		(((v) >> 32) & 0xffff)
#define TB_DEFAULT		TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT

#define TRACEBACK(name) \
LT_LABEL(name): ; \
	.long	0 ; \
	.quad	TB_DEFAULT ; \
	.long	LT_LABEL(name)-BODY_LABEL(name) ; \
	.short	LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
LT_LABELSUFFIX(name,_name_start): ;\
	.ascii	stringify(name) ; \
LT_LABELSUFFIX(name,_name_end): ; \
	.p2align 2

#define TRACEBACK_MASK(name,mask) \
LT_LABEL(name): ; \
	.long	0 ; \
	.quad	TB_DEFAULT | mask ; \
	.long	LT_LABEL(name)-BODY_LABEL(name) ; \
	.short	LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
LT_LABELSUFFIX(name,_name_start): ;\
	.ascii	stringify(name) ; \
LT_LABELSUFFIX(name,_name_end): ; \
	.p2align 2

/* END generates Traceback tables */
#undef	END
#define END(name) \
  cfi_endproc;			\
  TRACEBACK(name);		\
  END_2(name)

/* This form supports more informative traceback tables */
#define END_GEN_TB(name,mask)	\
  cfi_endproc;			\
  TRACEBACK_MASK(name,mask);	\
  END_2(name)

/* We will allocate a new frame to save LR and the non-volatile register used to
   read the TCB when checking for scv support on syscall code.  We actually just
   need the minimum frame size plus room for 1 reg (8 bytes).  But the ABI
   mandates stack frames should be aligned at 16 Bytes, so we end up allocating
   a bit more space then what will actually be used.  */
#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE

/* Allocate frame and save register */
#define NVOLREG_SAVE \
    stdu r1,-SCV_FRAME_SIZE(r1); \
    cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \
    std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
    cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE);

/* Restore register and destroy frame */
#define NVOLREG_RESTORE	\
    ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
    cfi_restore(r31); \
    addi r1,r1,SCV_FRAME_SIZE; \
    cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);

/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB.  If it is not set, scv is
   not available, then go to JUMPFALSE (label given by the macro's caller).  We
   save the value we read from the TCB in a non-volatile register so we can
   reuse it later when exiting from the syscall in PSEUDO_RET.  Note that for
   the static case we need an extra check to guarantee the thread pointer has
   already been initialized, otherwise we may try to access an invalid address
   if a syscall is called before the TLS has been setup.  */
    .macro CHECK_SCV_SUPPORT REG JUMPFALSE

#ifndef SHARED
    /* Check if thread pointer has already been setup.  */
    cmpdi r13,0
    beq \JUMPFALSE
#endif

    /* Read PPC_FEATURE2_SCV from TCB and store it in REG */
    ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
    andis. \REG,\REG,PPC_FEATURE2_SCV>>16

    beq \JUMPFALSE
    .endm

#if !defined(USE_PPC_SCV) || IS_IN(rtld)
# define DO_CALL(syscall) \
    li r0,syscall; \
    DO_CALL_SC
#else
/* Before doing the syscall, check if we can use scv.  scv is supported by P9
   and later with Linux v5.9 and later.  If so, use it.  Otherwise, fallback to
   sc.  We use a non-volatile register to save hwcap2 from the TCB, so we need
   to save its content beforehand.  */
# define DO_CALL(syscall) \
    li r0,syscall; \
    NVOLREG_SAVE; \
    CHECK_SCV_SUPPORT r31 0f; \
    DO_CALL_SCV; \
    b 1f; \
0:  DO_CALL_SC; \
1:
#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */

/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0.  */
#define DO_CALL_SC \
    sc

#define DO_CALL_SCV \
    mflr r9; \
    std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1);   \
    cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \
    .machine "push"; \
    .machine "power9"; \
    scv 0; \
    .machine "pop"; \
    ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1);      \
    mtlr r9; \
    cfi_restore(lr);

/* ppc64 is always PIC */
#undef JUMPTARGET
#define JUMPTARGET(name) FUNC_LABEL(name)

#define PSEUDO(name, syscall_name, args) \
  .section ".text";				\
  ENTRY (name);					\
  DO_CALL (SYS_ify (syscall_name))

#ifdef SHARED
#define TAIL_CALL_SYSCALL_ERROR \
    b JUMPTARGET (NOTOC (__syscall_error))
#else
/* Static version might be linked into a large app with a toc exceeding
   64k.  We can't put a toc adjusting stub on a plain branch, so can't
   tail call __syscall_error.  */
#define TAIL_CALL_SYSCALL_ERROR \
    .ifdef .Local_syscall_error; \
    b .Local_syscall_error; \
    .else; \
.Local_syscall_error: \
    mflr 0; \
    std 0,FRAME_LR_SAVE(1); \
    stdu 1,-FRAME_MIN_SIZE(1); \
    cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
    cfi_offset(lr,FRAME_LR_SAVE); \
    bl JUMPTARGET(__syscall_error); \
    nop; \
    ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
    addi 1,1,FRAME_MIN_SIZE; \
    cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
    mtlr 0; \
    cfi_restore(lr); \
    blr; \
    .endif
#endif

#if !defined(USE_PPC_SCV) || IS_IN(rtld)
# define PSEUDO_RET \
    RET_SC; \
    TAIL_CALL_SYSCALL_ERROR
#else
/* This should only be called after a DO_CALL.  In such cases, r31 contains the
   value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT.  If it is
   set, we know we have entered the kernel using scv, so handle the return code
   accordingly.  */
# define PSEUDO_RET \
    cmpdi cr5,r31,0; \
    NVOLREG_RESTORE; \
    beq cr5,0f; \
    RET_SCV; \
    b 1f; \
0:  RET_SC; \
1:  TAIL_CALL_SYSCALL_ERROR
#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */

#define RET_SCV \
    li r9,-4095; \
    cmpld r3,r9; \
    bltlr+; \
    neg r3,r3;

#define RET_SC \
    bnslr+;

#define ret PSEUDO_RET

#undef	PSEUDO_END
#define	PSEUDO_END(name) \
  END (name)

#define PSEUDO_NOERRNO(name, syscall_name, args) \
  .section ".text";					\
  ENTRY (name);						\
  DO_CALL (SYS_ify (syscall_name))

#if !defined(USE_PPC_SCV) || IS_IN(rtld)
# define PSEUDO_RET_NOERRNO \
    blr
#else
/* This should only be called after a DO_CALL.  */
# define PSEUDO_RET_NOERRNO \
    NVOLREG_RESTORE; \
    blr
#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */

#define ret_NOERRNO PSEUDO_RET_NOERRNO

#undef	PSEUDO_END_NOERRNO
#define	PSEUDO_END_NOERRNO(name) \
  END (name)

#define PSEUDO_ERRVAL(name, syscall_name, args) \
  .section ".text";					\
  ENTRY (name);						\
  DO_CALL (SYS_ify (syscall_name))

#if !defined(USE_PPC_SCV) || IS_IN(rtld)
# define PSEUDO_RET_ERRVAL \
    blr
#else
/* This should only be called after a DO_CALL.  */
# define PSEUDO_RET_ERRVAL \
    NVOLREG_RESTORE; \
    blr
#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */

#define ret_ERRVAL PSEUDO_RET_ERRVAL

#undef	PSEUDO_END_ERRVAL
#define	PSEUDO_END_ERRVAL(name) \
  END (name)

#ifdef SHARED
# if IS_IN (rtld)
	 /* Inside ld.so we use the local alias to avoid runtime GOT
	    relocations.  */
#  define __GLRO_DEF(var)				\
.LC__ ## var:						\
	.tc _rtld_local_ro[TC],_rtld_local_ro
# else
#  define __GLRO_DEF(var)				\
.LC__ ## var:						\
	.tc _rtld_global_ro[TC],_rtld_global_ro
# endif
# define __GLRO(rOUT, var, offset)		\
	addis	rOUT,r2,.LC__ ## var@toc@ha;	\
	ld	rOUT,.LC__ ## var@toc@l(rOUT);	\
	lwz	rOUT,offset(rOUT)
#else
# define __GLRO_DEF(var)			\
.LC__ ## var:					\
	.tc _ ## var[TC],_ ## var
# define __GLRO(rOUT, var, offset)		\
	addis	rOUT,r2,.LC__ ## var@toc@ha;	\
	ld	rOUT,.LC__ ## var@toc@l(rOUT);	\
	lwz	rOUT,0(rOUT)
#endif

#ifdef USE_PPC64_NOTOC
# define NOTOC(l) l@notoc
#else
# define NOTOC(l) l
#endif

#else /* !__ASSEMBLER__ */

#if _CALL_ELF != 2

#define PPC64_LOAD_FUNCPTR(ptr) \
	"ld 	12,0(" #ptr ")\n"					\
	"ld	2,8(" #ptr ")\n"					\
	"mtctr	12\n"							\
	"ld	11,16(" #ptr ")"

#ifdef USE_PPC64_OVERLAPPING_OPD
# define OPD_ENT(name)	".quad " BODY_PREFIX #name ", .TOC.@tocbase"
#else
# define OPD_ENT(name)	".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
#endif

#define ENTRY_1(name)	\
	".type   " BODY_PREFIX #name ",@function\n"			\
	".globl " #name "\n"						\
	".pushsection \".opd\",\"aw\"\n"				\
	".p2align 3\n"							\
#name ":\n"								\
	OPD_ENT (name) "\n"						\
	".popsection"

#define DOT_PREFIX ""
#define BODY_PREFIX ".LY"
#define ENTRY_2(name)	\
	".type " #name ",@function\n"					\
	ENTRY_1(name)
#define END_2(name)	\
	".size " #name ",.-" BODY_PREFIX #name "\n"			\
	".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
#define LOCALENTRY(name)

#else /* _CALL_ELF */

#define PPC64_LOAD_FUNCPTR(ptr) \
	"mr	12," #ptr "\n"						\
	"mtctr 	12"

#define DOT_PREFIX ""
#define BODY_PREFIX ""
#define ENTRY_2(name)	\
	".type " #name ",@function\n"					\
	".globl " #name
#define END_2(name)	\
	".size " #name ",.-" #name
#define LOCALENTRY(name)	\
	"1: addis 2,12,.TOC.-1b@ha\n"					\
	"addi	2,2,.TOC.-1b@l\n"					\
	".localentry " #name ",.-" #name

#endif /* _CALL_ELF */

#endif	/* __ASSEMBLER__ */
Commit	Line	Data
b80a3db0	1	/* Assembly macros for 64-bit PowerPC.
6d7e8eda	2	Copyright (C) 2002-2023 Free Software Foundation, Inc.
b80a3db0 RM	3	This file is part of the GNU C Library.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6	16	License along with the GNU C Library; if not, see
5a82c748	17	<https://www.gnu.org/licenses/>. */
b80a3db0	18
b80a3db0	19	#include <sysdeps/powerpc/sysdep.h>
68ab82f5	20	#include <tls.h>
b80a3db0	21
afe01786 UD	22	#ifdef __ASSEMBLER__
afe01786 UD	23
8b8a692c	24	/* Stack frame offsets. */
8b8a692c UW	25	#define FRAME_BACKCHAIN 0
	26	#define FRAME_CR_SAVE 8
	27	#define FRAME_LR_SAVE 16
fb499eb0 AM	28	#if _CALL_ELF != 2
	29	#define FRAME_MIN_SIZE 112
	30	#define FRAME_MIN_SIZE_PARM 112
8b8a692c UW	31	#define FRAME_TOC_SAVE 40
8b8a692c UW	32	#define FRAME_PARM_SAVE 48
8b8a692c UW	33	#else
	34	#define FRAME_MIN_SIZE 32
	35	#define FRAME_MIN_SIZE_PARM 96
8b8a692c UW	36	#define FRAME_TOC_SAVE 24
8b8a692c UW	37	#define FRAME_PARM_SAVE 32
8b8a692c UW	38	#endif
8b8a692c UW	39
d7d06f79 UD	40	/* Support macros for CALL_MCOUNT. */
	41	.macro SAVE_ARG NARG
	42	.if \NARG
	43	SAVE_ARG \NARG-1
dd885436	44	std 2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
d7d06f79 UD	45	.endif
	46	.endm
	47
	48	.macro REST_ARG NARG
	49	.if \NARG
	50	REST_ARG \NARG-1
dd885436	51	ld 2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
bebff237 AM	52	.endif
	53	.endm
	54
	55	.macro CFI_SAVE_ARG NARG
	56	.if \NARG
	57	CFI_SAVE_ARG \NARG-1
dd885436	58	cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
bebff237 AM	59	.endif
	60	.endm
	61
	62	.macro CFI_REST_ARG NARG
	63	.if \NARG
	64	CFI_REST_ARG \NARG-1
	65	cfi_restore(2+\NARG)
d7d06f79 UD	66	.endif
	67	.endm
	68
b80a3db0 RM	69	/* If compiled for profiling, call `_mcount' at the start of each function.
b80a3db0 RM	70	see ppc-mcount.S for more details. */
d7d06f79	71	.macro CALL_MCOUNT NARG
b80a3db0	72	#ifdef PROF
d7d06f79 UD	73	mflr r0
d7d06f79 UD	74	SAVE_ARG \NARG
8b8a692c UW	75	std r0,FRAME_LR_SAVE(r1)
	76	stdu r1,-FRAME_MIN_SIZE_PARM(r1)
	77	cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
	78	cfi_offset(lr,FRAME_LR_SAVE)
bebff237	79	CFI_SAVE_ARG \NARG
d7d06f79	80	bl JUMPTARGET (_mcount)
bebff237 AM	81	#ifndef SHARED
	82	nop
	83	#endif
8b8a692c	84	ld r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
d7d06f79	85	REST_ARG \NARG
d7d06f79	86	mtlr r0
8b8a692c UW	87	addi r1,r1,FRAME_MIN_SIZE_PARM
8b8a692c UW	88	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
bebff237 AM	89	cfi_restore(lr)
bebff237 AM	90	CFI_REST_ARG \NARG
d7d06f79 UD	91	#endif
d7d06f79 UD	92	.endm
b80a3db0	93
696caf1d UW	94	#if _CALL_ELF != 2
696caf1d UW	95
d31beafa UW	96	/* Macro to prepare for calling via a function pointer. */
	97	.macro PPC64_LOAD_FUNCPTR PTR
	98	ld r12,0(\PTR)
	99	ld r2,8(\PTR)
	100	mtctr r12
	101	ld r11,16(\PTR)
	102	.endm
	103
590b40f7 UD	104	#ifdef USE_PPC64_OVERLAPPING_OPD
	105	# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase
	106	#else
	107	# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase, 0
	108	#endif
b80a3db0	109
4b6e7667	110	#define ENTRY_1(name) \
590b40f7 UD	111	.type BODY_LABEL(name),@function; \
	112	.globl name; \
	113	.section ".opd","aw"; \
4b6e7667 AM	114	.p2align 3;FUNC_LABEL(name): \
	115	OPD_ENT (name); \
	116	.previous
590b40f7	117
4b6e7667	118	#define FUNC_LABEL(X) X
5ca10a0c	119	#define BODY_LABEL(X) .LY##X
4b6e7667	120	#define ENTRY_2(name) \
590b40f7 UD	121	.type name,@function; \
590b40f7 UD	122	ENTRY_1(name)
4b6e7667	123	#define END_2(name) \
590b40f7	124	.size name,.-BODY_LABEL(name); \
4b6e7667	125	.size BODY_LABEL(name),.-BODY_LABEL(name)
696caf1d UW	126	#define LOCALENTRY(name)
696caf1d UW	127
4b6e7667	128	#else /* _CALL_ELF == 2 */
696caf1d UW	129
	130	/* Macro to prepare for calling via a function pointer. */
	131	.macro PPC64_LOAD_FUNCPTR PTR
	132	mr r12,\PTR
	133	mtctr r12
	134	.endm
	135
4b6e7667	136	#define FUNC_LABEL(X) X
696caf1d	137	#define BODY_LABEL(X) X
4b6e7667	138	#define ENTRY_2(name) \
696caf1d	139	.globl name; \
4b6e7667 AM	140	.type name,@function
	141	#define END_2(name) \
	142	.size name,.-name
	143	#define LOCALENTRY(name) \
	144	1: addis r2,r12,.TOC.-1b@ha; \
	145	addi r2,r2,.TOC.-1b@l; \
	146	.localentry name,.-name
696caf1d UW	147
696caf1d UW	148	#endif /* _CALL_ELF */
b80a3db0	149
d5b41185 AM	150	.macro NOPS NARG
	151	.if \NARG
	152	NOPS \NARG-1
	153	nop
	154	.endif
	155	.endm
b80a3db0	156
d5b41185 AM	157	.macro ENTRY_3 name, alignp2=2, nopwords=0
	158	.text
	159	ENTRY_2(\name)
	160	.p2align \alignp2
	161	NOPS \nopwords
	162	BODY_LABEL(\name):
	163	.endm
	164
	165	/* Use ENTRY_TOCLESS for functions that make no use of r2 and
	166	guarantee r2 is unchanged on exit. Any function that has @toc or
	167	@got relocs uses r2. Functions that call other functions via the
	168	PLT use r2. Use ENTRY for functions that may use or change r2.
	169	The first argument is the function name.
	170	The optional second argument specifies alignment of the function's
	171	code, as the logarithm base two of the byte alignment. For
	172	example, a value of four aligns to a sixteen byte boundary.
	173	The optional third argument specifies the number of NOPs to emit
	174	before the start of the function's code. */
	175	#ifndef PROF
	176	#define ENTRY_TOCLESS(name, ...) \
	177	ENTRY_3 name, ## __VA_ARGS__; \
	178	cfi_startproc
	179
	180	#define ENTRY(name, ...) \
	181	ENTRY_TOCLESS(name, ## __VA_ARGS__); \
	182	LOCALENTRY(name)
	183	#else
	184	/* The call to _mcount is potentially via the plt, so profiling code
	185	is never free of an r2 use. */
	186	#define ENTRY_TOCLESS(name, ...) \
	187	ENTRY_3 name, ## __VA_ARGS__; \
696caf1d UW	188	cfi_startproc; \
696caf1d UW	189	LOCALENTRY(name)
b80a3db0	190
d5b41185 AM	191	#define ENTRY(name, ...) \
	192	ENTRY_TOCLESS(name, ## __VA_ARGS__)
	193	#endif
	194
b80a3db0 RM	195	/* Local labels stripped out by the linker. */
	196	#undef L
	197	#define L(x) .L##x
	198
	199	#define tostring(s) #s
	200	#define stringify(s) tostring(s)
	201	#define XGLUE(a,b) a##b
	202	#define GLUE(a,b) XGLUE(a,b)
	203	#define LT_LABEL(name) GLUE(.LT,name)
	204	#define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)
	205
	206	/* Support Traceback tables */
	207	#define TB_ASM 0x000c000000000000
590b40f7	208	#define TB_GLOBALLINK 0x0000800000000000
b80a3db0	209	#define TB_IS_EPROL 0x0000400000000000
590b40f7	210	#define TB_HAS_TBOFF 0x0000200000000000
b80a3db0 RM	211	#define TB_INT_PROC 0x0000100000000000
	212	#define TB_HAS_CTL 0x0000080000000000
	213	#define TB_TOCLESS 0x0000040000000000
590b40f7 UD	214	#define TB_FP_PRESENT 0x0000020000000000
	215	#define TB_LOG_ABORT 0x0000010000000000
	216	#define TB_INT_HANDL 0x0000008000000000
	217	#define TB_NAME_PRESENT 0x0000004000000000
	218	#define TB_USES_ALLOCA 0x0000002000000000
b80a3db0 RM	219	#define TB_SAVES_CR 0x0000000200000000
b80a3db0 RM	220	#define TB_SAVES_LR 0x0000000100000000
590b40f7	221	#define TB_STORES_BC 0x0000000080000000
b80a3db0 RM	222	#define TB_FIXUP 0x0000000040000000
	223	#define TB_FP_SAVED(fprs) (((fprs) & 0x3f) << 24)
	224	#define TB_GPR_SAVED(gprs) (((fprs) & 0x3f) << 16)
	225	#define TB_FIXEDPARMS(parms) (((parms) & 0xff) << 8)
	226	#define TB_FLOATPARMS(parms) (((parms) & 0x7f) << 1)
590b40f7	227	#define TB_PARMSONSTK 0x0000000000000001
b80a3db0	228
590b40f7 UD	229	#define PPC_HIGHER(v) (((v) >> 32) & 0xffff)
590b40f7 UD	230	#define TB_DEFAULT TB_ASM \| TB_HAS_TBOFF \| TB_NAME_PRESENT
b80a3db0 RM	231
	232	#define TRACEBACK(name) \
	233	LT_LABEL(name): ; \
	234	.long 0 ; \
	235	.quad TB_DEFAULT ; \
590b40f7	236	.long LT_LABEL(name)-BODY_LABEL(name) ; \
b80a3db0 RM	237	.short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
	238	LT_LABELSUFFIX(name,_name_start): ;\
	239	.ascii stringify(name) ; \
	240	LT_LABELSUFFIX(name,_name_end): ; \
4b6e7667	241	.p2align 2
b80a3db0 RM	242
	243	#define TRACEBACK_MASK(name,mask) \
	244	LT_LABEL(name): ; \
	245	.long 0 ; \
	246	.quad TB_DEFAULT \| mask ; \
590b40f7	247	.long LT_LABEL(name)-BODY_LABEL(name) ; \
b80a3db0 RM	248	.short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
	249	LT_LABELSUFFIX(name,_name_start): ;\
	250	.ascii stringify(name) ; \
	251	LT_LABELSUFFIX(name,_name_end): ; \
4b6e7667	252	.p2align 2
b80a3db0 RM	253
	254	/* END generates Traceback tables */
	255	#undef END
	256	#define END(name) \
9759bbf1	257	cfi_endproc; \
4b6e7667	258	TRACEBACK(name); \
590b40f7	259	END_2(name)
b80a3db0 RM	260
	261	/* This form supports more informative traceback tables */
	262	#define END_GEN_TB(name,mask) \
9759bbf1	263	cfi_endproc; \
4b6e7667	264	TRACEBACK_MASK(name,mask); \
590b40f7	265	END_2(name)
b80a3db0	266
68ab82f5 MC	267	/* We will allocate a new frame to save LR and the non-volatile register used to
	268	read the TCB when checking for scv support on syscall code. We actually just
	269	need the minimum frame size plus room for 1 reg (8 bytes). But the ABI
	270	mandates stack frames should be aligned at 16 Bytes, so we end up allocating
	271	a bit more space then what will actually be used. */
	272	#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
	273	#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE
	274
	275	/* Allocate frame and save register */
	276	#define NVOLREG_SAVE \
	277	stdu r1,-SCV_FRAME_SIZE(r1); \
d120fb99	278	cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \
68ab82f5	279	std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
d120fb99	280	cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE);
68ab82f5 MC	281
	282	/* Restore register and destroy frame */
	283	#define NVOLREG_RESTORE \
	284	ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
d120fb99	285	cfi_restore(r31); \
68ab82f5 MC	286	addi r1,r1,SCV_FRAME_SIZE; \
	287	cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);
	288
	289	/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is
	290	not available, then go to JUMPFALSE (label given by the macro's caller). We
	291	save the value we read from the TCB in a non-volatile register so we can
	292	reuse it later when exiting from the syscall in PSEUDO_RET. Note that for
	293	the static case we need an extra check to guarantee the thread pointer has
	294	already been initialized, otherwise we may try to access an invalid address
	295	if a syscall is called before the TLS has been setup. */
	296	.macro CHECK_SCV_SUPPORT REG JUMPFALSE
	297
	298	#ifndef SHARED
	299	/* Check if thread pointer has already been setup. */
	300	cmpdi r13,0
	301	beq \JUMPFALSE
	302	#endif
	303
	304	/* Read PPC_FEATURE2_SCV from TCB and store it in REG */
	305	ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
	306	andis. \REG,\REG,PPC_FEATURE2_SCV>>16
	307
	308	beq \JUMPFALSE
	309	.endm
	310
ebae2f5a	311	#if !defined(USE_PPC_SCV) \|\| IS_IN(rtld)
68ab82f5 MC	312	# define DO_CALL(syscall) \
	313	li r0,syscall; \
	314	DO_CALL_SC
	315	#else
	316	/* Before doing the syscall, check if we can use scv. scv is supported by P9
	317	and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to
	318	sc. We use a non-volatile register to save hwcap2 from the TCB, so we need
	319	to save its content beforehand. */
	320	# define DO_CALL(syscall) \
	321	li r0,syscall; \
	322	NVOLREG_SAVE; \
	323	CHECK_SCV_SUPPORT r31 0f; \
	324	DO_CALL_SCV; \
	325	b 1f; \
	326	0: DO_CALL_SC; \
	327	1:
ebae2f5a	328	#endif /* !defined(USE_PPC_SCV) \|\| IS_IN(rtld) */
68ab82f5 MC	329
	330	/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */
	331	#define DO_CALL_SC \
b80a3db0 RM	332	sc
b80a3db0 RM	333
68ab82f5 MC	334	#define DO_CALL_SCV \
68ab82f5 MC	335	mflr r9; \
d120fb99 MC	336	std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
d120fb99 MC	337	cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \
527c89cd FW	338	.machine "push"; \
527c89cd FW	339	.machine "power9"; \
68ab82f5	340	scv 0; \
527c89cd	341	.machine "pop"; \
d120fb99	342	ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
68ab82f5 MC	343	mtlr r9; \
	344	cfi_restore(lr);
	345
b80a3db0	346	/* ppc64 is always PIC */
29d9a17d	347	#undef JUMPTARGET
4b6e7667	348	#define JUMPTARGET(name) FUNC_LABEL(name)
b80a3db0 RM	349
b80a3db0 RM	350	#define PSEUDO(name, syscall_name, args) \
4b6e7667 AM	351	.section ".text"; \
	352	ENTRY (name); \
	353	DO_CALL (SYS_ify (syscall_name))
b80a3db0	354
bebff237 AM	355	#ifdef SHARED
bebff237 AM	356	#define TAIL_CALL_SYSCALL_ERROR \
7c7bcf36	357	b JUMPTARGET (NOTOC (__syscall_error))
bebff237 AM	358	#else
	359	/* Static version might be linked into a large app with a toc exceeding
	360	64k. We can't put a toc adjusting stub on a plain branch, so can't
	361	tail call __syscall_error. */
	362	#define TAIL_CALL_SYSCALL_ERROR \
	363	.ifdef .Local_syscall_error; \
	364	b .Local_syscall_error; \
	365	.else; \
	366	.Local_syscall_error: \
	367	mflr 0; \
8b8a692c UW	368	std 0,FRAME_LR_SAVE(1); \
	369	stdu 1,-FRAME_MIN_SIZE(1); \
	370	cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
	371	cfi_offset(lr,FRAME_LR_SAVE); \
bebff237 AM	372	bl JUMPTARGET(__syscall_error); \
bebff237 AM	373	nop; \
8b8a692c UW	374	ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
	375	addi 1,1,FRAME_MIN_SIZE; \
	376	cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
bebff237 AM	377	mtlr 0; \
	378	cfi_restore(lr); \
	379	blr; \
	380	.endif
	381	#endif
	382
ebae2f5a	383	#if !defined(USE_PPC_SCV) \|\| IS_IN(rtld)
68ab82f5 MC	384	# define PSEUDO_RET \
68ab82f5 MC	385	RET_SC; \
bebff237	386	TAIL_CALL_SYSCALL_ERROR
68ab82f5 MC	387	#else
	388	/* This should only be called after a DO_CALL. In such cases, r31 contains the
	389	value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is
	390	set, we know we have entered the kernel using scv, so handle the return code
	391	accordingly. */
	392	# define PSEUDO_RET \
	393	cmpdi cr5,r31,0; \
	394	NVOLREG_RESTORE; \
	395	beq cr5,0f; \
	396	RET_SCV; \
	397	b 1f; \
	398	0: RET_SC; \
	399	1: TAIL_CALL_SYSCALL_ERROR
ebae2f5a	400	#endif /* !defined(USE_PPC_SCV) \|\| IS_IN(rtld) */
68ab82f5 MC	401
68ab82f5 MC	402	#define RET_SCV \
7de36744 NP	403	li r9,-4095; \
	404	cmpld r3,r9; \
	405	bltlr+; \
68ab82f5 MC	406	neg r3,r3;
	407
	408	#define RET_SC \
	409	bnslr+;
b80a3db0 RM	410
	411	#define ret PSEUDO_RET
	412
	413	#undef PSEUDO_END
	414	#define PSEUDO_END(name) \
	415	END (name)
	416
9eb88290	417	#define PSEUDO_NOERRNO(name, syscall_name, args) \
4b6e7667 AM	418	.section ".text"; \
	419	ENTRY (name); \
	420	DO_CALL (SYS_ify (syscall_name))
9eb88290	421
ebae2f5a	422	#if !defined(USE_PPC_SCV) \|\| IS_IN(rtld)
68ab82f5	423	# define PSEUDO_RET_NOERRNO \
9eb88290	424	blr
68ab82f5 MC	425	#else
	426	/* This should only be called after a DO_CALL. */
	427	# define PSEUDO_RET_NOERRNO \
	428	NVOLREG_RESTORE; \
	429	blr
ebae2f5a	430	#endif /* !defined(USE_PPC_SCV) \|\| IS_IN(rtld) */
9eb88290 UD	431
	432	#define ret_NOERRNO PSEUDO_RET_NOERRNO
	433
	434	#undef PSEUDO_END_NOERRNO
c6289757	435	#define PSEUDO_END_NOERRNO(name) \
9eb88290 UD	436	END (name)
9eb88290 UD	437
137ffcdc	438	#define PSEUDO_ERRVAL(name, syscall_name, args) \
4b6e7667 AM	439	.section ".text"; \
	440	ENTRY (name); \
	441	DO_CALL (SYS_ify (syscall_name))
137ffcdc	442
ebae2f5a	443	#if !defined(USE_PPC_SCV) \|\| IS_IN(rtld)
68ab82f5 MC	444	# define PSEUDO_RET_ERRVAL \
	445	blr
	446	#else
	447	/* This should only be called after a DO_CALL. */
	448	# define PSEUDO_RET_ERRVAL \
	449	NVOLREG_RESTORE; \
137ffcdc	450	blr
ebae2f5a	451	#endif /* !defined(USE_PPC_SCV) \|\| IS_IN(rtld) */
137ffcdc UD	452
	453	#define ret_ERRVAL PSEUDO_RET_ERRVAL
	454
	455	#undef PSEUDO_END_ERRVAL
	456	#define PSEUDO_END_ERRVAL(name) \
	457	END (name)
	458
18363b4f TMQMF	459	#ifdef SHARED
	460	# if IS_IN (rtld)
	461	/* Inside ld.so we use the local alias to avoid runtime GOT
	462	relocations. */
	463	# define __GLRO_DEF(var) \
	464	.LC__ ## var: \
	465	.tc _rtld_local_ro[TC],_rtld_local_ro
	466	# else
	467	# define __GLRO_DEF(var) \
	468	.LC__ ## var: \
	469	.tc _rtld_global_ro[TC],_rtld_global_ro
	470	# endif
	471	# define __GLRO(rOUT, var, offset) \
d6efcc11 AM	472	addis rOUT,r2,.LC__ ## var@toc@ha; \
d6efcc11 AM	473	ld rOUT,.LC__ ## var@toc@l(rOUT); \
18363b4f TMQMF	474	lwz rOUT,offset(rOUT)
	475	#else
	476	# define __GLRO_DEF(var) \
	477	.LC__ ## var: \
	478	.tc _ ## var[TC],_ ## var
	479	# define __GLRO(rOUT, var, offset) \
d6efcc11 AM	480	addis rOUT,r2,.LC__ ## var@toc@ha; \
d6efcc11 AM	481	ld rOUT,.LC__ ## var@toc@l(rOUT); \
18363b4f TMQMF	482	lwz rOUT,0(rOUT)
	483	#endif
	484
7c7bcf36 TMQMF	485	#ifdef USE_PPC64_NOTOC
	486	# define NOTOC(l) l@notoc
	487	#else
	488	# define NOTOC(l) l
	489	#endif
	490
590b40f7	491	#else /* !__ASSEMBLER__ */
b80a3db0	492
696caf1d UW	493	#if _CALL_ELF != 2
696caf1d UW	494
d31beafa	495	#define PPC64_LOAD_FUNCPTR(ptr) \
4b6e7667 AM	496	"ld 12,0(" #ptr ")\n" \
	497	"ld 2,8(" #ptr ")\n" \
	498	"mtctr 12\n" \
	499	"ld 11,16(" #ptr ")"
d31beafa	500
590b40f7	501	#ifdef USE_PPC64_OVERLAPPING_OPD
4b6e7667	502	# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase"
590b40f7	503	#else
4b6e7667	504	# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
590b40f7 UD	505	#endif
590b40f7 UD	506
d31beafa	507	#define ENTRY_1(name) \
4b6e7667 AM	508	".type " BODY_PREFIX #name ",@function\n" \
	509	".globl " #name "\n" \
	510	".pushsection \".opd\",\"aw\"\n" \
	511	".p2align 3\n" \
d31beafa UW	512	#name ":\n" \
d31beafa UW	513	OPD_ENT (name) "\n" \
4b6e7667	514	".popsection"
d31beafa	515
5ca10a0c AZ	516	#define DOT_PREFIX ""
	517	#define BODY_PREFIX ".LY"
	518	#define ENTRY_2(name) \
4b6e7667	519	".type " #name ",@function\n" \
d31beafa	520	ENTRY_1(name)
5ca10a0c	521	#define END_2(name) \
4b6e7667 AM	522	".size " #name ",.-" BODY_PREFIX #name "\n" \
4b6e7667 AM	523	".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
696caf1d UW	524	#define LOCALENTRY(name)
	525
	526	#else /* _CALL_ELF */
	527
	528	#define PPC64_LOAD_FUNCPTR(ptr) \
4b6e7667 AM	529	"mr 12," #ptr "\n" \
4b6e7667 AM	530	"mtctr 12"
696caf1d UW	531
	532	#define DOT_PREFIX ""
	533	#define BODY_PREFIX ""
	534	#define ENTRY_2(name) \
4b6e7667 AM	535	".type " #name ",@function\n" \
4b6e7667 AM	536	".globl " #name
696caf1d	537	#define END_2(name) \
4b6e7667	538	".size " #name ",.-" #name
696caf1d	539	#define LOCALENTRY(name) \
4b6e7667 AM	540	"1: addis 2,12,.TOC.-1b@ha\n" \
	541	"addi 2,2,.TOC.-1b@l\n" \
	542	".localentry " #name ",.-" #name
696caf1d UW	543
696caf1d UW	544	#endif /* _CALL_ELF */
b80a3db0 RM	545
b80a3db0 RM	546	#endif /* __ASSEMBLER__ */