From e951d64c085267a30036907d02f94a70dee4c9d9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 10:12:25 -0700 Subject: [PATCH 01/58] Add entry points for interacting with Go A "ffi_go_closure" is intended to be compatible with the function descriptors used by Go, and ffi_call_go sets up the static chain parameter for calling a Go function. The entry points are disabled when a backend has not been updated, much like we do for "normal" closures. --- include/ffi.h.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/ffi.h.in b/include/ffi.h.in index f403ae0..c43d52f 100644 --- a/include/ffi.h.in +++ b/include/ffi.h.in @@ -428,6 +428,22 @@ ffi_prep_java_raw_closure_loc (ffi_java_raw_closure*, #endif /* FFI_CLOSURES */ +#if FFI_GO_CLOSURES + +typedef struct { + void *tramp; + ffi_cif *cif; + void (*fun)(ffi_cif*,void*,void**,void*); +} ffi_go_closure; + +ffi_status ffi_prep_go_closure (ffi_go_closure*, ffi_cif *, + void (*fun)(ffi_cif*,void*,void**,void*)); + +void ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure); + +#endif /* FFI_GO_CLOSURES */ + /* ---- Public interface definition -------------------------------------- */ ffi_status ffi_prep_cif(ffi_cif *cif, From 89bbde8b4e802625bdbf830ea14e938beea79835 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 10:55:11 -0700 Subject: [PATCH 02/58] Add ffi_cfi.h Have one copy of the HAVE_AS_CFI_PSEUDO_OP code to share between all backends. --- include/ffi_cfi.h | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 include/ffi_cfi.h diff --git a/include/ffi_cfi.h b/include/ffi_cfi.h new file mode 100644 index 0000000..244ce57 --- /dev/null +++ b/include/ffi_cfi.h @@ -0,0 +1,55 @@ +/* ----------------------------------------------------------------------- + ffi_cfi.h - Copyright (c) 2014 Red Hat, Inc. + + Conditionally assemble cfi directives. Only necessary for building libffi. + ----------------------------------------------------------------------- */ + +#ifndef FFI_CFI_H +#define FFI_CFI_H + +#ifdef HAVE_AS_CFI_PSEUDO_OP + +# define cfi_startproc .cfi_startproc +# define cfi_endproc .cfi_endproc +# define cfi_def_cfa(reg, off) .cfi_def_cfa reg, off +# define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg +# define cfi_def_cfa_offset(off) .cfi_def_cfa_offset off +# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off +# define cfi_offset(reg, off) .cfi_offset reg, off +# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off +# define cfi_register(r1, r2) .cfi_register r1, r2 +# define cfi_return_column(reg) .cfi_return_column reg +# define cfi_restore(reg) .cfi_restore reg +# define cfi_same_value(reg) .cfi_same_value reg +# define cfi_undefined(reg) .cfi_undefined reg +# define cfi_remember_state .cfi_remember_state +# define cfi_restore_state .cfi_restore_state +# define cfi_window_save .cfi_window_save +# define cfi_personality(enc, exp) .cfi_personality enc, exp +# define cfi_lsda(enc, exp) .cfi_lsda enc, exp +# define cfi_escape(...) .cfi_escape __VA_ARGS__ + +#else + +# define cfi_startproc +# define cfi_endproc +# define cfi_def_cfa(reg, off) +# define cfi_def_cfa_register(reg) +# define cfi_def_cfa_offset(off) +# define cfi_adjust_cfa_offset(off) +# define cfi_offset(reg, off) +# define cfi_rel_offset(reg, off) +# define cfi_register(r1, r2) +# define cfi_return_column(reg) +# define cfi_restore(reg) +# define cfi_same_value(reg) +# define cfi_undefined(reg) +# define cfi_remember_state +# define cfi_restore_state +# define cfi_window_save +# define cfi_personality(enc, exp) +# define cfi_lsda(enc, exp) +# define cfi_escape(...) + +#endif /* HAVE_AS_CFI_PSEUDO_OP */ +#endif /* FFI_CFI_H */ From 6b62fb4a26d7363548a92f1a156562cfa648726e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 11:11:58 -0700 Subject: [PATCH 03/58] x86-64: Support go closures Dumps all of the hand-coded unwind info for gas generated. Move jump table data into .rodata. Adjust ffi_call_unix64 to load the static chain. Split out sse portions of ffi_closure_unix64 to ffi_closure_unix64_sse rather than test cif->flags at runtime. --- src/x86/ffi64.c | 103 ++++++++++----- src/x86/ffitarget.h | 2 + src/x86/unix64.S | 309 +++++++++++++++++++++----------------------- 3 files changed, 223 insertions(+), 191 deletions(-) diff --git a/src/x86/ffi64.c b/src/x86/ffi64.c index 5a5e043..384a93a 100644 --- a/src/x86/ffi64.c +++ b/src/x86/ffi64.c @@ -32,6 +32,7 @@ #include #include +#include #ifdef __x86_64__ @@ -62,10 +63,12 @@ struct register_args /* Registers for argument passing. */ UINT64 gpr[MAX_GPR_REGS]; union big_int_union sse[MAX_SSE_REGS]; + UINT64 rax; /* ssecount */ + UINT64 r10; /* static chain */ }; extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, - void *raddr, void (*fnaddr)(void), unsigned ssecount); + void *raddr, void (*fnaddr)(void)) FFI_HIDDEN; /* All reference to register classes here is identical to the code in gcc/config/i386/i386.c. Do *not* change one without the other. */ @@ -358,6 +361,9 @@ ffi_prep_cif_machdep (ffi_cif *cif) enum x86_64_reg_class classes[MAX_CLASSES]; size_t bytes, n; + if (cif->abi != FFI_UNIX64) + return FFI_BAD_ABI; + gprcount = ssecount = 0; flags = cif->rtype->type; @@ -419,8 +425,9 @@ ffi_prep_cif_machdep (ffi_cif *cif) return FFI_OK; } -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) { enum x86_64_reg_class classes[MAX_CLASSES]; char *stack, *argp; @@ -445,6 +452,8 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) reg_args = (struct register_args *) stack; argp = stack + sizeof (struct register_args); + reg_args->r10 = (uintptr_t) closure; + gprcount = ssecount = 0; /* If the return value is passed in memory, add the pointer as the @@ -521,13 +530,27 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) } } } + reg_args->rax = ssecount; ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args), - cif->flags, rvalue, fn, ssecount); + cif->flags, rvalue, fn); } +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} -extern void ffi_closure_unix64(void); +void +ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); +} + +extern void ffi_closure_unix64(void) FFI_HIDDEN; +extern void ffi_closure_unix64_sse(void) FFI_HIDDEN; ffi_status ffi_prep_closure_loc (ffi_closure* closure, @@ -536,29 +559,26 @@ ffi_prep_closure_loc (ffi_closure* closure, void *user_data, void *codeloc) { - volatile unsigned short *tramp; + static const unsigned char trampoline[16] = { + /* leaq -0x7(%rip),%r10 # 0x0 */ + 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff, + /* jmpq *0x3(%rip) # 0x10 */ + 0xff, 0x25, 0x03, 0x00, 0x00, 0x00, + /* nopl (%rax) */ + 0x0f, 0x1f, 0x00 + }; + void (*dest)(void); - /* Sanity check on the cif ABI. */ - { - int abi = cif->abi; - if (UNLIKELY (! (abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI))) - return FFI_BAD_ABI; - } + if (cif->abi != FFI_UNIX64) + return FFI_BAD_ABI; - tramp = (volatile unsigned short *) &closure->tramp[0]; + if (cif->flags & (1 << 11)) + dest = ffi_closure_unix64_sse; + else + dest = ffi_closure_unix64; - tramp[0] = 0xbb49; /* mov , %r11 */ - *((unsigned long long * volatile) &tramp[1]) - = (unsigned long) ffi_closure_unix64; - tramp[5] = 0xba49; /* mov , %r10 */ - *((unsigned long long * volatile) &tramp[6]) - = (unsigned long) codeloc; - - /* Set the carry bit iff the function uses any sse registers. - This is clc or stc, together with the first byte of the jmp. */ - tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8; - - tramp[11] = 0xe3ff; /* jmp *%r11 */ + memcpy (closure->tramp, trampoline, sizeof(trampoline)); + *(UINT64 *)(closure->tramp + 16) = (uintptr_t)dest; closure->cif = cif; closure->fun = fun; @@ -567,18 +587,20 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } -int -ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue, - struct register_args *reg_args, char *argp) +int FFI_HIDDEN +ffi_closure_unix64_inner(ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, + void *rvalue, + struct register_args *reg_args, + char *argp) { - ffi_cif *cif; void **avalue; ffi_type **arg_types; long i, avn; int gprcount, ssecount, ngpr, nsse; int ret; - cif = closure->cif; avalue = alloca(cif->nargs * sizeof(void *)); gprcount = ssecount = 0; @@ -667,10 +689,29 @@ ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue, } /* Invoke the closure. */ - closure->fun (cif, rvalue, avalue, closure->user_data); + fun (cif, rvalue, avalue, user_data); /* Tell assembly how to perform return type promotions. */ return ret; } +extern void ffi_go_closure_unix64(void) FFI_HIDDEN; +extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN; + +ffi_status +ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*)) +{ + if (cif->abi != FFI_UNIX64) + return FFI_BAD_ABI; + + closure->tramp = (cif->flags & (1 << 11) + ? ffi_go_closure_unix64_sse + : ffi_go_closure_unix64); + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + #endif /* __x86_64__ */ diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index a236677..0d295e0 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -121,6 +121,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 + #define FFI_TYPE_SMALL_STRUCT_1B (FFI_TYPE_LAST + 1) #define FFI_TYPE_SMALL_STRUCT_2B (FFI_TYPE_LAST + 2) #define FFI_TYPE_SMALL_STRUCT_4B (FFI_TYPE_LAST + 3) @@ -129,6 +130,7 @@ typedef enum ffi_abi { #if defined (X86_64) || (defined (__x86_64__) && defined (X86_DARWIN)) #define FFI_TRAMPOLINE_SIZE 24 #define FFI_NATIVE_RAW_API 0 +#define FFI_GO_CLOSURES 1 #else #ifdef X86_WIN32 #define FFI_TRAMPOLINE_SIZE 52 diff --git a/src/x86/unix64.S b/src/x86/unix64.S index dcd6bc7..134cb3d 100644 --- a/src/x86/unix64.S +++ b/src/x86/unix64.S @@ -30,6 +30,7 @@ #define LIBFFI_ASM #include #include +#include .text @@ -43,9 +44,10 @@ .align 2 .globl ffi_call_unix64 .type ffi_call_unix64,@function + FFI_HIDDEN(ffi_call_unix64) ffi_call_unix64: -.LUW0: + cfi_startproc movq (%rsp), %r10 /* Load return address. */ leaq (%rdi, %rsi), %rax /* Find local stack base. */ movq %rdx, (%rax) /* Save flags. */ @@ -53,24 +55,36 @@ ffi_call_unix64: movq %rbp, 16(%rax) /* Save old frame pointer. */ movq %r10, 24(%rax) /* Relocate return address. */ movq %rax, %rbp /* Finalize local stack frame. */ -.LUW1: + + /* New stack frame based off rbp. This is a itty bit of unwind + trickery in that the CFA *has* changed. There is no easy way + to describe it correctly on entry to the function. Fortunately, + it doesn't matter too much since at all points we can correctly + unwind back to ffi_call. Note that the location to which we + moved the return address is (the new) CFA-8, so from the + perspective of the unwind info, it hasn't moved. */ + cfi_def_cfa(%rbp, 32) + cfi_rel_offset(%rbp, 16) + movq %rdi, %r10 /* Save a copy of the register area. */ movq %r8, %r11 /* Save a copy of the target fn. */ movl %r9d, %eax /* Set number of SSE registers. */ /* Load up all argument registers. */ movq (%r10), %rdi - movq 8(%r10), %rsi - movq 16(%r10), %rdx - movq 24(%r10), %rcx - movq 32(%r10), %r8 - movq 40(%r10), %r9 + movq 0x08(%r10), %rsi + movq 0x10(%r10), %rdx + movq 0x18(%r10), %rcx + movq 0x20(%r10), %r8 + movq 0x28(%r10), %r9 + movl 0xb0(%r10), %eax testl %eax, %eax jnz .Lload_sse .Lret_from_load_sse: - /* Deallocate the reg arg area. */ - leaq 176(%r10), %rsp + /* Deallocate the reg arg area, except for r10, then load via pop. */ + leaq 0xb8(%r10), %rsp + popq %r10 /* Call the user function. */ call *%r11 @@ -81,7 +95,9 @@ ffi_call_unix64: movq 0(%rbp), %rcx /* Reload flags. */ movq 8(%rbp), %rdi /* Reload raddr. */ movq 16(%rbp), %rbp /* Reload old frame pointer. */ -.LUW2: + cfi_remember_state + cfi_def_cfa(%rsp, 8) + cfi_restore(%rbp) /* The first byte of the flags contains the FFI_TYPE. */ movzbl %cl, %r10d @@ -90,6 +106,8 @@ ffi_call_unix64: addq %r11, %r10 jmp *%r10 + .section .rodata + .align 2 .Lstore_table: .long .Lst_void-.Lstore_table /* FFI_TYPE_VOID */ .long .Lst_sint32-.Lstore_table /* FFI_TYPE_INT */ @@ -106,6 +124,7 @@ ffi_call_unix64: .long .Lst_int64-.Lstore_table /* FFI_TYPE_SINT64 */ .long .Lst_struct-.Lstore_table /* FFI_TYPE_STRUCT */ .long .Lst_int64-.Lstore_table /* FFI_TYPE_POINTER */ + .previous .align 2 .Lst_void: @@ -188,49 +207,83 @@ ffi_call_unix64: It's not worth an indirect jump to load the exact set of SSE registers needed; zero or all is a good compromise. */ .align 2 -.LUW3: + cfi_restore_state .Lload_sse: - movdqa 48(%r10), %xmm0 - movdqa 64(%r10), %xmm1 - movdqa 80(%r10), %xmm2 - movdqa 96(%r10), %xmm3 - movdqa 112(%r10), %xmm4 - movdqa 128(%r10), %xmm5 - movdqa 144(%r10), %xmm6 - movdqa 160(%r10), %xmm7 + movdqa 0x30(%r10), %xmm0 + movdqa 0x40(%r10), %xmm1 + movdqa 0x50(%r10), %xmm2 + movdqa 0x60(%r10), %xmm3 + movdqa 0x70(%r10), %xmm4 + movdqa 0x80(%r10), %xmm5 + movdqa 0x90(%r10), %xmm6 + movdqa 0xa0(%r10), %xmm7 jmp .Lret_from_load_sse -.LUW4: + cfi_endproc .size ffi_call_unix64,.-ffi_call_unix64 +/* 6 general registers, 8 vector registers, + 16 bytes of rvalue, 8 bytes of alignment. */ +#define ffi_closure_OFS_G 0 +#define ffi_closure_OFS_V (6*8) +#define ffi_closure_OFS_RVALUE (ffi_closure_OFS_V + 8*16) +#define ffi_closure_FS (ffi_closure_OFS_RVALUE + 16 + 8) + +/* The location of rvalue within the red zone after deallocating the frame. */ +#define ffi_closure_RED_RVALUE (ffi_closure_OFS_RVALUE - ffi_closure_FS) + .align 2 - .globl ffi_closure_unix64 + .globl ffi_closure_unix64_sse + .type ffi_closure_unix64_sse,@function + FFI_HIDDEN(ffi_closure_unix64_sse) + +ffi_closure_unix64_sse: + cfi_startproc + subq $ffi_closure_FS, %rsp + cfi_adjust_cfa_offset(ffi_closure_FS) + + movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp) + movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp) + movdqa %xmm2, ffi_closure_OFS_V+0x20(%rsp) + movdqa %xmm3, ffi_closure_OFS_V+0x30(%rsp) + movdqa %xmm4, ffi_closure_OFS_V+0x40(%rsp) + movdqa %xmm5, ffi_closure_OFS_V+0x50(%rsp) + movdqa %xmm6, ffi_closure_OFS_V+0x60(%rsp) + movdqa %xmm7, ffi_closure_OFS_V+0x70(%rsp) + jmp 0f + + cfi_endproc + .size ffi_closure_unix64_sse,.-ffi_closure_unix64_sse + + .align 2 + .globl ffi_closure_unix64 .type ffi_closure_unix64,@function + FFI_HIDDEN(ffi_closure_unix64) ffi_closure_unix64: -.LUW5: - /* The carry flag is set by the trampoline iff SSE registers - are used. Don't clobber it before the branch instruction. */ - leaq -200(%rsp), %rsp -.LUW6: - movq %rdi, (%rsp) - movq %rsi, 8(%rsp) - movq %rdx, 16(%rsp) - movq %rcx, 24(%rsp) - movq %r8, 32(%rsp) - movq %r9, 40(%rsp) - jc .Lsave_sse -.Lret_from_save_sse: + cfi_startproc + subq $ffi_closure_FS, %rsp + cfi_adjust_cfa_offset(ffi_closure_FS) +0: + movq %rdi, ffi_closure_OFS_G+0x00(%rsp) + movq %rsi, ffi_closure_OFS_G+0x08(%rsp) + movq %rdx, ffi_closure_OFS_G+0x10(%rsp) + movq %rcx, ffi_closure_OFS_G+0x18(%rsp) + movq %r8, ffi_closure_OFS_G+0x20(%rsp) + movq %r9, ffi_closure_OFS_G+0x28(%rsp) - movq %r10, %rdi - leaq 176(%rsp), %rsi - movq %rsp, %rdx - leaq 208(%rsp), %rcx - call ffi_closure_unix64_inner@PLT + movq 24(%r10), %rdi /* Load cif */ + movq 32(%r10), %rsi /* Load fun */ + movq 40(%r10), %rdx /* Load user_data */ +.Ldo_closure: + leaq ffi_closure_OFS_RVALUE(%rsp), %rcx /* Load rvalue */ + movq %rsp, %r8 /* Load reg_args */ + leaq ffi_closure_FS+8(%rsp), %r9 /* Load argp */ + call ffi_closure_unix64_inner /* Deallocate stack frame early; return value is now in redzone. */ - addq $200, %rsp -.LUW7: + addq $ffi_closure_FS, %rsp + cfi_adjust_cfa_offset(-ffi_closure_FS) /* The first byte of the return value contains the FFI_TYPE. */ movzbl %al, %r10d @@ -239,6 +292,8 @@ ffi_closure_unix64: addq %r11, %r10 jmp *%r10 + .section .rodata + .align 2 .Lload_table: .long .Lld_void-.Lload_table /* FFI_TYPE_VOID */ .long .Lld_int32-.Lload_table /* FFI_TYPE_INT */ @@ -255,6 +310,7 @@ ffi_closure_unix64: .long .Lld_int64-.Lload_table /* FFI_TYPE_SINT64 */ .long .Lld_struct-.Lload_table /* FFI_TYPE_STRUCT */ .long .Lld_int64-.Lload_table /* FFI_TYPE_POINTER */ + .previous .align 2 .Lld_void: @@ -262,32 +318,32 @@ ffi_closure_unix64: .align 2 .Lld_int8: - movzbl -24(%rsp), %eax + movzbl ffi_closure_RED_RVALUE(%rsp), %eax ret .align 2 .Lld_int16: - movzwl -24(%rsp), %eax + movzwl ffi_closure_RED_RVALUE(%rsp), %eax ret .align 2 .Lld_int32: - movl -24(%rsp), %eax + movl ffi_closure_RED_RVALUE(%rsp), %eax ret .align 2 .Lld_int64: - movq -24(%rsp), %rax + movq ffi_closure_RED_RVALUE(%rsp), %rax ret .align 2 .Lld_float: - movss -24(%rsp), %xmm0 + movss ffi_closure_RED_RVALUE(%rsp), %xmm0 ret .align 2 .Lld_double: - movsd -24(%rsp), %xmm0 + movsd ffi_closure_RED_RVALUE(%rsp), %xmm0 ret .align 2 .Lld_ldouble: - fldt -24(%rsp) + fldt ffi_closure_RED_RVALUE(%rsp) ret .align 2 @@ -297,136 +353,69 @@ ffi_closure_unix64: both rdx and xmm1 with the second word. For the remaining, bit 8 set means xmm0 gets the second word, and bit 9 means that rax gets the second word. */ - movq -24(%rsp), %rcx - movq -16(%rsp), %rdx - movq -16(%rsp), %xmm1 + movq ffi_closure_RED_RVALUE(%rsp), %rcx + movq ffi_closure_RED_RVALUE+8(%rsp), %rdx + movq ffi_closure_RED_RVALUE+8(%rsp), %xmm1 testl $0x100, %eax cmovnz %rdx, %rcx movd %rcx, %xmm0 testl $0x200, %eax - movq -24(%rsp), %rax + movq ffi_closure_RED_RVALUE(%rsp), %rax cmovnz %rdx, %rax ret - /* See the comment above .Lload_sse; the same logic applies here. */ - .align 2 -.LUW8: -.Lsave_sse: - movdqa %xmm0, 48(%rsp) - movdqa %xmm1, 64(%rsp) - movdqa %xmm2, 80(%rsp) - movdqa %xmm3, 96(%rsp) - movdqa %xmm4, 112(%rsp) - movdqa %xmm5, 128(%rsp) - movdqa %xmm6, 144(%rsp) - movdqa %xmm7, 160(%rsp) - jmp .Lret_from_save_sse - -.LUW9: + cfi_endproc .size ffi_closure_unix64,.-ffi_closure_unix64 -#ifdef __GNUC__ -/* Only emit DWARF unwind info when building with the GNU toolchain. */ + .align 2 + .globl ffi_go_closure_unix64_sse + .type ffi_go_closure_unix64_sse,@function + FFI_HIDDEN(ffi_go_closure_unix64_sse) -#ifdef HAVE_AS_X86_64_UNWIND_SECTION_TYPE - .section .eh_frame,"a",@unwind -#else - .section .eh_frame,"a",@progbits -#endif -.Lframe1: - .long .LECIE1-.LSCIE1 /* CIE Length */ -.LSCIE1: - .long 0 /* CIE Identifier Tag */ - .byte 1 /* CIE Version */ - .ascii "zR\0" /* CIE Augmentation */ - .uleb128 1 /* CIE Code Alignment Factor */ - .sleb128 -8 /* CIE Data Alignment Factor */ - .byte 0x10 /* CIE RA Column */ - .uleb128 1 /* Augmentation size */ - .byte 0x1b /* FDE Encoding (pcrel sdata4) */ - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .uleb128 7 - .uleb128 8 - .byte 0x80+16 /* DW_CFA_offset, %rip offset 1*-8 */ - .uleb128 1 - .align 8 -.LECIE1: -.LSFDE1: - .long .LEFDE1-.LASFDE1 /* FDE Length */ -.LASFDE1: - .long .LASFDE1-.Lframe1 /* FDE CIE offset */ -#if HAVE_AS_X86_PCREL - .long .LUW0-. /* FDE initial location */ -#else - .long .LUW0@rel -#endif - .long .LUW4-.LUW0 /* FDE address range */ - .uleb128 0x0 /* Augmentation size */ +ffi_go_closure_unix64_sse: + cfi_startproc + subq $ffi_closure_FS, %rsp + cfi_adjust_cfa_offset(ffi_closure_FS) - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW1-.LUW0 + movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp) + movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp) + movdqa %xmm2, ffi_closure_OFS_V+0x20(%rsp) + movdqa %xmm3, ffi_closure_OFS_V+0x30(%rsp) + movdqa %xmm4, ffi_closure_OFS_V+0x40(%rsp) + movdqa %xmm5, ffi_closure_OFS_V+0x50(%rsp) + movdqa %xmm6, ffi_closure_OFS_V+0x60(%rsp) + movdqa %xmm7, ffi_closure_OFS_V+0x70(%rsp) + jmp 0f - /* New stack frame based off rbp. This is a itty bit of unwind - trickery in that the CFA *has* changed. There is no easy way - to describe it correctly on entry to the function. Fortunately, - it doesn't matter too much since at all points we can correctly - unwind back to ffi_call. Note that the location to which we - moved the return address is (the new) CFA-8, so from the - perspective of the unwind info, it hasn't moved. */ - .byte 0xc /* DW_CFA_def_cfa, %rbp offset 32 */ - .uleb128 6 - .uleb128 32 - .byte 0x80+6 /* DW_CFA_offset, %rbp offset 2*-8 */ - .uleb128 2 - .byte 0xa /* DW_CFA_remember_state */ + cfi_endproc + .size ffi_go_closure_unix64_sse,.-ffi_go_closure_unix64_sse - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW2-.LUW1 - .byte 0xc /* DW_CFA_def_cfa, %rsp offset 8 */ - .uleb128 7 - .uleb128 8 - .byte 0xc0+6 /* DW_CFA_restore, %rbp */ + .align 2 + .globl ffi_go_closure_unix64 + .type ffi_go_closure_unix64,@function + FFI_HIDDEN(ffi_go_closure_unix64) - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW3-.LUW2 - .byte 0xb /* DW_CFA_restore_state */ +ffi_go_closure_unix64: + cfi_startproc + subq $ffi_closure_FS, %rsp + cfi_adjust_cfa_offset(ffi_closure_FS) +0: + movq %rdi, ffi_closure_OFS_G+0x00(%rsp) + movq %rsi, ffi_closure_OFS_G+0x08(%rsp) + movq %rdx, ffi_closure_OFS_G+0x10(%rsp) + movq %rcx, ffi_closure_OFS_G+0x18(%rsp) + movq %r8, ffi_closure_OFS_G+0x20(%rsp) + movq %r9, ffi_closure_OFS_G+0x28(%rsp) - .align 8 -.LEFDE1: -.LSFDE3: - .long .LEFDE3-.LASFDE3 /* FDE Length */ -.LASFDE3: - .long .LASFDE3-.Lframe1 /* FDE CIE offset */ -#if HAVE_AS_X86_PCREL - .long .LUW5-. /* FDE initial location */ -#else - .long .LUW5@rel -#endif - .long .LUW9-.LUW5 /* FDE address range */ - .uleb128 0x0 /* Augmentation size */ + movq 8(%r10), %rdi /* Load cif */ + movq 16(%r10), %rsi /* Load fun */ + movq %r10, %rdx /* Load closure (user_data) */ + jmp .Ldo_closure - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW6-.LUW5 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .uleb128 208 - .byte 0xa /* DW_CFA_remember_state */ + cfi_endproc + .size ffi_go_closure_unix64,.-ffi_go_closure_unix64 - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW7-.LUW6 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .uleb128 8 - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LUW8-.LUW7 - .byte 0xb /* DW_CFA_restore_state */ - - .align 8 -.LEFDE3: - -#endif /* __GNUC__ */ - #endif /* __x86_64__ */ - #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits #endif From 99db4d42acc7ee6d2ad206848be38083d7b5e094 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 23 Oct 2014 14:12:18 -0700 Subject: [PATCH 04/58] win64: Rewrite It's way too different from the 32-bit ABIs with which it is currently associated. As seen from all of the existing XFAILs. --- Makefile.am | 4 +- src/x86/ffitarget.h | 29 +- src/x86/ffiw64.c | 281 +++++++ src/x86/win64.S | 685 +++++------------- testsuite/libffi.call/call.exp | 13 +- .../libffi.call/cls_align_longdouble_split.c | 2 - .../libffi.call/cls_align_longdouble_split2.c | 2 - testsuite/libffi.call/cls_longdouble.c | 2 - testsuite/libffi.call/float2.c | 3 - testsuite/libffi.call/huge_struct.c | 2 - testsuite/libffi.call/return_ldl.c | 1 - 11 files changed, 492 insertions(+), 532 deletions(-) create mode 100644 src/x86/ffiw64.c diff --git a/Makefile.am b/Makefile.am index 0e40451..3d1ecae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,7 +37,7 @@ EXTRA_DIST = LICENSE ChangeLog.v1 ChangeLog.libgcj \ src/sh64/sysv.S src/sh64/ffitarget.h src/sparc/v8.S \ src/sparc/v9.S src/sparc/ffitarget.h src/sparc/ffi.c \ src/x86/darwin64.S src/x86/ffi.c src/x86/sysv.S \ - src/x86/win32.S src/x86/darwin.S src/x86/win64.S \ + src/x86/win32.S src/x86/darwin.S src/x86/ffiw64.c src/x86/win64.S \ src/x86/freebsd.S src/x86/ffi64.c src/x86/unix64.S \ src/x86/ffitarget.h src/pa/ffitarget.h src/pa/ffi.c \ src/pa/linux.S src/pa/hpux32.S src/frv/ffi.c src/bfin/ffi.c \ @@ -135,7 +135,7 @@ if X86_WIN32 nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/win32.S endif if X86_WIN64 -nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/win64.S +nodist_libffi_la_SOURCES += src/x86/ffiw64.c src/x86/win64.S endif if X86_DARWIN nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/darwin.S src/x86/ffi64.c src/x86/darwin64.S diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index 0d295e0..8c52573 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -127,25 +127,18 @@ typedef enum ffi_abi { #define FFI_TYPE_SMALL_STRUCT_4B (FFI_TYPE_LAST + 3) #define FFI_TYPE_MS_STRUCT (FFI_TYPE_LAST + 4) -#if defined (X86_64) || (defined (__x86_64__) && defined (X86_DARWIN)) -#define FFI_TRAMPOLINE_SIZE 24 -#define FFI_NATIVE_RAW_API 0 -#define FFI_GO_CLOSURES 1 +#if defined (X86_64) || defined(X86_WIN64) \ + || (defined (__x86_64__) && defined (X86_DARWIN)) +# define FFI_TRAMPOLINE_SIZE 24 +# define FFI_NATIVE_RAW_API 0 +# define FFI_GO_CLOSURES 1 #else -#ifdef X86_WIN32 -#define FFI_TRAMPOLINE_SIZE 52 -#else -#ifdef X86_WIN64 -#define FFI_TRAMPOLINE_SIZE 29 -#define FFI_NATIVE_RAW_API 0 -#define FFI_NO_RAW_API 1 -#else -#define FFI_TRAMPOLINE_SIZE 10 -#endif -#endif -#ifndef X86_WIN64 -#define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ -#endif +# ifdef X86_WIN32 +# define FFI_TRAMPOLINE_SIZE 52 +# else +# define FFI_TRAMPOLINE_SIZE 10 +# endif +# define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ #endif #endif diff --git a/src/x86/ffiw64.c b/src/x86/ffiw64.c new file mode 100644 index 0000000..316f544 --- /dev/null +++ b/src/x86/ffiw64.c @@ -0,0 +1,281 @@ +/* ----------------------------------------------------------------------- + ffiw64.c - Copyright (c) 2014 Red Hat, Inc. + + x86 win64 Foreign Function Interface + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ----------------------------------------------------------------------- */ + +#include +#include +#include +#include + +#ifdef X86_WIN64 + +struct win64_call_frame +{ + UINT64 rbp; /* 0 */ + UINT64 retaddr; /* 8 */ + UINT64 fn; /* 16 */ + UINT64 flags; /* 24 */ + UINT64 rvalue; /* 32 */ +}; + +extern void ffi_call_win64 (void *stack, struct win64_call_frame *, + void *closure) FFI_HIDDEN; + +ffi_status +ffi_prep_cif_machdep (ffi_cif *cif) +{ + int flags, n; + + if (cif->abi != FFI_WIN64) + return FFI_BAD_ABI; + + flags = cif->rtype->type; + switch (flags) + { + default: + break; + case FFI_TYPE_LONGDOUBLE: + flags = FFI_TYPE_STRUCT; + break; + case FFI_TYPE_COMPLEX: + flags = FFI_TYPE_STRUCT; + /* FALLTHRU */ + case FFI_TYPE_STRUCT: + switch (cif->rtype->size) + { + case 8: + flags = FFI_TYPE_UINT64; + break; + case 4: + flags = FFI_TYPE_SMALL_STRUCT_4B; + break; + case 2: + flags = FFI_TYPE_SMALL_STRUCT_2B; + break; + case 1: + flags = FFI_TYPE_SMALL_STRUCT_1B; + break; + } + break; + } + cif->flags = flags; + + /* Each argument either fits in a register, an 8 byte slot, or is + passed by reference with the pointer in the 8 byte slot. */ + n = cif->nargs; + n += (flags == FFI_TYPE_STRUCT); + if (n < 4) + n = 4; + cif->bytes = n * 8; + + return FFI_OK; +} + +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + int i, j, n, flags; + UINT64 *stack; + size_t rsize; + struct win64_call_frame *frame; + + FFI_ASSERT(cif->abi == FFI_WIN64); + + flags = cif->flags; + rsize = 0; + + /* If we have no return value for a structure, we need to create one. + Otherwise we can ignore the return type entirely. */ + if (rvalue == NULL) + { + if (flags == FFI_TYPE_STRUCT) + rsize = cif->rtype->size; + else + flags = FFI_TYPE_VOID; + } + + stack = alloca(cif->bytes + sizeof(struct win64_call_frame) + rsize); + frame = (struct win64_call_frame *)((char *)stack + cif->bytes); + if (rsize) + rvalue = frame + 1; + + frame->fn = (uintptr_t)fn; + frame->flags = flags; + frame->rvalue = (uintptr_t)rvalue; + + j = 0; + if (flags == FFI_TYPE_STRUCT) + { + stack[0] = (uintptr_t)rvalue; + j = 1; + } + + for (i = 0, n = cif->nargs; i < n; ++i, ++j) + { + switch (cif->arg_types[i]->size) + { + case 8: + stack[j] = *(UINT64 *)avalue[i]; + break; + case 4: + stack[j] = *(UINT32 *)avalue[i]; + break; + case 2: + stack[j] = *(UINT16 *)avalue[i]; + break; + case 1: + stack[j] = *(UINT8 *)avalue[i]; + break; + default: + stack[j] = (uintptr_t)avalue[i]; + break; + } + } + + ffi_call_win64 (stack, frame, closure); +} + +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); +} + + +extern void ffi_closure_win64(void) FFI_HIDDEN; +extern void ffi_go_closure_win64(void) FFI_HIDDEN; + +ffi_status +ffi_prep_closure_loc (ffi_closure* closure, + ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, + void *codeloc) +{ + static const unsigned char trampoline[16] = { + /* leaq -0x7(%rip),%r10 # 0x0 */ + 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff, + /* jmpq *0x3(%rip) # 0x10 */ + 0xff, 0x25, 0x03, 0x00, 0x00, 0x00, + /* nopl (%rax) */ + 0x0f, 0x1f, 0x00 + }; + void *tramp = closure->tramp; + + if (cif->abi != FFI_WIN64) + return FFI_BAD_ABI; + + memcpy (tramp, trampoline, sizeof(trampoline)); + *(UINT64 *)(tramp + 16) = (uintptr_t)ffi_closure_win64; + + closure->cif = cif; + closure->fun = fun; + closure->user_data = user_data; + + return FFI_OK; +} + +ffi_status +ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*)) +{ + if (cif->abi != FFI_WIN64) + return FFI_BAD_ABI; + + closure->tramp = ffi_go_closure_win64; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + +struct win64_closure_frame +{ + UINT64 rvalue[2]; + UINT64 fargs[4]; + UINT64 retaddr; + UINT64 args[]; +}; + +int FFI_HIDDEN +ffi_closure_win64_inner(ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, + struct win64_closure_frame *frame) +{ + void **avalue; + void *rvalue; + int i, n, nreg, flags; + + avalue = alloca(cif->nargs * sizeof(void *)); + rvalue = frame->rvalue; + nreg = 0; + + /* When returning a structure, the address is in the first argument. + We must also be prepared to return the same address in eax, so + install that address in the frame and pretend we return a pointer. */ + flags = cif->flags; + if (flags == FFI_TYPE_STRUCT) + { + rvalue = (void *)(uintptr_t)frame->args[0]; + frame->rvalue[0] = frame->args[0]; + nreg = 1; + } + + for (i = 0, n = cif->nargs; i < n; ++i, ++nreg) + { + size_t size = cif->arg_types[i]->size; + size_t type = cif->arg_types[i]->type; + void *a; + + if (type == FFI_TYPE_DOUBLE || type == FFI_TYPE_FLOAT) + { + if (nreg < 4) + a = &frame->fargs[nreg]; + else + a = &frame->args[nreg]; + } + else if (size == 1 || size == 2 || size == 4 || size == 8) + a = &frame->args[nreg]; + else + a = (void *)(uintptr_t)frame->args[nreg]; + + avalue[i] = a; + } + + /* Invoke the closure. */ + fun (cif, rvalue, avalue, user_data); + return flags; +} + +#endif /* X86_WIN64 */ diff --git a/src/x86/win64.S b/src/x86/win64.S index 687f97c..a5a20b6 100644 --- a/src/x86/win64.S +++ b/src/x86/win64.S @@ -1,264 +1,16 @@ #define LIBFFI_ASM #include #include +#include -/* Constants for ffi_call_win64 */ -#define STACK 0 -#define PREP_ARGS_FN 32 -#define ECIF 40 -#define CIF_BYTES 48 -#define CIF_FLAGS 56 -#define RVALUE 64 -#define FN 72 +#if defined(HAVE_AS_CFI_PSEUDO_OP) + .cfi_sections .debug_frame +#endif -/* ffi_call_win64 (void (*prep_args_fn)(char *, extended_cif *), - extended_cif *ecif, unsigned bytes, unsigned flags, - unsigned *rvalue, void (*fn)()); - */ - -#ifdef _MSC_VER -PUBLIC ffi_call_win64 - -EXTRN __chkstk:NEAR -EXTRN ffi_closure_win64_inner:NEAR - -_TEXT SEGMENT - -;;; ffi_closure_win64 will be called with these registers set: -;;; rax points to 'closure' -;;; r11 contains a bit mask that specifies which of the -;;; first four parameters are float or double -;;; -;;; It must move the parameters passed in registers to their stack location, -;;; call ffi_closure_win64_inner for the actual work, then return the result. -;;; -ffi_closure_win64 PROC FRAME - ;; copy register arguments onto stack - test r11, 1 - jne first_is_float - mov QWORD PTR [rsp+8], rcx - jmp second -first_is_float: - movlpd QWORD PTR [rsp+8], xmm0 - -second: - test r11, 2 - jne second_is_float - mov QWORD PTR [rsp+16], rdx - jmp third -second_is_float: - movlpd QWORD PTR [rsp+16], xmm1 - -third: - test r11, 4 - jne third_is_float - mov QWORD PTR [rsp+24], r8 - jmp fourth -third_is_float: - movlpd QWORD PTR [rsp+24], xmm2 - -fourth: - test r11, 8 - jne fourth_is_float - mov QWORD PTR [rsp+32], r9 - jmp done -fourth_is_float: - movlpd QWORD PTR [rsp+32], xmm3 - -done: - .ALLOCSTACK 40 - sub rsp, 40 - .ENDPROLOG - mov rcx, rax ; context is first parameter - mov rdx, rsp ; stack is second parameter - add rdx, 48 ; point to start of arguments - mov rax, ffi_closure_win64_inner - call rax ; call the real closure function - add rsp, 40 - movd xmm0, rax ; If the closure returned a float, - ; ffi_closure_win64_inner wrote it to rax - ret 0 -ffi_closure_win64 ENDP - -ffi_call_win64 PROC FRAME - ;; copy registers onto stack - mov QWORD PTR [rsp+32], r9 - mov QWORD PTR [rsp+24], r8 - mov QWORD PTR [rsp+16], rdx - mov QWORD PTR [rsp+8], rcx - .PUSHREG rbp - push rbp - .ALLOCSTACK 48 - sub rsp, 48 ; 00000030H - .SETFRAME rbp, 32 - lea rbp, QWORD PTR [rsp+32] - .ENDPROLOG - - mov eax, DWORD PTR CIF_BYTES[rbp] - add rax, 15 - and rax, -16 - call __chkstk - sub rsp, rax - lea rax, QWORD PTR [rsp+32] - mov QWORD PTR STACK[rbp], rax - - mov rdx, QWORD PTR ECIF[rbp] - mov rcx, QWORD PTR STACK[rbp] - call QWORD PTR PREP_ARGS_FN[rbp] - - mov rsp, QWORD PTR STACK[rbp] - - movlpd xmm3, QWORD PTR [rsp+24] - movd r9, xmm3 - - movlpd xmm2, QWORD PTR [rsp+16] - movd r8, xmm2 - - movlpd xmm1, QWORD PTR [rsp+8] - movd rdx, xmm1 - - movlpd xmm0, QWORD PTR [rsp] - movd rcx, xmm0 - - call QWORD PTR FN[rbp] -ret_struct4b$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SMALL_STRUCT_4B - jne ret_struct2b$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov DWORD PTR [rcx], eax - jmp ret_void$ - -ret_struct2b$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SMALL_STRUCT_2B - jne ret_struct1b$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov WORD PTR [rcx], ax - jmp ret_void$ - -ret_struct1b$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SMALL_STRUCT_1B - jne ret_uint8$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov BYTE PTR [rcx], al - jmp ret_void$ - -ret_uint8$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_UINT8 - jne ret_sint8$ - - mov rcx, QWORD PTR RVALUE[rbp] - movzx rax, al - mov QWORD PTR [rcx], rax - jmp ret_void$ - -ret_sint8$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SINT8 - jne ret_uint16$ - - mov rcx, QWORD PTR RVALUE[rbp] - movsx rax, al - mov QWORD PTR [rcx], rax - jmp ret_void$ - -ret_uint16$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_UINT16 - jne ret_sint16$ - - mov rcx, QWORD PTR RVALUE[rbp] - movzx rax, ax - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_sint16$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SINT16 - jne ret_uint32$ - - mov rcx, QWORD PTR RVALUE[rbp] - movsx rax, ax - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_uint32$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_UINT32 - jne ret_sint32$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov eax, eax - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_sint32$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SINT32 - jne ret_float$ - - mov rcx, QWORD PTR RVALUE[rbp] - cdqe - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_float$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_FLOAT - jne SHORT ret_double$ - - mov rax, QWORD PTR RVALUE[rbp] - movss DWORD PTR [rax], xmm0 - jmp SHORT ret_void$ - -ret_double$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_DOUBLE - jne SHORT ret_uint64$ - - mov rax, QWORD PTR RVALUE[rbp] - movlpd QWORD PTR [rax], xmm0 - jmp SHORT ret_void$ - -ret_uint64$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_UINT64 - jne SHORT ret_sint64$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_sint64$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_SINT64 - jne SHORT ret_pointer$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_pointer$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_POINTER - jne SHORT ret_int$ - - mov rcx, QWORD PTR RVALUE[rbp] - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_int$: - cmp DWORD PTR CIF_FLAGS[rbp], FFI_TYPE_INT - jne SHORT ret_void$ - - mov rcx, QWORD PTR RVALUE[rbp] - cdqe - mov QWORD PTR [rcx], rax - jmp SHORT ret_void$ - -ret_void$: - xor rax, rax - - lea rsp, QWORD PTR [rbp+16] - pop rbp - ret 0 -ffi_call_win64 ENDP -_TEXT ENDS -END - -#else +#define arg0 %rcx +#define arg1 %rdx +#define arg2 %r8 +#define arg3 %r9 #ifdef SYMBOL_UNDERSCORE #define SYMBOL_NAME(name) _##name @@ -266,255 +18,202 @@ END #define SYMBOL_NAME(name) name #endif -.text +.macro E which + .align 8 + .org 0b + \which * 8 +.endm -.extern SYMBOL_NAME(ffi_closure_win64_inner) + .text -# ffi_closure_win64 will be called with these registers set: -# rax points to 'closure' -# r11 contains a bit mask that specifies which of the -# first four parameters are float or double -# -# It must move the parameters passed in registers to their stack location, -# call ffi_closure_win64_inner for the actual work, then return the result. -# - .balign 16 - .globl SYMBOL_NAME(ffi_closure_win64) - .seh_proc SYMBOL_NAME(ffi_closure_win64) -SYMBOL_NAME(ffi_closure_win64): - # copy register arguments onto stack - test $1,%r11 - jne .Lfirst_is_float - mov %rcx, 8(%rsp) - jmp .Lsecond -.Lfirst_is_float: - movlpd %xmm0, 8(%rsp) +/* ffi_call_win64 (void *stack, struct win64_call_frame *frame, void *r10) -.Lsecond: - test $2, %r11 - jne .Lsecond_is_float - mov %rdx, 16(%rsp) - jmp .Lthird -.Lsecond_is_float: - movlpd %xmm1, 16(%rsp) + Bit o trickiness here -- FRAME is the base of the stack frame + for this function. This has been allocated by ffi_call. We also + deallocate some of the stack that has been alloca'd. */ -.Lthird: - test $4, %r11 - jne .Lthird_is_float - mov %r8,24(%rsp) - jmp .Lfourth -.Lthird_is_float: - movlpd %xmm2, 24(%rsp) + .align 8 + .globl ffi_call_win64 -.Lfourth: - test $8, %r11 - jne .Lfourth_is_float - mov %r9, 32(%rsp) - jmp .Ldone -.Lfourth_is_float: - movlpd %xmm3, 32(%rsp) - -.Ldone: - .seh_stackalloc 40 - sub $40, %rsp + .seh_proc ffi_call_win64 +ffi_call_win64: + cfi_startproc + /* Set up the local stack frame and install it in rbp/rsp. */ + movq (%rsp), %rax + movq %rbp, (arg1) + movq %rax, 8(arg1) + movq arg1, %rbp + cfi_def_cfa(%rbp, 16) + cfi_rel_offset(%rbp, 0) + .seh_pushreg %rbp + .seh_setframe %rbp, 0 .seh_endprologue - mov %rax, %rcx # context is first parameter - mov %rsp, %rdx # stack is second parameter - add $48, %rdx # point to start of arguments - leaq SYMBOL_NAME(ffi_closure_win64_inner)(%rip), %rax - callq *%rax # call the real closure function - add $40, %rsp - movq %rax, %xmm0 # If the closure returned a float, - # ffi_closure_win64_inner wrote it to rax - retq + movq arg0, %rsp + + movq arg2, %r10 + + /* Load all slots into both general and xmm registers. */ + movq (%rsp), %rcx + movsd (%rsp), %xmm0 + movq 8(%rsp), %rdx + movsd 8(%rsp), %xmm1 + movq 16(%rsp), %r8 + movsd 16(%rsp), %xmm2 + movq 24(%rsp), %r9 + movsd 24(%rsp), %xmm3 + + call *16(%rbp) + + movl 24(%rbp), %ecx + movq 32(%rbp), %r8 + leaq 0f(%rip), %r10 + cmpl $FFI_TYPE_SMALL_STRUCT_4B, %ecx + leaq (%r10, %rcx, 8), %r10 + ja 99f + jmp *%r10 + +/* Below, we're space constrained most of the time. Thus we eschew the + modern "mov, pop, ret" sequence (5 bytes) for "leave, ret" (2 bytes). */ +.macro epilogue + leaveq + cfi_remember_state + cfi_def_cfa(%rsp, 8) + cfi_restore(%rbp) + ret + cfi_restore_state +.endm + + .align 8 +0: +E FFI_TYPE_VOID + epilogue +E FFI_TYPE_INT + movslq %eax, %rax + movq %rax, (%r8) + epilogue +E FFI_TYPE_FLOAT + movss %xmm0, (%r8) + epilogue +E FFI_TYPE_DOUBLE + movsd %xmm0, (%r8) + epilogue +E FFI_TYPE_LONGDOUBLE + call abort +E FFI_TYPE_UINT8 + movzbl %al, %eax + movq %rax, (%r8) + epilogue +E FFI_TYPE_SINT8 + movsbq %al, %rax + jmp 98f +E FFI_TYPE_UINT16 + movzwl %ax, %eax + movq %rax, (%r8) + epilogue +E FFI_TYPE_SINT16 + movswq %ax, %rax + jmp 98f +E FFI_TYPE_UINT32 + movl %eax, %eax + movq %rax, (%r8) + epilogue +E FFI_TYPE_SINT32 + movslq %eax, %rax + movq %rax, (%r8) + epilogue +E FFI_TYPE_UINT64 +98: movq %rax, (%r8) + epilogue +E FFI_TYPE_SINT64 + movq %rax, (%r8) + epilogue +E FFI_TYPE_STRUCT + epilogue +E FFI_TYPE_POINTER + movq %rax, (%r8) + epilogue +E FFI_TYPE_COMPLEX + call abort +E FFI_TYPE_SMALL_STRUCT_1B + movb %al, (%r8) + epilogue +E FFI_TYPE_SMALL_STRUCT_2B + movw %ax, (%r8) + epilogue +E FFI_TYPE_SMALL_STRUCT_4B + movl %eax, (%r8) + epilogue + + .align 8 +99: call abort + +.purgem epilogue + + cfi_endproc .seh_endproc - .balign 16 - .globl SYMBOL_NAME(ffi_call_win64) - .seh_proc SYMBOL_NAME(ffi_call_win64) -SYMBOL_NAME(ffi_call_win64): - # copy registers onto stack - mov %r9,32(%rsp) - mov %r8,24(%rsp) - mov %rdx,16(%rsp) - mov %rcx,8(%rsp) - .seh_pushreg rbp - push %rbp - .seh_stackalloc 48 - sub $48,%rsp - .seh_setframe rbp, 32 - lea 32(%rsp),%rbp + +/* 32 bytes of outgoing register stack space, 8 bytes of alignment, + 16 bytes of result, 32 bytes of xmm registers. */ +#define ffi_clo_FS (32+8+16+32) +#define ffi_clo_OFF_R (32+8) +#define ffi_clo_OFF_X (32+8+16) + + .align 8 + .globl ffi_go_closure_win64 + + .seh_proc ffi_go_closure_win64 +ffi_go_closure_win64: + cfi_startproc + /* Save all integer arguments into the incoming reg stack space. */ + movq arg0, 8(%rsp) + movq arg1, 16(%rsp) + movq arg2, 24(%rsp) + movq arg3, 32(%rsp) + + movq 8(%r10), arg0 /* load cif */ + movq 16(%r10), arg1 /* load fun */ + movq %r10, arg2 /* closure is user_data */ + jmp 0f + cfi_endproc + .seh_endproc + + .align 8 + .globl ffi_closure_win64 + + .seh_proc ffi_closure_win64 +ffi_closure_win64: + cfi_startproc + /* Save all integer arguments into the incoming reg stack space. */ + movq arg0, 8(%rsp) + movq arg1, 16(%rsp) + movq arg2, 24(%rsp) + movq arg3, 32(%rsp) + + movq FFI_TRAMPOLINE_SIZE(%r10), arg0 /* load cif */ + movq FFI_TRAMPOLINE_SIZE+8(%r10), arg1 /* load fun */ + movq FFI_TRAMPOLINE_SIZE+16(%r10), arg2 /* load user_data */ +0: + subq $ffi_clo_FS, %rsp + cfi_adjust_cfa_offset(ffi_clo_FS) + .seh_stackalloc ffi_clo_FS .seh_endprologue - mov CIF_BYTES(%rbp),%eax - add $15, %rax - and $-16, %rax - cmpq $0x1000, %rax - jb Lch_done -Lch_probe: - subq $0x1000,%rsp - orl $0x0, (%rsp) - subq $0x1000,%rax - cmpq $0x1000,%rax - ja Lch_probe -Lch_done: - subq %rax, %rsp - orl $0x0, (%rsp) - lea 32(%rsp), %rax - mov %rax, STACK(%rbp) + /* Save all sse arguments into the stack frame. */ + movsd %xmm0, ffi_clo_OFF_X(%rsp) + movsd %xmm1, ffi_clo_OFF_X+8(%rsp) + movsd %xmm2, ffi_clo_OFF_X+16(%rsp) + movsd %xmm3, ffi_clo_OFF_X+24(%rsp) - mov ECIF(%rbp), %rdx - mov STACK(%rbp), %rcx - callq *PREP_ARGS_FN(%rbp) + leaq ffi_clo_OFF_R(%rsp), arg3 + call ffi_closure_win64_inner - mov STACK(%rbp), %rsp + /* Load the result into both possible result registers. */ + movq ffi_clo_OFF_R(%rsp), %rax + movsd ffi_clo_OFF_R(%rsp), %xmm0 - movlpd 24(%rsp), %xmm3 - movd %xmm3, %r9 + addq $ffi_clo_FS, %rsp + cfi_adjust_cfa_offset(-ffi_clo_FS) + ret - movlpd 16(%rsp), %xmm2 - movd %xmm2, %r8 - - movlpd 8(%rsp), %xmm1 - movd %xmm1, %rdx - - movlpd (%rsp), %xmm0 - movd %xmm0, %rcx - - callq *FN(%rbp) -.Lret_struct4b: - cmpl $FFI_TYPE_SMALL_STRUCT_4B, CIF_FLAGS(%rbp) - jne .Lret_struct2b - - mov RVALUE(%rbp), %rcx - mov %eax, (%rcx) - jmp .Lret_void - -.Lret_struct2b: - cmpl $FFI_TYPE_SMALL_STRUCT_2B, CIF_FLAGS(%rbp) - jne .Lret_struct1b - - mov RVALUE(%rbp), %rcx - mov %ax, (%rcx) - jmp .Lret_void - -.Lret_struct1b: - cmpl $FFI_TYPE_SMALL_STRUCT_1B, CIF_FLAGS(%rbp) - jne .Lret_uint8 - - mov RVALUE(%rbp), %rcx - mov %al, (%rcx) - jmp .Lret_void - -.Lret_uint8: - cmpl $FFI_TYPE_UINT8, CIF_FLAGS(%rbp) - jne .Lret_sint8 - - mov RVALUE(%rbp), %rcx - movzbq %al, %rax - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_sint8: - cmpl $FFI_TYPE_SINT8, CIF_FLAGS(%rbp) - jne .Lret_uint16 - - mov RVALUE(%rbp), %rcx - movsbq %al, %rax - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_uint16: - cmpl $FFI_TYPE_UINT16, CIF_FLAGS(%rbp) - jne .Lret_sint16 - - mov RVALUE(%rbp), %rcx - movzwq %ax, %rax - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_sint16: - cmpl $FFI_TYPE_SINT16, CIF_FLAGS(%rbp) - jne .Lret_uint32 - - mov RVALUE(%rbp), %rcx - movswq %ax, %rax - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_uint32: - cmpl $FFI_TYPE_UINT32, CIF_FLAGS(%rbp) - jne .Lret_sint32 - - mov RVALUE(%rbp), %rcx - movl %eax, %eax - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_sint32: - cmpl $FFI_TYPE_SINT32, CIF_FLAGS(%rbp) - jne .Lret_float - - mov RVALUE(%rbp), %rcx - cltq - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_float: - cmpl $FFI_TYPE_FLOAT, CIF_FLAGS(%rbp) - jne .Lret_double - - mov RVALUE(%rbp), %rax - movss %xmm0, (%rax) - jmp .Lret_void - -.Lret_double: - cmpl $FFI_TYPE_DOUBLE, CIF_FLAGS(%rbp) - jne .Lret_uint64 - - mov RVALUE(%rbp), %rax - movlpd %xmm0, (%rax) - jmp .Lret_void - -.Lret_uint64: - cmpl $FFI_TYPE_UINT64, CIF_FLAGS(%rbp) - jne .Lret_sint64 - - mov RVALUE(%rbp), %rcx - mov %rax, (%rcx) - jmp .Lret_void - -.Lret_sint64: - cmpl $FFI_TYPE_SINT64, CIF_FLAGS(%rbp) - jne .Lret_pointer - - mov RVALUE(%rbp), %rcx - mov %rax, (%rcx) - jmp .Lret_void - -.Lret_pointer: - cmpl $FFI_TYPE_POINTER, CIF_FLAGS(%rbp) - jne .Lret_int - - mov RVALUE(%rbp), %rcx - mov %rax, (%rcx) - jmp .Lret_void - -.Lret_int: - cmpl $FFI_TYPE_INT, CIF_FLAGS(%rbp) - jne .Lret_void - - mov RVALUE(%rbp), %rcx - cltq - movq %rax, (%rcx) - jmp .Lret_void - -.Lret_void: - xor %rax, %rax - - lea 16(%rbp), %rsp - pop %rbp - retq + cfi_endproc .seh_endproc -#endif /* !_MSC_VER */ - diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index 5177f07..55de25c 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -24,16 +24,15 @@ set ctlist [lsearch -inline -all -glob [lsort [glob -nocomplain -- $srcdir/$subd run-many-tests $tlist "" -if { ![istarget s390*] } { - +# ??? We really should preprocess ffi.h and grep +# for FFI_TARGET_HAS_COMPLEX_TYPE. +if { [istarget s390*] + || [istarget x86_64*] } { + run-many-tests $ctlist "" +} else { foreach test $ctlist { unsupported "$test" } - -} else { - - run-many-tests $ctlist "" - } dg-finish diff --git a/testsuite/libffi.call/cls_align_longdouble_split.c b/testsuite/libffi.call/cls_align_longdouble_split.c index 15f9365..cc1c43b 100644 --- a/testsuite/libffi.call/cls_align_longdouble_split.c +++ b/testsuite/libffi.call/cls_align_longdouble_split.c @@ -4,10 +4,8 @@ PR: none. Originator: 20031203 */ -/* { dg-excess-errors "no long double format" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ /* { dg-do run { xfail strongarm*-*-* xscale*-*-* } } */ /* { dg-options -mlong-double-128 { target powerpc64*-*-linux* } } */ -/* { dg-output "" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ #include "ffitest.h" diff --git a/testsuite/libffi.call/cls_align_longdouble_split2.c b/testsuite/libffi.call/cls_align_longdouble_split2.c index ca1c356..5d3bec0 100644 --- a/testsuite/libffi.call/cls_align_longdouble_split2.c +++ b/testsuite/libffi.call/cls_align_longdouble_split2.c @@ -5,10 +5,8 @@ Originator: Blake Chaffin 6/18/2007 */ -/* { dg-excess-errors "no long double format" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ /* { dg-do run { xfail strongarm*-*-* } } */ /* { dg-options -mlong-double-128 { target powerpc64*-*-linux* } } */ -/* { dg-output "" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ #include "ffitest.h" diff --git a/testsuite/libffi.call/cls_longdouble.c b/testsuite/libffi.call/cls_longdouble.c index 5dc9ac7..d24e72e 100644 --- a/testsuite/libffi.call/cls_longdouble.c +++ b/testsuite/libffi.call/cls_longdouble.c @@ -4,12 +4,10 @@ PR: none. Originator: Blake Chaffin */ -/* { dg-excess-errors "no long double format" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ /* This test is known to PASS on armv7l-unknown-linux-gnueabihf, so I have remove the xfail for arm*-*-* below, until we know more. */ /* { dg-do run { xfail strongarm*-*-* xscale*-*-* } } */ /* { dg-options -mlong-double-128 { target powerpc64*-*-linux* } } */ -/* { dg-output "" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ #include "ffitest.h" diff --git a/testsuite/libffi.call/float2.c b/testsuite/libffi.call/float2.c index a0b296c..aae1abf 100644 --- a/testsuite/libffi.call/float2.c +++ b/testsuite/libffi.call/float2.c @@ -4,9 +4,6 @@ PR: none. Originator: From the original ffitest.c */ -/* { dg-excess-errors "fails" { target x86_64-*-mingw* x86_64-*-cygwin* } } */ -/* { dg-do run { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ - #include "ffitest.h" #include "float.h" diff --git a/testsuite/libffi.call/huge_struct.c b/testsuite/libffi.call/huge_struct.c index 657fe54..187c42c 100644 --- a/testsuite/libffi.call/huge_struct.c +++ b/testsuite/libffi.call/huge_struct.c @@ -5,11 +5,9 @@ Originator: Blake Chaffin 6/18/2007 */ -/* { dg-excess-errors "" { target x86_64-*-mingw* x86_64-*-cygwin* } } */ /* { dg-do run { xfail strongarm*-*-* xscale*-*-* } } */ /* { dg-options -mlong-double-128 { target powerpc64*-*-linux* } } */ /* { dg-options -Wformat=0 { target moxie*-*-elf } } */ -/* { dg-output "" { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ #include "ffitest.h" diff --git a/testsuite/libffi.call/return_ldl.c b/testsuite/libffi.call/return_ldl.c index 5c2fe65..520e710 100644 --- a/testsuite/libffi.call/return_ldl.c +++ b/testsuite/libffi.call/return_ldl.c @@ -4,7 +4,6 @@ PR: none. Originator: 20071113 */ -/* { dg-do run { xfail x86_64-*-mingw* x86_64-*-cygwin* } } */ #include "ffitest.h" static long double return_ldl(long double ldl) From ebd82769de5fc909ec9678aafca674ece394d173 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 23 Oct 2014 23:57:06 -0700 Subject: [PATCH 05/58] win64: Remove support from ffi.c --- src/x86/ffi.c | 212 ++------------------------------------------------ 1 file changed, 8 insertions(+), 204 deletions(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 006c95d..c387fb5 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -28,18 +28,12 @@ DEALINGS IN THE SOFTWARE. ----------------------------------------------------------------------- */ -#if !defined(__x86_64__) || defined(_WIN64) || defined(__CYGWIN__) - -#ifdef _WIN64 -#include -#endif +#ifndef __x86_64__ #include #include - #include - /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments */ @@ -50,26 +44,17 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) register void **p_argv; register char *argp; register ffi_type **p_arg; -#ifndef X86_WIN64 const int cabi = ecif->cif->abi; const int dir = (cabi == FFI_PASCAL || cabi == FFI_REGISTER) ? -1 : +1; unsigned int stack_args_count = 0; void *p_stack_data[3]; char *argp2 = stack; -#else - #define dir 1 -#endif argp = stack; if ((ecif->cif->flags == FFI_TYPE_STRUCT - || ecif->cif->flags == FFI_TYPE_MS_STRUCT) -#ifdef X86_WIN64 - && ((ecif->cif->rtype->size & (1 | 2 | 4 | 8)) == 0) -#endif - ) + || ecif->cif->flags == FFI_TYPE_MS_STRUCT)) { -#ifndef X86_WIN64 /* For fastcall/thiscall/register this is first register-passed argument. */ if (cabi == FFI_THISCALL || cabi == FFI_FASTCALL || cabi == FFI_REGISTER) @@ -77,7 +62,6 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) p_stack_data[stack_args_count] = argp; ++stack_args_count; } -#endif *(void **) argp = ecif->rvalue; argp += sizeof(void*); @@ -105,24 +89,6 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) size_t z = (*p_arg)->size; -#ifdef X86_WIN64 - if (z > FFI_SIZEOF_ARG - || ((*p_arg)->type == FFI_TYPE_STRUCT - && (z & (1 | 2 | 4 | 8)) == 0) -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - || ((*p_arg)->type == FFI_TYPE_LONGDOUBLE) -#endif - ) - { - z = FFI_SIZEOF_ARG; - *(void **)argp = *p_argv; - } - else if ((*p_arg)->type == FFI_TYPE_FLOAT) - { - memcpy(argp, *p_argv, z); - } - else -#endif if (z < FFI_SIZEOF_ARG) { z = FFI_SIZEOF_ARG; @@ -165,7 +131,6 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) memcpy(argp, *p_argv, z); } -#ifndef X86_WIN64 /* For thiscall/fastcall/register convention register-passed arguments are the first two none-floating-point arguments with a size smaller or equal to sizeof (void*). */ @@ -188,18 +153,13 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) p_stack_data[stack_args_count] = argp; ++stack_args_count; } -#endif -#ifdef X86_WIN64 - argp += (z + sizeof(void*) - 1) & ~(sizeof(void*) - 1); -#else argp += z; -#endif } -#ifndef X86_WIN64 - /* We need to move the register-passed arguments for thiscall/fastcall/register - on top of stack, so that those can be moved to registers by call-handler. */ + /* We need to move the register-passed arguments for thiscall, + fastcall, register on top of stack, so that those can be moved + to registers by call-handler. */ if (stack_args_count > 0) { if (dir < 0 && stack_args_count > 1) @@ -225,7 +185,6 @@ unsigned int ffi_prep_args(char *stack, extended_cif *ecif) } return stack_args_count; -#endif return 0; } @@ -243,25 +202,16 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_UINT16: case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: -#ifdef X86_WIN64 - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: -#endif case FFI_TYPE_SINT64: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#ifndef X86_WIN64 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif #endif cif->flags = (unsigned) cif->rtype->type; break; case FFI_TYPE_UINT64: -#ifdef X86_WIN64 - case FFI_TYPE_POINTER: -#endif cif->flags = FFI_TYPE_SINT64; break; @@ -277,11 +227,7 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) } else if (cif->rtype->size == 4) { -#ifdef X86_WIN64 - cif->flags = FFI_TYPE_SMALL_STRUCT_4B; -#else cif->flags = FFI_TYPE_INT; /* same as int type */ -#endif } else if (cif->rtype->size == 8) { @@ -302,14 +248,7 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) break; default: -#ifdef X86_WIN64 - cif->flags = FFI_TYPE_SINT64; - break; - case FFI_TYPE_INT: - cif->flags = FFI_TYPE_SINT32; -#else cif->flags = FFI_TYPE_INT; -#endif break; } @@ -320,32 +259,19 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) cif->bytes += (unsigned)ALIGN((*ptr)->size, FFI_SIZEOF_ARG); } -#ifdef X86_WIN64 - /* ensure space for storing four registers */ - cif->bytes += 4 * FFI_SIZEOF_ARG; -#endif - #ifndef X86_WIN32 -#ifndef X86_WIN64 if (cif->abi == FFI_SYSV || cif->abi == FFI_UNIX64) -#endif cif->bytes = (cif->bytes + 15) & ~0xF; #endif return FFI_OK; } -#ifdef X86_WIN64 -extern int -ffi_call_win64(unsigned int (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)(void)); -#else extern void ffi_call_win32(unsigned int (*)(char *, extended_cif *), extended_cif *, unsigned, unsigned, unsigned, unsigned *, void (*fn)(void)); extern void ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, unsigned, unsigned, unsigned *, void (*fn)(void)); -#endif void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { @@ -357,33 +283,18 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) /* If the return value is a struct and we don't have a return */ /* value address then we need to make one */ -#ifdef X86_WIN64 - if (rvalue == NULL - && cif->flags == FFI_TYPE_STRUCT - && ((cif->rtype->size & (1 | 2 | 4 | 8)) == 0)) - { - ecif.rvalue = alloca((cif->rtype->size + 0xF) & ~0xF); - } -#else if (rvalue == NULL && (cif->flags == FFI_TYPE_STRUCT || cif->flags == FFI_TYPE_MS_STRUCT)) { ecif.rvalue = alloca(cif->rtype->size); } -#endif else ecif.rvalue = rvalue; switch (cif->abi) { -#ifdef X86_WIN64 - case FFI_WIN64: - ffi_call_win64(ffi_prep_args, &ecif, cif->bytes, - cif->flags, ecif.rvalue, fn); - break; -#else #ifndef X86_WIN32 case FFI_SYSV: ffi_call_SYSV(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, @@ -401,7 +312,6 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) ffi_call_win32(ffi_prep_args, &ecif, cif->abi, cif->bytes, cif->flags, ecif.rvalue, fn); break; -#endif default: FFI_ASSERT(0); break; @@ -427,47 +337,13 @@ void FFI_HIDDEN ffi_closure_raw_SYSV (ffi_raw_closure *) void FFI_HIDDEN ffi_closure_raw_THISCALL (ffi_raw_closure *) __attribute__ ((regparm(1))); #endif -#ifndef X86_WIN64 void FFI_HIDDEN ffi_closure_STDCALL (ffi_closure *); void FFI_HIDDEN ffi_closure_THISCALL (ffi_closure *); void FFI_HIDDEN ffi_closure_FASTCALL (ffi_closure *); void FFI_HIDDEN ffi_closure_REGISTER (ffi_closure *); -#else -void FFI_HIDDEN ffi_closure_win64 (ffi_closure *); -#endif /* This function is jumped to by the trampoline */ -#ifdef X86_WIN64 -void * FFI_HIDDEN -ffi_closure_win64_inner (ffi_closure *closure, void *args) { - ffi_cif *cif; - void **arg_area; - void *result; - void *resp = &result; - - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); - - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will change RESP to point to the - * structure return address. */ - - ffi_prep_incoming_args(args, &resp, arg_area, cif); - - (closure->fun) (cif, resp, arg_area, closure->user_data); - - /* The result is returned in rax. This does the right thing for - result types except for floats; we have to 'mov xmm0, rax' in the - caller to correct this. - TODO: structure sizes of 3 5 6 7 are returned by reference, too!!! - */ - return cif->rtype->size > sizeof(void *) ? resp : *(void **)resp; -} - -#else unsigned int FFI_HIDDEN __attribute__ ((regparm(1))) ffi_closure_SYSV_inner (ffi_closure *closure, void **respp, void *args) { @@ -514,7 +390,6 @@ ffi_closure_WIN32_inner (ffi_closure *closure, void **respp, void *args) return ret; } -#endif /* !X86_WIN64 */ static unsigned int ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, @@ -524,7 +399,6 @@ ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, register void **p_argv; register char *argp; register ffi_type **p_arg; -#ifndef X86_WIN64 const int cabi = cif->abi; const int dir = (cabi == FFI_PASCAL || cabi == FFI_REGISTER) ? -1 : +1; const unsigned int max_stack_count = (cabi == FFI_THISCALL) ? 1 @@ -533,37 +407,25 @@ ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, : 0; unsigned int passed_regs = 0; void *p_stack_data[3] = { stack - 1 }; -#else - #define dir 1 -#endif argp = stack; -#ifndef X86_WIN64 argp += max_stack_count * FFI_SIZEOF_ARG; -#endif if ((cif->flags == FFI_TYPE_STRUCT - || cif->flags == FFI_TYPE_MS_STRUCT) -#ifdef X86_WIN64 - && ((cif->rtype->size & (1 | 2 | 4 | 8)) == 0) -#endif - ) + || cif->flags == FFI_TYPE_MS_STRUCT)) { -#ifndef X86_WIN64 if (passed_regs < max_stack_count) { *rvalue = *(void**) (stack + (passed_regs*FFI_SIZEOF_ARG)); ++passed_regs; } else -#endif { *rvalue = *(void **) argp; argp += sizeof(void *); } } -#ifndef X86_WIN64 /* Do register arguments first */ for (i = 0, p_arg = cif->arg_types; i < cif->nargs && passed_regs < max_stack_count; @@ -581,7 +443,6 @@ ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, avalue[i] = stack + (passed_regs*FFI_SIZEOF_ARG); ++passed_regs; } -#endif p_arg = cif->arg_types; p_argv = avalue; @@ -605,20 +466,6 @@ ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, size_t z = (*p_arg)->size; -#ifdef X86_WIN64 - if (z > FFI_SIZEOF_ARG - || ((*p_arg)->type == FFI_TYPE_STRUCT - && (z & (1 | 2 | 4 | 8)) == 0) -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - || ((*p_arg)->type == FFI_TYPE_LONGDOUBLE) -#endif - ) - { - z = FFI_SIZEOF_ARG; - *p_argv = *(void **)argp; - } - else -#else if (passed_regs > 0 && z <= FFI_SIZEOF_ARG && (p_argv == p_stack_data[0] @@ -629,40 +476,17 @@ ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, continue; } else -#endif { /* because we're little endian, this is what it turns into. */ *p_argv = (void*) argp; } -#ifdef X86_WIN64 - argp += (z + sizeof(void*) - 1) & ~(sizeof(void*) - 1); -#else argp += z; -#endif } return (size_t)argp - (size_t)stack; } -#define FFI_INIT_TRAMPOLINE_WIN64(TRAMP,FUN,CTX,MASK) \ -{ unsigned char *__tramp = (unsigned char*)(TRAMP); \ - void* __fun = (void*)(FUN); \ - void* __ctx = (void*)(CTX); \ - *(unsigned char*) &__tramp[0] = 0x41; \ - *(unsigned char*) &__tramp[1] = 0xbb; \ - *(unsigned int*) &__tramp[2] = MASK; /* mov $mask, %r11 */ \ - *(unsigned char*) &__tramp[6] = 0x48; \ - *(unsigned char*) &__tramp[7] = 0xb8; \ - *(void**) &__tramp[8] = __ctx; /* mov __ctx, %rax */ \ - *(unsigned char *) &__tramp[16] = 0x49; \ - *(unsigned char *) &__tramp[17] = 0xba; \ - *(void**) &__tramp[18] = __fun; /* mov __fun, %r10 */ \ - *(unsigned char *) &__tramp[26] = 0x41; \ - *(unsigned char *) &__tramp[27] = 0xff; \ - *(unsigned char *) &__tramp[28] = 0xe2; /* jmp %r10 */ \ - } - /* How to make a trampoline. Derived from gcc/config/i386/i386.c. */ #define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX) \ @@ -723,18 +547,6 @@ ffi_prep_closure_loc (ffi_closure* closure, void *user_data, void *codeloc) { -#ifdef X86_WIN64 -#define ISFLOAT(IDX) (cif->arg_types[IDX]->type == FFI_TYPE_FLOAT || cif->arg_types[IDX]->type == FFI_TYPE_DOUBLE) -#define FLAG(IDX) (cif->nargs>(IDX)&&ISFLOAT(IDX)?(1<<(IDX)):0) - if (cif->abi == FFI_WIN64) - { - int mask = FLAG(0)|FLAG(1)|FLAG(2)|FLAG(3); - FFI_INIT_TRAMPOLINE_WIN64 (&closure->tramp[0], - &ffi_closure_win64, - codeloc, mask); - /* make sure we can execute here */ - } -#else if (cif->abi == FFI_SYSV) { FFI_INIT_TRAMPOLINE (&closure->tramp[0], @@ -773,7 +585,6 @@ ffi_prep_closure_loc (ffi_closure* closure, (void*)codeloc); } #endif /* X86_WIN32 */ -#endif /* !X86_WIN64 */ else { return FFI_BAD_ABI; @@ -843,7 +654,6 @@ ffi_prep_args_raw(char *stack, extended_cif *ecif) const ffi_cif *cif = ecif->cif; unsigned int i, passed_regs = 0; -#ifndef X86_WIN64 const unsigned int abi = cif->abi; const unsigned int max_regs = (abi == FFI_THISCALL) ? 1 : (abi == FFI_FASTCALL) ? 2 @@ -865,7 +675,6 @@ ffi_prep_args_raw(char *stack, extended_cif *ecif) ++passed_regs; } -#endif memcpy (stack, ecif->avalue, cif->bytes); return passed_regs; @@ -909,7 +718,6 @@ ffi_raw_call(ffi_cif *cif, void (*fn)(void), void *rvalue, ffi_raw *fake_avalue) case FFI_SYSV: case FFI_MS_CDECL: #endif -#ifndef X86_WIN64 case FFI_STDCALL: case FFI_THISCALL: case FFI_FASTCALL: @@ -918,14 +726,10 @@ ffi_raw_call(ffi_cif *cif, void (*fn)(void), void *rvalue, ffi_raw *fake_avalue) ffi_call_win32(ffi_prep_args_raw, &ecif, cif->abi, cif->bytes, cif->flags, ecif.rvalue, fn); break; -#endif default: FFI_ASSERT(0); break; } } - -#endif - -#endif /* !__x86_64__ || X86_WIN64 */ - +#endif /* !FFI_NO_RAW_API */ +#endif /* !__x86_64__ */ From 2e9dc1655600eb8928ebab13fa246187b6205109 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 27 Oct 2014 13:41:39 -0700 Subject: [PATCH 06/58] x86_64: Fixups for x32 --- src/x86/ffi64.c | 5 +++-- src/x86/unix64.S | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/x86/ffi64.c b/src/x86/ffi64.c index 384a93a..65fb595 100644 --- a/src/x86/ffi64.c +++ b/src/x86/ffi64.c @@ -568,6 +568,7 @@ ffi_prep_closure_loc (ffi_closure* closure, 0x0f, 0x1f, 0x00 }; void (*dest)(void); + char *tramp = closure->tramp; if (cif->abi != FFI_UNIX64) return FFI_BAD_ABI; @@ -577,8 +578,8 @@ ffi_prep_closure_loc (ffi_closure* closure, else dest = ffi_closure_unix64; - memcpy (closure->tramp, trampoline, sizeof(trampoline)); - *(UINT64 *)(closure->tramp + 16) = (uintptr_t)dest; + memcpy (tramp, trampoline, sizeof(trampoline)); + *(UINT64 *)(tramp + 16) = (uintptr_t)dest; closure->cif = cif; closure->fun = fun; diff --git a/src/x86/unix64.S b/src/x86/unix64.S index 134cb3d..797b9d9 100644 --- a/src/x86/unix64.S +++ b/src/x86/unix64.S @@ -32,7 +32,7 @@ #include #include -.text + .text /* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, void *raddr, void (*fnaddr)(void)); @@ -272,9 +272,15 @@ ffi_closure_unix64: movq %r8, ffi_closure_OFS_G+0x20(%rsp) movq %r9, ffi_closure_OFS_G+0x28(%rsp) - movq 24(%r10), %rdi /* Load cif */ - movq 32(%r10), %rsi /* Load fun */ - movq 40(%r10), %rdx /* Load user_data */ +#ifdef __ILP32__ + movl FFI_TRAMPOLINE_SIZE(%r10), %edi /* Load cif */ + movl FFI_TRAMPOLINE_SIZE+4(%r10), %esi /* Load fun */ + movl FFI_TRAMPOLINE_SIZE+8(%r10), %edx /* Load user_data */ +#else + movq FFI_TRAMPOLINE_SIZE(%r10), %rdi /* Load cif */ + movq FFI_TRAMPOLINE_SIZE+8(%r10), %rsi /* Load fun */ + movq FFI_TRAMPOLINE_SIZE+16(%r10), %rdx /* Load user_data */ +#endif .Ldo_closure: leaq ffi_closure_OFS_RVALUE(%rsp), %rcx /* Load rvalue */ movq %rsp, %r8 /* Load reg_args */ @@ -407,9 +413,15 @@ ffi_go_closure_unix64: movq %r8, ffi_closure_OFS_G+0x20(%rsp) movq %r9, ffi_closure_OFS_G+0x28(%rsp) +#ifdef __ILP32__ + movl 4(%r10), %edi /* Load cif */ + movl 8(%r10), %esi /* Load fun */ + movl %r10d, %edx /* Load closure (user_data) */ +#else movq 8(%r10), %rdi /* Load cif */ movq 16(%r10), %rsi /* Load fun */ movq %r10, %rdx /* Load closure (user_data) */ +#endif jmp .Ldo_closure cfi_endproc From 32c56831636a489a1c5084e42ffbd935902b1033 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 28 Oct 2014 11:17:35 -0700 Subject: [PATCH 07/58] x86_64: Decouple return types from FFI_TYPE constants We can better support structure returns, and as prep for complex types. --- src/x86/ffi64.c | 146 ++++++++++++++++----------- src/x86/internal64.h | 20 ++++ src/x86/unix64.S | 234 ++++++++++++++++++------------------------- 3 files changed, 203 insertions(+), 197 deletions(-) create mode 100644 src/x86/internal64.h diff --git a/src/x86/ffi64.c b/src/x86/ffi64.c index 65fb595..a03061b 100644 --- a/src/x86/ffi64.c +++ b/src/x86/ffi64.c @@ -33,6 +33,7 @@ #include #include #include +#include "internal64.h" #ifdef __x86_64__ @@ -191,7 +192,7 @@ classify_argument (ffi_type *type, enum x86_64_reg_class classes[], } else if (size <= 16) { - classes[0] = classes[1] = X86_64_INTEGERSI_CLASS; + classes[0] = classes[1] = X86_64_INTEGER_CLASS; return 2; } else @@ -360,15 +361,55 @@ ffi_prep_cif_machdep (ffi_cif *cif) int gprcount, ssecount, i, avn, ngpr, nsse, flags; enum x86_64_reg_class classes[MAX_CLASSES]; size_t bytes, n; + ffi_type *rtype; if (cif->abi != FFI_UNIX64) return FFI_BAD_ABI; gprcount = ssecount = 0; - flags = cif->rtype->type; - if (flags != FFI_TYPE_VOID) + rtype = cif->rtype; + switch (rtype->type) { + case FFI_TYPE_VOID: + flags = UNIX64_RET_VOID; + break; + case FFI_TYPE_UINT8: + flags = UNIX64_RET_UINT8; + break; + case FFI_TYPE_SINT8: + flags = UNIX64_RET_SINT8; + break; + case FFI_TYPE_UINT16: + flags = UNIX64_RET_UINT16; + break; + case FFI_TYPE_SINT16: + flags = UNIX64_RET_SINT16; + break; + case FFI_TYPE_UINT32: + flags = UNIX64_RET_UINT32; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + flags = UNIX64_RET_SINT32; + break; + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + flags = UNIX64_RET_INT64; + break; + case FFI_TYPE_POINTER: + flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64); + break; + case FFI_TYPE_FLOAT: + flags = UNIX64_RET_XMM32; + break; + case FFI_TYPE_DOUBLE: + flags = UNIX64_RET_XMM64; + break; + case FFI_TYPE_LONGDOUBLE: + flags = UNIX64_RET_X87; + break; + case FFI_TYPE_STRUCT: n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); if (n == 0) { @@ -376,22 +417,24 @@ ffi_prep_cif_machdep (ffi_cif *cif) memory is the first argument. Allocate a register for it. */ gprcount++; /* We don't have to do anything in asm for the return. */ - flags = FFI_TYPE_VOID; + flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM; } - else if (flags == FFI_TYPE_STRUCT) + else { /* Mark which registers the result appears in. */ _Bool sse0 = SSE_CLASS_P (classes[0]); _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]); - if (sse0 && !sse1) - flags |= 1 << 8; - else if (!sse0 && sse1) - flags |= 1 << 9; - else if (sse0 && sse1) - flags |= 1 << 10; + if (sse0) + flags = (sse1 ? UNIX64_RET_ST_XMM0_XMM1 : UNIX64_RET_ST_XMM0_RAX); + else + flags = (sse1 ? UNIX64_RET_ST_RAX_XMM0 : UNIX64_RET_ST_RAX_RDX); + /* Mark the true size of the structure. */ - flags |= cif->rtype->size << 12; + flags |= rtype->size << UNIX64_SIZE_SHIFT; } + break; + default: + return FFI_BAD_TYPEDEF; } /* Go over all arguments and determine the way they should be passed. @@ -418,9 +461,10 @@ ffi_prep_cif_machdep (ffi_cif *cif) } } if (ssecount) - flags |= 1 << 11; + flags |= UNIX64_FLAG_XMM_ARGS; + cif->flags = flags; - cif->bytes = (unsigned)ALIGN (bytes, 8); + cif->bytes = ALIGN (bytes, 8); return FFI_OK; } @@ -432,20 +476,22 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, enum x86_64_reg_class classes[MAX_CLASSES]; char *stack, *argp; ffi_type **arg_types; - int gprcount, ssecount, ngpr, nsse, i, avn; - _Bool ret_in_memory; + int gprcount, ssecount, ngpr, nsse, i, avn, flags; struct register_args *reg_args; /* Can't call 32-bit mode from 64-bit mode. */ FFI_ASSERT (cif->abi == FFI_UNIX64); /* If the return value is a struct and we don't have a return value - address then we need to make one. Note the setting of flags to - VOID above in ffi_prep_cif_machdep. */ - ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT - && (cif->flags & 0xff) == FFI_TYPE_VOID); - if (rvalue == NULL && ret_in_memory) - rvalue = alloca (cif->rtype->size); + address then we need to make one. Otherwise we can ignore it. */ + flags = cif->flags; + if (rvalue == NULL) + { + if (flags & UNIX64_FLAG_RET_IN_MEM) + rvalue = alloca (cif->rtype->size); + else + flags = UNIX64_RET_VOID; + } /* Allocate the space for the arguments, plus 4 words of temp space. */ stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8); @@ -458,7 +504,7 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, /* If the return value is passed in memory, add the pointer as the first integer argument. */ - if (ret_in_memory) + if (flags & UNIX64_FLAG_RET_IN_MEM) reg_args->gpr[gprcount++] = (unsigned long) rvalue; avn = cif->nargs; @@ -503,17 +549,17 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, switch (arg_types[i]->type) { case FFI_TYPE_SINT8: - *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT8 *) a); + reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a); break; case FFI_TYPE_SINT16: - *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT16 *) a); + reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a); break; case FFI_TYPE_SINT32: - *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT32 *) a); + reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a); break; default: reg_args->gpr[gprcount] = 0; - memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8); + memcpy (®_args->gpr[gprcount], a, size); } gprcount++; break; @@ -533,7 +579,7 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, reg_args->rax = ssecount; ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args), - cif->flags, rvalue, fn); + flags, rvalue, fn); } void @@ -573,7 +619,7 @@ ffi_prep_closure_loc (ffi_closure* closure, if (cif->abi != FFI_UNIX64) return FFI_BAD_ABI; - if (cif->flags & (1 << 11)) + if (cif->flags & UNIX64_FLAG_XMM_ARGS) dest = ffi_closure_unix64_sse; else dest = ffi_closure_unix64; @@ -600,39 +646,17 @@ ffi_closure_unix64_inner(ffi_cif *cif, ffi_type **arg_types; long i, avn; int gprcount, ssecount, ngpr, nsse; - int ret; - - avalue = alloca(cif->nargs * sizeof(void *)); - gprcount = ssecount = 0; - - ret = cif->rtype->type; - if (ret != FFI_TYPE_VOID) - { - enum x86_64_reg_class classes[MAX_CLASSES]; - size_t n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); - if (n == 0) - { - /* The return value goes in memory. Arrange for the closure - return value to go directly back to the original caller. */ - rvalue = (void *) (unsigned long) reg_args->gpr[gprcount++]; - /* We don't have to do anything in asm for the return. */ - ret = FFI_TYPE_VOID; - } - else if (ret == FFI_TYPE_STRUCT && n == 2) - { - /* Mark which register the second word of the structure goes in. */ - _Bool sse0 = SSE_CLASS_P (classes[0]); - _Bool sse1 = SSE_CLASS_P (classes[1]); - if (!sse0 && sse1) - ret |= 1 << 8; - else if (sse0 && !sse1) - ret |= 1 << 9; - } - } + int flags; avn = cif->nargs; - arg_types = cif->arg_types; + flags = cif->flags; + avalue = alloca(avn * sizeof(void *)); + gprcount = ssecount = 0; + if (flags & UNIX64_FLAG_RET_IN_MEM) + rvalue = (void *)(uintptr_t)reg_args->gpr[gprcount++]; + + arg_types = cif->arg_types; for (i = 0; i < avn; ++i) { enum x86_64_reg_class classes[MAX_CLASSES]; @@ -693,7 +717,7 @@ ffi_closure_unix64_inner(ffi_cif *cif, fun (cif, rvalue, avalue, user_data); /* Tell assembly how to perform return type promotions. */ - return ret; + return flags; } extern void ffi_go_closure_unix64(void) FFI_HIDDEN; @@ -706,7 +730,7 @@ ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif, if (cif->abi != FFI_UNIX64) return FFI_BAD_ABI; - closure->tramp = (cif->flags & (1 << 11) + closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS ? ffi_go_closure_unix64_sse : ffi_go_closure_unix64); closure->cif = cif; diff --git a/src/x86/internal64.h b/src/x86/internal64.h new file mode 100644 index 0000000..07b1b10 --- /dev/null +++ b/src/x86/internal64.h @@ -0,0 +1,20 @@ +#define UNIX64_RET_VOID 0 +#define UNIX64_RET_UINT8 1 +#define UNIX64_RET_UINT16 2 +#define UNIX64_RET_UINT32 3 +#define UNIX64_RET_SINT8 4 +#define UNIX64_RET_SINT16 5 +#define UNIX64_RET_SINT32 6 +#define UNIX64_RET_INT64 7 +#define UNIX64_RET_XMM32 8 +#define UNIX64_RET_XMM64 9 +#define UNIX64_RET_X87 10 +#define UNIX64_RET_ST_RAX_RDX 11 +#define UNIX64_RET_ST_XMM0_RAX 12 +#define UNIX64_RET_ST_RAX_XMM0 13 +#define UNIX64_RET_ST_XMM0_XMM1 14 +#define UNIX64_RET_LAST 14 + +#define UNIX64_FLAG_RET_IN_MEM (1 << 10) +#define UNIX64_FLAG_XMM_ARGS (1 << 11) +#define UNIX64_SIZE_SHIFT 12 diff --git a/src/x86/unix64.S b/src/x86/unix64.S index 797b9d9..0151229 100644 --- a/src/x86/unix64.S +++ b/src/x86/unix64.S @@ -31,9 +31,15 @@ #include #include #include +#include "internal64.h" .text +.macro E index + .align 8 + .org 0b + \index * 8, 0x90 +.endm + /* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, void *raddr, void (*fnaddr)(void)); @@ -41,7 +47,7 @@ for this function. This has been allocated by ffi_call. We also deallocate some of the stack that has been alloca'd. */ - .align 2 + .align 8 .globl ffi_call_unix64 .type ffi_call_unix64,@function FFI_HIDDEN(ffi_call_unix64) @@ -100,108 +106,80 @@ ffi_call_unix64: cfi_restore(%rbp) /* The first byte of the flags contains the FFI_TYPE. */ + cmpb $UNIX64_RET_LAST, %cl movzbl %cl, %r10d - leaq .Lstore_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 + leaq 0f(%rip), %r11 + ja 9f + leaq (%r11, %r10, 8), %r10 + + /* Prep for the structure cases: scratch area in redzone. */ + leaq -20(%rsp), %rsi jmp *%r10 - .section .rodata - .align 2 -.Lstore_table: - .long .Lst_void-.Lstore_table /* FFI_TYPE_VOID */ - .long .Lst_sint32-.Lstore_table /* FFI_TYPE_INT */ - .long .Lst_float-.Lstore_table /* FFI_TYPE_FLOAT */ - .long .Lst_double-.Lstore_table /* FFI_TYPE_DOUBLE */ - .long .Lst_ldouble-.Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lst_uint8-.Lstore_table /* FFI_TYPE_UINT8 */ - .long .Lst_sint8-.Lstore_table /* FFI_TYPE_SINT8 */ - .long .Lst_uint16-.Lstore_table /* FFI_TYPE_UINT16 */ - .long .Lst_sint16-.Lstore_table /* FFI_TYPE_SINT16 */ - .long .Lst_uint32-.Lstore_table /* FFI_TYPE_UINT32 */ - .long .Lst_sint32-.Lstore_table /* FFI_TYPE_SINT32 */ - .long .Lst_int64-.Lstore_table /* FFI_TYPE_UINT64 */ - .long .Lst_int64-.Lstore_table /* FFI_TYPE_SINT64 */ - .long .Lst_struct-.Lstore_table /* FFI_TYPE_STRUCT */ - .long .Lst_int64-.Lstore_table /* FFI_TYPE_POINTER */ - .previous - - .align 2 -.Lst_void: + .align 8 +0: +E UNIX64_RET_VOID ret - .align 2 - -.Lst_uint8: - movzbq %al, %rax +E UNIX64_RET_UINT8 + movzbl %al, %eax movq %rax, (%rdi) ret - .align 2 -.Lst_sint8: +E UNIX64_RET_UINT16 + movzwl %ax, %eax + movq %rax, (%rdi) + ret +E UNIX64_RET_UINT32 + movl %eax, %eax + movq %rax, (%rdi) + ret +E UNIX64_RET_SINT8 movsbq %al, %rax movq %rax, (%rdi) ret - .align 2 -.Lst_uint16: - movzwq %ax, %rax - movq %rax, (%rdi) - .align 2 -.Lst_sint16: +E UNIX64_RET_SINT16 movswq %ax, %rax movq %rax, (%rdi) ret - .align 2 -.Lst_uint32: - movl %eax, %eax - movq %rax, (%rdi) - .align 2 -.Lst_sint32: +E UNIX64_RET_SINT32 cltq movq %rax, (%rdi) ret - .align 2 -.Lst_int64: +E UNIX64_RET_INT64 movq %rax, (%rdi) ret - - .align 2 -.Lst_float: - movss %xmm0, (%rdi) +E UNIX64_RET_XMM32 + movd %xmm0, (%rdi) ret - .align 2 -.Lst_double: - movsd %xmm0, (%rdi) +E UNIX64_RET_XMM64 + movq %xmm0, (%rdi) ret -.Lst_ldouble: +E UNIX64_RET_X87 fstpt (%rdi) ret - - .align 2 -.Lst_struct: - leaq -20(%rsp), %rsi /* Scratch area in redzone. */ - - /* We have to locate the values now, and since we don't want to - write too much data into the user's return value, we spill the - value to a 16 byte scratch area first. Bits 8, 9, and 10 - control where the values are located. Only one of the three - bits will be set; see ffi_prep_cif_machdep for the pattern. */ - movd %xmm0, %r10 - movd %xmm1, %r11 - testl $0x100, %ecx - cmovnz %rax, %rdx - cmovnz %r10, %rax - testl $0x200, %ecx - cmovnz %r10, %rdx - testl $0x400, %ecx - cmovnz %r10, %rax - cmovnz %r11, %rdx - movq %rax, (%rsi) +E UNIX64_RET_ST_RAX_RDX movq %rdx, 8(%rsi) + jmp 2f +E UNIX64_RET_ST_XMM0_RAX + movq %rax, 8(%rsi) + jmp 3f +E UNIX64_RET_ST_RAX_XMM0 + movq %xmm0, 8(%rsi) + jmp 2f +E UNIX64_RET_ST_XMM0_XMM1 + movq %xmm1, 8(%rsi) - /* Bits 12-31 contain the true size of the structure. Copy from - the scratch area to the true destination. */ - shrl $12, %ecx + .align 8 +3: movq %xmm0, (%rsi) + shrl $UNIX64_SIZE_SHIFT, %ecx rep movsb ret + .align 8 +2: movq %rax, (%rsi) + shrl $UNIX64_SIZE_SHIFT, %ecx + rep movsb + ret + +9: call abort@PLT /* Many times we can avoid loading any SSE registers at all. It's not worth an indirect jump to load the exact set of @@ -292,83 +270,67 @@ ffi_closure_unix64: cfi_adjust_cfa_offset(-ffi_closure_FS) /* The first byte of the return value contains the FFI_TYPE. */ + cmpb $UNIX64_RET_LAST, %al movzbl %al, %r10d - leaq .Lload_table(%rip), %r11 - movslq (%r11, %r10, 4), %r10 - addq %r11, %r10 + leaq 0f(%rip), %r11 + ja 9f + leaq (%r11, %r10, 8), %r10 jmp *%r10 - .section .rodata - .align 2 -.Lload_table: - .long .Lld_void-.Lload_table /* FFI_TYPE_VOID */ - .long .Lld_int32-.Lload_table /* FFI_TYPE_INT */ - .long .Lld_float-.Lload_table /* FFI_TYPE_FLOAT */ - .long .Lld_double-.Lload_table /* FFI_TYPE_DOUBLE */ - .long .Lld_ldouble-.Lload_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lld_int8-.Lload_table /* FFI_TYPE_UINT8 */ - .long .Lld_int8-.Lload_table /* FFI_TYPE_SINT8 */ - .long .Lld_int16-.Lload_table /* FFI_TYPE_UINT16 */ - .long .Lld_int16-.Lload_table /* FFI_TYPE_SINT16 */ - .long .Lld_int32-.Lload_table /* FFI_TYPE_UINT32 */ - .long .Lld_int32-.Lload_table /* FFI_TYPE_SINT32 */ - .long .Lld_int64-.Lload_table /* FFI_TYPE_UINT64 */ - .long .Lld_int64-.Lload_table /* FFI_TYPE_SINT64 */ - .long .Lld_struct-.Lload_table /* FFI_TYPE_STRUCT */ - .long .Lld_int64-.Lload_table /* FFI_TYPE_POINTER */ - .previous - - .align 2 -.Lld_void: + .align 8 +0: +E UNIX64_RET_VOID ret - - .align 2 -.Lld_int8: +E UNIX64_RET_UINT8 movzbl ffi_closure_RED_RVALUE(%rsp), %eax ret - .align 2 -.Lld_int16: +E UNIX64_RET_UINT16 movzwl ffi_closure_RED_RVALUE(%rsp), %eax ret - .align 2 -.Lld_int32: +E UNIX64_RET_UINT32 movl ffi_closure_RED_RVALUE(%rsp), %eax ret - .align 2 -.Lld_int64: +E UNIX64_RET_SINT8 + movsbl ffi_closure_RED_RVALUE(%rsp), %eax + ret +E UNIX64_RET_SINT16 + movswl ffi_closure_RED_RVALUE(%rsp), %eax + ret +E UNIX64_RET_SINT32 + movl ffi_closure_RED_RVALUE(%rsp), %eax + ret +E UNIX64_RET_INT64 movq ffi_closure_RED_RVALUE(%rsp), %rax ret - - .align 2 -.Lld_float: - movss ffi_closure_RED_RVALUE(%rsp), %xmm0 +E UNIX64_RET_XMM32 + movd ffi_closure_RED_RVALUE(%rsp), %xmm0 ret - .align 2 -.Lld_double: - movsd ffi_closure_RED_RVALUE(%rsp), %xmm0 +E UNIX64_RET_XMM64 + movq ffi_closure_RED_RVALUE(%rsp), %xmm0 ret - .align 2 -.Lld_ldouble: +E UNIX64_RET_X87 fldt ffi_closure_RED_RVALUE(%rsp) ret - - .align 2 -.Lld_struct: - /* There are four possibilities here, %rax/%rdx, %xmm0/%rax, - %rax/%xmm0, %xmm0/%xmm1. We collapse two by always loading - both rdx and xmm1 with the second word. For the remaining, - bit 8 set means xmm0 gets the second word, and bit 9 means - that rax gets the second word. */ - movq ffi_closure_RED_RVALUE(%rsp), %rcx +E UNIX64_RET_ST_RAX_RDX movq ffi_closure_RED_RVALUE+8(%rsp), %rdx + jmp 2f +E UNIX64_RET_ST_XMM0_RAX + movq ffi_closure_RED_RVALUE+8(%rsp), %rax + jmp 3f +E UNIX64_RET_ST_RAX_XMM0 + movq ffi_closure_RED_RVALUE+8(%rsp), %xmm0 + jmp 2f +E UNIX64_RET_ST_XMM0_XMM1 movq ffi_closure_RED_RVALUE+8(%rsp), %xmm1 - testl $0x100, %eax - cmovnz %rdx, %rcx - movd %rcx, %xmm0 - testl $0x200, %eax - movq ffi_closure_RED_RVALUE(%rsp), %rax - cmovnz %rdx, %rax + + .align 8 +3: movq ffi_closure_RED_RVALUE(%rsp), %xmm0 ret + .align 8 +2: movq ffi_closure_RED_RVALUE(%rsp), %rax + ret + +9: call abort@PLT cfi_endproc .size ffi_closure_unix64,.-ffi_closure_unix64 From 610c90bf7131de70318dc981a529a63ae36981b8 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 28 Oct 2014 11:21:50 -0700 Subject: [PATCH 08/58] x86_64: Add support for complex types --- src/x86/ffi64.c | 95 ++++++++++++++++++++++++++++++++++++++------ src/x86/internal64.h | 6 ++- src/x86/unix64.S | 63 +++++++++++++++-------------- 3 files changed, 121 insertions(+), 43 deletions(-) diff --git a/src/x86/ffi64.c b/src/x86/ffi64.c index a03061b..650f7bb 100644 --- a/src/x86/ffi64.c +++ b/src/x86/ffi64.c @@ -171,6 +171,7 @@ classify_argument (ffi_type *type, enum x86_64_reg_class classes[], case FFI_TYPE_UINT64: case FFI_TYPE_SINT64: case FFI_TYPE_POINTER: + do_integer: { size_t size = byte_offset + type->size; @@ -301,11 +302,42 @@ classify_argument (ffi_type *type, enum x86_64_reg_class classes[], } return words; } + case FFI_TYPE_COMPLEX: + { + ffi_type *inner = type->elements[0]; + switch (inner->type) + { + case FFI_TYPE_INT: + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + goto do_integer; - default: - FFI_ASSERT(0); + case FFI_TYPE_FLOAT: + classes[0] = X86_64_SSE_CLASS; + if (byte_offset % 8) + { + classes[1] = X86_64_SSESF_CLASS; + return 2; + } + return 1; + case FFI_TYPE_DOUBLE: + classes[0] = classes[1] = X86_64_SSEDF_CLASS; + return 2; +#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE + case FFI_TYPE_LONGDOUBLE: + classes[0] = X86_64_COMPLEX_X87_CLASS; + return 1; +#endif + } + } } - return 0; /* Never reached. */ + abort(); } /* Examine the argument and return set number of register required in each @@ -360,7 +392,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) { int gprcount, ssecount, i, avn, ngpr, nsse, flags; enum x86_64_reg_class classes[MAX_CLASSES]; - size_t bytes, n; + size_t bytes, n, rtype_size; ffi_type *rtype; if (cif->abi != FFI_UNIX64) @@ -369,6 +401,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) gprcount = ssecount = 0; rtype = cif->rtype; + rtype_size = rtype->size; switch (rtype->type) { case FFI_TYPE_VOID: @@ -421,16 +454,54 @@ ffi_prep_cif_machdep (ffi_cif *cif) } else { - /* Mark which registers the result appears in. */ _Bool sse0 = SSE_CLASS_P (classes[0]); - _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]); - if (sse0) - flags = (sse1 ? UNIX64_RET_ST_XMM0_XMM1 : UNIX64_RET_ST_XMM0_RAX); - else - flags = (sse1 ? UNIX64_RET_ST_RAX_XMM0 : UNIX64_RET_ST_RAX_RDX); - /* Mark the true size of the structure. */ - flags |= rtype->size << UNIX64_SIZE_SHIFT; + if (rtype_size == 4 && sse0) + flags = UNIX64_RET_XMM32; + else if (rtype_size == 8) + flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64; + else + { + _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]); + if (sse0 && sse1) + flags = UNIX64_RET_ST_XMM0_XMM1; + else if (sse0) + flags = UNIX64_RET_ST_XMM0_RAX; + else if (sse1) + flags = UNIX64_RET_ST_RAX_XMM0; + else + flags = UNIX64_RET_ST_RAX_RDX; + flags |= rtype_size << UNIX64_SIZE_SHIFT; + } + } + break; + case FFI_TYPE_COMPLEX: + switch (rtype->elements[0]->type) + { + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + case FFI_TYPE_INT: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + flags = UNIX64_RET_ST_RAX_RDX | (rtype_size << UNIX64_SIZE_SHIFT); + break; + case FFI_TYPE_FLOAT: + flags = UNIX64_RET_XMM64; + break; + case FFI_TYPE_DOUBLE: + flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT); + break; +#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE + case FFI_TYPE_LONGDOUBLE: + flags = UNIX64_RET_X87_2; + break; +#endif + default: + return FFI_BAD_TYPEDEF; } break; default: diff --git a/src/x86/internal64.h b/src/x86/internal64.h index 07b1b10..512e955 100644 --- a/src/x86/internal64.h +++ b/src/x86/internal64.h @@ -9,11 +9,13 @@ #define UNIX64_RET_XMM32 8 #define UNIX64_RET_XMM64 9 #define UNIX64_RET_X87 10 -#define UNIX64_RET_ST_RAX_RDX 11 +#define UNIX64_RET_X87_2 11 #define UNIX64_RET_ST_XMM0_RAX 12 #define UNIX64_RET_ST_RAX_XMM0 13 #define UNIX64_RET_ST_XMM0_XMM1 14 -#define UNIX64_RET_LAST 14 +#define UNIX64_RET_ST_RAX_RDX 15 + +#define UNIX64_RET_LAST 15 #define UNIX64_FLAG_RET_IN_MEM (1 << 10) #define UNIX64_FLAG_XMM_ARGS (1 << 11) diff --git a/src/x86/unix64.S b/src/x86/unix64.S index 0151229..6066bbf 100644 --- a/src/x86/unix64.S +++ b/src/x86/unix64.S @@ -156,9 +156,10 @@ E UNIX64_RET_XMM64 E UNIX64_RET_X87 fstpt (%rdi) ret -E UNIX64_RET_ST_RAX_RDX - movq %rdx, 8(%rsi) - jmp 2f +E UNIX64_RET_X87_2 + fstpt (%rdi) + fstpt 16(%rdi) + ret E UNIX64_RET_ST_XMM0_RAX movq %rax, 8(%rsi) jmp 3f @@ -167,14 +168,15 @@ E UNIX64_RET_ST_RAX_XMM0 jmp 2f E UNIX64_RET_ST_XMM0_XMM1 movq %xmm1, 8(%rsi) - - .align 8 -3: movq %xmm0, (%rsi) + jmp 3f +E UNIX64_RET_ST_RAX_RDX + movq %rdx, 8(%rsi) +2: movq %rax, (%rsi) shrl $UNIX64_SIZE_SHIFT, %ecx rep movsb ret .align 8 -2: movq %rax, (%rsi) +3: movq %xmm0, (%rsi) shrl $UNIX64_SIZE_SHIFT, %ecx rep movsb ret @@ -201,11 +203,11 @@ E UNIX64_RET_ST_XMM0_XMM1 .size ffi_call_unix64,.-ffi_call_unix64 /* 6 general registers, 8 vector registers, - 16 bytes of rvalue, 8 bytes of alignment. */ + 32 bytes of rvalue, 8 bytes of alignment. */ #define ffi_closure_OFS_G 0 #define ffi_closure_OFS_V (6*8) #define ffi_closure_OFS_RVALUE (ffi_closure_OFS_V + 8*16) -#define ffi_closure_FS (ffi_closure_OFS_RVALUE + 16 + 8) +#define ffi_closure_FS (ffi_closure_OFS_RVALUE + 32 + 8) /* The location of rvalue within the red zone after deallocating the frame. */ #define ffi_closure_RED_RVALUE (ffi_closure_OFS_RVALUE - ffi_closure_FS) @@ -275,6 +277,7 @@ ffi_closure_unix64: leaq 0f(%rip), %r11 ja 9f leaq (%r11, %r10, 8), %r10 + leaq ffi_closure_RED_RVALUE(%rsp), %rsi jmp *%r10 .align 8 @@ -282,52 +285,54 @@ ffi_closure_unix64: E UNIX64_RET_VOID ret E UNIX64_RET_UINT8 - movzbl ffi_closure_RED_RVALUE(%rsp), %eax + movzbl (%rsi), %eax ret E UNIX64_RET_UINT16 - movzwl ffi_closure_RED_RVALUE(%rsp), %eax + movzwl (%rsi), %eax ret E UNIX64_RET_UINT32 - movl ffi_closure_RED_RVALUE(%rsp), %eax + movl (%rsi), %eax ret E UNIX64_RET_SINT8 - movsbl ffi_closure_RED_RVALUE(%rsp), %eax + movsbl (%rsi), %eax ret E UNIX64_RET_SINT16 - movswl ffi_closure_RED_RVALUE(%rsp), %eax + movswl (%rsi), %eax ret E UNIX64_RET_SINT32 - movl ffi_closure_RED_RVALUE(%rsp), %eax + movl (%rsi), %eax ret E UNIX64_RET_INT64 - movq ffi_closure_RED_RVALUE(%rsp), %rax + movq (%rsi), %rax ret E UNIX64_RET_XMM32 - movd ffi_closure_RED_RVALUE(%rsp), %xmm0 + movd (%rsi), %xmm0 ret E UNIX64_RET_XMM64 - movq ffi_closure_RED_RVALUE(%rsp), %xmm0 + movq (%rsi), %xmm0 ret E UNIX64_RET_X87 - fldt ffi_closure_RED_RVALUE(%rsp) + fldt (%rsi) + ret +E UNIX64_RET_X87_2 + fldt 16(%rsi) + fldt (%rsi) ret -E UNIX64_RET_ST_RAX_RDX - movq ffi_closure_RED_RVALUE+8(%rsp), %rdx - jmp 2f E UNIX64_RET_ST_XMM0_RAX - movq ffi_closure_RED_RVALUE+8(%rsp), %rax + movq 8(%rsi), %rax jmp 3f E UNIX64_RET_ST_RAX_XMM0 - movq ffi_closure_RED_RVALUE+8(%rsp), %xmm0 + movq 8(%rsi), %xmm0 jmp 2f E UNIX64_RET_ST_XMM0_XMM1 - movq ffi_closure_RED_RVALUE+8(%rsp), %xmm1 - - .align 8 -3: movq ffi_closure_RED_RVALUE(%rsp), %xmm0 + movq 8(%rsi), %xmm1 + jmp 3f +E UNIX64_RET_ST_RAX_RDX + movq 8(%rsi), %rdx +2: movq (%rsi), %rax ret .align 8 -2: movq ffi_closure_RED_RVALUE(%rsp), %rax +3: movq (%rsi), %xmm0 ret 9: call abort@PLT From fc501750f41e2ee7f4268f490f590ce97a25a818 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 5 Nov 2014 16:33:44 +0100 Subject: [PATCH 09/58] testsuite: Fix return_complex2 vs excessive precision Use the previously computed rc2 to validate, rather than recomputing a floating point result with excess precision. --- testsuite/libffi.call/return_complex2.inc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/testsuite/libffi.call/return_complex2.inc b/testsuite/libffi.call/return_complex2.inc index dad4a0f..265170b 100644 --- a/testsuite/libffi.call/return_complex2.inc +++ b/testsuite/libffi.call/return_complex2.inc @@ -2,10 +2,14 @@ #include "ffitest.h" #include -static _Complex T_C_TYPE return_c(_Complex T_C_TYPE c1, _Complex T_C_TYPE c2, unsigned int in3, _Complex T_C_TYPE c4) +_Complex T_C_TYPE +return_c(_Complex T_C_TYPE c1, _Complex T_C_TYPE c2, + unsigned int in3, _Complex T_C_TYPE c4) { - return c1 + c2 + in3 + c4; + volatile _Complex T_C_TYPE r = c1 + c2 + in3 + c4; + return r; } + int main (void) { ffi_cif cif; @@ -35,6 +39,6 @@ int main (void) printf ("%f,%fi vs %f,%fi\n", T_CONV creal (rc), T_CONV cimag (rc), T_CONV creal (rc2), T_CONV cimag (rc2)); - CHECK(rc == c1 + c2 + in3 + c4); + CHECK(rc == rc2); exit(0); } From 7cf841324f3d15c46833e0f2ddf8a98c4b24711e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 7 Nov 2014 06:56:55 -0800 Subject: [PATCH 10/58] testsuite: Add two dg-do run markers Caught by clang warning about unused -L parameter. --- testsuite/libffi.call/float2.c | 1 + testsuite/libffi.call/return_ldl.c | 1 + 2 files changed, 2 insertions(+) diff --git a/testsuite/libffi.call/float2.c b/testsuite/libffi.call/float2.c index aae1abf..419c2bd 100644 --- a/testsuite/libffi.call/float2.c +++ b/testsuite/libffi.call/float2.c @@ -3,6 +3,7 @@ Limitations: none. PR: none. Originator: From the original ffitest.c */ +/* { dg-do run } */ #include "ffitest.h" #include "float.h" diff --git a/testsuite/libffi.call/return_ldl.c b/testsuite/libffi.call/return_ldl.c index 520e710..52a92fe 100644 --- a/testsuite/libffi.call/return_ldl.c +++ b/testsuite/libffi.call/return_ldl.c @@ -3,6 +3,7 @@ Limitations: none. PR: none. Originator: 20071113 */ +/* { dg-do run } */ #include "ffitest.h" From ef76205647bca77796882d31f6ab5e889f461f07 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 30 Oct 2014 12:13:31 -0700 Subject: [PATCH 11/58] x86: Tidy ffi_abi The x86_64 unix port only handles one ABI; don't define all of the other symbols. The UNIX64 symbol retains the same value. The i386 ports ought to have the same symbols, even if we can't yet unify the values without incrementing the libffi soname. --- src/x86/ffi.c | 2 +- src/x86/ffitarget.h | 62 ++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index c387fb5..90e3f79 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -260,7 +260,7 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) } #ifndef X86_WIN32 - if (cif->abi == FFI_SYSV || cif->abi == FFI_UNIX64) + if (cif->abi == FFI_SYSV) cif->bytes = (cif->bytes + 15) & ~0xF; #endif diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index 8c52573..a4c9573 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -76,44 +76,44 @@ typedef signed long ffi_sarg; #endif typedef enum ffi_abi { +#if defined(X86_WIN64) FFI_FIRST_ABI = 0, - - /* ---- Intel x86 Win32 ---------- */ -#ifdef X86_WIN32 - FFI_SYSV, - FFI_STDCALL, - FFI_THISCALL, - FFI_FASTCALL, - FFI_MS_CDECL, - FFI_PASCAL, - FFI_REGISTER, - FFI_LAST_ABI, -#ifdef _MSC_VER - FFI_DEFAULT_ABI = FFI_MS_CDECL -#else - FFI_DEFAULT_ABI = FFI_SYSV -#endif - -#elif defined(X86_WIN64) FFI_WIN64, FFI_LAST_ABI, FFI_DEFAULT_ABI = FFI_WIN64 -#else - /* ---- Intel x86 and AMD x86-64 - */ - FFI_SYSV, - FFI_UNIX64, /* Unix variants all use the same ABI for x86-64 */ - FFI_THISCALL, - FFI_FASTCALL, - FFI_STDCALL, - FFI_PASCAL, - FFI_REGISTER, +#elif defined(X86_64) || (defined (__x86_64__) && defined (X86_DARWIN)) + FFI_FIRST_ABI = 1, + FFI_UNIX64, FFI_LAST_ABI, -#if defined(__i386__) || defined(__i386) - FFI_DEFAULT_ABI = FFI_SYSV -#else FFI_DEFAULT_ABI = FFI_UNIX64 -#endif + +#elif defined(X86_WIN32) + FFI_FIRST_ABI = 0, + FFI_SYSV = 1, + FFI_STDCALL = 2, + FFI_THISCALL = 3, + FFI_FASTCALL = 4, + FFI_MS_CDECL = 5, + FFI_PASCAL = 6, + FFI_REGISTER = 7, + FFI_LAST_ABI, +# ifdef _MSC_VER + FFI_DEFAULT_ABI = FFI_MS_CDECL +# else + FFI_DEFAULT_ABI = FFI_SYSV +# endif +#else + FFI_FIRST_ABI = 0, + FFI_SYSV = 1, + FFI_THISCALL = 3, + FFI_FASTCALL = 4, + FFI_STDCALL = 5, + FFI_PASCAL = 6, + FFI_REGISTER = 7, + FFI_MS_CDECL = 8, + FFI_LAST_ABI, + FFI_DEFAULT_ABI = FFI_SYSV #endif } ffi_abi; #endif From 4b2fad8fd583f1d9c36db0044c4d46316207aa49 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 30 Oct 2014 12:41:31 -0700 Subject: [PATCH 12/58] x86: Remove some conditional compilation Removal of ifdefs made possible to due to ffi_abi unification. --- src/x86/ffi.c | 81 ++++++++++++++++++++------------------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 90e3f79..98aa008 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -217,34 +217,25 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_STRUCT: #ifndef X86 + /* ??? This should be a different ABI rather than an ifdef. */ if (cif->rtype->size == 1) - { - cif->flags = FFI_TYPE_SMALL_STRUCT_1B; /* same as char size */ - } + cif->flags = FFI_TYPE_SMALL_STRUCT_1B; /* same as char size */ else if (cif->rtype->size == 2) - { - cif->flags = FFI_TYPE_SMALL_STRUCT_2B; /* same as short size */ - } + cif->flags = FFI_TYPE_SMALL_STRUCT_2B; /* same as short size */ else if (cif->rtype->size == 4) - { - cif->flags = FFI_TYPE_INT; /* same as int type */ - } + cif->flags = FFI_TYPE_INT; /* same as int type */ else if (cif->rtype->size == 8) - { - cif->flags = FFI_TYPE_SINT64; /* same as int64 type */ - } + cif->flags = FFI_TYPE_SINT64; /* same as int64 type */ else #endif - { -#ifdef X86_WIN32 - if (cif->abi == FFI_MS_CDECL) - cif->flags = FFI_TYPE_MS_STRUCT; - else -#endif - cif->flags = FFI_TYPE_STRUCT; - /* allocate space for return value pointer */ - cif->bytes += ALIGN(sizeof(void*), FFI_SIZEOF_ARG); - } + { + if (cif->abi == FFI_MS_CDECL) + cif->flags = FFI_TYPE_MS_STRUCT; + else + cif->flags = FFI_TYPE_STRUCT; + /* Allocate space for return value pointer. */ + cif->bytes += ALIGN(sizeof(void*), FFI_SIZEOF_ARG); + } break; default: @@ -259,10 +250,8 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) cif->bytes += (unsigned)ALIGN((*ptr)->size, FFI_SIZEOF_ARG); } -#ifndef X86_WIN32 if (cif->abi == FFI_SYSV) - cif->bytes = (cif->bytes + 15) & ~0xF; -#endif + cif->bytes = ALIGN (cif->bytes, 15); return FFI_OK; } @@ -577,14 +566,12 @@ ffi_prep_closure_loc (ffi_closure* closure, &ffi_closure_STDCALL, (void*)codeloc); } -#ifdef X86_WIN32 else if (cif->abi == FFI_MS_CDECL) { FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, (void*)codeloc); } -#endif /* X86_WIN32 */ else { return FFI_BAD_ABI; @@ -610,40 +597,36 @@ ffi_prep_raw_closure_loc (ffi_raw_closure* closure, { int i; - if (cif->abi != FFI_SYSV -#ifdef X86_WIN32 - && cif->abi != FFI_THISCALL -#endif - ) - return FFI_BAD_ABI; - - /* we currently don't support certain kinds of arguments for raw + /* We currently don't support certain kinds of arguments for raw closures. This should be implemented by a separate assembly language routine, since it would require argument processing, something we don't do now for performance. */ - for (i = cif->nargs-1; i >= 0; i--) { FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_STRUCT); FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_LONGDOUBLE); } - -#ifdef X86_WIN32 - if (cif->abi == FFI_SYSV) + + switch (cif->abi) { -#endif - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, - codeloc); #ifdef X86_WIN32 - } - else if (cif->abi == FFI_THISCALL) - { - FFI_INIT_TRAMPOLINE_RAW_THISCALL (&closure->tramp[0], &ffi_closure_raw_THISCALL, codeloc, cif->bytes); - } + case FFI_THISCALL: + FFI_INIT_TRAMPOLINE_RAW_THISCALL (&closure->tramp[0], + &ffi_closure_raw_THISCALL, + codeloc, cif->bytes); + break; #endif - closure->cif = cif; + case FFI_SYSV: + FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, + codeloc); + break; + default: + return FFI_BAD_ABI; + } + + closure->cif = cif; + closure->fun = fun; closure->user_data = user_data; - closure->fun = fun; return FFI_OK; } From e7b0056d60f29675072454a575eed98053fc63d6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 30 Oct 2014 13:57:39 -0700 Subject: [PATCH 13/58] x86: Force FFI_TYPE_LONGDOUBLE different from FFI_TYPE_DOUBLE There are few abis that set double = long double. Eliminate the conditional compilation and let this code simply be unused there. --- src/x86/ffi.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 98aa008..339ca89 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -34,6 +34,18 @@ #include #include +/* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; + all further uses in this file will refer to the 80-bit type. */ +#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE +# if FFI_TYPE_LONGDOUBLE != 4 +# error FFI_TYPE_LONGDOUBLE out of date +# endif +#else +# undef FFI_TYPE_LONGDOUBLE +# define FFI_TYPE_LONGDOUBLE 4 +#endif + + /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments */ @@ -205,9 +217,7 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_SINT64: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif cif->flags = (unsigned) cif->rtype->type; break; From 159d3788eb52b89ed6bc5dd6ebb85c02cd150232 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 31 Oct 2014 12:07:02 -0700 Subject: [PATCH 14/58] x86: Convert to gas generated unwind info --- src/x86/sysv.S | 173 ++++++------------------------------------------- 1 file changed, 20 insertions(+), 153 deletions(-) diff --git a/src/x86/sysv.S b/src/x86/sysv.S index 3bd5477..fd13bc0 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -30,6 +30,7 @@ #define LIBFFI_ASM #include #include +#include .text @@ -40,11 +41,12 @@ .type ffi_call_SYSV,@function ffi_call_SYSV: -.LFB1: + cfi_startproc pushl %ebp -.LCFI0: + cfi_adjust_cfa_offset(4) + cfi_rel_offset(%ebp, 0) movl %esp,%ebp -.LCFI1: + cfi_def_cfa_register(%ebp) /* Make room for all of the new args. */ movl 16(%ebp),%ecx subl %ecx,%esp @@ -163,7 +165,7 @@ epilogue: movl %ebp,%esp popl %ebp ret -.LFE1: + cfi_endproc .ffi_call_SYSV_end: .size ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV @@ -173,11 +175,12 @@ FFI_HIDDEN (ffi_closure_SYSV) .type ffi_closure_SYSV, @function ffi_closure_SYSV: -.LFB2: + cfi_startproc pushl %ebp -.LCFI2: + cfi_adjust_cfa_offset(4) + cfi_rel_offset(%ebp, 0) movl %esp, %ebp -.LCFI3: + cfi_def_cfa_register(%ebp) subl $40, %esp leal -24(%ebp), %edx movl %edx, -12(%ebp) /* resp */ @@ -199,12 +202,13 @@ ffi_closure_SYSV: call ffi_closure_SYSV_inner #else movl %ebx, 8(%esp) -.LCFI7: + cfi_offset(%ebx, -40) call 1f 1: popl %ebx addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx call ffi_closure_SYSV_inner@PLT movl 8(%esp), %ebx + cfi_restore(%ebx) #endif movl -12(%ebp), %ecx cmpl $FFI_TYPE_INT, %eax @@ -251,7 +255,7 @@ ffi_closure_SYSV: movl %ebp, %esp popl %ebp ret $4 -.LFE2: + cfi_endproc .size ffi_closure_SYSV, .-ffi_closure_SYSV #if !FFI_NO_RAW_API @@ -278,13 +282,14 @@ FFI_HIDDEN (ffi_closure_raw_SYSV) .type ffi_closure_raw_SYSV, @function ffi_closure_raw_SYSV: -.LFB3: + cfi_startproc pushl %ebp -.LCFI4: + cfi_adjust_cfa_offset(4) + cfi_rel_offset(%ebp, 0) movl %esp, %ebp -.LCFI5: + cfi_def_cfa_register(%ebp) pushl %esi -.LCFI6: + cfi_offset(%esi, -12) subl $36, %esp movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ @@ -335,149 +340,11 @@ ffi_closure_raw_SYSV: movl -24(%ebp), %eax movl -20(%ebp), %edx jmp .Lrcls_epilogue -.LFE3: + cfi_endproc .size ffi_closure_raw_SYSV, .-ffi_closure_raw_SYSV -#endif - -#if defined __GNUC__ -/* Only emit dwarf unwind info when building with GNU toolchain. */ - -#if defined __PIC__ -# if defined __sun__ && defined __svr4__ -/* 32-bit Solaris 2/x86 uses datarel encoding for PIC. GNU ld before 2.22 - doesn't correctly sort .eh_frame_hdr with mixed encodings, so match this. */ -# define FDE_ENCODING 0x30 /* datarel */ -# define FDE_ENCODE(X) X@GOTOFF -# else -# define FDE_ENCODING 0x1b /* pcrel sdata4 */ -# if defined HAVE_AS_X86_PCREL -# define FDE_ENCODE(X) X-. -# else -# define FDE_ENCODE(X) X@rel -# endif -# endif -#else -# define FDE_ENCODING 0 /* absolute */ -# define FDE_ENCODE(X) X -#endif - - .section .eh_frame,EH_FRAME_FLAGS,@progbits -.Lframe1: - .long .LECIE1-.LSCIE1 /* Length of Common Information Entry */ -.LSCIE1: - .long 0x0 /* CIE Identifier Tag */ - .byte 0x1 /* CIE Version */ -#ifdef HAVE_AS_ASCII_PSEUDO_OP -#ifdef __PIC__ - .ascii "zR\0" /* CIE Augmentation */ -#else - .ascii "\0" /* CIE Augmentation */ -#endif -#elif defined HAVE_AS_STRING_PSEUDO_OP -#ifdef __PIC__ - .string "zR" /* CIE Augmentation */ -#else - .string "" /* CIE Augmentation */ -#endif -#else -#error missing .ascii/.string -#endif - .byte 0x1 /* .uleb128 0x1; CIE Code Alignment Factor */ - .byte 0x7c /* .sleb128 -4; CIE Data Alignment Factor */ - .byte 0x8 /* CIE RA Column */ -#ifdef __PIC__ - .byte 0x1 /* .uleb128 0x1; Augmentation size */ - .byte FDE_ENCODING -#endif - .byte 0xc /* DW_CFA_def_cfa */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x88 /* DW_CFA_offset, column 0x8 */ - .byte 0x1 /* .uleb128 0x1 */ - .align 4 -.LECIE1: -.LSFDE1: - .long .LEFDE1-.LASFDE1 /* FDE Length */ -.LASFDE1: - .long .LASFDE1-.Lframe1 /* FDE CIE offset */ - .long FDE_ENCODE(.LFB1) /* FDE initial location */ - .long .LFE1-.LFB1 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI0-.LFB1 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI1-.LCFI0 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ - .align 4 -.LEFDE1: -.LSFDE2: - .long .LEFDE2-.LASFDE2 /* FDE Length */ -.LASFDE2: - .long .LASFDE2-.Lframe1 /* FDE CIE offset */ - .long FDE_ENCODE(.LFB2) /* FDE initial location */ - .long .LFE2-.LFB2 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI2-.LFB2 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI3-.LCFI2 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ -#if !defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE && defined __PIC__ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI7-.LCFI3 - .byte 0x83 /* DW_CFA_offset, column 0x3 */ - .byte 0xa /* .uleb128 0xa */ -#endif - .align 4 -.LEFDE2: - -#if !FFI_NO_RAW_API - -.LSFDE3: - .long .LEFDE3-.LASFDE3 /* FDE Length */ -.LASFDE3: - .long .LASFDE3-.Lframe1 /* FDE CIE offset */ - .long FDE_ENCODE(.LFB3) /* FDE initial location */ - .long .LFE3-.LFB3 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI4-.LFB3 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI5-.LCFI4 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI6-.LCFI5 - .byte 0x86 /* DW_CFA_offset, column 0x6 */ - .byte 0x3 /* .uleb128 0x3 */ - .align 4 -.LEFDE3: - -#endif -#endif +#endif /* !FFI_NO_RAW_API */ #endif /* ifndef __x86_64__ */ - #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits #endif From b9ac94f3af9b1ca1853850911b7d3bc058296c71 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Nov 2014 15:10:34 -0700 Subject: [PATCH 15/58] x86: Rewrite ffi_call Decouple the assembly from FFI_TYPE_*. Merge prep_args with ffi_call, passing the frame and the stack to the assembly. Note that this patch isn't really standalone, as this breaks closures. --- src/x86/ffi.c | 609 ++++++++++++++++++++++---------------------- src/x86/ffitarget.h | 4 - src/x86/internal.h | 23 ++ src/x86/sysv.S | 243 +++++++++--------- 4 files changed, 452 insertions(+), 427 deletions(-) create mode 100644 src/x86/internal.h diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 339ca89..1c77bb8 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -29,10 +29,10 @@ ----------------------------------------------------------------------- */ #ifndef __x86_64__ - #include #include #include +#include "internal.h" /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; all further uses in this file will refer to the 80-bit type. */ @@ -45,276 +45,277 @@ # define FFI_TYPE_LONGDOUBLE 4 #endif +#if defined(__GNUC__) && !defined(__declspec) +# define __declspec(x) __attribute__((x)) +#endif -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments */ - -unsigned int ffi_prep_args(char *stack, extended_cif *ecif); -unsigned int ffi_prep_args(char *stack, extended_cif *ecif) +/* Perform machine dependent cif processing. */ +ffi_status FFI_HIDDEN +ffi_prep_cif_machdep(ffi_cif *cif) { - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - const int cabi = ecif->cif->abi; - const int dir = (cabi == FFI_PASCAL || cabi == FFI_REGISTER) ? -1 : +1; - unsigned int stack_args_count = 0; - void *p_stack_data[3]; - char *argp2 = stack; + size_t bytes = 0; + int i, n, flags, cabi = cif->abi; - argp = stack; - - if ((ecif->cif->flags == FFI_TYPE_STRUCT - || ecif->cif->flags == FFI_TYPE_MS_STRUCT)) + switch (cabi) { - /* For fastcall/thiscall/register this is first register-passed - argument. */ - if (cabi == FFI_THISCALL || cabi == FFI_FASTCALL || cabi == FFI_REGISTER) - { - p_stack_data[stack_args_count] = argp; - ++stack_args_count; - } - - *(void **) argp = ecif->rvalue; - argp += sizeof(void*); + case FFI_SYSV: + case FFI_STDCALL: + case FFI_THISCALL: + case FFI_FASTCALL: + case FFI_MS_CDECL: + case FFI_PASCAL: + case FFI_REGISTER: + break; + default: + return FFI_BAD_ABI; } - p_arg = ecif->cif->arg_types; - p_argv = ecif->avalue; - if (dir < 0) - { - const int nargs = ecif->cif->nargs - 1; - if (nargs > 0) - { - p_arg += nargs; - p_argv += nargs; - } - } - - for (i = ecif->cif->nargs; - i != 0; - i--, p_arg += dir, p_argv += dir) - { - /* Align if necessary */ - if ((sizeof(void*) - 1) & (size_t) argp) - argp = (char *) ALIGN(argp, sizeof(void*)); - - size_t z = (*p_arg)->size; - - if (z < FFI_SIZEOF_ARG) - { - z = FFI_SIZEOF_ARG; - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(ffi_sarg *) argp = (ffi_sarg)*(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(ffi_arg *) argp = (ffi_arg)*(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(ffi_sarg *) argp = (ffi_sarg)*(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(ffi_arg *) argp = (ffi_arg)*(UINT16 *)(* p_argv); - break; - - case FFI_TYPE_SINT32: - *(ffi_sarg *) argp = (ffi_sarg)*(SINT32 *)(* p_argv); - break; - - case FFI_TYPE_UINT32: - *(ffi_arg *) argp = (ffi_arg)*(UINT32 *)(* p_argv); - break; - - case FFI_TYPE_STRUCT: - *(ffi_arg *) argp = *(ffi_arg *)(* p_argv); - break; - - default: - FFI_ASSERT(0); - } - } - else - { - memcpy(argp, *p_argv, z); - } - - /* For thiscall/fastcall/register convention register-passed arguments - are the first two none-floating-point arguments with a size - smaller or equal to sizeof (void*). */ - if ((z == FFI_SIZEOF_ARG) - && ((cabi == FFI_REGISTER) - || (cabi == FFI_THISCALL && stack_args_count < 1) - || (cabi == FFI_FASTCALL && stack_args_count < 2)) - && ((*p_arg)->type != FFI_TYPE_FLOAT && (*p_arg)->type != FFI_TYPE_STRUCT) - ) - { - if (dir < 0 && stack_args_count > 2) - { - /* Iterating arguments backwards, so first register-passed argument - will be passed last. Shift temporary values to make place. */ - p_stack_data[0] = p_stack_data[1]; - p_stack_data[1] = p_stack_data[2]; - stack_args_count = 2; - } - - p_stack_data[stack_args_count] = argp; - ++stack_args_count; - } - - argp += z; - } - - /* We need to move the register-passed arguments for thiscall, - fastcall, register on top of stack, so that those can be moved - to registers by call-handler. */ - if (stack_args_count > 0) - { - if (dir < 0 && stack_args_count > 1) - { - /* Reverse order if iterating arguments backwards */ - ffi_arg tmp = *(ffi_arg*) p_stack_data[0]; - *(ffi_arg*) p_stack_data[0] = *(ffi_arg*) p_stack_data[stack_args_count - 1]; - *(ffi_arg*) p_stack_data[stack_args_count - 1] = tmp; - } - - int i; - for (i = 0; i < stack_args_count; i++) - { - if (p_stack_data[i] != argp2) - { - ffi_arg tmp = *(ffi_arg*) p_stack_data[i]; - memmove (argp2 + FFI_SIZEOF_ARG, argp2, (size_t) ((char*) p_stack_data[i] - (char*)argp2)); - *(ffi_arg *) argp2 = tmp; - } - - argp2 += FFI_SIZEOF_ARG; - } - } - - return stack_args_count; - return 0; -} - -/* Perform machine dependent cif processing */ -ffi_status ffi_prep_cif_machdep(ffi_cif *cif) -{ - unsigned int i; - ffi_type **ptr; - - /* Set the return type flag */ switch (cif->rtype->type) { case FFI_TYPE_VOID: - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT64: + flags = X86_RET_VOID; + break; case FFI_TYPE_FLOAT: + flags = X86_RET_FLOAT; + break; case FFI_TYPE_DOUBLE: + flags = X86_RET_DOUBLE; + break; case FFI_TYPE_LONGDOUBLE: - cif->flags = (unsigned) cif->rtype->type; + flags = X86_RET_LDOUBLE; break; - + case FFI_TYPE_UINT8: + flags = X86_RET_UINT8; + break; + case FFI_TYPE_UINT16: + flags = X86_RET_UINT16; + break; + case FFI_TYPE_SINT8: + flags = X86_RET_SINT8; + break; + case FFI_TYPE_SINT16: + flags = X86_RET_SINT16; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + case FFI_TYPE_POINTER: + flags = X86_RET_INT32; + break; + case FFI_TYPE_SINT64: case FFI_TYPE_UINT64: - cif->flags = FFI_TYPE_SINT64; + flags = X86_RET_INT64; break; - case FFI_TYPE_STRUCT: #ifndef X86 /* ??? This should be a different ABI rather than an ifdef. */ if (cif->rtype->size == 1) - cif->flags = FFI_TYPE_SMALL_STRUCT_1B; /* same as char size */ + flags = X86_RET_STRUCT_1B; else if (cif->rtype->size == 2) - cif->flags = FFI_TYPE_SMALL_STRUCT_2B; /* same as short size */ + flags = X86_RET_STRUCT_2B; else if (cif->rtype->size == 4) - cif->flags = FFI_TYPE_INT; /* same as int type */ + flags = X86_RET_INT32; else if (cif->rtype->size == 8) - cif->flags = FFI_TYPE_SINT64; /* same as int64 type */ + flags = X86_RET_INT64; else #endif { - if (cif->abi == FFI_MS_CDECL) - cif->flags = FFI_TYPE_MS_STRUCT; - else - cif->flags = FFI_TYPE_STRUCT; + switch (cabi) + { + case FFI_THISCALL: + case FFI_FASTCALL: + case FFI_STDCALL: + case FFI_MS_CDECL: + flags = X86_RET_STRUCTARG; + break; + default: + flags = X86_RET_STRUCTPOP; + break; + } /* Allocate space for return value pointer. */ - cif->bytes += ALIGN(sizeof(void*), FFI_SIZEOF_ARG); + bytes += ALIGN (sizeof(void*), FFI_SIZEOF_ARG); } break; - default: - cif->flags = FFI_TYPE_INT; - break; + return FFI_BAD_TYPEDEF; } + cif->flags = flags; - for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++) + for (i = 0, n = cif->nargs; i < n; i++) { - if (((*ptr)->alignment - 1) & cif->bytes) - cif->bytes = ALIGN(cif->bytes, (*ptr)->alignment); - cif->bytes += (unsigned)ALIGN((*ptr)->size, FFI_SIZEOF_ARG); - } + ffi_type *t = cif->arg_types[i]; - if (cif->abi == FFI_SYSV) - cif->bytes = ALIGN (cif->bytes, 15); + bytes = ALIGN (bytes, t->alignment); + bytes += ALIGN (t->size, FFI_SIZEOF_ARG); + } + cif->bytes = ALIGN (bytes, 16); return FFI_OK; } -extern void -ffi_call_win32(unsigned int (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned, unsigned *, void (*fn)(void)); -extern void ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, - unsigned, unsigned, unsigned *, void (*fn)(void)); - -void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static ffi_arg +extend_basic_type(void *arg, int type) { - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ - - if (rvalue == NULL - && (cif->flags == FFI_TYPE_STRUCT - || cif->flags == FFI_TYPE_MS_STRUCT)) + switch (type) { - ecif.rvalue = alloca(cif->rtype->size); - } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) - { -#ifndef X86_WIN32 - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, - fn); - break; -#else - case FFI_SYSV: - case FFI_MS_CDECL: -#endif - case FFI_STDCALL: - case FFI_THISCALL: - case FFI_FASTCALL: - case FFI_PASCAL: - case FFI_REGISTER: - ffi_call_win32(ffi_prep_args, &ecif, cif->abi, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; + case FFI_TYPE_SINT8: + return *(SINT8 *)arg; + case FFI_TYPE_UINT8: + return *(UINT8 *)arg; + case FFI_TYPE_SINT16: + return *(SINT16 *)arg; + case FFI_TYPE_UINT16: + return *(UINT16 *)arg; + + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + case FFI_TYPE_POINTER: + case FFI_TYPE_FLOAT: + return *(UINT32 *)arg; + default: - FFI_ASSERT(0); + abort(); + } +} + +struct call_frame +{ + void *ebp; /* 0 */ + void *retaddr; /* 4 */ + void (*fn)(void); /* 8 */ + int flags; /* 12 */ + void *rvalue; /* 16 */ + unsigned regs[3]; /* 20-28 */ +}; + +struct abi_params +{ + int dir; /* parameter growth direction */ + int nregs; /* number of register parameters */ + int regs[3]; +}; + +static const struct abi_params abi_params[FFI_LAST_ABI] = { + [FFI_SYSV] = { 1, 0 }, + [FFI_THISCALL] = { 1, 1, { R_ECX } }, + [FFI_FASTCALL] = { 1, 2, { R_ECX, R_EDX } }, + [FFI_STDCALL] = { 1, 0 }, + [FFI_PASCAL] = { -1, 0 }, + [FFI_REGISTER] = { -1, 3, { R_EAX, R_EDX, R_ECX } }, + [FFI_MS_CDECL] = { 1, 0 } +}; + +extern void ffi_call_i386(struct call_frame *, char *) + FFI_HIDDEN __declspec(fastcall); + +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + size_t rsize, bytes; + struct call_frame *frame; + char *stack, *argp; + ffi_type **arg_types; + int flags, cabi, i, n, dir, narg_reg; + const struct abi_params *pabi; + + flags = cif->flags; + cabi = cif->abi; + pabi = &abi_params[cabi]; + dir = pabi->dir; + + rsize = 0; + if (rvalue == NULL) + { + switch (flags) + { + case X86_RET_FLOAT: + case X86_RET_DOUBLE: + case X86_RET_LDOUBLE: + case X86_RET_STRUCTPOP: + case X86_RET_STRUCTARG: + /* The float cases need to pop the 387 stack. + The struct cases need to pass a valid pointer to the callee. */ + rsize = cif->rtype->size; + break; + default: + /* We can pretend that the callee returns nothing. */ + flags = X86_RET_VOID; + break; + } + } + + bytes = cif->bytes; + stack = alloca(bytes + sizeof(*frame) + rsize); + argp = (dir < 0 ? stack + bytes : stack); + frame = (struct call_frame *)(stack + bytes); + if (rsize) + rvalue = frame + 1; + + frame->fn = fn; + frame->flags = flags; + frame->rvalue = rvalue; + + narg_reg = 0; + switch (flags) + { + case X86_RET_STRUCTARG: + /* The pointer is passed as the first argument. */ + if (pabi->nregs > 0) + { + frame->regs[pabi->regs[0]] = (unsigned)rvalue; + narg_reg = 1; + break; + } + /* fallthru */ + case X86_RET_STRUCTPOP: + *(void **)argp = rvalue; + argp += sizeof(void *); break; } + + arg_types = cif->arg_types; + for (i = 0, n = cif->nargs; i < n; i++) + { + ffi_type *ty = arg_types[i]; + void *valp = avalue[i]; + size_t z = ty->size; + int t = ty->type; + + if (z <= FFI_SIZEOF_ARG && t != FFI_TYPE_STRUCT) + { + ffi_arg val = extend_basic_type (valp, t); + + if (t != FFI_TYPE_FLOAT && narg_reg < pabi->nregs) + frame->regs[pabi->regs[narg_reg++]] = val; + else if (dir < 0) + { + argp -= 4; + *(ffi_arg *)argp = val; + } + else + { + *(ffi_arg *)argp = val; + argp += 4; + } + } + else + { + size_t za = ALIGN (z, FFI_SIZEOF_ARG); + if (dir < 0) + { + argp -= za; + memcpy (argp, valp, z); + } + else + { + memcpy (argp, valp, z); + argp += za; + } + } + } + FFI_ASSERT (dir > 0 || argp == stack); + + ffi_call_i386 (frame, stack); } @@ -641,88 +642,92 @@ ffi_prep_raw_closure_loc (ffi_raw_closure* closure, return FFI_OK; } -static unsigned int -ffi_prep_args_raw(char *stack, extended_cif *ecif) -{ - const ffi_cif *cif = ecif->cif; - unsigned int i, passed_regs = 0; - - const unsigned int abi = cif->abi; - const unsigned int max_regs = (abi == FFI_THISCALL) ? 1 - : (abi == FFI_FASTCALL) ? 2 - : (abi == FFI_REGISTER) ? 3 - : 0; - - if (cif->flags == FFI_TYPE_STRUCT) - ++passed_regs; - - for (i = 0; i < cif->nargs && passed_regs <= max_regs; i++) - { - if (cif->arg_types[i]->type == FFI_TYPE_FLOAT - || cif->arg_types[i]->type == FFI_TYPE_STRUCT) - continue; - - size_t sz = cif->arg_types[i]->size; - if (sz == 0 || sz > FFI_SIZEOF_ARG) - continue; - - ++passed_regs; - } - - memcpy (stack, ecif->avalue, cif->bytes); - return passed_regs; -} - -/* we borrow this routine from libffi (it must be changed, though, to - * actually call the function passed in the first argument. as of - * libffi-1.20, this is not the case.) - */ - void -ffi_raw_call(ffi_cif *cif, void (*fn)(void), void *rvalue, ffi_raw *fake_avalue) +ffi_raw_call(ffi_cif *cif, void (*fn)(void), void *rvalue, ffi_raw *avalue) { - extended_cif ecif; - void **avalue = (void **)fake_avalue; + size_t rsize, bytes; + struct call_frame *frame; + char *stack, *argp; + ffi_type **arg_types; + int flags, cabi, i, n, narg_reg; + const struct abi_params *pabi; - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ + flags = cif->flags; + cabi = cif->abi; + pabi = &abi_params[cabi]; - if (rvalue == NULL - && (cif->flags == FFI_TYPE_STRUCT - || cif->flags == FFI_TYPE_MS_STRUCT)) + rsize = 0; + if (rvalue == NULL) { - ecif.rvalue = alloca(cif->rtype->size); + switch (flags) + { + case X86_RET_FLOAT: + case X86_RET_DOUBLE: + case X86_RET_LDOUBLE: + case X86_RET_STRUCTPOP: + case X86_RET_STRUCTARG: + /* The float cases need to pop the 387 stack. + The struct cases need to pass a valid pointer to the callee. */ + rsize = cif->rtype->size; + break; + default: + /* We can pretend that the callee returns nothing. */ + flags = X86_RET_VOID; + break; + } } - else - ecif.rvalue = rvalue; - - - switch (cif->abi) + + bytes = cif->bytes; + argp = stack = alloca(bytes + sizeof(*frame) + rsize); + frame = (struct call_frame *)(stack + bytes); + if (rsize) + rvalue = frame + 1; + + narg_reg = 0; + switch (flags) { -#ifndef X86_WIN32 - case FFI_SYSV: - ffi_call_SYSV(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; -#else - case FFI_SYSV: - case FFI_MS_CDECL: -#endif - case FFI_STDCALL: - case FFI_THISCALL: - case FFI_FASTCALL: - case FFI_PASCAL: - case FFI_REGISTER: - ffi_call_win32(ffi_prep_args_raw, &ecif, cif->abi, cif->bytes, cif->flags, - ecif.rvalue, fn); - break; - default: - FFI_ASSERT(0); + case X86_RET_STRUCTARG: + /* The pointer is passed as the first argument. */ + if (pabi->nregs > 0) + { + frame->regs[pabi->regs[0]] = (unsigned)rvalue; + narg_reg = 1; + break; + } + /* fallthru */ + case X86_RET_STRUCTPOP: + *(void **)argp = rvalue; + argp += sizeof(void *); + bytes -= sizeof(void *); break; } + + arg_types = cif->arg_types; + for (i = 0, n = cif->nargs; narg_reg < pabi->nregs && i < n; i++) + { + ffi_type *ty = arg_types[i]; + size_t z = ty->size; + int t = ty->type; + + if (z <= FFI_SIZEOF_ARG && t != FFI_TYPE_STRUCT && t != FFI_TYPE_FLOAT) + { + ffi_arg val = extend_basic_type (avalue, t); + frame->regs[pabi->regs[narg_reg++]] = val; + z = FFI_SIZEOF_ARG; + } + else + { + memcpy (argp, avalue, z); + z = ALIGN (z, FFI_SIZEOF_ARG); + argp += z; + } + avalue += z; + bytes -= z; + } + if (i < n) + memcpy (argp, avalue, bytes); + + ffi_call_i386 (frame, stack); } #endif /* !FFI_NO_RAW_API */ #endif /* !__x86_64__ */ diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index a4c9573..91e429c 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -98,11 +98,7 @@ typedef enum ffi_abi { FFI_PASCAL = 6, FFI_REGISTER = 7, FFI_LAST_ABI, -# ifdef _MSC_VER FFI_DEFAULT_ABI = FFI_MS_CDECL -# else - FFI_DEFAULT_ABI = FFI_SYSV -# endif #else FFI_FIRST_ABI = 0, FFI_SYSV = 1, diff --git a/src/x86/internal.h b/src/x86/internal.h new file mode 100644 index 0000000..480c1d0 --- /dev/null +++ b/src/x86/internal.h @@ -0,0 +1,23 @@ +#define X86_RET_FLOAT 0 +#define X86_RET_DOUBLE 1 +#define X86_RET_LDOUBLE 2 +#define X86_RET_SINT8 3 +#define X86_RET_SINT16 4 +#define X86_RET_UINT8 5 +#define X86_RET_UINT16 6 +#define X86_RET_INT64 7 +#define X86_RET_INT32 8 +#define X86_RET_VOID 9 +#define X86_RET_STRUCTPOP 10 +#define X86_RET_STRUCTARG 11 +#define X86_RET_STRUCT_1B 12 +#define X86_RET_STRUCT_2B 13 +#define X86_RET_UNUSED14 14 +#define X86_RET_UNUSED15 15 + +#define X86_RET_TYPE_MASK 15 +#define X86_RET_POP_SHIFT 4 + +#define R_EAX 0 +#define R_EDX 1 +#define R_ECX 2 diff --git a/src/x86/sysv.S b/src/x86/sysv.S index fd13bc0..d0b8417 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -31,143 +31,144 @@ #include #include #include +#include "internal.h" -.text +#define C2(X, Y) X ## Y +#define C1(X, Y) C2(X, Y) +#ifdef __USER_LABEL_PREFIX__ +# define C(X) C1(__USER_LABEL_PREFIX__, X) +#else +# define C(X) X +#endif -.globl ffi_prep_args +#ifdef __ELF__ +# define ENDF(X) .type X,@function; .size X, . - X +#else +# define ENDF(X) +#endif - .align 4 -.globl ffi_call_SYSV - .type ffi_call_SYSV,@function +/* This macro allows the safe creation of jump tables without an + actual table. The entry points into the table are all 8 bytes. + The use of ORG asserts that we're at the correct location. */ +#define E(X) .align 8; .org 0b + X * 8 -ffi_call_SYSV: + .text + .align 16 + .globl C(ffi_call_i386) + FFI_HIDDEN(C(ffi_call_i386)) + +/* This is declared as + + void ffi_call_i386(struct ffi_call_frame *frame, char *argp) + __attribute__((fastcall)); + + This the arguments are present in + + ecx: frame + edx: argp +*/ + +C(ffi_call_i386): cfi_startproc - pushl %ebp - cfi_adjust_cfa_offset(4) + movl (%esp), %eax /* move the return address */ + movl %ebp, (%ecx) /* store %ebp into local frame */ + movl %eax, 4(%ecx) /* store retaddr into local frame */ + + /* New stack frame based off ebp. This is a itty bit of unwind + trickery in that the CFA *has* changed. There is no easy way + to describe it correctly on entry to the function. Fortunately, + it doesn't matter too much since at all points we can correctly + unwind back to ffi_call. Note that the location to which we + moved the return address is (the new) CFA-4, so from the + perspective of the unwind info, it hasn't moved. */ + movl %ecx, %ebp + cfi_def_cfa(%ebp, 8) cfi_rel_offset(%ebp, 0) - movl %esp,%ebp - cfi_def_cfa_register(%ebp) - /* Make room for all of the new args. */ - movl 16(%ebp),%ecx - subl %ecx,%esp - /* Align the stack pointer to 16-bytes */ - andl $0xfffffff0, %esp + movl %edx, %esp /* set outgoing argument stack */ + movl 20+R_EAX*4(%ebp), %eax /* set register arguments */ + movl 20+R_EDX*4(%ebp), %edx + movl 20+R_ECX*4(%ebp), %ecx - movl %esp,%eax + call *8(%ebp) - /* Place all of the ffi_prep_args in position */ - pushl 12(%ebp) - pushl %eax - call *8(%ebp) + movl 12(%ebp), %ecx /* load return type code */ + movl %ebx, 8(%ebp) /* preserve %ebx */ + cfi_rel_offset(%ebp, 8) - /* Return stack to previous state and call the function */ - addl $8,%esp - - call *28(%ebp) - - /* Load %ecx with the return type code */ - movl 20(%ebp),%ecx - - /* Protect %esi. We're going to pop it in the epilogue. */ - pushl %esi - - /* If the return value pointer is NULL, assume no return value. */ - cmpl $0,24(%ebp) - jne 0f - - /* Even if there is no space for the return value, we are - obliged to handle floating-point values. */ - cmpl $FFI_TYPE_FLOAT,%ecx - jne noretval - fstp %st(0) - - jmp epilogue + andl $X86_RET_TYPE_MASK, %ecx +#ifdef __PIC__ + call __x86.get_pc_thunk.bx +1: leal 0f-1b(%ebx, %ecx, 8), %ebx +#else + leal 0f(,%ecx, 8), %ebx +#endif + movl 16(%ebp), %ecx /* load result address */ + jmp *%ebx + .align 8 0: - call 1f +E(X86_RET_FLOAT) + fstps (%ecx) + jmp 9f +E(X86_RET_DOUBLE) + fstpl (%ecx) + jmp 9f +E(X86_RET_LDOUBLE) + fstpt (%ecx) + jmp 9f +E(X86_RET_SINT8) + movsbl %al, %eax + mov %eax, (%ecx) + jmp 9f +E(X86_RET_SINT16) + movswl %ax, %eax + mov %eax, (%ecx) + jmp 9f +E(X86_RET_UINT8) + movzbl %al, %eax + mov %eax, (%ecx) + jmp 9f +E(X86_RET_UINT16) + movzwl %ax, %eax + mov %eax, (%ecx) + jmp 9f +E(X86_RET_INT64) + movl %edx, 4(%ecx) + /* fallthru */ +E(X86_RET_INT32) + movl %eax, (%ecx) + /* fallthru */ +E(X86_RET_VOID) +9: movl 8(%ebp), %ebx + movl %ebp, %esp + popl %ebp + cfi_remember_state + cfi_def_cfa(%esp, 4) + cfi_restore(%ebx) + cfi_restore(%ebp) + ret + cfi_restore_state -.Lstore_table: - .long noretval-.Lstore_table /* FFI_TYPE_VOID */ - .long retint-.Lstore_table /* FFI_TYPE_INT */ - .long retfloat-.Lstore_table /* FFI_TYPE_FLOAT */ - .long retdouble-.Lstore_table /* FFI_TYPE_DOUBLE */ - .long retlongdouble-.Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long retuint8-.Lstore_table /* FFI_TYPE_UINT8 */ - .long retsint8-.Lstore_table /* FFI_TYPE_SINT8 */ - .long retuint16-.Lstore_table /* FFI_TYPE_UINT16 */ - .long retsint16-.Lstore_table /* FFI_TYPE_SINT16 */ - .long retint-.Lstore_table /* FFI_TYPE_UINT32 */ - .long retint-.Lstore_table /* FFI_TYPE_SINT32 */ - .long retint64-.Lstore_table /* FFI_TYPE_UINT64 */ - .long retint64-.Lstore_table /* FFI_TYPE_SINT64 */ - .long retstruct-.Lstore_table /* FFI_TYPE_STRUCT */ - .long retint-.Lstore_table /* FFI_TYPE_POINTER */ +E(X86_RET_STRUCTPOP) + jmp 9b +E(X86_RET_STRUCTARG) + jmp 9b +E(X86_RET_STRUCT_1B) + movb %al, (%ecx) + jmp 9b +E(X86_RET_STRUCT_2B) + movw %ax, (%ecx) + jmp 9b -1: - pop %esi - add (%esi, %ecx, 4), %esi - jmp *%esi + /* Fill out the table so that bad values are predictable. */ +E(X86_RET_UNUSED14) + ud2 +E(X86_RET_UNUSED15) + ud2 - /* Sign/zero extend as appropriate. */ -retsint8: - movsbl %al, %eax - jmp retint - -retsint16: - movswl %ax, %eax - jmp retint - -retuint8: - movzbl %al, %eax - jmp retint - -retuint16: - movzwl %ax, %eax - jmp retint - -retfloat: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstps (%ecx) - jmp epilogue - -retdouble: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpl (%ecx) - jmp epilogue - -retlongdouble: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpt (%ecx) - jmp epilogue - -retint64: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - movl %edx,4(%ecx) - jmp epilogue - -retint: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - -retstruct: - /* Nothing to do! */ - -noretval: -epilogue: - popl %esi - movl %ebp,%esp - popl %ebp - ret cfi_endproc -.ffi_call_SYSV_end: - .size ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV +ENDF(C(ffi_call_i386)) .align 4 FFI_HIDDEN (ffi_closure_SYSV) From b21ec1ce783f09335c0e6c6f96ccc7e89da5a1b4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 5 Nov 2014 10:15:25 +0100 Subject: [PATCH 16/58] x86: Rewrite closures Move everything into sysv.S, removing win32.S and freebsd.S. Handle all abis with a single ffi_closure_inner function. Move complexity of the raw THISCALL trampoline into assembly instead of the trampoline itself. Only push the context for the REGISTER abi; let the rest receive it in a register. --- Makefile.am | 13 +- src/x86/ffi.c | 399 +++++-------- src/x86/ffitarget.h | 6 +- src/x86/freebsd.S | 463 --------------- src/x86/sysv.S | 611 ++++++++++++++----- src/x86/win32.S | 1351 ------------------------------------------- 6 files changed, 600 insertions(+), 2243 deletions(-) delete mode 100644 src/x86/freebsd.S delete mode 100644 src/x86/win32.S diff --git a/Makefile.am b/Makefile.am index 3d1ecae..97a7bd3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,8 +37,8 @@ EXTRA_DIST = LICENSE ChangeLog.v1 ChangeLog.libgcj \ src/sh64/sysv.S src/sh64/ffitarget.h src/sparc/v8.S \ src/sparc/v9.S src/sparc/ffitarget.h src/sparc/ffi.c \ src/x86/darwin64.S src/x86/ffi.c src/x86/sysv.S \ - src/x86/win32.S src/x86/darwin.S src/x86/ffiw64.c src/x86/win64.S \ - src/x86/freebsd.S src/x86/ffi64.c src/x86/unix64.S \ + src/x86/darwin.S src/x86/ffiw64.c src/x86/win64.S \ + src/x86/ffi64.c src/x86/unix64.S \ src/x86/ffitarget.h src/pa/ffitarget.h src/pa/ffi.c \ src/pa/linux.S src/pa/hpux32.S src/frv/ffi.c src/bfin/ffi.c \ src/bfin/ffitarget.h src/bfin/sysv.S src/frv/eabi.S \ @@ -126,22 +126,19 @@ if BFIN nodist_libffi_la_SOURCES += src/bfin/ffi.c src/bfin/sysv.S endif if X86 -nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/sysv.S src/x86/win32.S +nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/sysv.S endif if X86_FREEBSD -nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/freebsd.S src/x86/win32.S +nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/sysv.S endif if X86_WIN32 -nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/win32.S +nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/sysv.S endif if X86_WIN64 nodist_libffi_la_SOURCES += src/x86/ffiw64.c src/x86/win64.S endif if X86_DARWIN nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/darwin.S src/x86/ffi64.c src/x86/darwin64.S -if X86_DARWIN32 -nodist_libffi_la_SOURCES += src/x86/win32.S -endif endif if SPARC nodist_libffi_la_SOURCES += src/sparc/ffi.c src/sparc/v8.S src/sparc/v9.S diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 1c77bb8..40e47d2 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -321,225 +321,106 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) /** private members **/ -/* The following __attribute__((regparm(1))) decorations will have no effect - on MSVC or SUNPRO_C -- standard conventions apply. */ -static unsigned int ffi_prep_incoming_args (char *stack, void **ret, - void** args, ffi_cif* cif); -void FFI_HIDDEN ffi_closure_SYSV (ffi_closure *) - __attribute__ ((regparm(1))); -unsigned int FFI_HIDDEN ffi_closure_SYSV_inner (ffi_closure *, void **, void *) - __attribute__ ((regparm(1))); -unsigned int FFI_HIDDEN ffi_closure_WIN32_inner (ffi_closure *, void **, void *) - __attribute__ ((regparm(1))); -void FFI_HIDDEN ffi_closure_raw_SYSV (ffi_raw_closure *) - __attribute__ ((regparm(1))); -#ifdef X86_WIN32 -void FFI_HIDDEN ffi_closure_raw_THISCALL (ffi_raw_closure *) - __attribute__ ((regparm(1))); -#endif -void FFI_HIDDEN ffi_closure_STDCALL (ffi_closure *); -void FFI_HIDDEN ffi_closure_THISCALL (ffi_closure *); -void FFI_HIDDEN ffi_closure_FASTCALL (ffi_closure *); -void FFI_HIDDEN ffi_closure_REGISTER (ffi_closure *); +void FFI_HIDDEN ffi_closure_i386(void); +void FFI_HIDDEN ffi_closure_STDCALL(void); +void FFI_HIDDEN ffi_closure_REGISTER(void); -/* This function is jumped to by the trampoline */ - -unsigned int FFI_HIDDEN __attribute__ ((regparm(1))) -ffi_closure_SYSV_inner (ffi_closure *closure, void **respp, void *args) +struct closure_frame { - /* our various things... */ - ffi_cif *cif; - void **arg_area; + unsigned rettemp[4]; /* 0 */ + unsigned regs[3]; /* 16-24 */ + ffi_cif *cif; /* 28 */ + void (*fun)(ffi_cif*,void*,void**,void*); /* 32 */ + void *user_data; /* 36 */ +}; - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); - - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will change RESP to point to the - * structure return address. */ - - ffi_prep_incoming_args(args, respp, arg_area, cif); - - (closure->fun) (cif, *respp, arg_area, closure->user_data); - - return cif->flags; -} - -unsigned int FFI_HIDDEN __attribute__ ((regparm(1))) -ffi_closure_WIN32_inner (ffi_closure *closure, void **respp, void *args) +int FFI_HIDDEN __declspec(fastcall) +ffi_closure_inner (struct closure_frame *frame, char *stack) { - /* our various things... */ - ffi_cif *cif; - void **arg_area; - unsigned int ret; + ffi_cif *cif = frame->cif; + int cabi, i, n, flags, dir, narg_reg; + const struct abi_params *pabi; + ffi_type **arg_types; + char *argp; + void *rvalue; + void **avalue; - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); + cabi = cif->abi; + flags = cif->flags; + narg_reg = 0; + rvalue = frame->rettemp; + pabi = &abi_params[cabi]; + dir = pabi->dir; + argp = (dir < 0 ? stack + cif->bytes : stack); - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will change RESP to point to the - * structure return address. */ - - ret = ffi_prep_incoming_args(args, respp, arg_area, cif); - - (closure->fun) (cif, *respp, arg_area, closure->user_data); - - return ret; -} - -static unsigned int -ffi_prep_incoming_args(char *stack, void **rvalue, void **avalue, - ffi_cif *cif) -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - const int cabi = cif->abi; - const int dir = (cabi == FFI_PASCAL || cabi == FFI_REGISTER) ? -1 : +1; - const unsigned int max_stack_count = (cabi == FFI_THISCALL) ? 1 - : (cabi == FFI_FASTCALL) ? 2 - : (cabi == FFI_REGISTER) ? 3 - : 0; - unsigned int passed_regs = 0; - void *p_stack_data[3] = { stack - 1 }; - - argp = stack; - argp += max_stack_count * FFI_SIZEOF_ARG; - - if ((cif->flags == FFI_TYPE_STRUCT - || cif->flags == FFI_TYPE_MS_STRUCT)) + switch (flags) { - if (passed_regs < max_stack_count) - { - *rvalue = *(void**) (stack + (passed_regs*FFI_SIZEOF_ARG)); - ++passed_regs; - } + case X86_RET_STRUCTARG: + if (pabi->nregs > 0) + { + rvalue = (void *)frame->regs[pabi->regs[0]]; + narg_reg = 1; + frame->rettemp[0] = (unsigned)rvalue; + break; + } + /* fallthru */ + case X86_RET_STRUCTPOP: + rvalue = *(void **)argp; + argp += sizeof(void *); + break; + } + + n = cif->nargs; + avalue = alloca(sizeof(void *) * n); + + arg_types = cif->arg_types; + for (i = 0; i < n; ++i) + { + size_t z = arg_types[i]->size; + int t = arg_types[i]->type; + void *valp; + + if (z <= FFI_SIZEOF_ARG && t != FFI_TYPE_STRUCT) + { + if (t != FFI_TYPE_FLOAT && narg_reg < pabi->nregs) + valp = &frame->regs[pabi->regs[narg_reg++]]; + else if (dir < 0) + { + argp -= 4; + valp = argp; + } + else + { + valp = argp; + argp += 4; + } + } else - { - *rvalue = *(void **) argp; - argp += sizeof(void *); - } + { + size_t za = ALIGN (z, FFI_SIZEOF_ARG); + if (dir < 0) + { + argp -= za; + valp = argp; + } + else + { + valp = argp; + argp += za; + } + } + + avalue[i] = valp; } - /* Do register arguments first */ - for (i = 0, p_arg = cif->arg_types; - i < cif->nargs && passed_regs < max_stack_count; - i++, p_arg++) - { - if ((*p_arg)->type == FFI_TYPE_FLOAT - || (*p_arg)->type == FFI_TYPE_STRUCT) - continue; + frame->fun (cif, rvalue, avalue, frame->user_data); - size_t sz = (*p_arg)->size; - if(sz == 0 || sz > FFI_SIZEOF_ARG) - continue; - - p_stack_data[passed_regs] = avalue + i; - avalue[i] = stack + (passed_regs*FFI_SIZEOF_ARG); - ++passed_regs; - } - - p_arg = cif->arg_types; - p_argv = avalue; - if (dir < 0) - { - const int nargs = cif->nargs - 1; - if (nargs > 0) - { - p_arg += nargs; - p_argv += nargs; - } - } - - for (i = cif->nargs; - i != 0; - i--, p_arg += dir, p_argv += dir) - { - /* Align if necessary */ - if ((sizeof(void*) - 1) & (size_t) argp) - argp = (char *) ALIGN(argp, sizeof(void*)); - - size_t z = (*p_arg)->size; - - if (passed_regs > 0 - && z <= FFI_SIZEOF_ARG - && (p_argv == p_stack_data[0] - || p_argv == p_stack_data[1] - || p_argv == p_stack_data[2])) - { - /* Already assigned a register value */ - continue; - } - else - { - /* because we're little endian, this is what it turns into. */ - *p_argv = (void*) argp; - } - - argp += z; - } - - return (size_t)argp - (size_t)stack; + if (cabi == FFI_STDCALL) + return flags + (cif->bytes << X86_RET_POP_SHIFT); + else + return flags; } -/* How to make a trampoline. Derived from gcc/config/i386/i386.c. */ - -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX) \ -{ unsigned char *__tramp = (unsigned char*)(TRAMP); \ - unsigned int __fun = (unsigned int)(FUN); \ - unsigned int __ctx = (unsigned int)(CTX); \ - unsigned int __dis = __fun - (__ctx + 10); \ - *(unsigned char*) &__tramp[0] = 0xb8; \ - *(unsigned int*) &__tramp[1] = __ctx; /* movl __ctx, %eax */ \ - *(unsigned char*) &__tramp[5] = 0xe9; \ - *(unsigned int*) &__tramp[6] = __dis; /* jmp __fun */ \ - } - -#define FFI_INIT_TRAMPOLINE_RAW_THISCALL(TRAMP,FUN,CTX,SIZE) \ -{ unsigned char *__tramp = (unsigned char*)(TRAMP); \ - unsigned int __fun = (unsigned int)(FUN); \ - unsigned int __ctx = (unsigned int)(CTX); \ - unsigned int __dis = __fun - (__ctx + 49); \ - unsigned short __size = (unsigned short)(SIZE); \ - *(unsigned int *) &__tramp[0] = 0x8324048b; /* mov (%esp), %eax */ \ - *(unsigned int *) &__tramp[4] = 0x4c890cec; /* sub $12, %esp */ \ - *(unsigned int *) &__tramp[8] = 0x04890424; /* mov %ecx, 4(%esp) */ \ - *(unsigned char*) &__tramp[12] = 0x24; /* mov %eax, (%esp) */ \ - *(unsigned char*) &__tramp[13] = 0xb8; \ - *(unsigned int *) &__tramp[14] = __size; /* mov __size, %eax */ \ - *(unsigned int *) &__tramp[18] = 0x08244c8d; /* lea 8(%esp), %ecx */ \ - *(unsigned int *) &__tramp[22] = 0x4802e8c1; /* shr $2, %eax ; dec %eax */ \ - *(unsigned short*) &__tramp[26] = 0x0b74; /* jz 1f */ \ - *(unsigned int *) &__tramp[28] = 0x8908518b; /* 2b: mov 8(%ecx), %edx */ \ - *(unsigned int *) &__tramp[32] = 0x04c18311; /* mov %edx, (%ecx) ; add $4, %ecx */ \ - *(unsigned char*) &__tramp[36] = 0x48; /* dec %eax */ \ - *(unsigned short*) &__tramp[37] = 0xf575; /* jnz 2b ; 1f: */ \ - *(unsigned char*) &__tramp[39] = 0xb8; \ - *(unsigned int*) &__tramp[40] = __ctx; /* movl __ctx, %eax */ \ - *(unsigned char *) &__tramp[44] = 0xe8; \ - *(unsigned int*) &__tramp[45] = __dis; /* call __fun */ \ - *(unsigned char*) &__tramp[49] = 0xc2; /* ret */ \ - *(unsigned short*) &__tramp[50] = (__size + 8); /* ret (__size + 8) */ \ - } - -#define FFI_INIT_TRAMPOLINE_WIN32(TRAMP,FUN,CTX) \ -{ unsigned char *__tramp = (unsigned char*)(TRAMP); \ - unsigned int __fun = (unsigned int)(FUN); \ - unsigned int __ctx = (unsigned int)(CTX); \ - unsigned int __dis = __fun - (__ctx + 10); \ - *(unsigned char*) &__tramp[0] = 0x68; \ - *(unsigned int*) &__tramp[1] = __ctx; /* push __ctx */ \ - *(unsigned char*) &__tramp[5] = 0xe9; \ - *(unsigned int*) &__tramp[6] = __dis; /* jmp __fun */ \ - } - -/* the cif must already be prep'ed */ - ffi_status ffi_prep_closure_loc (ffi_closure* closure, ffi_cif* cif, @@ -547,50 +428,40 @@ ffi_prep_closure_loc (ffi_closure* closure, void *user_data, void *codeloc) { - if (cif->abi == FFI_SYSV) - { - FFI_INIT_TRAMPOLINE (&closure->tramp[0], - &ffi_closure_SYSV, - (void*)codeloc); - } - else if (cif->abi == FFI_REGISTER) - { - FFI_INIT_TRAMPOLINE_WIN32 (&closure->tramp[0], - &ffi_closure_REGISTER, - (void*)codeloc); - } - else if (cif->abi == FFI_FASTCALL) - { - FFI_INIT_TRAMPOLINE_WIN32 (&closure->tramp[0], - &ffi_closure_FASTCALL, - (void*)codeloc); - } - else if (cif->abi == FFI_THISCALL) - { - FFI_INIT_TRAMPOLINE_WIN32 (&closure->tramp[0], - &ffi_closure_THISCALL, - (void*)codeloc); - } - else if (cif->abi == FFI_STDCALL || cif->abi == FFI_PASCAL) - { - FFI_INIT_TRAMPOLINE_WIN32 (&closure->tramp[0], - &ffi_closure_STDCALL, - (void*)codeloc); - } - else if (cif->abi == FFI_MS_CDECL) - { - FFI_INIT_TRAMPOLINE (&closure->tramp[0], - &ffi_closure_SYSV, - (void*)codeloc); - } - else + char *tramp = closure->tramp; + void (*dest)(void); + int op = 0xb8; /* movl imm, %eax */ + + switch (cif->abi) { + case FFI_SYSV: + case FFI_THISCALL: + case FFI_FASTCALL: + case FFI_MS_CDECL: + dest = ffi_closure_i386; + break; + case FFI_STDCALL: + case FFI_PASCAL: + dest = ffi_closure_STDCALL; + break; + case FFI_REGISTER: + dest = ffi_closure_REGISTER; + op = 0x68; /* pushl imm */ + default: return FFI_BAD_ABI; } - - closure->cif = cif; + + /* movl or pushl immediate. */ + tramp[0] = op; + *(void **)(tramp + 1) = codeloc; + + /* jmp dest */ + tramp[5] = 0xe9; + *(unsigned *)(tramp + 6) = (unsigned)dest - ((unsigned)codeloc + 10); + + closure->cif = cif; + closure->fun = fun; closure->user_data = user_data; - closure->fun = fun; return FFI_OK; } @@ -599,13 +470,18 @@ ffi_prep_closure_loc (ffi_closure* closure, #if !FFI_NO_RAW_API +void FFI_HIDDEN ffi_closure_raw_SYSV(void); +void FFI_HIDDEN ffi_closure_raw_THISCALL(void); + ffi_status -ffi_prep_raw_closure_loc (ffi_raw_closure* closure, - ffi_cif* cif, +ffi_prep_raw_closure_loc (ffi_raw_closure *closure, + ffi_cif *cif, void (*fun)(ffi_cif*,void*,ffi_raw*,void*), void *user_data, void *codeloc) { + char *tramp = closure->tramp; + void (*dest)(void); int i; /* We currently don't support certain kinds of arguments for raw @@ -613,28 +489,33 @@ ffi_prep_raw_closure_loc (ffi_raw_closure* closure, language routine, since it would require argument processing, something we don't do now for performance. */ for (i = cif->nargs-1; i >= 0; i--) - { - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_STRUCT); - FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_LONGDOUBLE); - } + switch (cif->arg_types[i]->type) + { + case FFI_TYPE_STRUCT: + case FFI_TYPE_LONGDOUBLE: + return FFI_BAD_TYPEDEF; + } switch (cif->abi) { -#ifdef X86_WIN32 case FFI_THISCALL: - FFI_INIT_TRAMPOLINE_RAW_THISCALL (&closure->tramp[0], - &ffi_closure_raw_THISCALL, - codeloc, cif->bytes); + dest = ffi_closure_raw_THISCALL; break; -#endif case FFI_SYSV: - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, - codeloc); + dest = ffi_closure_raw_SYSV; break; default: return FFI_BAD_ABI; } + /* movl imm, %eax. */ + tramp[0] = 0xb8; + *(void **)(tramp + 1) = codeloc; + + /* jmp dest */ + tramp[5] = 0xe9; + *(unsigned *)(tramp + 6) = (unsigned)dest - ((unsigned)codeloc + 10); + closure->cif = cif; closure->fun = fun; closure->user_data = user_data; diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index 91e429c..8fff29f 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -129,11 +129,7 @@ typedef enum ffi_abi { # define FFI_NATIVE_RAW_API 0 # define FFI_GO_CLOSURES 1 #else -# ifdef X86_WIN32 -# define FFI_TRAMPOLINE_SIZE 52 -# else -# define FFI_TRAMPOLINE_SIZE 10 -# endif +# define FFI_TRAMPOLINE_SIZE 12 # define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ #endif diff --git a/src/x86/freebsd.S b/src/x86/freebsd.S deleted file mode 100644 index 97e0b4e..0000000 --- a/src/x86/freebsd.S +++ /dev/null @@ -1,463 +0,0 @@ -/* ----------------------------------------------------------------------- - freebsd.S - Copyright (c) 1996, 1998, 2001, 2002, 2003, 2005 Red Hat, Inc. - Copyright (c) 2008 Björn König - - X86 Foreign Function Interface for FreeBSD - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------ */ - -#ifndef __x86_64__ - -#define LIBFFI_ASM -#include -#include - -.text - -.globl ffi_prep_args - - .align 4 -.globl ffi_call_SYSV - .type ffi_call_SYSV,@function - -ffi_call_SYSV: -.LFB1: - pushl %ebp -.LCFI0: - movl %esp,%ebp -.LCFI1: - /* Make room for all of the new args. */ - movl 16(%ebp),%ecx - subl %ecx,%esp - - /* Align the stack pointer to 16-bytes */ - andl $0xfffffff0, %esp - - movl %esp,%eax - - /* Place all of the ffi_prep_args in position */ - pushl 12(%ebp) - pushl %eax - call *8(%ebp) - - /* Return stack to previous state and call the function */ - addl $8,%esp - - call *28(%ebp) - - /* Load %ecx with the return type code */ - movl 20(%ebp),%ecx - - /* Protect %esi. We're going to pop it in the epilogue. */ - pushl %esi - - /* If the return value pointer is NULL, assume no return value. */ - cmpl $0,24(%ebp) - jne 0f - - /* Even if there is no space for the return value, we are - obliged to handle floating-point values. */ - cmpl $FFI_TYPE_FLOAT,%ecx - jne noretval - fstp %st(0) - - jmp epilogue - -0: - call 1f - -.Lstore_table: - .long noretval-.Lstore_table /* FFI_TYPE_VOID */ - .long retint-.Lstore_table /* FFI_TYPE_INT */ - .long retfloat-.Lstore_table /* FFI_TYPE_FLOAT */ - .long retdouble-.Lstore_table /* FFI_TYPE_DOUBLE */ - .long retlongdouble-.Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long retuint8-.Lstore_table /* FFI_TYPE_UINT8 */ - .long retsint8-.Lstore_table /* FFI_TYPE_SINT8 */ - .long retuint16-.Lstore_table /* FFI_TYPE_UINT16 */ - .long retsint16-.Lstore_table /* FFI_TYPE_SINT16 */ - .long retint-.Lstore_table /* FFI_TYPE_UINT32 */ - .long retint-.Lstore_table /* FFI_TYPE_SINT32 */ - .long retint64-.Lstore_table /* FFI_TYPE_UINT64 */ - .long retint64-.Lstore_table /* FFI_TYPE_SINT64 */ - .long retstruct-.Lstore_table /* FFI_TYPE_STRUCT */ - .long retint-.Lstore_table /* FFI_TYPE_POINTER */ - .long retstruct1b-.Lstore_table /* FFI_TYPE_SMALL_STRUCT_1B */ - .long retstruct2b-.Lstore_table /* FFI_TYPE_SMALL_STRUCT_2B */ - -1: - pop %esi - add (%esi, %ecx, 4), %esi - jmp *%esi - - /* Sign/zero extend as appropriate. */ -retsint8: - movsbl %al, %eax - jmp retint - -retsint16: - movswl %ax, %eax - jmp retint - -retuint8: - movzbl %al, %eax - jmp retint - -retuint16: - movzwl %ax, %eax - jmp retint - -retfloat: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstps (%ecx) - jmp epilogue - -retdouble: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpl (%ecx) - jmp epilogue - -retlongdouble: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - fstpt (%ecx) - jmp epilogue - -retint64: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - movl %edx,4(%ecx) - jmp epilogue - -retstruct1b: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movb %al,0(%ecx) - jmp epilogue - -retstruct2b: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movw %ax,0(%ecx) - jmp epilogue - -retint: - /* Load %ecx with the pointer to storage for the return value */ - movl 24(%ebp),%ecx - movl %eax,0(%ecx) - -retstruct: - /* Nothing to do! */ - -noretval: -epilogue: - popl %esi - movl %ebp,%esp - popl %ebp - ret -.LFE1: -.ffi_call_SYSV_end: - .size ffi_call_SYSV,.ffi_call_SYSV_end-ffi_call_SYSV - - .align 4 -FFI_HIDDEN (ffi_closure_SYSV) -.globl ffi_closure_SYSV - .type ffi_closure_SYSV, @function - -ffi_closure_SYSV: -.LFB2: - pushl %ebp -.LCFI2: - movl %esp, %ebp -.LCFI3: - subl $40, %esp - leal -24(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 8(%ebp), %edx - movl %edx, 4(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ -#if defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE || !defined __PIC__ - call ffi_closure_SYSV_inner -#else - movl %ebx, 8(%esp) -.LCFI7: - call 1f -1: popl %ebx - addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx - call ffi_closure_SYSV_inner@PLT - movl 8(%esp), %ebx -#endif - movl -12(%ebp), %ecx - cmpl $FFI_TYPE_INT, %eax - je .Lcls_retint - - /* Handle FFI_TYPE_UINT8, FFI_TYPE_SINT8, FFI_TYPE_UINT16, - FFI_TYPE_SINT16, FFI_TYPE_UINT32, FFI_TYPE_SINT32. */ - cmpl $FFI_TYPE_UINT64, %eax - jge 0f - cmpl $FFI_TYPE_UINT8, %eax - jge .Lcls_retint - -0: cmpl $FFI_TYPE_FLOAT, %eax - je .Lcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je .Lcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je .Lcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je .Lcls_retllong - cmpl $FFI_TYPE_SMALL_STRUCT_1B, %eax - je .Lcls_retstruct1b - cmpl $FFI_TYPE_SMALL_STRUCT_2B, %eax - je .Lcls_retstruct2b - cmpl $FFI_TYPE_STRUCT, %eax - je .Lcls_retstruct -.Lcls_epilogue: - movl %ebp, %esp - popl %ebp - ret -.Lcls_retint: - movl (%ecx), %eax - jmp .Lcls_epilogue -.Lcls_retfloat: - flds (%ecx) - jmp .Lcls_epilogue -.Lcls_retdouble: - fldl (%ecx) - jmp .Lcls_epilogue -.Lcls_retldouble: - fldt (%ecx) - jmp .Lcls_epilogue -.Lcls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp .Lcls_epilogue -.Lcls_retstruct1b: - movsbl (%ecx), %eax - jmp .Lcls_epilogue -.Lcls_retstruct2b: - movswl (%ecx), %eax - jmp .Lcls_epilogue -.Lcls_retstruct: - movl %ebp, %esp - popl %ebp - ret $4 -.LFE2: - .size ffi_closure_SYSV, .-ffi_closure_SYSV - -#if !FFI_NO_RAW_API - -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) -#define CIF_FLAGS_OFFSET 20 - - .align 4 -FFI_HIDDEN (ffi_closure_raw_SYSV) -.globl ffi_closure_raw_SYSV - .type ffi_closure_raw_SYSV, @function - -ffi_closure_raw_SYSV: -.LFB3: - pushl %ebp -.LCFI4: - movl %esp, %ebp -.LCFI5: - pushl %esi -.LCFI6: - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 8(%ebp), %edx /* __builtin_dwarf_cfa () */ - movl %edx, 8(%esp) /* raw_args */ - leal -24(%ebp), %edx - movl %edx, 4(%esp) /* &res */ - movl %esi, (%esp) /* cif */ - call *RAW_CLOSURE_FUN_OFFSET(%eax) /* closure->fun */ - movl CIF_FLAGS_OFFSET(%esi), %eax /* rtype */ - cmpl $FFI_TYPE_INT, %eax - je .Lrcls_retint - - /* Handle FFI_TYPE_UINT8, FFI_TYPE_SINT8, FFI_TYPE_UINT16, - FFI_TYPE_SINT16, FFI_TYPE_UINT32, FFI_TYPE_SINT32. */ - cmpl $FFI_TYPE_UINT64, %eax - jge 0f - cmpl $FFI_TYPE_UINT8, %eax - jge .Lrcls_retint -0: - cmpl $FFI_TYPE_FLOAT, %eax - je .Lrcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je .Lrcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je .Lrcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je .Lrcls_retllong -.Lrcls_epilogue: - addl $36, %esp - popl %esi - popl %ebp - ret -.Lrcls_retint: - movl -24(%ebp), %eax - jmp .Lrcls_epilogue -.Lrcls_retfloat: - flds -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retdouble: - fldl -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retldouble: - fldt -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retllong: - movl -24(%ebp), %eax - movl -20(%ebp), %edx - jmp .Lrcls_epilogue -.LFE3: - .size ffi_closure_raw_SYSV, .-ffi_closure_raw_SYSV -#endif - - .section .eh_frame,EH_FRAME_FLAGS,@progbits -.Lframe1: - .long .LECIE1-.LSCIE1 /* Length of Common Information Entry */ -.LSCIE1: - .long 0x0 /* CIE Identifier Tag */ - .byte 0x1 /* CIE Version */ -#ifdef __PIC__ - .ascii "zR\0" /* CIE Augmentation */ -#else - .ascii "\0" /* CIE Augmentation */ -#endif - .byte 0x1 /* .uleb128 0x1; CIE Code Alignment Factor */ - .byte 0x7c /* .sleb128 -4; CIE Data Alignment Factor */ - .byte 0x8 /* CIE RA Column */ -#ifdef __PIC__ - .byte 0x1 /* .uleb128 0x1; Augmentation size */ - .byte 0x1b /* FDE Encoding (pcrel sdata4) */ -#endif - .byte 0xc /* DW_CFA_def_cfa */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x88 /* DW_CFA_offset, column 0x8 */ - .byte 0x1 /* .uleb128 0x1 */ - .align 4 -.LECIE1: -.LSFDE1: - .long .LEFDE1-.LASFDE1 /* FDE Length */ -.LASFDE1: - .long .LASFDE1-.Lframe1 /* FDE CIE offset */ -#ifdef __PIC__ - .long .LFB1-. /* FDE initial location */ -#else - .long .LFB1 /* FDE initial location */ -#endif - .long .LFE1-.LFB1 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI0-.LFB1 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI1-.LCFI0 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ - .align 4 -.LEFDE1: -.LSFDE2: - .long .LEFDE2-.LASFDE2 /* FDE Length */ -.LASFDE2: - .long .LASFDE2-.Lframe1 /* FDE CIE offset */ -#ifdef __PIC__ - .long .LFB2-. /* FDE initial location */ -#else - .long .LFB2 -#endif - .long .LFE2-.LFB2 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI2-.LFB2 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI3-.LCFI2 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ -#if !defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE && defined __PIC__ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI7-.LCFI3 - .byte 0x83 /* DW_CFA_offset, column 0x3 */ - .byte 0xa /* .uleb128 0xa */ -#endif - .align 4 -.LEFDE2: - -#if !FFI_NO_RAW_API - -.LSFDE3: - .long .LEFDE3-.LASFDE3 /* FDE Length */ -.LASFDE3: - .long .LASFDE3-.Lframe1 /* FDE CIE offset */ -#ifdef __PIC__ - .long .LFB3-. /* FDE initial location */ -#else - .long .LFB3 -#endif - .long .LFE3-.LFB3 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI4-.LFB3 - .byte 0xe /* DW_CFA_def_cfa_offset */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 */ - .byte 0x2 /* .uleb128 0x2 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI5-.LCFI4 - .byte 0xd /* DW_CFA_def_cfa_register */ - .byte 0x5 /* .uleb128 0x5 */ - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI6-.LCFI5 - .byte 0x86 /* DW_CFA_offset, column 0x6 */ - .byte 0x3 /* .uleb128 0x3 */ - .align 4 -.LEFDE3: - -#endif - -#endif /* ifndef __x86_64__ */ - - .section .note.GNU-stack,"",%progbits diff --git a/src/x86/sysv.S b/src/x86/sysv.S index d0b8417..7b898ae 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -59,10 +59,10 @@ /* This is declared as - void ffi_call_i386(struct ffi_call_frame *frame, char *argp) + void ffi_call_i386(struct call_frame *frame, char *argp) __attribute__((fastcall)); - This the arguments are present in + Thus the arguments are present in ecx: frame edx: argp @@ -170,181 +170,478 @@ E(X86_RET_UNUSED15) cfi_endproc ENDF(C(ffi_call_i386)) - .align 4 -FFI_HIDDEN (ffi_closure_SYSV) -.globl ffi_closure_SYSV - .type ffi_closure_SYSV, @function +/* The inner helper is declared as -ffi_closure_SYSV: - cfi_startproc - pushl %ebp - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%ebp, 0) - movl %esp, %ebp - cfi_def_cfa_register(%ebp) - subl $40, %esp - leal -24(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 8(%ebp), %edx -#ifdef __SUNPRO_C - /* The SUNPRO compiler doesn't support GCC's regparm function - attribute, so we have to pass all three arguments to - ffi_closure_SYSV_inner on the stack. */ - movl %edx, 8(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, 4(%esp) /* &resp */ - movl %eax, (%esp) /* closure */ -#else - movl %edx, 4(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ + void ffi_closure_inner(struct closure_frame *frame, char *argp) + __attribute_((fastcall)) + + Thus the arguments are placed in + + ecx: frame + edx: argp +*/ + +/* Macros to help setting up the closure_data structure. */ + +#define closure_FS (16 + 3*4 + 3*4 + 4) + +.macro FFI_CLOSURE_SAVE_REGS + movl %eax, 16+R_EAX*4(%esp) + movl %edx, 16+R_EDX*4(%esp) + movl %ecx, 16+R_ECX*4(%esp) +.endm + +.macro FFI_CLOSURE_COPY_TRAMP_DATA chain + movl FFI_TRAMPOLINE_SIZE(%eax), %edx /* copy cif */ + movl FFI_TRAMPOLINE_SIZE+4(%eax), %ecx /* copy fun */ + movl FFI_TRAMPOLINE_SIZE+8(%eax), %eax /* copy user_data */ + movl %edx, 28(%esp) + movl %ecx, 32(%esp) + movl %eax, 36(%esp) +.endm + +.macro FFI_CLOSURE_CALL_INNER + movl %esp, %ecx /* load closure_data */ + leal closure_FS+4(%esp), %edx /* load incoming stack */ +#ifdef __PIC__ + movl %ebx, 40(%esp) /* save ebx */ + cfi_rel_offset(%ebx, 40) + call __x86.get_pc_thunk.bx /* load got register */ + addl $C(_GLOBAL_OFFSET_TABLE_), %ebx #endif #if defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE || !defined __PIC__ - call ffi_closure_SYSV_inner + call C(ffi_closure_inner) #else - movl %ebx, 8(%esp) - cfi_offset(%ebx, -40) - call 1f -1: popl %ebx - addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx - call ffi_closure_SYSV_inner@PLT - movl 8(%esp), %ebx - cfi_restore(%ebx) + call C(ffi_closure_inner)@PLT #endif - movl -12(%ebp), %ecx - cmpl $FFI_TYPE_INT, %eax - je .Lcls_retint +.endm - /* Handle FFI_TYPE_UINT8, FFI_TYPE_SINT8, FFI_TYPE_UINT16, - FFI_TYPE_SINT16, FFI_TYPE_UINT32, FFI_TYPE_SINT32. */ - cmpl $FFI_TYPE_UINT64, %eax - jge 0f - cmpl $FFI_TYPE_UINT8, %eax - jge .Lcls_retint - -0: cmpl $FFI_TYPE_FLOAT, %eax - je .Lcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je .Lcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je .Lcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je .Lcls_retllong - cmpl $FFI_TYPE_STRUCT, %eax - je .Lcls_retstruct -.Lcls_epilogue: - movl %ebp, %esp - popl %ebp +.macro FFI_CLOSURE_MASK_AND_JUMP + andl $X86_RET_TYPE_MASK, %eax +#ifdef __PIC__ + leal 0f@GOTOFF(%ebx, %eax, 8), %eax + movl 40(%esp), %ebx /* restore ebx */ + cfi_restore(%ebx) +#else + leal 0f(, %eax, 8), %eax +#endif + jmp *%eax +.endm + +/* The closure entry points are reached from the ffi_closure trampoline. + On entry, %eax contains the address of the ffi_closure. */ + + .align 16 + .globl C(ffi_closure_i386) + FFI_HIDDEN(C(ffi_closure_i386)) + +C(ffi_closure_i386): + cfi_startproc + subl $closure_FS, %esp + cfi_adjust_cfa_offset(closure_FS) + + FFI_CLOSURE_SAVE_REGS + FFI_CLOSURE_COPY_TRAMP_DATA + FFI_CLOSURE_CALL_INNER + FFI_CLOSURE_MASK_AND_JUMP + + .align 8 +0: +E(X86_RET_FLOAT) + flds (%esp) + jmp 9f +E(X86_RET_DOUBLE) + fldl (%esp) + jmp 9f +E(X86_RET_LDOUBLE) + fldt (%esp) + jmp 9f +E(X86_RET_SINT8) + movsbl (%esp), %eax + jmp 9f +E(X86_RET_SINT16) + movswl (%esp), %eax + jmp 9f +E(X86_RET_UINT8) + movzbl (%esp), %eax + jmp 9f +E(X86_RET_UINT16) + movzwl (%esp), %eax + jmp 9f +E(X86_RET_INT64) + movl 4(%esp), %edx + /* fallthru */ +E(X86_RET_INT32) + movl (%esp), %eax + /* fallthru */ +E(X86_RET_VOID) +9: addl $closure_FS, %esp + cfi_adjust_cfa_offset(-closure_FS) ret -.Lcls_retint: - movl (%ecx), %eax - jmp .Lcls_epilogue -.Lcls_retfloat: - flds (%ecx) - jmp .Lcls_epilogue -.Lcls_retdouble: - fldl (%ecx) - jmp .Lcls_epilogue -.Lcls_retldouble: - fldt (%ecx) - jmp .Lcls_epilogue -.Lcls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp .Lcls_epilogue -.Lcls_retstruct: - movl %ebp, %esp - popl %ebp + cfi_adjust_cfa_offset(closure_FS) +E(X86_RET_STRUCTPOP) + addl $closure_FS, %esp + cfi_adjust_cfa_offset(-closure_FS) ret $4 + cfi_adjust_cfa_offset(closure_FS) +E(X86_RET_STRUCTARG) + movl (%esp), %eax + jmp 9b +E(X86_RET_STRUCT_1B) + movzbl (%esp), %eax + jmp 9b +E(X86_RET_STRUCT_2B) + movzwl (%esp), %eax + jmp 9b + + /* Fill out the table so that bad values are predictable. */ +E(X86_RET_UNUSED14) + ud2 +E(X86_RET_UNUSED15) + ud2 + cfi_endproc - .size ffi_closure_SYSV, .-ffi_closure_SYSV +ENDF(C(ffi_closure_i386)) + +/* For REGISTER, we have no available parameter registers, and so we + enter here having pushed the closure onto the stack. */ + + .align 16 + .globl C(ffi_closure_REGISTER) + FFI_HIDDEN(C(ffi_closure_REGISTER)) +C(ffi_closure_REGISTER): + cfi_startproc + cfi_def_cfa(%esp, 8) + cfi_offset(%eip, -8) + subl $closure_FS-4, %esp + cfi_adjust_cfa_offset(closure_FS-4) + + FFI_CLOSURE_SAVE_REGS + + movl closure_FS-4(%esp), %ecx /* load retaddr */ + movl closure_FS(%esp), %eax /* load closure */ + movl %ecx, closure_FS(%esp) /* move retaddr */ + jmp 0f + + cfi_endproc +ENDF(C(ffi_closure_REGISTER)) + +/* For STDCALL (and others), we need to pop N bytes of arguments off + the stack following the closure. The amount needing to be popped + is returned to us from ffi_closure_inner. */ + + .align 16 + .globl C(ffi_closure_STDCALL) + FFI_HIDDEN(C(ffi_closure_STDCALL)) +C(ffi_closure_STDCALL): + cfi_startproc + subl $closure_FS, %esp + cfi_adjust_cfa_offset(closure_FS) + + FFI_CLOSURE_SAVE_REGS +0: + FFI_CLOSURE_COPY_TRAMP_DATA + FFI_CLOSURE_CALL_INNER + + movl %eax, %ecx + shrl $X86_RET_POP_SHIFT, %ecx /* isolate pop count */ + leal closure_FS(%esp, %ecx), %ecx /* compute popped esp */ + movl closure_FS(%esp), %edx /* move return address */ + movl %edx, (%ecx) + + /* New pseudo-stack frame based off ecx. This is unwind trickery + in that the CFA *has* changed, to the proper popped stack address. + Note that the location to which we moved the return address + is the new CFA-4, so that's unchanged. */ + cfi_def_cfa(%ecx, 4) + /* Normally esp is unwound to CFA + the caller's ARGS_SIZE. + We've just set the CFA to that final value. Tell the unwinder + to restore esp from CFA without the ARGS_SIZE: + DW_CFA_val_expression %esp, DW_OP_call_frame_cfa. */ + cfi_escape(0x16, 4, 1, 0x9c) + + FFI_CLOSURE_MASK_AND_JUMP + + .align 8 +0: +E(X86_RET_FLOAT) + flds (%esp) + movl %ecx, %esp + ret +E(X86_RET_DOUBLE) + fldl (%esp) + movl %ecx, %esp + ret +E(X86_RET_LDOUBLE) + fldt (%esp) + movl %ecx, %esp + ret +E(X86_RET_SINT8) + movsbl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_SINT16) + movswl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_UINT8) + movzbl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_UINT16) + movzwl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_INT64) + popl %eax + popl %edx + movl %ecx, %esp + ret +E(X86_RET_INT32) + movl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_VOID) + movl %ecx, %esp + ret +E(X86_RET_STRUCTPOP) + movl %ecx, %esp + ret +E(X86_RET_STRUCTARG) + movl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_STRUCT_1B) + movzbl (%esp), %eax + movl %ecx, %esp + ret +E(X86_RET_STRUCT_2B) + movzwl (%esp), %eax + movl %ecx, %esp + ret + + /* Fill out the table so that bad values are predictable. */ +E(X86_RET_UNUSED14) + ud2 +E(X86_RET_UNUSED15) + ud2 + + cfi_endproc +ENDF(C(ffi_closure_STDCALL)) #if !FFI_NO_RAW_API -/* Precalculate for e.g. the Solaris 10/x86 assembler. */ -#if FFI_TRAMPOLINE_SIZE == 10 -#define RAW_CLOSURE_CIF_OFFSET 12 -#define RAW_CLOSURE_FUN_OFFSET 16 -#define RAW_CLOSURE_USER_DATA_OFFSET 20 -#elif FFI_TRAMPOLINE_SIZE == 24 -#define RAW_CLOSURE_CIF_OFFSET 24 -#define RAW_CLOSURE_FUN_OFFSET 28 -#define RAW_CLOSURE_USER_DATA_OFFSET 32 -#else -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) -#endif -#define CIF_FLAGS_OFFSET 20 +#define raw_closure_S_FS (16+16+12) - .align 4 -FFI_HIDDEN (ffi_closure_raw_SYSV) -.globl ffi_closure_raw_SYSV - .type ffi_closure_raw_SYSV, @function - -ffi_closure_raw_SYSV: + .align 16 + .globl C(ffi_closure_raw_SYSV) + FFI_HIDDEN(C(ffi_closure_raw_SYSV)) +C(ffi_closure_raw_SYSV): cfi_startproc - pushl %ebp - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%ebp, 0) - movl %esp, %ebp - cfi_def_cfa_register(%ebp) - pushl %esi - cfi_offset(%esi, -12) - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 8(%ebp), %edx /* __builtin_dwarf_cfa () */ - movl %edx, 8(%esp) /* raw_args */ - leal -24(%ebp), %edx - movl %edx, 4(%esp) /* &res */ - movl %esi, (%esp) /* cif */ - call *RAW_CLOSURE_FUN_OFFSET(%eax) /* closure->fun */ - movl CIF_FLAGS_OFFSET(%esi), %eax /* rtype */ - cmpl $FFI_TYPE_INT, %eax - je .Lrcls_retint + subl $raw_closure_S_FS, %esp + cfi_adjust_cfa_offset(raw_closure_S_FS) + movl %ebx, raw_closure_S_FS-4(%esp) + cfi_rel_offset(%ebx, raw_closure_S_FS-4) - /* Handle FFI_TYPE_UINT8, FFI_TYPE_SINT8, FFI_TYPE_UINT16, - FFI_TYPE_SINT16, FFI_TYPE_UINT32, FFI_TYPE_SINT32. */ - cmpl $FFI_TYPE_UINT64, %eax - jge 0f - cmpl $FFI_TYPE_UINT8, %eax - jge .Lrcls_retint + movl FFI_TRAMPOLINE_SIZE+8(%eax), %edx /* load cl->user_data */ + movl %edx, 12(%esp) + leal raw_closure_S_FS+4(%esp), %edx /* load raw_args */ + movl %edx, 8(%esp) + leal 16(%esp), %edx /* load &res */ + movl %edx, 4(%esp) + movl FFI_TRAMPOLINE_SIZE(%eax), %ebx /* load cl->cif */ + movl %ebx, (%esp) + call *FFI_TRAMPOLINE_SIZE+4(%eax) /* call cl->fun */ + + movl 20(%ebx), %eax /* load cif->flags */ + andl $X86_RET_TYPE_MASK, %eax +#ifdef __PIC__ + call __x86.get_pc_thunk.bx +1: leal 0f-1b(%ebx, %eax, 8), %eax +#else + leal 0f(,%eax, 8), %eax +#endif + movl raw_closure_S_FS-4(%esp), %ebx + cfi_restore(%ebx) + jmp *%eax + + .align 8 0: - cmpl $FFI_TYPE_FLOAT, %eax - je .Lrcls_retfloat - cmpl $FFI_TYPE_DOUBLE, %eax - je .Lrcls_retdouble - cmpl $FFI_TYPE_LONGDOUBLE, %eax - je .Lrcls_retldouble - cmpl $FFI_TYPE_SINT64, %eax - je .Lrcls_retllong -.Lrcls_epilogue: - addl $36, %esp - popl %esi - popl %ebp +E(X86_RET_FLOAT) + flds 16(%esp) + jmp 9f +E(X86_RET_DOUBLE) + fldl 16(%esp) + jmp 9f +E(X86_RET_LDOUBLE) + fldt 16(%esp) + jmp 9f +E(X86_RET_SINT8) + movsbl 16(%esp), %eax + jmp 9f +E(X86_RET_SINT16) + movswl 16(%esp), %eax + jmp 9f +E(X86_RET_UINT8) + movzbl 16(%esp), %eax + jmp 9f +E(X86_RET_UINT16) + movzwl 16(%esp), %eax + jmp 9f +E(X86_RET_INT64) + movl 16+4(%esp), %edx + /* fallthru */ +E(X86_RET_INT32) + movl 16(%esp), %eax + /* fallthru */ +E(X86_RET_VOID) +9: addl $raw_closure_S_FS, %esp + cfi_adjust_cfa_offset(-raw_closure_S_FS) ret -.Lrcls_retint: - movl -24(%ebp), %eax - jmp .Lrcls_epilogue -.Lrcls_retfloat: - flds -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retdouble: - fldl -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retldouble: - fldt -24(%ebp) - jmp .Lrcls_epilogue -.Lrcls_retllong: - movl -24(%ebp), %eax - movl -20(%ebp), %edx - jmp .Lrcls_epilogue + cfi_adjust_cfa_offset(raw_closure_S_FS) +E(X86_RET_STRUCTPOP) + addl $raw_closure_S_FS, %esp + cfi_adjust_cfa_offset(-raw_closure_S_FS) + ret $4 + cfi_adjust_cfa_offset(raw_closure_S_FS) +E(X86_RET_STRUCTARG) + movl 16(%esp), %eax + jmp 9b +E(X86_RET_STRUCT_1B) + movzbl 16(%esp), %eax + jmp 9b +E(X86_RET_STRUCT_2B) + movzwl 16(%esp), %eax + jmp 9b + + /* Fill out the table so that bad values are predictable. */ +E(X86_RET_UNUSED14) + ud2 +E(X86_RET_UNUSED15) + ud2 + cfi_endproc - .size ffi_closure_raw_SYSV, .-ffi_closure_raw_SYSV +ENDF(C(ffi_closure_raw_SYSV)) + +#undef raw_closure_S_FS +#define raw_closure_T_FS (16+16+8) + + .align 16 + .globl C(ffi_closure_raw_THISCALL) + FFI_HIDDEN(C(ffi_closure_raw_THISCALL)) +C(ffi_closure_raw_THISCALL): + cfi_startproc + /* Rearrange the stack such that %ecx is the first argument. + This means moving the return address. */ + popl %edx + cfi_adjust_cfa_offset(-4) + cfi_register(%eip, %edx) + pushl %ecx + cfi_adjust_cfa_offset(4) + pushl %edx + cfi_adjust_cfa_offset(4) + cfi_rel_offset(%eip, 0) + subl $raw_closure_T_FS, %esp + cfi_adjust_cfa_offset(raw_closure_T_FS) + movl %ebx, raw_closure_T_FS-4(%esp) + cfi_offset(%ebx, raw_closure_T_FS-4) + + movl FFI_TRAMPOLINE_SIZE+8(%eax), %edx /* load cl->user_data */ + movl %edx, 12(%esp) + leal raw_closure_T_FS+4(%esp), %edx /* load raw_args */ + movl %edx, 8(%esp) + leal 16(%esp), %edx /* load &res */ + movl %edx, 4(%esp) + movl FFI_TRAMPOLINE_SIZE(%eax), %ebx /* load cl->cif */ + movl %ebx, (%esp) + call *FFI_TRAMPOLINE_SIZE+4(%eax) /* call cl->fun */ + + movl 20(%ebx), %eax /* load cif->flags */ + andl $X86_RET_TYPE_MASK, %eax +#ifdef __PIC__ + call __x86.get_pc_thunk.bx +1: leal 0f-1b(%ebx, %eax, 8), %eax +#else + leal 0f(,%eax, 8), %eax +#endif + movl raw_closure_T_FS-4(%esp), %ebx + cfi_restore(%ebx) + jmp *%eax + + .align 8 +0: +E(X86_RET_FLOAT) + flds 16(%esp) + jmp 9f +E(X86_RET_DOUBLE) + fldl 16(%esp) + jmp 9f +E(X86_RET_LDOUBLE) + fldt 16(%esp) + jmp 9f +E(X86_RET_SINT8) + movsbl 16(%esp), %eax + jmp 9f +E(X86_RET_SINT16) + movswl 16(%esp), %eax + jmp 9f +E(X86_RET_UINT8) + movzbl 16(%esp), %eax + jmp 9f +E(X86_RET_UINT16) + movzwl 16(%esp), %eax + jmp 9f +E(X86_RET_INT64) + movl 16+4(%esp), %edx + /* fallthru */ +E(X86_RET_INT32) + movl 16(%esp), %eax + /* fallthru */ +E(X86_RET_VOID) +9: addl $raw_closure_T_FS, %esp + cfi_adjust_cfa_offset(-raw_closure_T_FS) + /* Remove the extra %ecx argument we pushed. */ + ret $4 + cfi_adjust_cfa_offset(raw_closure_T_FS) +E(X86_RET_STRUCTPOP) + addl $raw_closure_T_FS, %esp + cfi_adjust_cfa_offset(-raw_closure_T_FS) + ret $8 + cfi_adjust_cfa_offset(raw_closure_T_FS) +E(X86_RET_STRUCTARG) + movl 16(%esp), %eax + jmp 9b +E(X86_RET_STRUCT_1B) + movzbl 16(%esp), %eax + jmp 9b +E(X86_RET_STRUCT_2B) + movzwl 16(%esp), %eax + jmp 9b + + /* Fill out the table so that bad values are predictable. */ +E(X86_RET_UNUSED14) + ud2 +E(X86_RET_UNUSED15) + ud2 + + cfi_endproc +ENDF(C(ffi_closure_raw_THISCALL)) #endif /* !FFI_NO_RAW_API */ + +#if defined(__PIC__) + .section .text.__x86.get_pc_thunk.bx,"axG",@progbits,__x86.get_pc_thunk.bx,comdat + .globl __x86.get_pc_thunk.bx + .hidden __x86.get_pc_thunk.bx + .type __x86.get_pc_thunk.bx,@function +__x86.get_pc_thunk.bx: + cfi_startproc + movl (%esp), %ebx + ret + cfi_endproc + .size __x86.get_pc_thunk.bx, . - __x86.get_pc_thunk.bx +#endif /* __PIC__ */ + #endif /* ifndef __x86_64__ */ #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits diff --git a/src/x86/win32.S b/src/x86/win32.S deleted file mode 100644 index 3680bf5..0000000 --- a/src/x86/win32.S +++ /dev/null @@ -1,1351 +0,0 @@ -/* ----------------------------------------------------------------------- - win32.S - Copyright (c) 2014 Anthony Green - Copyright (c) 1996, 1998, 2001, 2002, 2009 Red Hat, Inc. - Copyright (c) 2001 John Beniton - Copyright (c) 2002 Ranjit Mathew - Copyright (c) 2009 Daniel Witte - - - X86 Foreign Function Interface - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- - */ - -#define LIBFFI_ASM -#include -#include - -#define CIF_BYTES_OFFSET 16 -#define CIF_FLAGS_OFFSET 20 - -#ifdef _MSC_VER - -#define CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) AND NOT 3) - -.386 -.MODEL FLAT, C - -EXTRN ffi_closure_SYSV_inner:NEAR -EXTRN ffi_closure_WIN32_inner:NEAR - -_TEXT SEGMENT - -ffi_call_win32 PROC NEAR, - ffi_prep_args : NEAR PTR DWORD, - ecif : NEAR PTR DWORD, - cif_abi : DWORD, - cif_bytes : DWORD, - cif_flags : DWORD, - rvalue : NEAR PTR DWORD, - fn : NEAR PTR DWORD - - ;; Make room for all of the new args. - mov ecx, cif_bytes - sub esp, ecx - - mov eax, esp - - ;; Call ffi_prep_args - push ecif - push eax - call ffi_prep_args - add esp, 8 - - ;; Prepare registers - ;; EAX stores the number of register arguments - cmp eax, 0 - je fun - cmp eax, 3 - jl prepr_two_cmp - - mov ecx, esp - add esp, 12 - mov eax, DWORD PTR [ecx+8] - jmp prepr_two -prepr_two_cmp: - cmp eax, 2 - jl prepr_one_prep - mov ecx, esp - add esp, 8 -prepr_two: - mov edx, DWORD PTR [ecx+4] - jmp prepr_one -prepr_one_prep: - mov ecx, esp - add esp, 4 -prepr_one: - mov ecx, DWORD PTR [ecx] - cmp cif_abi, 7 ;; FFI_REGISTER - jne fun - - xchg ecx, eax - -fun: - ;; Call function - call fn - - ;; Load ecx with the return type code - mov ecx, cif_flags - - ;; If the return value pointer is NULL, assume no return value. - cmp rvalue, 0 - jne ca_jumptable - - ;; Even if there is no space for the return value, we are - ;; obliged to handle floating-point values. - cmp ecx, FFI_TYPE_FLOAT - jne ca_epilogue - fstp st(0) - - jmp ca_epilogue - -ca_jumptable: - jmp [ca_jumpdata + 4 * ecx] -ca_jumpdata: - ;; Do not insert anything here between label and jump table. - dd offset ca_epilogue ;; FFI_TYPE_VOID - dd offset ca_retint ;; FFI_TYPE_INT - dd offset ca_retfloat ;; FFI_TYPE_FLOAT - dd offset ca_retdouble ;; FFI_TYPE_DOUBLE - dd offset ca_retlongdouble ;; FFI_TYPE_LONGDOUBLE - dd offset ca_retuint8 ;; FFI_TYPE_UINT8 - dd offset ca_retsint8 ;; FFI_TYPE_SINT8 - dd offset ca_retuint16 ;; FFI_TYPE_UINT16 - dd offset ca_retsint16 ;; FFI_TYPE_SINT16 - dd offset ca_retint ;; FFI_TYPE_UINT32 - dd offset ca_retint ;; FFI_TYPE_SINT32 - dd offset ca_retint64 ;; FFI_TYPE_UINT64 - dd offset ca_retint64 ;; FFI_TYPE_SINT64 - dd offset ca_epilogue ;; FFI_TYPE_STRUCT - dd offset ca_retint ;; FFI_TYPE_POINTER - dd offset ca_retstruct1b ;; FFI_TYPE_SMALL_STRUCT_1B - dd offset ca_retstruct2b ;; FFI_TYPE_SMALL_STRUCT_2B - dd offset ca_retint ;; FFI_TYPE_SMALL_STRUCT_4B - dd offset ca_epilogue ;; FFI_TYPE_MS_STRUCT - - /* Sign/zero extend as appropriate. */ -ca_retuint8: - movzx eax, al - jmp ca_retint - -ca_retsint8: - movsx eax, al - jmp ca_retint - -ca_retuint16: - movzx eax, ax - jmp ca_retint - -ca_retsint16: - movsx eax, ax - jmp ca_retint - -ca_retint: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - mov [ecx + 0], eax - jmp ca_epilogue - -ca_retint64: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - mov [ecx + 0], eax - mov [ecx + 4], edx - jmp ca_epilogue - -ca_retfloat: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - fstp DWORD PTR [ecx] - jmp ca_epilogue - -ca_retdouble: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - fstp QWORD PTR [ecx] - jmp ca_epilogue - -ca_retlongdouble: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - fstp TBYTE PTR [ecx] - jmp ca_epilogue - -ca_retstruct1b: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - mov [ecx + 0], al - jmp ca_epilogue - -ca_retstruct2b: - ;; Load %ecx with the pointer to storage for the return value - mov ecx, rvalue - mov [ecx + 0], ax - jmp ca_epilogue - -ca_epilogue: - ;; Epilogue code is autogenerated. - ret -ffi_call_win32 ENDP - -ffi_closure_THISCALL PROC NEAR - ;; Insert the register argument on the stack as the first argument - xchg DWORD PTR [esp+4], ecx - xchg DWORD PTR [esp], ecx - push ecx - jmp ffi_closure_STDCALL -ffi_closure_THISCALL ENDP - -ffi_closure_FASTCALL PROC NEAR - ;; Insert the 2 register arguments on the stack as the first argument - xchg DWORD PTR [esp+4], edx - xchg DWORD PTR [esp], ecx - push edx - push ecx - jmp ffi_closure_STDCALL -ffi_closure_FASTCALL ENDP - -ffi_closure_REGISTER PROC NEAR - ;; Insert the 3 register arguments on the stack as the first argument - push eax - xchg DWORD PTR [esp+8], ecx - xchg DWORD PTR [esp+4], edx - push ecx - push edx - jmp ffi_closure_STDCALL -ffi_closure_REGISTER ENDP - -ffi_closure_SYSV PROC NEAR FORCEFRAME - ;; the ffi_closure ctx is passed in eax by the trampoline. - - sub esp, 40 - lea edx, [ebp - 24] - mov [ebp - 12], edx ;; resp - lea edx, [ebp + 8] -stub:: - mov [esp + 8], edx ;; args - lea edx, [ebp - 12] - mov [esp + 4], edx ;; &resp - mov [esp], eax ;; closure - call ffi_closure_SYSV_inner - mov ecx, [ebp - 12] - -cs_jumptable: - jmp [cs_jumpdata + 4 * eax] -cs_jumpdata: - ;; Do not insert anything here between the label and jump table. - dd offset cs_epilogue ;; FFI_TYPE_VOID - dd offset cs_retint ;; FFI_TYPE_INT - dd offset cs_retfloat ;; FFI_TYPE_FLOAT - dd offset cs_retdouble ;; FFI_TYPE_DOUBLE - dd offset cs_retlongdouble ;; FFI_TYPE_LONGDOUBLE - dd offset cs_retuint8 ;; FFI_TYPE_UINT8 - dd offset cs_retsint8 ;; FFI_TYPE_SINT8 - dd offset cs_retuint16 ;; FFI_TYPE_UINT16 - dd offset cs_retsint16 ;; FFI_TYPE_SINT16 - dd offset cs_retint ;; FFI_TYPE_UINT32 - dd offset cs_retint ;; FFI_TYPE_SINT32 - dd offset cs_retint64 ;; FFI_TYPE_UINT64 - dd offset cs_retint64 ;; FFI_TYPE_SINT64 - dd offset cs_retstruct ;; FFI_TYPE_STRUCT - dd offset cs_retint ;; FFI_TYPE_POINTER - dd offset cs_retsint8 ;; FFI_TYPE_SMALL_STRUCT_1B - dd offset cs_retsint16 ;; FFI_TYPE_SMALL_STRUCT_2B - dd offset cs_retint ;; FFI_TYPE_SMALL_STRUCT_4B - dd offset cs_retmsstruct ;; FFI_TYPE_MS_STRUCT - -cs_retuint8: - movzx eax, BYTE PTR [ecx] - jmp cs_epilogue - -cs_retsint8: - movsx eax, BYTE PTR [ecx] - jmp cs_epilogue - -cs_retuint16: - movzx eax, WORD PTR [ecx] - jmp cs_epilogue - -cs_retsint16: - movsx eax, WORD PTR [ecx] - jmp cs_epilogue - -cs_retint: - mov eax, [ecx] - jmp cs_epilogue - -cs_retint64: - mov eax, [ecx + 0] - mov edx, [ecx + 4] - jmp cs_epilogue - -cs_retfloat: - fld DWORD PTR [ecx] - jmp cs_epilogue - -cs_retdouble: - fld QWORD PTR [ecx] - jmp cs_epilogue - -cs_retlongdouble: - fld TBYTE PTR [ecx] - jmp cs_epilogue - -cs_retstruct: - ;; Caller expects us to pop struct return value pointer hidden arg. - ;; Epilogue code is autogenerated. - ret 4 - -cs_retmsstruct: - ;; Caller expects us to return a pointer to the real return value. - mov eax, ecx - ;; Caller doesn't expects us to pop struct return value pointer hidden arg. - jmp cs_epilogue - -cs_epilogue: - ;; Epilogue code is autogenerated. - ret -ffi_closure_SYSV ENDP - -#if !FFI_NO_RAW_API - -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) AND NOT 3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) - -ffi_closure_raw_THISCALL PROC NEAR USES esi FORCEFRAME - sub esp, 36 - mov esi, [eax + RAW_CLOSURE_CIF_OFFSET] ;; closure->cif - mov edx, [eax + RAW_CLOSURE_USER_DATA_OFFSET] ;; closure->user_data - mov [esp + 12], edx - lea edx, [ebp + 12] - jmp stubraw -ffi_closure_raw_THISCALL ENDP - -ffi_closure_raw_SYSV PROC NEAR USES esi FORCEFRAME - ;; the ffi_closure ctx is passed in eax by the trampoline. - - sub esp, 40 - mov esi, [eax + RAW_CLOSURE_CIF_OFFSET] ;; closure->cif - mov edx, [eax + RAW_CLOSURE_USER_DATA_OFFSET] ;; closure->user_data - mov [esp + 12], edx ;; user_data - lea edx, [ebp + 8] -stubraw:: - mov [esp + 8], edx ;; raw_args - lea edx, [ebp - 24] - mov [esp + 4], edx ;; &res - mov [esp], esi ;; cif - call DWORD PTR [eax + RAW_CLOSURE_FUN_OFFSET] ;; closure->fun - mov eax, [esi + CIF_FLAGS_OFFSET] ;; cif->flags - lea ecx, [ebp - 24] - -cr_jumptable: - jmp [cr_jumpdata + 4 * eax] -cr_jumpdata: - ;; Do not insert anything here between the label and jump table. - dd offset cr_epilogue ;; FFI_TYPE_VOID - dd offset cr_retint ;; FFI_TYPE_INT - dd offset cr_retfloat ;; FFI_TYPE_FLOAT - dd offset cr_retdouble ;; FFI_TYPE_DOUBLE - dd offset cr_retlongdouble ;; FFI_TYPE_LONGDOUBLE - dd offset cr_retuint8 ;; FFI_TYPE_UINT8 - dd offset cr_retsint8 ;; FFI_TYPE_SINT8 - dd offset cr_retuint16 ;; FFI_TYPE_UINT16 - dd offset cr_retsint16 ;; FFI_TYPE_SINT16 - dd offset cr_retint ;; FFI_TYPE_UINT32 - dd offset cr_retint ;; FFI_TYPE_SINT32 - dd offset cr_retint64 ;; FFI_TYPE_UINT64 - dd offset cr_retint64 ;; FFI_TYPE_SINT64 - dd offset cr_epilogue ;; FFI_TYPE_STRUCT - dd offset cr_retint ;; FFI_TYPE_POINTER - dd offset cr_retsint8 ;; FFI_TYPE_SMALL_STRUCT_1B - dd offset cr_retsint16 ;; FFI_TYPE_SMALL_STRUCT_2B - dd offset cr_retint ;; FFI_TYPE_SMALL_STRUCT_4B - dd offset cr_epilogue ;; FFI_TYPE_MS_STRUCT - -cr_retuint8: - movzx eax, BYTE PTR [ecx] - jmp cr_epilogue - -cr_retsint8: - movsx eax, BYTE PTR [ecx] - jmp cr_epilogue - -cr_retuint16: - movzx eax, WORD PTR [ecx] - jmp cr_epilogue - -cr_retsint16: - movsx eax, WORD PTR [ecx] - jmp cr_epilogue - -cr_retint: - mov eax, [ecx] - jmp cr_epilogue - -cr_retint64: - mov eax, [ecx + 0] - mov edx, [ecx + 4] - jmp cr_epilogue - -cr_retfloat: - fld DWORD PTR [ecx] - jmp cr_epilogue - -cr_retdouble: - fld QWORD PTR [ecx] - jmp cr_epilogue - -cr_retlongdouble: - fld TBYTE PTR [ecx] - jmp cr_epilogue - -cr_epilogue: - ;; Epilogue code is autogenerated. - ret -ffi_closure_raw_SYSV ENDP - -#endif /* !FFI_NO_RAW_API */ - -ffi_closure_STDCALL PROC NEAR FORCEFRAME - mov eax, [esp] ;; the ffi_closure ctx passed by the trampoline. - - sub esp, 40 - lea edx, [ebp - 24] - mov [ebp - 12], edx ;; resp - lea edx, [ebp + 12] ;; account for stub return address on stack - mov [esp + 8], edx ;; args - lea edx, [ebp - 12] - mov [esp + 4], edx ;; &resp - mov [esp], eax ;; closure - call ffi_closure_WIN32_inner - mov ecx, [ebp - 12] - - xchg [ebp + 4], eax ;;xchg size of stack parameters and ffi_closure ctx - mov eax, DWORD PTR [eax + CLOSURE_CIF_OFFSET] - mov eax, DWORD PTR [eax + CIF_FLAGS_OFFSET] - -cd_jumptable: - jmp [cd_jumpdata + 4 * eax] -cd_jumpdata: - ;; Do not insert anything here between the label and jump table. - dd offset cd_epilogue ;; FFI_TYPE_VOID - dd offset cd_retint ;; FFI_TYPE_INT - dd offset cd_retfloat ;; FFI_TYPE_FLOAT - dd offset cd_retdouble ;; FFI_TYPE_DOUBLE - dd offset cd_retlongdouble ;; FFI_TYPE_LONGDOUBLE - dd offset cd_retuint8 ;; FFI_TYPE_UINT8 - dd offset cd_retsint8 ;; FFI_TYPE_SINT8 - dd offset cd_retuint16 ;; FFI_TYPE_UINT16 - dd offset cd_retsint16 ;; FFI_TYPE_SINT16 - dd offset cd_retint ;; FFI_TYPE_UINT32 - dd offset cd_retint ;; FFI_TYPE_SINT32 - dd offset cd_retint64 ;; FFI_TYPE_UINT64 - dd offset cd_retint64 ;; FFI_TYPE_SINT64 - dd offset cd_epilogue ;; FFI_TYPE_STRUCT - dd offset cd_retint ;; FFI_TYPE_POINTER - dd offset cd_retsint8 ;; FFI_TYPE_SMALL_STRUCT_1B - dd offset cd_retsint16 ;; FFI_TYPE_SMALL_STRUCT_2B - dd offset cd_retint ;; FFI_TYPE_SMALL_STRUCT_4B - -cd_retuint8: - movzx eax, BYTE PTR [ecx] - jmp cd_epilogue - -cd_retsint8: - movsx eax, BYTE PTR [ecx] - jmp cd_epilogue - -cd_retuint16: - movzx eax, WORD PTR [ecx] - jmp cd_epilogue - -cd_retsint16: - movsx eax, WORD PTR [ecx] - jmp cd_epilogue - -cd_retint: - mov eax, [ecx] - jmp cd_epilogue - -cd_retint64: - mov eax, [ecx + 0] - mov edx, [ecx + 4] - jmp cd_epilogue - -cd_retfloat: - fld DWORD PTR [ecx] - jmp cd_epilogue - -cd_retdouble: - fld QWORD PTR [ecx] - jmp cd_epilogue - -cd_retlongdouble: - fld TBYTE PTR [ecx] - jmp cd_epilogue - -cd_epilogue: - mov esp, ebp - pop ebp - mov ecx, [esp + 4] ;; Return address - add esp, [esp] ;; Parameters stack size - add esp, 8 - jmp ecx -ffi_closure_STDCALL ENDP - -_TEXT ENDS -END - -#else - -#define CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) - -#if defined(SYMBOL_UNDERSCORE) -#define USCORE_SYMBOL(x) _##x -#else -#define USCORE_SYMBOL(x) x -#endif - .text - - # This assumes we are using gas. - .balign 16 -FFI_HIDDEN(ffi_call_win32) - .globl USCORE_SYMBOL(ffi_call_win32) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_call_win32; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_call_win32): -.LFB1: - pushl %ebp -.LCFI0: - movl %esp,%ebp -.LCFI1: - # Make room for all of the new args. - movl 20(%ebp),%ecx - subl %ecx,%esp - - movl %esp,%eax - - # Call ffi_prep_args - pushl 12(%ebp) - pushl %eax - call *8(%ebp) - addl $8,%esp - - # Prepare registers - # EAX stores the number of register arguments - cmpl $0, %eax - je .fun - cmpl $3, %eax - jl .prepr_two_cmp - - movl %esp, %ecx - addl $12, %esp - movl 8(%ecx), %eax - jmp .prepr_two -.prepr_two_cmp: - cmpl $2, %eax - jl .prepr_one_prep - movl %esp, %ecx - addl $8, %esp -.prepr_two: - movl 4(%ecx), %edx - jmp .prepr_one -.prepr_one_prep: - movl %esp, %ecx - addl $4, %esp -.prepr_one: - movl (%ecx), %ecx - cmpl $7, 16(%ebp) # FFI_REGISTER - jne .fun - - xchgl %eax, %ecx - -.fun: - # FIXME: Align the stack to a 128-bit boundary to avoid - # potential performance hits. - - # Call function - call *32(%ebp) - - # stdcall functions pop arguments off the stack themselves - - # Load %ecx with the return type code - movl 24(%ebp),%ecx - - # If the return value pointer is NULL, assume no return value. - cmpl $0,28(%ebp) - jne 0f - - # Even if there is no space for the return value, we are - # obliged to handle floating-point values. - cmpl $FFI_TYPE_FLOAT,%ecx - jne .Lnoretval - fstp %st(0) - - jmp .Lepilogue - -0: - call 1f - # Do not insert anything here between the call and the jump table. -.Lstore_table: - .long .Lnoretval-.Lstore_table /* FFI_TYPE_VOID */ - .long .Lretint-.Lstore_table /* FFI_TYPE_INT */ - .long .Lretfloat-.Lstore_table /* FFI_TYPE_FLOAT */ - .long .Lretdouble-.Lstore_table /* FFI_TYPE_DOUBLE */ - .long .Lretlongdouble-.Lstore_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lretuint8-.Lstore_table /* FFI_TYPE_UINT8 */ - .long .Lretsint8-.Lstore_table /* FFI_TYPE_SINT8 */ - .long .Lretuint16-.Lstore_table /* FFI_TYPE_UINT16 */ - .long .Lretsint16-.Lstore_table /* FFI_TYPE_SINT16 */ - .long .Lretint-.Lstore_table /* FFI_TYPE_UINT32 */ - .long .Lretint-.Lstore_table /* FFI_TYPE_SINT32 */ - .long .Lretint64-.Lstore_table /* FFI_TYPE_UINT64 */ - .long .Lretint64-.Lstore_table /* FFI_TYPE_SINT64 */ - .long .Lretstruct-.Lstore_table /* FFI_TYPE_STRUCT */ - .long .Lretint-.Lstore_table /* FFI_TYPE_POINTER */ - .long .Lretstruct1b-.Lstore_table /* FFI_TYPE_SMALL_STRUCT_1B */ - .long .Lretstruct2b-.Lstore_table /* FFI_TYPE_SMALL_STRUCT_2B */ - .long .Lretstruct4b-.Lstore_table /* FFI_TYPE_SMALL_STRUCT_4B */ - .long .Lretstruct-.Lstore_table /* FFI_TYPE_MS_STRUCT */ -1: - shl $2, %ecx - add (%esp),%ecx - mov (%ecx),%ecx - add (%esp),%ecx - add $4, %esp - jmp *%ecx - - /* Sign/zero extend as appropriate. */ -.Lretsint8: - movsbl %al, %eax - jmp .Lretint - -.Lretsint16: - movswl %ax, %eax - jmp .Lretint - -.Lretuint8: - movzbl %al, %eax - jmp .Lretint - -.Lretuint16: - movzwl %ax, %eax - jmp .Lretint - -.Lretint: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - movl %eax,0(%ecx) - jmp .Lepilogue - -.Lretfloat: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - fstps (%ecx) - jmp .Lepilogue - -.Lretdouble: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - fstpl (%ecx) - jmp .Lepilogue - -.Lretlongdouble: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - fstpt (%ecx) - jmp .Lepilogue - -.Lretint64: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - movl %eax,0(%ecx) - movl %edx,4(%ecx) - jmp .Lepilogue - -.Lretstruct1b: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - movb %al,0(%ecx) - jmp .Lepilogue - -.Lretstruct2b: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - movw %ax,0(%ecx) - jmp .Lepilogue - -.Lretstruct4b: - # Load %ecx with the pointer to storage for the return value - movl 28(%ebp),%ecx - movl %eax,0(%ecx) - jmp .Lepilogue - -.Lretstruct: - # Nothing to do! - -.Lnoretval: -.Lepilogue: - movl %ebp,%esp - popl %ebp - ret -.ffi_call_win32_end: - .balign 16 -FFI_HIDDEN(ffi_closure_THISCALL) - .globl USCORE_SYMBOL(ffi_closure_THISCALL) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_THISCALL; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_THISCALL): - /* Insert the register argument on the stack as the first argument */ - xchg %ecx, 4(%esp) - xchg %ecx, (%esp) - push %ecx - jmp .ffi_closure_STDCALL_internal - - .balign 16 -FFI_HIDDEN(ffi_closure_FASTCALL) - .globl USCORE_SYMBOL(ffi_closure_FASTCALL) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_FASTCALL; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_FASTCALL): - /* Insert the 2 register arguments on the stack as the first two arguments */ - xchg %edx, 4(%esp) - xchg %ecx, (%esp) - push %edx - push %ecx - jmp .ffi_closure_STDCALL_internal -FFI_HIDDEN(ffi_closure_REGISTER) - .globl USCORE_SYMBOL(ffi_closure_REGISTER) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_REGISTER; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_REGISTER): - /* Insert the 3 register arguments on the stack as the first two arguments */ - push %eax - xchg %ecx, 8(%esp) - xchg %edx, 4(%esp) - push %ecx - push %edx - jmp .ffi_closure_STDCALL_internal - -.LFE1: - # This assumes we are using gas. - .balign 16 -FFI_HIDDEN(ffi_closure_SYSV) -#if defined(X86_WIN32) - .globl USCORE_SYMBOL(ffi_closure_SYSV) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_SYSV; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_SYSV): -#endif -.LFB3: - pushl %ebp -.LCFI4: - movl %esp, %ebp -.LCFI5: - subl $40, %esp - leal -24(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 8(%ebp), %edx - movl %edx, 4(%esp) /* args = __builtin_dwarf_cfa () */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ -#if defined(HAVE_HIDDEN_VISIBILITY_ATTRIBUTE) || !defined(__PIC__) - call USCORE_SYMBOL(ffi_closure_SYSV_inner) -#elif defined(X86_DARWIN) - calll L_ffi_closure_SYSV_inner$stub -#else - movl %ebx, 8(%esp) - call 1f -1: popl %ebx - addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx - call ffi_closure_SYSV_inner@PLT - movl 8(%esp), %ebx -#endif - movl -12(%ebp), %ecx - -0: - call 1f - # Do not insert anything here between the call and the jump table. -.Lcls_store_table: - .long .Lcls_noretval-.Lcls_store_table /* FFI_TYPE_VOID */ - .long .Lcls_retint-.Lcls_store_table /* FFI_TYPE_INT */ - .long .Lcls_retfloat-.Lcls_store_table /* FFI_TYPE_FLOAT */ - .long .Lcls_retdouble-.Lcls_store_table /* FFI_TYPE_DOUBLE */ - .long .Lcls_retldouble-.Lcls_store_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lcls_retuint8-.Lcls_store_table /* FFI_TYPE_UINT8 */ - .long .Lcls_retsint8-.Lcls_store_table /* FFI_TYPE_SINT8 */ - .long .Lcls_retuint16-.Lcls_store_table /* FFI_TYPE_UINT16 */ - .long .Lcls_retsint16-.Lcls_store_table /* FFI_TYPE_SINT16 */ - .long .Lcls_retint-.Lcls_store_table /* FFI_TYPE_UINT32 */ - .long .Lcls_retint-.Lcls_store_table /* FFI_TYPE_SINT32 */ - .long .Lcls_retllong-.Lcls_store_table /* FFI_TYPE_UINT64 */ - .long .Lcls_retllong-.Lcls_store_table /* FFI_TYPE_SINT64 */ - .long .Lcls_retstruct-.Lcls_store_table /* FFI_TYPE_STRUCT */ - .long .Lcls_retint-.Lcls_store_table /* FFI_TYPE_POINTER */ - .long .Lcls_retstruct1-.Lcls_store_table /* FFI_TYPE_SMALL_STRUCT_1B */ - .long .Lcls_retstruct2-.Lcls_store_table /* FFI_TYPE_SMALL_STRUCT_2B */ - .long .Lcls_retstruct4-.Lcls_store_table /* FFI_TYPE_SMALL_STRUCT_4B */ - .long .Lcls_retmsstruct-.Lcls_store_table /* FFI_TYPE_MS_STRUCT */ - -1: - shl $2, %eax - add (%esp),%eax - mov (%eax),%eax - add (%esp),%eax - add $4, %esp - jmp *%eax - - /* Sign/zero extend as appropriate. */ -.Lcls_retsint8: - movsbl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retsint16: - movswl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retuint8: - movzbl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retuint16: - movzwl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retint: - movl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retfloat: - flds (%ecx) - jmp .Lcls_epilogue - -.Lcls_retdouble: - fldl (%ecx) - jmp .Lcls_epilogue - -.Lcls_retldouble: - fldt (%ecx) - jmp .Lcls_epilogue - -.Lcls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp .Lcls_epilogue - -.Lcls_retstruct1: - movsbl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retstruct2: - movswl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retstruct4: - movl (%ecx), %eax - jmp .Lcls_epilogue - -.Lcls_retstruct: - # Caller expects us to pop struct return value pointer hidden arg. - movl %ebp, %esp - popl %ebp - ret $0x4 - -.Lcls_retmsstruct: - # Caller expects us to return a pointer to the real return value. - mov %ecx, %eax - # Caller doesn't expects us to pop struct return value pointer hidden arg. - jmp .Lcls_epilogue - -.Lcls_noretval: -.Lcls_epilogue: - movl %ebp, %esp - popl %ebp - ret -.ffi_closure_SYSV_end: -.LFE3: - -#if !FFI_NO_RAW_API - -#define RAW_CLOSURE_CIF_OFFSET ((FFI_TRAMPOLINE_SIZE + 3) & ~3) -#define RAW_CLOSURE_FUN_OFFSET (RAW_CLOSURE_CIF_OFFSET + 4) -#define RAW_CLOSURE_USER_DATA_OFFSET (RAW_CLOSURE_FUN_OFFSET + 4) - -#ifdef X86_WIN32 - .balign 16 -FFI_HIDDEN(ffi_closure_raw_THISCALL) - .globl USCORE_SYMBOL(ffi_closure_raw_THISCALL) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_raw_THISCALL; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_raw_THISCALL): - pushl %ebp - movl %esp, %ebp - pushl %esi - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 12(%ebp), %edx /* __builtin_dwarf_cfa () */ - jmp .stubraw -#endif /* X86_WIN32 */ - - # This assumes we are using gas. - .balign 16 -#if defined(X86_WIN32) - .globl USCORE_SYMBOL(ffi_closure_raw_SYSV) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_raw_SYSV; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_raw_SYSV): -#endif /* defined(X86_WIN32) */ -.LFB4: - pushl %ebp -.LCFI6: - movl %esp, %ebp -.LCFI7: - pushl %esi -.LCFI8: - subl $36, %esp - movl RAW_CLOSURE_CIF_OFFSET(%eax), %esi /* closure->cif */ - movl RAW_CLOSURE_USER_DATA_OFFSET(%eax), %edx /* closure->user_data */ - movl %edx, 12(%esp) /* user_data */ - leal 8(%ebp), %edx /* __builtin_dwarf_cfa () */ -.stubraw: - movl %edx, 8(%esp) /* raw_args */ - leal -24(%ebp), %edx - movl %edx, 4(%esp) /* &res */ - movl %esi, (%esp) /* cif */ - call *RAW_CLOSURE_FUN_OFFSET(%eax) /* closure->fun */ - movl CIF_FLAGS_OFFSET(%esi), %eax /* rtype */ -0: - call 1f - # Do not insert anything here between the call and the jump table. -.Lrcls_store_table: - .long .Lrcls_noretval-.Lrcls_store_table /* FFI_TYPE_VOID */ - .long .Lrcls_retint-.Lrcls_store_table /* FFI_TYPE_INT */ - .long .Lrcls_retfloat-.Lrcls_store_table /* FFI_TYPE_FLOAT */ - .long .Lrcls_retdouble-.Lrcls_store_table /* FFI_TYPE_DOUBLE */ - .long .Lrcls_retldouble-.Lrcls_store_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lrcls_retuint8-.Lrcls_store_table /* FFI_TYPE_UINT8 */ - .long .Lrcls_retsint8-.Lrcls_store_table /* FFI_TYPE_SINT8 */ - .long .Lrcls_retuint16-.Lrcls_store_table /* FFI_TYPE_UINT16 */ - .long .Lrcls_retsint16-.Lrcls_store_table /* FFI_TYPE_SINT16 */ - .long .Lrcls_retint-.Lrcls_store_table /* FFI_TYPE_UINT32 */ - .long .Lrcls_retint-.Lrcls_store_table /* FFI_TYPE_SINT32 */ - .long .Lrcls_retllong-.Lrcls_store_table /* FFI_TYPE_UINT64 */ - .long .Lrcls_retllong-.Lrcls_store_table /* FFI_TYPE_SINT64 */ - .long .Lrcls_retstruct-.Lrcls_store_table /* FFI_TYPE_STRUCT */ - .long .Lrcls_retint-.Lrcls_store_table /* FFI_TYPE_POINTER */ - .long .Lrcls_retstruct1-.Lrcls_store_table /* FFI_TYPE_SMALL_STRUCT_1B */ - .long .Lrcls_retstruct2-.Lrcls_store_table /* FFI_TYPE_SMALL_STRUCT_2B */ - .long .Lrcls_retstruct4-.Lrcls_store_table /* FFI_TYPE_SMALL_STRUCT_4B */ - .long .Lrcls_retstruct-.Lrcls_store_table /* FFI_TYPE_MS_STRUCT */ -1: - shl $2, %eax - add (%esp),%eax - mov (%eax),%eax - add (%esp),%eax - add $4, %esp - jmp *%eax - - /* Sign/zero extend as appropriate. */ -.Lrcls_retsint8: - movsbl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retsint16: - movswl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retuint8: - movzbl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retuint16: - movzwl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retint: - movl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retfloat: - flds -24(%ebp) - jmp .Lrcls_epilogue - -.Lrcls_retdouble: - fldl -24(%ebp) - jmp .Lrcls_epilogue - -.Lrcls_retldouble: - fldt -24(%ebp) - jmp .Lrcls_epilogue - -.Lrcls_retllong: - movl -24(%ebp), %eax - movl -20(%ebp), %edx - jmp .Lrcls_epilogue - -.Lrcls_retstruct1: - movsbl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retstruct2: - movswl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retstruct4: - movl -24(%ebp), %eax - jmp .Lrcls_epilogue - -.Lrcls_retstruct: - # Nothing to do! - -.Lrcls_noretval: -.Lrcls_epilogue: - addl $36, %esp - popl %esi - popl %ebp - ret -.ffi_closure_raw_SYSV_end: -.LFE4: - -#endif /* !FFI_NO_RAW_API */ - - # This assumes we are using gas. - .balign 16 -FFI_HIDDEN(ffi_closure_STDCALL) - .globl USCORE_SYMBOL(ffi_closure_STDCALL) -#if defined(X86_WIN32) && !defined(__OS2__) - .def _ffi_closure_STDCALL; .scl 2; .type 32; .endef -#endif -USCORE_SYMBOL(ffi_closure_STDCALL): -.ffi_closure_STDCALL_internal: - /* ffi_closure ctx is at top of the stack */ - movl (%esp), %eax -.LFB5: - pushl %ebp -.LCFI9: - movl %esp, %ebp -.LCFI10: - subl $40, %esp - leal -24(%ebp), %edx - movl %edx, -12(%ebp) /* resp */ - leal 12(%ebp), %edx /* account for stub return address on stack */ - movl %edx, 4(%esp) /* args */ - leal -12(%ebp), %edx - movl %edx, (%esp) /* &resp */ -#if defined(HAVE_HIDDEN_VISIBILITY_ATTRIBUTE) || !defined(__PIC__) - call USCORE_SYMBOL(ffi_closure_WIN32_inner) -#elif defined(X86_DARWIN) - calll L_ffi_closure_WIN32_inner$stub -#else - movl %ebx, 8(%esp) - call 1f -1: popl %ebx - addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx - call ffi_closure_WIN32_inner@PLT - movl 8(%esp), %ebx -#endif - movl -12(%ebp), %ecx -0: - xchgl 4(%ebp), %eax /* xchg size of stack parameters and ffi_closure ctx */ - movl CLOSURE_CIF_OFFSET(%eax), %eax - movl CIF_FLAGS_OFFSET(%eax), %eax - - call 1f - # Do not insert anything here between the call and the jump table. -.Lscls_store_table: - .long .Lscls_noretval-.Lscls_store_table /* FFI_TYPE_VOID */ - .long .Lscls_retint-.Lscls_store_table /* FFI_TYPE_INT */ - .long .Lscls_retfloat-.Lscls_store_table /* FFI_TYPE_FLOAT */ - .long .Lscls_retdouble-.Lscls_store_table /* FFI_TYPE_DOUBLE */ - .long .Lscls_retldouble-.Lscls_store_table /* FFI_TYPE_LONGDOUBLE */ - .long .Lscls_retuint8-.Lscls_store_table /* FFI_TYPE_UINT8 */ - .long .Lscls_retsint8-.Lscls_store_table /* FFI_TYPE_SINT8 */ - .long .Lscls_retuint16-.Lscls_store_table /* FFI_TYPE_UINT16 */ - .long .Lscls_retsint16-.Lscls_store_table /* FFI_TYPE_SINT16 */ - .long .Lscls_retint-.Lscls_store_table /* FFI_TYPE_UINT32 */ - .long .Lscls_retint-.Lscls_store_table /* FFI_TYPE_SINT32 */ - .long .Lscls_retllong-.Lscls_store_table /* FFI_TYPE_UINT64 */ - .long .Lscls_retllong-.Lscls_store_table /* FFI_TYPE_SINT64 */ - .long .Lscls_retstruct-.Lscls_store_table /* FFI_TYPE_STRUCT */ - .long .Lscls_retint-.Lscls_store_table /* FFI_TYPE_POINTER */ - .long .Lscls_retstruct1-.Lscls_store_table /* FFI_TYPE_SMALL_STRUCT_1B */ - .long .Lscls_retstruct2-.Lscls_store_table /* FFI_TYPE_SMALL_STRUCT_2B */ - .long .Lscls_retstruct4-.Lscls_store_table /* FFI_TYPE_SMALL_STRUCT_4B */ -1: - shl $2, %eax - add (%esp),%eax - mov (%eax),%eax - add (%esp),%eax - add $4, %esp - jmp *%eax - - /* Sign/zero extend as appropriate. */ -.Lscls_retsint8: - movsbl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retsint16: - movswl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retuint8: - movzbl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retuint16: - movzwl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retint: - movl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retfloat: - flds (%ecx) - jmp .Lscls_epilogue - -.Lscls_retdouble: - fldl (%ecx) - jmp .Lscls_epilogue - -.Lscls_retldouble: - fldt (%ecx) - jmp .Lscls_epilogue - -.Lscls_retllong: - movl (%ecx), %eax - movl 4(%ecx), %edx - jmp .Lscls_epilogue - -.Lscls_retstruct1: - movsbl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retstruct2: - movswl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retstruct4: - movl (%ecx), %eax - jmp .Lscls_epilogue - -.Lscls_retstruct: - # Nothing to do! - -.Lscls_noretval: -.Lscls_epilogue: - movl %ebp, %esp - popl %ebp - movl 4(%esp), %ecx /* Return address */ - addl (%esp), %esp /* Parameters stack size */ - addl $8, %esp - jmp *%ecx -.ffi_closure_STDCALL_end: -.LFE5: - -#if defined(X86_DARWIN) -.section __IMPORT,__jump_table,symbol_stubs,self_modifying_code+pure_instructions,5 -L_ffi_closure_SYSV_inner$stub: - .indirect_symbol _ffi_closure_SYSV_inner - hlt ; hlt ; hlt ; hlt ; hlt -L_ffi_closure_WIN32_inner$stub: - .indirect_symbol _ffi_closure_WIN32_inner - hlt ; hlt ; hlt ; hlt ; hlt -#endif - -#if defined(X86_WIN32) && !defined(__OS2__) - .section .eh_frame,"w" -#endif -.Lframe1: -.LSCIE1: - .long .LECIE1-.LASCIE1 /* Length of Common Information Entry */ -.LASCIE1: - .long 0x0 /* CIE Identifier Tag */ - .byte 0x1 /* CIE Version */ -#ifdef __PIC__ - .ascii "zR\0" /* CIE Augmentation */ -#else - .ascii "\0" /* CIE Augmentation */ -#endif - .byte 0x1 /* .uleb128 0x1; CIE Code Alignment Factor */ - .byte 0x7c /* .sleb128 -4; CIE Data Alignment Factor */ - .byte 0x8 /* CIE RA Column */ -#ifdef __PIC__ - .byte 0x1 /* .uleb128 0x1; Augmentation size */ - .byte 0x1b /* FDE Encoding (pcrel sdata4) */ -#endif - .byte 0xc /* DW_CFA_def_cfa CFA = r4 + 4 = 4(%esp) */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x4 /* .uleb128 0x4 */ - .byte 0x88 /* DW_CFA_offset, column 0x8 %eip at CFA + 1 * -4 */ - .byte 0x1 /* .uleb128 0x1 */ - .align 4 -.LECIE1: - -.LSFDE1: - .long .LEFDE1-.LASFDE1 /* FDE Length */ -.LASFDE1: - .long .LASFDE1-.Lframe1 /* FDE CIE offset */ -#if defined __PIC__ && defined HAVE_AS_X86_PCREL - .long .LFB1-. /* FDE initial location */ -#else - .long .LFB1 -#endif - .long .LFE1-.LFB1 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - /* DW_CFA_xxx CFI instructions go here. */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI0-.LFB1 - .byte 0xe /* DW_CFA_def_cfa_offset CFA = r4 + 8 = 8(%esp) */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 %ebp at CFA + 2 * -4 */ - .byte 0x2 /* .uleb128 0x2 */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI1-.LCFI0 - .byte 0xd /* DW_CFA_def_cfa_register CFA = r5 = %ebp */ - .byte 0x5 /* .uleb128 0x5 */ - - /* End of DW_CFA_xxx CFI instructions. */ - .align 4 -.LEFDE1: - -.LSFDE3: - .long .LEFDE3-.LASFDE3 /* FDE Length */ -.LASFDE3: - .long .LASFDE3-.Lframe1 /* FDE CIE offset */ -#if defined __PIC__ && defined HAVE_AS_X86_PCREL - .long .LFB3-. /* FDE initial location */ -#else - .long .LFB3 -#endif - .long .LFE3-.LFB3 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - /* DW_CFA_xxx CFI instructions go here. */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI4-.LFB3 - .byte 0xe /* DW_CFA_def_cfa_offset CFA = r4 + 8 = 8(%esp) */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 %ebp at CFA + 2 * -4 */ - .byte 0x2 /* .uleb128 0x2 */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI5-.LCFI4 - .byte 0xd /* DW_CFA_def_cfa_register CFA = r5 = %ebp */ - .byte 0x5 /* .uleb128 0x5 */ - - /* End of DW_CFA_xxx CFI instructions. */ - .align 4 -.LEFDE3: - -#if !FFI_NO_RAW_API - -.LSFDE4: - .long .LEFDE4-.LASFDE4 /* FDE Length */ -.LASFDE4: - .long .LASFDE4-.Lframe1 /* FDE CIE offset */ -#if defined __PIC__ && defined HAVE_AS_X86_PCREL - .long .LFB4-. /* FDE initial location */ -#else - .long .LFB4 -#endif - .long .LFE4-.LFB4 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - /* DW_CFA_xxx CFI instructions go here. */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI6-.LFB4 - .byte 0xe /* DW_CFA_def_cfa_offset CFA = r4 + 8 = 8(%esp) */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 %ebp at CFA + 2 * -4 */ - .byte 0x2 /* .uleb128 0x2 */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI7-.LCFI6 - .byte 0xd /* DW_CFA_def_cfa_register CFA = r5 = %ebp */ - .byte 0x5 /* .uleb128 0x5 */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI8-.LCFI7 - .byte 0x86 /* DW_CFA_offset, column 0x6 %esi at CFA + 3 * -4 */ - .byte 0x3 /* .uleb128 0x3 */ - - /* End of DW_CFA_xxx CFI instructions. */ - .align 4 -.LEFDE4: - -#endif /* !FFI_NO_RAW_API */ - -.LSFDE5: - .long .LEFDE5-.LASFDE5 /* FDE Length */ -.LASFDE5: - .long .LASFDE5-.Lframe1 /* FDE CIE offset */ -#if defined __PIC__ && defined HAVE_AS_X86_PCREL - .long .LFB5-. /* FDE initial location */ -#else - .long .LFB5 -#endif - .long .LFE5-.LFB5 /* FDE address range */ -#ifdef __PIC__ - .byte 0x0 /* .uleb128 0x0; Augmentation size */ -#endif - /* DW_CFA_xxx CFI instructions go here. */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI9-.LFB5 - .byte 0xe /* DW_CFA_def_cfa_offset CFA = r4 + 8 = 8(%esp) */ - .byte 0x8 /* .uleb128 0x8 */ - .byte 0x85 /* DW_CFA_offset, column 0x5 %ebp at CFA + 2 * -4 */ - .byte 0x2 /* .uleb128 0x2 */ - - .byte 0x4 /* DW_CFA_advance_loc4 */ - .long .LCFI10-.LCFI9 - .byte 0xd /* DW_CFA_def_cfa_register CFA = r5 = %ebp */ - .byte 0x5 /* .uleb128 0x5 */ - - /* End of DW_CFA_xxx CFI instructions. */ - .align 4 -.LEFDE5: - -#endif /* !_MSC_VER */ - -#if defined __ELF__ && defined __linux__ - .section .note.GNU-stack,"",@progbits -#endif From 198f469e91076f84ab6ac38055ad12a5490101cd Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 5 Nov 2014 16:34:41 +0100 Subject: [PATCH 17/58] x86: Add support for Complex --- src/x86/ffi.c | 27 +++++++++++++++++++++++++++ testsuite/libffi.call/call.exp | 3 ++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index 40e47d2..a0d0cf3 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -120,6 +120,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) else #endif { + do_struct: switch (cabi) { case FFI_THISCALL: @@ -136,6 +137,32 @@ ffi_prep_cif_machdep(ffi_cif *cif) bytes += ALIGN (sizeof(void*), FFI_SIZEOF_ARG); } break; + case FFI_TYPE_COMPLEX: + switch (cif->rtype->elements[0]->type) + { + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + goto do_struct; + case FFI_TYPE_FLOAT: + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + flags = X86_RET_INT64; + break; + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT16: + flags = X86_RET_INT32; + break; + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT8: + flags = X86_RET_STRUCT_2B; + break; + default: + return FFI_BAD_TYPEDEF; + } + break; default: return FFI_BAD_TYPEDEF; } diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index 55de25c..d42dae5 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -27,7 +27,8 @@ run-many-tests $tlist "" # ??? We really should preprocess ffi.h and grep # for FFI_TARGET_HAS_COMPLEX_TYPE. if { [istarget s390*] - || [istarget x86_64*] } { + || [istarget x86_64*] + || [istarget i?86*] } { run-many-tests $ctlist "" } else { foreach test $ctlist { From f8c64e2486c956109bf5255a0c0a85f24cc86c72 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 5 Nov 2014 17:04:29 +0100 Subject: [PATCH 18/58] x86: Add support for Go closures --- src/x86/ffi.c | 70 +++++++++++++++++++++++++++++++++++++++------ src/x86/ffitarget.h | 2 +- src/x86/sysv.S | 34 +++++++++++++++++++++- 3 files changed, 95 insertions(+), 11 deletions(-) diff --git a/src/x86/ffi.c b/src/x86/ffi.c index a0d0cf3..359a864 100644 --- a/src/x86/ffi.c +++ b/src/x86/ffi.c @@ -218,25 +218,28 @@ struct call_frame struct abi_params { int dir; /* parameter growth direction */ + int static_chain; /* the static chain register used by gcc */ int nregs; /* number of register parameters */ int regs[3]; }; static const struct abi_params abi_params[FFI_LAST_ABI] = { - [FFI_SYSV] = { 1, 0 }, - [FFI_THISCALL] = { 1, 1, { R_ECX } }, - [FFI_FASTCALL] = { 1, 2, { R_ECX, R_EDX } }, - [FFI_STDCALL] = { 1, 0 }, - [FFI_PASCAL] = { -1, 0 }, - [FFI_REGISTER] = { -1, 3, { R_EAX, R_EDX, R_ECX } }, - [FFI_MS_CDECL] = { 1, 0 } + [FFI_SYSV] = { 1, R_ECX, 0 }, + [FFI_THISCALL] = { 1, R_EAX, 1, { R_ECX } }, + [FFI_FASTCALL] = { 1, R_EAX, 2, { R_ECX, R_EDX } }, + [FFI_STDCALL] = { 1, R_ECX, 0 }, + [FFI_PASCAL] = { -1, R_ECX, 0 }, + /* ??? No defined static chain; gcc does not support REGISTER. */ + [FFI_REGISTER] = { -1, R_ECX, 3, { R_EAX, R_EDX, R_ECX } }, + [FFI_MS_CDECL] = { 1, R_ECX, 0 } }; extern void ffi_call_i386(struct call_frame *, char *) FFI_HIDDEN __declspec(fastcall); -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) { size_t rsize, bytes; struct call_frame *frame; @@ -281,6 +284,7 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) frame->fn = fn; frame->flags = flags; frame->rvalue = rvalue; + frame->regs[pabi->static_chain] = (unsigned)closure; narg_reg = 0; switch (flags) @@ -345,6 +349,18 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) ffi_call_i386 (frame, stack); } +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); +} /** private members **/ @@ -493,6 +509,42 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } +void FFI_HIDDEN ffi_go_closure_EAX(void); +void FFI_HIDDEN ffi_go_closure_ECX(void); +void FFI_HIDDEN ffi_go_closure_STDCALL(void); + +ffi_status +ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif, + void (*fun)(ffi_cif*,void*,void**,void*)) +{ + void (*dest)(void); + + switch (cif->abi) + { + case FFI_SYSV: + case FFI_MS_CDECL: + dest = ffi_go_closure_ECX; + break; + case FFI_THISCALL: + case FFI_FASTCALL: + dest = ffi_go_closure_ECX; + break; + case FFI_STDCALL: + case FFI_PASCAL: + dest = ffi_go_closure_STDCALL; + break; + case FFI_REGISTER: + default: + return FFI_BAD_ABI; + } + + closure->tramp = dest; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + /* ------- Native raw API support -------------------------------- */ #if !FFI_NO_RAW_API diff --git a/src/x86/ffitarget.h b/src/x86/ffitarget.h index 8fff29f..580522f 100644 --- a/src/x86/ffitarget.h +++ b/src/x86/ffitarget.h @@ -117,6 +117,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 +#define FFI_GO_CLOSURES 1 #define FFI_TYPE_SMALL_STRUCT_1B (FFI_TYPE_LAST + 1) #define FFI_TYPE_SMALL_STRUCT_2B (FFI_TYPE_LAST + 2) @@ -127,7 +128,6 @@ typedef enum ffi_abi { || (defined (__x86_64__) && defined (X86_DARWIN)) # define FFI_TRAMPOLINE_SIZE 24 # define FFI_NATIVE_RAW_API 0 -# define FFI_GO_CLOSURES 1 #else # define FFI_TRAMPOLINE_SIZE 12 # define FFI_NATIVE_RAW_API 1 /* x86 has native raw api support */ diff --git a/src/x86/sysv.S b/src/x86/sysv.S index 7b898ae..f412b7a 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -228,6 +228,28 @@ ENDF(C(ffi_call_i386)) jmp *%eax .endm +.macro FFI_GO_CLOSURE suffix, chain, t1, t2 + .align 16 + .globl C(ffi_go_closure_\suffix) + FFI_HIDDEN(C(ffi_go_closure_\suffix)) +C(ffi_go_closure_\suffix): + cfi_startproc + subl $closure_FS, %esp + cfi_adjust_cfa_offset(closure_FS) + FFI_CLOSURE_SAVE_REGS + movl 4(\chain), \t1 /* copy cif */ + movl 8(\chain), \t2 /* copy fun */ + movl \t1, 28(%esp) + movl \t2, 32(%esp) + movl \chain, 36(%esp) /* closure is user_data */ + jmp 88f + cfi_endproc +ENDF(C(ffi_go_closure_\suffix)) +.endm + +FFI_GO_CLOSURE EAX, %eax, %edx, %ecx +FFI_GO_CLOSURE ECX, %ecx, %edx, %eax + /* The closure entry points are reached from the ffi_closure trampoline. On entry, %eax contains the address of the ffi_closure. */ @@ -242,6 +264,9 @@ C(ffi_closure_i386): FFI_CLOSURE_SAVE_REGS FFI_CLOSURE_COPY_TRAMP_DATA + +88: /* Entry point from preceeding Go closures. */ + FFI_CLOSURE_CALL_INNER FFI_CLOSURE_MASK_AND_JUMP @@ -303,6 +328,8 @@ E(X86_RET_UNUSED15) cfi_endproc ENDF(C(ffi_closure_i386)) +FFI_GO_CLOSURE STDCALL, %ecx, %edx, %eax + /* For REGISTER, we have no available parameter registers, and so we enter here having pushed the closure onto the stack. */ @@ -339,8 +366,13 @@ C(ffi_closure_STDCALL): cfi_adjust_cfa_offset(closure_FS) FFI_CLOSURE_SAVE_REGS -0: + +0: /* Entry point from ffi_closure_REGISTER. */ + FFI_CLOSURE_COPY_TRAMP_DATA + +88: /* Entry point from preceeding Go closure. */ + FFI_CLOSURE_CALL_INNER movl %eax, %ecx From 2650f47fe8388f3281ec00e2d1e3bc777c1d3544 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 6 Nov 2014 10:57:04 -0800 Subject: [PATCH 19/58] x86: Use win32 name mangling for fastcall functions --- src/x86/sysv.S | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/x86/sysv.S b/src/x86/sysv.S index f412b7a..e6a8c1e 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -47,6 +47,15 @@ # define ENDF(X) #endif +/* Handle win32 fastcall name mangling. */ +#ifdef X86_WIN32 +# define ffi_call_i386 @ffi_call_i386@8 +# define ffi_closure_inner @ffi_closure_inner@8 +#else +# define ffi_call_i386 C(ffi_call_i386) +# define ffi_closure_inner C(ffi_closure_inner) +#endif + /* This macro allows the safe creation of jump tables without an actual table. The entry points into the table are all 8 bytes. The use of ORG asserts that we're at the correct location. */ @@ -54,8 +63,8 @@ .text .align 16 - .globl C(ffi_call_i386) - FFI_HIDDEN(C(ffi_call_i386)) + .globl ffi_call_i386 + FFI_HIDDEN(ffi_call_i386) /* This is declared as @@ -68,7 +77,7 @@ edx: argp */ -C(ffi_call_i386): +ffi_call_i386: cfi_startproc movl (%esp), %eax /* move the return address */ movl %ebp, (%ecx) /* store %ebp into local frame */ @@ -168,7 +177,7 @@ E(X86_RET_UNUSED15) ud2 cfi_endproc -ENDF(C(ffi_call_i386)) +ENDF(ffi_call_i386) /* The inner helper is declared as @@ -210,9 +219,9 @@ ENDF(C(ffi_call_i386)) addl $C(_GLOBAL_OFFSET_TABLE_), %ebx #endif #if defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE || !defined __PIC__ - call C(ffi_closure_inner) + call ffi_closure_inner #else - call C(ffi_closure_inner)@PLT + call ffi_closure_inner@PLT #endif .endm From 18b74ce54afab45fcf2a7d4eb86bb2ce9db8cec8 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 13:00:34 -0400 Subject: [PATCH 20/58] aarch64: Fix non-apple compilation --- src/aarch64/ffi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index 5369ea4..cdb7816 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -782,7 +782,9 @@ ffi_prep_cif_machdep (ffi_cif *cif) } } +#if defined (__APPLE__) cif->aarch64_nfixedargs = 0; +#endif return FFI_OK; } From 38b54b9c180af13a3371e70a151a1a97e105b03f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 13:17:39 -0400 Subject: [PATCH 21/58] aarch64: Improve is_hfa The set of functions get_homogeneous_type, element_count, and is_hfa are all intertwined and recompute data. Return a compound quantity from is_hfa that contains all the data and avoids the recomputation. --- src/aarch64/ffi.c | 236 ++++++++++++++++++++++++++++------------------ 1 file changed, 143 insertions(+), 93 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index cdb7816..0834614 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -242,88 +242,132 @@ is_floating_type (unsigned short type) || type == FFI_TYPE_LONGDOUBLE); } -/* Test for a homogeneous structure. */ - -static unsigned short -get_homogeneous_type (ffi_type *ty) -{ - if (ty->type == FFI_TYPE_STRUCT && ty->elements) - { - unsigned i; - unsigned short candidate_type - = get_homogeneous_type (ty->elements[0]); - for (i =1; ty->elements[i]; i++) - { - unsigned short iteration_type = 0; - /* If we have a nested struct, we must find its homogeneous type. - If that fits with our candidate type, we are still - homogeneous. */ - if (ty->elements[i]->type == FFI_TYPE_STRUCT - && ty->elements[i]->elements) - { - iteration_type = get_homogeneous_type (ty->elements[i]); - } - else - { - iteration_type = ty->elements[i]->type; - } - - /* If we are not homogeneous, return FFI_TYPE_STRUCT. */ - if (candidate_type != iteration_type) - return FFI_TYPE_STRUCT; - } - return candidate_type; - } - - /* Base case, we have no more levels of nesting, so we - are a basic type, and so, trivially homogeneous in that type. */ - return ty->type; -} - -/* Determine the number of elements within a STRUCT. - - Note, we must handle nested structs. - - If ty is not a STRUCT this function will return 0. */ - -static unsigned -element_count (ffi_type *ty) -{ - if (ty->type == FFI_TYPE_STRUCT && ty->elements) - { - unsigned n; - unsigned elems = 0; - for (n = 0; ty->elements[n]; n++) - { - if (ty->elements[n]->type == FFI_TYPE_STRUCT - && ty->elements[n]->elements) - elems += element_count (ty->elements[n]); - else - elems++; - } - return elems; - } - return 0; -} - -/* Test for a homogeneous floating point aggregate. - - A homogeneous floating point aggregate is a homogeneous aggregate of - a half- single- or double- precision floating point type with one - to four elements. Note that this includes nested structs of the - basic type. */ +/* A subroutine of is_hfa. Given a structure type, return the type code + of the first non-structure element. Recurse for structure elements. + Return -1 if the structure is in fact empty, i.e. no nested elements. */ static int -is_hfa (ffi_type *ty) +is_hfa0 (const ffi_type *ty) { - if (ty->type == FFI_TYPE_STRUCT - && ty->elements[0] - && is_floating_type (get_homogeneous_type (ty))) + ffi_type **elements = ty->elements; + int i, ret = -1; + + if (elements != NULL) + for (i = 0; elements[i]; ++i) + { + ret = elements[i]->type; + if (ret == FFI_TYPE_STRUCT) + { + ret = is_hfa0 (elements[i]); + if (ret < 0) + continue; + } + break; + } + + return ret; +} + +/* A subroutine of is_hfa. Given a structure type, return true if all + of the non-structure elements are the same as CANDIDATE. */ + +static int +is_hfa1 (const ffi_type *ty, int candidate) +{ + ffi_type **elements = ty->elements; + int i; + + if (elements != NULL) + for (i = 0; elements[i]; ++i) + { + int t = elements[i]->type; + if (t == FFI_TYPE_STRUCT) + { + if (!is_hfa1 (elements[i], candidate)) + return 0; + } + else if (t != candidate) + return 0; + } + + return 1; +} + +/* Determine if TY is an homogenous floating point aggregate (HFA). + That is, a structure consisting of 1 to 4 members of all the same type, + where that type is a floating point scalar. + + Returns non-zero iff TY is an HFA. The result is an encoded value where + bits 0-7 contain the type code, and bits 8-10 contain the element count. */ + +static int +is_hfa(const ffi_type *ty) +{ + ffi_type **elements; + int candidate, i; + size_t size, ele_count; + + /* Quickest tests first. */ + if (ty->type != FFI_TYPE_STRUCT) + return 0; + + /* No HFA types are smaller than 4 bytes, or larger than 64 bytes. */ + size = ty->size; + if (size < 4 || size > 64) + return 0; + + /* Find the type of the first non-structure member. */ + elements = ty->elements; + candidate = elements[0]->type; + if (candidate == FFI_TYPE_STRUCT) { - unsigned n = element_count (ty); - return n >= 1 && n <= 4; + for (i = 0; ; ++i) + { + candidate = is_hfa0 (elements[i]); + if (candidate >= 0) + break; + } } - return 0; + + /* If the first member is not a floating point type, it's not an HFA. + Also quickly re-check the size of the structure. */ + switch (candidate) + { + case FFI_TYPE_FLOAT: + ele_count = size / sizeof(float); + if (size != ele_count * sizeof(float)) + return 0; + break; + case FFI_TYPE_DOUBLE: + ele_count = size / sizeof(double); + if (size != ele_count * sizeof(double)) + return 0; + break; + case FFI_TYPE_LONGDOUBLE: + ele_count = size / sizeof(long double); + if (size != ele_count * sizeof(long double)) + return 0; + break; + default: + return 0; + } + if (ele_count > 4) + return 0; + + /* Finally, make sure that all scalar elements are the same type. */ + for (i = 0; elements[i]; ++i) + { + if (elements[i]->type == FFI_TYPE_STRUCT) + { + if (!is_hfa1 (elements[i], candidate)) + return 0; + } + else if (elements[i]->type != candidate) + return 0; + } + + /* All tests succeeded. Encode the result. */ + return (ele_count << 8) | candidate; } /* Test if an ffi_type is a candidate for passing in a register. @@ -559,7 +603,10 @@ copy_hfa_to_reg_or_stack (void *memory, unsigned char *stack, struct arg_state *state) { - unsigned elems = element_count (ty); + int h = is_hfa (ty); + int type = h & 0xff; + unsigned elems = h >> 8; + if (available_v (state) < elems) { /* There are insufficient V registers. Further V register allocations @@ -573,7 +620,6 @@ copy_hfa_to_reg_or_stack (void *memory, else { int i; - unsigned short type = get_homogeneous_type (ty); for (i = 0; i < elems; i++) { void *reg = allocate_to_v (context, state); @@ -813,6 +859,7 @@ void ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { extended_cif ecif; + int h; ecif.cif = cif; ecif.avalue = avalue; @@ -861,11 +908,12 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) } case FFI_TYPE_STRUCT: - if (is_hfa (cif->rtype)) + h = is_hfa (cif->rtype); + if (h) { int j; - unsigned short type = get_homogeneous_type (cif->rtype); - unsigned elems = element_count (cif->rtype); + int type = h & 0xff; + int elems = h >> 8; for (j = 0; j < elems; j++) { void *reg = get_basic_type_addr (type, &context, j); @@ -967,7 +1015,7 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, ffi_cif *cif = closure->cif; void **avalue = (void**) alloca (cif->nargs * sizeof (void*)); void *rvalue = NULL; - int i; + int i, h; struct arg_state state; arg_init (&state, ALIGN(cif->bytes, 16)); @@ -1002,9 +1050,10 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, #endif case FFI_TYPE_STRUCT: - if (is_hfa (ty)) + h = is_hfa (ty); + if (h) { - unsigned n = element_count (ty); + unsigned n = h >> 8; if (available_v (&state) < n) { state.nsrn = N_V_ARG_REG; @@ -1013,7 +1062,7 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, } else { - switch (get_homogeneous_type (ty)) + switch (h & 0xff) { case FFI_TYPE_FLOAT: { @@ -1027,9 +1076,9 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, correctly. The fake can be tossed once the closure function has returned hence alloca() is sufficient. */ - int j; + unsigned j; UINT32 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < element_count (ty); j++) + for (j = 0; j < n; j++) memcpy (&p[j], allocate_to_s (context, &state), sizeof (*p)); @@ -1048,9 +1097,9 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, correctly. The fake can be tossed once the closure function has returned hence alloca() is sufficient. */ - int j; + unsigned j; UINT64 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < element_count (ty); j++) + for (j = 0; j < n; j++) memcpy (&p[j], allocate_to_d (context, &state), sizeof (*p)); @@ -1143,11 +1192,12 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, break; } case FFI_TYPE_STRUCT: - if (is_hfa (cif->rtype)) + h = is_hfa (cif->rtype); + if (h) { int j; - unsigned short type = get_homogeneous_type (cif->rtype); - unsigned elems = element_count (cif->rtype); + int type = h & 0xff; + int elems = h >> 8; for (j = 0; j < elems; j++) { void *reg = get_basic_type_addr (type, context, j); From b5f147d84761dc673ffe01d6af82bcde4ea47928 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 13:27:57 -0400 Subject: [PATCH 22/58] aarch64: Always distinguish LONGDOUBLE Avoid if-deffery by forcing FFI_TYPE_LONGDOUBLE different from FFI_TYPE_DOUBLE. This will simply be unused on hosts that define them identically. --- src/aarch64/ffi.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index 0834614..f065be5 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -20,11 +20,20 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include - +#include #include #include -#include +/* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; + all further uses in this file will refer to the 128-bit type. */ +#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE +# if FFI_TYPE_LONGDOUBLE != 4 +# error FFI_TYPE_LONGDOUBLE out of date +# endif +#else +# undef FFI_TYPE_LONGDOUBLE +# define FFI_TYPE_LONGDOUBLE 4 +#endif /* Stack alignment requirement in bytes */ #if defined (__APPLE__) @@ -115,10 +124,8 @@ get_basic_type_addr (unsigned short type, struct call_context *context, return get_s_addr (context, n); case FFI_TYPE_DOUBLE: return get_d_addr (context, n); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: return get_v_addr (context, n); -#endif case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: @@ -151,10 +158,8 @@ get_basic_type_alignment (unsigned short type) #endif case FFI_TYPE_DOUBLE: return sizeof (UINT64); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: return sizeof (long double); -#endif case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: #if defined (__APPLE__) @@ -193,10 +198,8 @@ get_basic_type_size (unsigned short type) return sizeof (UINT32); case FFI_TYPE_DOUBLE: return sizeof (UINT64); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: return sizeof (long double); -#endif case FFI_TYPE_UINT8: return sizeof (UINT8); case FFI_TYPE_SINT8: @@ -390,9 +393,7 @@ is_register_candidate (ffi_type *ty) case FFI_TYPE_VOID: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: case FFI_TYPE_UINT32: @@ -557,11 +558,9 @@ copy_basic_type (void *dest, void *source, unsigned short type) case FFI_TYPE_DOUBLE: *(double *) dest = *(double *) source; break; -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: *(long double *) dest = *(long double *) source; break; -#endif case FFI_TYPE_UINT8: *(ffi_arg *) dest = *(UINT8 *) source; break; @@ -653,13 +652,11 @@ allocate_to_register_or_stack (struct call_context *context, return allocate_to_d (context, state); state->nsrn = N_V_ARG_REG; break; -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: if (state->nsrn < N_V_ARG_REG) return allocate_to_v (context, state); state->nsrn = N_V_ARG_REG; break; -#endif case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: @@ -722,9 +719,7 @@ aarch64_prep_args (struct call_context *context, unsigned char *stack, appropriate register, or if none are available, to the stack. */ case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: @@ -887,9 +882,7 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) case FFI_TYPE_VOID: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: @@ -1040,14 +1033,12 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, case FFI_TYPE_POINTER: case FFI_TYPE_UINT64: case FFI_TYPE_SINT64: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: avalue[i] = allocate_to_register_or_stack (context, stack, &state, ty->type); break; -#endif case FFI_TYPE_STRUCT: h = is_hfa (ty); @@ -1106,13 +1097,11 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, break; } -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: memcpy (&avalue[i], allocate_to_v (context, &state), sizeof (*avalue)); break; -#endif default: FFI_ASSERT (0); @@ -1183,9 +1172,7 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, case FFI_TYPE_SINT64: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE case FFI_TYPE_LONGDOUBLE: -#endif { void *addr = get_basic_type_addr (cif->rtype->type, context, 0); copy_basic_type (addr, rvalue, cif->rtype->type); From 77c4cddca6aeb6e545e21f235e29323e05f5a3a3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 13:30:40 -0400 Subject: [PATCH 23/58] aarch64: Simplify AARCH64_STACK_ALIGN The iOS abi doesn't require padding between arguments, but that's not what AARCH64_STACK_ALIGN meant. The hardware will in fact trap if the SP register is not 16 byte aligned. --- src/aarch64/ffi.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index f065be5..a6fcc11 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -35,13 +35,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ # define FFI_TYPE_LONGDOUBLE 4 #endif -/* Stack alignment requirement in bytes */ -#if defined (__APPLE__) -#define AARCH64_STACK_ALIGN 1 -#else -#define AARCH64_STACK_ALIGN 16 -#endif - #define N_X_ARG_REG 8 #define N_V_ARG_REG 8 @@ -799,8 +792,7 @@ ffi_status ffi_prep_cif_machdep (ffi_cif *cif) { /* Round the stack up to a multiple of the stack alignment requirement. */ - cif->bytes = - (cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1); + cif->bytes = ALIGN(cif->bytes, 16); /* Initialize our flags. We are interested if this CIF will touch a vector register, if so we will enable context save and load to From 95a04af134431ccc8230aca1641541a5e8fcbdc9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 22:41:07 -0400 Subject: [PATCH 24/58] aarch64: Reduce the size of register_context We don't need to store 32 general and vector registers. Only 8 of each are used for parameter passing. --- src/aarch64/ffi.c | 35 +++++++------- src/aarch64/ffitarget.h | 6 --- src/aarch64/internal.h | 26 +++++++++++ src/aarch64/sysv.S | 100 +++++++++++++++++++--------------------- 4 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 src/aarch64/internal.h diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index a6fcc11..58d088b 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -21,8 +21,10 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include #include +#include #include #include +#include "internal.h" /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; all further uses in this file will refer to the 128-bit type. */ @@ -35,38 +37,35 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ # define FFI_TYPE_LONGDOUBLE 4 #endif -#define N_X_ARG_REG 8 -#define N_V_ARG_REG 8 - -#define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT) - union _d { UINT64 d; UINT32 s[2]; }; +struct _v +{ + union _d d[2] __attribute__((aligned(16))); +}; + struct call_context { - UINT64 x [AARCH64_N_XREG]; - struct - { - union _d d[2]; - } v [AARCH64_N_VREG]; + struct _v v[N_V_ARG_REG]; + UINT64 x[N_X_ARG_REG]; + UINT64 x8; }; #if defined (__clang__) && defined (__APPLE__) -extern void -sys_icache_invalidate (void *start, size_t len); +extern void sys_icache_invalidate (void *start, size_t len); #endif static inline void ffi_clear_cache (void *start, void *end) { #if defined (__clang__) && defined (__APPLE__) - sys_icache_invalidate (start, (char *)end - (char *)start); + sys_icache_invalidate (start, (char *)end - (char *)start); #elif defined (__GNUC__) - __builtin___clear_cache (start, end); + __builtin___clear_cache (start, end); #else #error "Missing builtin to flush instruction cache" #endif @@ -802,7 +801,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) if (is_v_register_candidate (cif->rtype)) { - cif->aarch64_flags |= AARCH64_FFI_WITH_V; + cif->aarch64_flags |= AARCH64_FLAG_ARG_V; } else { @@ -810,7 +809,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) for (i = 0; i < cif->nargs; i++) if (is_v_register_candidate (cif->arg_types[i])) { - cif->aarch64_flags |= AARCH64_FFI_WITH_V; + cif->aarch64_flags |= AARCH64_FLAG_ARG_V; break; } } @@ -924,7 +923,7 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) } else { - memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64)); + context.x8 = (uintptr_t)rvalue; ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); } @@ -1201,7 +1200,7 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, } else { - memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64)); + rvalue = (void *)(uintptr_t)context->x8; (closure->fun) (cif, rvalue, avalue, closure->user_data); } } diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index 4bbced2..336f28a 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -54,10 +54,4 @@ typedef enum ffi_abi #define FFI_EXTRA_CIF_FIELDS unsigned aarch64_flags #endif -#define AARCH64_FFI_WITH_V_BIT 0 - -#define AARCH64_N_XREG 32 -#define AARCH64_N_VREG 32 -#define AARCH64_CALL_CONTEXT_SIZE (AARCH64_N_XREG * 8 + AARCH64_N_VREG * 16) - #endif diff --git a/src/aarch64/internal.h b/src/aarch64/internal.h new file mode 100644 index 0000000..b6b6104 --- /dev/null +++ b/src/aarch64/internal.h @@ -0,0 +1,26 @@ +/* +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +``Software''), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ + +#define AARCH64_FLAG_ARG_V_BIT 0 +#define AARCH64_FLAG_ARG_V (1 << AARCH64_FLAG_ARG_V_BIT) + +#define N_X_ARG_REG 8 +#define N_V_ARG_REG 8 +#define CALL_CONTEXT_SIZE (N_V_ARG_REG * 16 + N_X_ARG_REG * 8 + 16) diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index 169eab8..70870db 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -22,6 +22,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #define LIBFFI_ASM #include #include +#include "internal.h" #ifdef HAVE_MACHINE_ASM_H #include @@ -43,13 +44,12 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg .text + .align 2 + .globl CNAME(ffi_call_SYSV) #ifdef __ELF__ .type CNAME(ffi_call_SYSV), #function #endif -#ifdef __APPLE__ - .align 2 -#endif /* ffi_call_SYSV() @@ -142,42 +142,40 @@ CNAME(ffi_call_SYSV): mov x23, x0 /* Figure out if we should touch the vector registers. */ - tbz x23, #AARCH64_FFI_WITH_V_BIT, 1f + tbz x23, #AARCH64_FLAG_ARG_V_BIT, 1f /* Load the vector argument passing registers. */ - ldp q0, q1, [x21, #8*32 + 0] - ldp q2, q3, [x21, #8*32 + 32] - ldp q4, q5, [x21, #8*32 + 64] - ldp q6, q7, [x21, #8*32 + 96] + ldp q0, q1, [x21, #0] + ldp q2, q3, [x21, #32] + ldp q4, q5, [x21, #64] + ldp q6, q7, [x21, #96] 1: - /* Load the core argument passing registers. */ - ldp x0, x1, [x21, #0] - ldp x2, x3, [x21, #16] - ldp x4, x5, [x21, #32] - ldp x6, x7, [x21, #48] - - /* Don't forget x8 which may be holding the address of a return buffer. - */ - ldr x8, [x21, #8*8] + /* Load the core argument passing registers, including + the structure return pointer. */ + ldp x0, x1, [x21, #16*N_V_ARG_REG + 0] + ldp x2, x3, [x21, #16*N_V_ARG_REG + 16] + ldp x4, x5, [x21, #16*N_V_ARG_REG + 32] + ldp x6, x7, [x21, #16*N_V_ARG_REG + 48] + ldr x8, [x21, #16*N_V_ARG_REG + 64] blr x24 /* Save the core argument passing registers. */ - stp x0, x1, [x21, #0] - stp x2, x3, [x21, #16] - stp x4, x5, [x21, #32] - stp x6, x7, [x21, #48] + stp x0, x1, [x21, #16*N_V_ARG_REG + 0] + stp x2, x3, [x21, #16*N_V_ARG_REG + 16] + stp x4, x5, [x21, #16*N_V_ARG_REG + 32] + stp x6, x7, [x21, #16*N_V_ARG_REG + 48] /* Note nothing useful ever comes back in x8! */ /* Figure out if we should touch the vector registers. */ - tbz x23, #AARCH64_FFI_WITH_V_BIT, 1f + tbz x23, #AARCH64_FLAG_ARG_V_BIT, 1f /* Save the vector argument passing registers. */ - stp q0, q1, [x21, #8*32 + 0] - stp q2, q3, [x21, #8*32 + 32] - stp q4, q5, [x21, #8*32 + 64] - stp q6, q7, [x21, #8*32 + 96] + stp q0, q1, [x21, #0] + stp q2, q3, [x21, #32] + stp q4, q5, [x21, #64] + stp q6, q7, [x21, #96] 1: /* All done, unwind our stack frame. */ ldp x21, x22, [x29, # - ffi_call_SYSV_FS] @@ -203,7 +201,7 @@ CNAME(ffi_call_SYSV): .size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV) #endif -#define ffi_closure_SYSV_FS (8 * 2 + AARCH64_CALL_CONTEXT_SIZE) +#define ffi_closure_SYSV_FS (8 * 2 + CALL_CONTEXT_SIZE) /* ffi_closure_SYSV @@ -243,10 +241,9 @@ CNAME(ffi_call_SYSV): Voila! */ .text - .globl CNAME(ffi_closure_SYSV) -#ifdef __APPLE__ .align 2 -#endif + + .globl CNAME(ffi_closure_SYSV) .cfi_startproc CNAME(ffi_closure_SYSV): stp x29, x30, [sp, #-16]! @@ -268,24 +265,23 @@ CNAME(ffi_closure_SYSV): /* Preserve our struct trampoline_data * */ mov x22, x17 - /* Save the rest of the argument passing registers. */ - stp x0, x1, [x21, #0] - stp x2, x3, [x21, #16] - stp x4, x5, [x21, #32] - stp x6, x7, [x21, #48] - /* Don't forget we may have been given a result scratch pad address. - */ - str x8, [x21, #64] + /* Save the rest of the argument passing registers, including + the structure return pointer. */ + stp x0, x1, [x21, #16*N_V_ARG_REG + 0] + stp x2, x3, [x21, #16*N_V_ARG_REG + 16] + stp x4, x5, [x21, #16*N_V_ARG_REG + 32] + stp x6, x7, [x21, #16*N_V_ARG_REG + 48] + str x8, [x21, #16*N_V_ARG_REG + 64] /* Figure out if we should touch the vector registers. */ ldr x0, [x22, #8] - tbz x0, #AARCH64_FFI_WITH_V_BIT, 1f + tbz x0, #AARCH64_FLAG_ARG_V_BIT, 1f /* Save the argument passing vector registers. */ - stp q0, q1, [x21, #8*32 + 0] - stp q2, q3, [x21, #8*32 + 32] - stp q4, q5, [x21, #8*32 + 64] - stp q6, q7, [x21, #8*32 + 96] + stp q0, q1, [x21, #0] + stp q2, q3, [x21, #32] + stp q4, q5, [x21, #64] + stp q6, q7, [x21, #96] 1: /* Load &ffi_closure.. */ ldr x0, [x22, #0] @@ -298,19 +294,19 @@ CNAME(ffi_closure_SYSV): /* Figure out if we should touch the vector registers. */ ldr x0, [x22, #8] - tbz x0, #AARCH64_FFI_WITH_V_BIT, 1f + tbz x0, #AARCH64_FLAG_ARG_V_BIT, 1f /* Load the result passing vector registers. */ - ldp q0, q1, [x21, #8*32 + 0] - ldp q2, q3, [x21, #8*32 + 32] - ldp q4, q5, [x21, #8*32 + 64] - ldp q6, q7, [x21, #8*32 + 96] + ldp q0, q1, [x21, #0] + ldp q2, q3, [x21, #32] + ldp q4, q5, [x21, #64] + ldp q6, q7, [x21, #96] 1: /* Load the result passing core registers. */ - ldp x0, x1, [x21, #0] - ldp x2, x3, [x21, #16] - ldp x4, x5, [x21, #32] - ldp x6, x7, [x21, #48] + ldp x0, x1, [x21, #16*N_V_ARG_REG + 0] + ldp x2, x3, [x21, #16*N_V_ARG_REG + 16] + ldp x4, x5, [x21, #16*N_V_ARG_REG + 32] + ldp x6, x7, [x21, #16*N_V_ARG_REG + 48] /* Note nothing useful is returned in x8. */ /* We are done, unwind our frame. */ From 2e32f9bf6ffa6782b70fafe96bdf21c5aa44f534 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 22:49:05 -0400 Subject: [PATCH 25/58] aarch64: Use correct return registers There are fewer return registers than argument registers. --- src/aarch64/sysv.S | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index 70870db..fa7ff5b 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -160,22 +160,15 @@ CNAME(ffi_call_SYSV): blr x24 - /* Save the core argument passing registers. */ - stp x0, x1, [x21, #16*N_V_ARG_REG + 0] - stp x2, x3, [x21, #16*N_V_ARG_REG + 16] - stp x4, x5, [x21, #16*N_V_ARG_REG + 32] - stp x6, x7, [x21, #16*N_V_ARG_REG + 48] - - /* Note nothing useful ever comes back in x8! */ + /* Save the core return registers. */ + stp x0, x1, [x21, #16*N_V_ARG_REG] /* Figure out if we should touch the vector registers. */ tbz x23, #AARCH64_FLAG_ARG_V_BIT, 1f - /* Save the vector argument passing registers. */ + /* Save the vector return registers. */ stp q0, q1, [x21, #0] stp q2, q3, [x21, #32] - stp q4, q5, [x21, #64] - stp q6, q7, [x21, #96] 1: /* All done, unwind our stack frame. */ ldp x21, x22, [x29, # - ffi_call_SYSV_FS] @@ -299,15 +292,9 @@ CNAME(ffi_closure_SYSV): /* Load the result passing vector registers. */ ldp q0, q1, [x21, #0] ldp q2, q3, [x21, #32] - ldp q4, q5, [x21, #64] - ldp q6, q7, [x21, #96] 1: /* Load the result passing core registers. */ ldp x0, x1, [x21, #16*N_V_ARG_REG + 0] - ldp x2, x3, [x21, #16*N_V_ARG_REG + 16] - ldp x4, x5, [x21, #16*N_V_ARG_REG + 32] - ldp x6, x7, [x21, #16*N_V_ARG_REG + 48] - /* Note nothing useful is returned in x8. */ /* We are done, unwind our frame. */ ldp x21, x22, [x29, #-16] From b55e03665ddf2423df9baee0d3172892ba781c26 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 12:33:59 -0400 Subject: [PATCH 26/58] aarch64: Treat void return as not passed in registers This lets us do less post-processing when there's no return value. --- src/aarch64/ffi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index 58d088b..6c338e1 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -383,6 +383,7 @@ is_register_candidate (ffi_type *ty) switch (ty->type) { case FFI_TYPE_VOID: + return 0; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: From 8c8161cb623585d5d0c783b9d494b9b74ada6ced Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 12:52:07 -0400 Subject: [PATCH 27/58] aarch64: Tidy up abi manipulation Avoid false abstraction, like get_x_addr. Avoid recomputing data about the type being manipulated. Use NEON insns for HFA manipulation. Note that some of the inline assembly will go away in a subsequent patch. --- src/aarch64/ffi.c | 954 ++++++++++++++++++---------------------------- 1 file changed, 378 insertions(+), 576 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index 6c338e1..d19384b 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -71,152 +71,6 @@ ffi_clear_cache (void *start, void *end) #endif } -static void * -get_x_addr (struct call_context *context, unsigned n) -{ - return &context->x[n]; -} - -static void * -get_s_addr (struct call_context *context, unsigned n) -{ -#if defined __AARCH64EB__ - return &context->v[n].d[1].s[1]; -#else - return &context->v[n].d[0].s[0]; -#endif -} - -static void * -get_d_addr (struct call_context *context, unsigned n) -{ -#if defined __AARCH64EB__ - return &context->v[n].d[1]; -#else - return &context->v[n].d[0]; -#endif -} - -static void * -get_v_addr (struct call_context *context, unsigned n) -{ - return &context->v[n]; -} - -/* Return the memory location at which a basic type would reside - were it to have been stored in register n. */ - -static void * -get_basic_type_addr (unsigned short type, struct call_context *context, - unsigned n) -{ - switch (type) - { - case FFI_TYPE_FLOAT: - return get_s_addr (context, n); - case FFI_TYPE_DOUBLE: - return get_d_addr (context, n); - case FFI_TYPE_LONGDOUBLE: - return get_v_addr (context, n); - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - return get_x_addr (context, n); - case FFI_TYPE_VOID: - return NULL; - default: - FFI_ASSERT (0); - return NULL; - } -} - -/* Return the alignment width for each of the basic types. */ - -static size_t -get_basic_type_alignment (unsigned short type) -{ - switch (type) - { - case FFI_TYPE_FLOAT: -#if defined (__APPLE__) - return sizeof (UINT32); -#endif - case FFI_TYPE_DOUBLE: - return sizeof (UINT64); - case FFI_TYPE_LONGDOUBLE: - return sizeof (long double); - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: -#if defined (__APPLE__) - return sizeof (UINT8); -#endif - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: -#if defined (__APPLE__) - return sizeof (UINT16); -#endif - case FFI_TYPE_UINT32: - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: -#if defined (__APPLE__) - return sizeof (UINT32); -#endif - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - return sizeof (UINT64); - - default: - FFI_ASSERT (0); - return 0; - } -} - -/* Return the size in bytes for each of the basic types. */ - -static size_t -get_basic_type_size (unsigned short type) -{ - switch (type) - { - case FFI_TYPE_FLOAT: - return sizeof (UINT32); - case FFI_TYPE_DOUBLE: - return sizeof (UINT64); - case FFI_TYPE_LONGDOUBLE: - return sizeof (long double); - case FFI_TYPE_UINT8: - return sizeof (UINT8); - case FFI_TYPE_SINT8: - return sizeof (SINT8); - case FFI_TYPE_UINT16: - return sizeof (UINT16); - case FFI_TYPE_SINT16: - return sizeof (SINT16); - case FFI_TYPE_UINT32: - return sizeof (UINT32); - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - return sizeof (SINT32); - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - return sizeof (UINT64); - case FFI_TYPE_SINT64: - return sizeof (SINT64); - - default: - FFI_ASSERT (0); - return 0; - } -} - extern void ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *, extended_cif *), @@ -468,157 +322,195 @@ arg_init (struct arg_state *state, size_t call_frame_size) #endif } -/* Return the number of available consecutive core argument - registers. */ - -static unsigned -available_x (struct arg_state *state) -{ - return N_X_ARG_REG - state->ngrn; -} - -/* Return the number of available consecutive vector argument - registers. */ - -static unsigned -available_v (struct arg_state *state) -{ - return N_V_ARG_REG - state->nsrn; -} - -static void * -allocate_to_x (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->ngrn < N_X_ARG_REG); - return get_x_addr (context, (state->ngrn)++); -} - -static void * -allocate_to_s (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_s_addr (context, (state->nsrn)++); -} - -static void * -allocate_to_d (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_d_addr (context, (state->nsrn)++); -} - -static void * -allocate_to_v (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_v_addr (context, (state->nsrn)++); -} - /* Allocate an aligned slot on the stack and return a pointer to it. */ static void * -allocate_to_stack (struct arg_state *state, void *stack, size_t alignment, - size_t size) +allocate_to_stack (struct arg_state *state, void *stack, + size_t alignment, size_t size) { - void *allocation; + size_t nsaa = state->nsaa; /* Round up the NSAA to the larger of 8 or the natural alignment of the argument's type. */ - state->nsaa = ALIGN (state->nsaa, alignment); - state->nsaa = ALIGN (state->nsaa, alignment); #if defined (__APPLE__) - if (state->allocating_variadic) - state->nsaa = ALIGN (state->nsaa, 8); + if (state->allocating_variadic && alignment < 8) + alignment = 8; #else - state->nsaa = ALIGN (state->nsaa, 8); + if (alignment < 8) + alignment = 8; #endif + + nsaa = ALIGN (nsaa, alignment); + state->nsaa = nsaa + size; - allocation = stack + state->nsaa; + return (char *)stack + nsaa; +} - state->nsaa += size; - return allocation; +static ffi_arg +extend_integer_type (void *source, int type) +{ + switch (type) + { + case FFI_TYPE_UINT8: + return *(UINT8 *) source; + case FFI_TYPE_SINT8: + return *(SINT8 *) source; + case FFI_TYPE_UINT16: + return *(UINT16 *) source; + case FFI_TYPE_SINT16: + return *(SINT16 *) source; + case FFI_TYPE_UINT32: + return *(UINT32 *) source; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + return *(SINT32 *) source; + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + return *(UINT64 *) source; + break; + case FFI_TYPE_POINTER: + return *(uintptr_t *) source; + default: + abort(); + } } static void -copy_basic_type (void *dest, void *source, unsigned short type) +extend_hfa_type (void *dest, void *src, int h) { - /* This is necessary to ensure that basic types are copied - sign extended to 64-bits as libffi expects. */ - switch (type) + int n = (h >> 8); + int t = h & 0xff; + int f = (t - FFI_TYPE_FLOAT) * 4 + 4 - n; + void *x0; + + asm volatile ( + "adr %0, 0f\n" +" add %0, %0, %1\n" +" br %0\n" +"0: ldp s16, s17, [%3]\n" /* S4 */ +" ldp s18, s19, [%3, #8]\n" +" b 4f\n" +" ldp s16, s17, [%3]\n" /* S3 */ +" ldr s18, [%3, #8]\n" +" b 3f\n" +" ldp s16, s17, [%3]\n" /* S2 */ +" b 2f\n" +" nop\n" +" ldr s16, [%3]\n" /* S1 */ +" b 1f\n" +" nop\n" +" ldp d16, d17, [%3]\n" /* D4 */ +" ldp d18, d19, [%3, #16]\n" +" b 4f\n" +" ldp d16, d17, [%3]\n" /* D3 */ +" ldr d18, [%3, #16]\n" +" b 3f\n" +" ldp d16, d17, [%3]\n" /* D2 */ +" b 2f\n" +" nop\n" +" ldr d16, [%3]\n" /* D1 */ +" b 1f\n" +" nop\n" +" ldp q16, q17, [%3]\n" /* Q4 */ +" ldp q18, q19, [%3, #16]\n" +" b 4f\n" +" ldp q16, q17, [%3]\n" /* Q3 */ +" ldr q18, [%3, #16]\n" +" b 3f\n" +" ldp q16, q17, [%3]\n" /* Q2 */ +" b 2f\n" +" nop\n" +" ldr q16, [%3]\n" /* Q1 */ +" b 1f\n" +"4: str q19, [%2, #48]\n" +"3: str q18, [%2, #32]\n" +"2: str q17, [%2, #16]\n" +"1: str q16, [%2]" + : "=&r"(x0) + : "r"(f * 12), "r"(dest), "r"(src) + : "memory", "v16", "v17", "v18", "v19"); +} + +static void * +compress_hfa_type (void *dest, void *reg, int h) +{ + int n = h >> 8; + switch (h & 0xff) { case FFI_TYPE_FLOAT: - *(float *) dest = *(float *) source; + switch (n) + { + default: + if (dest == reg) + { +#ifdef __AARCH64EB__ + dest += 12; +#endif + } + else + *(float *)dest = *(float *)reg; + break; + case 2: + asm("ldp q16, q17, [%1]\n\t" + "st2 { v16.s, v17.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); + break; + case 3: + asm("ldp q16, q17, [%1]\n\t" + "ldr q18, [%1, #32]\n\t" + "st3 { v16.s, v17.s, v18.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); + break; + case 4: + asm("ldp q16, q17, [%1]\n\t" + "ldp q18, q19, [%1, #32]\n\t" + "st4 { v16.s, v17.s, v18.s, v19.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); + break; + } break; + case FFI_TYPE_DOUBLE: - *(double *) dest = *(double *) source; + switch (n) + { + default: + if (dest == reg) + { +#ifdef __AARCH64EB__ + dest += 8; +#endif + } + else + *(double *)dest = *(double *)reg; + break; + case 2: + asm("ldp q16, q17, [%1]\n\t" + "st2 { v16.d, v17.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); + break; + case 3: + asm("ldp q16, q17, [%1]\n\t" + "ldr q18, [%1, #32]\n\t" + "st3 { v16.d, v17.d, v18.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); + break; + case 4: + asm("ldp q16, q17, [%1]\n\t" + "ldp q18, q19, [%1, #32]\n\t" + "st4 { v16.d, v17.d, v18.d, v19.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); + break; + } break; + case FFI_TYPE_LONGDOUBLE: - *(long double *) dest = *(long double *) source; - break; - case FFI_TYPE_UINT8: - *(ffi_arg *) dest = *(UINT8 *) source; - break; - case FFI_TYPE_SINT8: - *(ffi_sarg *) dest = *(SINT8 *) source; - break; - case FFI_TYPE_UINT16: - *(ffi_arg *) dest = *(UINT16 *) source; - break; - case FFI_TYPE_SINT16: - *(ffi_sarg *) dest = *(SINT16 *) source; - break; - case FFI_TYPE_UINT32: - *(ffi_arg *) dest = *(UINT32 *) source; - break; - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - *(ffi_sarg *) dest = *(SINT32 *) source; - break; - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - *(ffi_arg *) dest = *(UINT64 *) source; - break; - case FFI_TYPE_SINT64: - *(ffi_sarg *) dest = *(SINT64 *) source; - break; - case FFI_TYPE_VOID: + if (dest != reg) + return memcpy (dest, reg, 16 * n); break; default: FFI_ASSERT (0); } -} - -static void -copy_hfa_to_reg_or_stack (void *memory, - ffi_type *ty, - struct call_context *context, - unsigned char *stack, - struct arg_state *state) -{ - int h = is_hfa (ty); - int type = h & 0xff; - unsigned elems = h >> 8; - - if (available_v (state) < elems) - { - /* There are insufficient V registers. Further V register allocations - are prevented, the NSAA is adjusted (by allocate_to_stack ()) - and the argument is copied to memory at the adjusted NSAA. */ - state->nsrn = N_V_ARG_REG; - memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size), - memory, - ty->size); - } - else - { - int i; - for (i = 0; i < elems; i++) - { - void *reg = allocate_to_v (context, state); - copy_basic_type (reg, memory, type); - memory += get_basic_type_size (type); - } - } + return dest; } /* Either allocate an appropriate register for the argument type, or if @@ -626,65 +518,15 @@ copy_hfa_to_reg_or_stack (void *memory, to the allocated space. */ static void * -allocate_to_register_or_stack (struct call_context *context, - unsigned char *stack, - struct arg_state *state, - unsigned short type) +allocate_int_to_reg_or_stack (struct call_context *context, + struct arg_state *state, + void *stack, size_t size) { - size_t alignment = get_basic_type_alignment (type); - size_t size = alignment; - switch (type) - { - case FFI_TYPE_FLOAT: - /* This is the only case for which the allocated stack size - should not match the alignment of the type. */ - size = sizeof (UINT32); - /* Fall through. */ - case FFI_TYPE_DOUBLE: - if (state->nsrn < N_V_ARG_REG) - return allocate_to_d (context, state); - state->nsrn = N_V_ARG_REG; - break; - case FFI_TYPE_LONGDOUBLE: - if (state->nsrn < N_V_ARG_REG) - return allocate_to_v (context, state); - state->nsrn = N_V_ARG_REG; - break; - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - if (state->ngrn < N_X_ARG_REG) - return allocate_to_x (context, state); - state->ngrn = N_X_ARG_REG; - break; - default: - FFI_ASSERT (0); - } + if (state->ngrn < N_X_ARG_REG) + return &context->x[state->ngrn++]; - return allocate_to_stack (state, stack, alignment, size); -} - -/* Copy a value to an appropriate register, or if none are - available, to the stack. */ - -static void -copy_to_register_or_stack (struct call_context *context, - unsigned char *stack, - struct arg_state *state, - void *value, - unsigned short type) -{ - copy_basic_type ( - allocate_to_register_or_stack (context, stack, state, type), - value, - type); + state->ngrn = N_X_ARG_REG; + return allocate_to_stack (state, stack, size, size); } /* Marshall the arguments from FFI representation to procedure call @@ -694,15 +536,21 @@ static unsigned aarch64_prep_args (struct call_context *context, unsigned char *stack, extended_cif *ecif) { - int i; + ffi_cif *cif = ecif->cif; + void **avalue = ecif->avalue; + int i, nargs = cif->nargs; struct arg_state state; - arg_init (&state, ALIGN(ecif->cif->bytes, 16)); + arg_init (&state, cif->bytes); - for (i = 0; i < ecif->cif->nargs; i++) + for (i = 0; i < nargs; i++) { - ffi_type *ty = ecif->cif->arg_types[i]; - switch (ty->type) + ffi_type *ty = cif->arg_types[i]; + size_t s = ty->size; + int h, t = ty->type; + void *a = avalue[i]; + + switch (t) { case FFI_TYPE_VOID: FFI_ASSERT (0); @@ -710,82 +558,114 @@ aarch64_prep_args (struct call_context *context, unsigned char *stack, /* If the argument is a basic type the argument is allocated to an appropriate register, or if none are available, to the stack. */ - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: + case FFI_TYPE_INT: case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: case FFI_TYPE_SINT16: case FFI_TYPE_UINT32: - case FFI_TYPE_INT: case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: case FFI_TYPE_UINT64: case FFI_TYPE_SINT64: - copy_to_register_or_stack (context, stack, &state, - ecif->avalue[i], ty->type); + case FFI_TYPE_POINTER: + do_pointer: + { + ffi_arg ext = extend_integer_type (a, t); + if (state.ngrn < N_X_ARG_REG) + context->x[state.ngrn++] = ext; + else + { + void *d = allocate_to_stack (&state, stack, ty->alignment, s); + state.ngrn = N_X_ARG_REG; + /* Note that the default abi extends each argument + to a full 64-bit slot, while the iOS abi allocates + only enough space. */ +#ifdef __APPLE__ + memcpy(d, a, s); +#else + *(ffi_arg *)d = ext; +#endif + } + } break; + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + /* Scalar float is a degenerate case of HFA. */ + h = t + 0x100; + goto do_hfa; + case FFI_TYPE_STRUCT: - if (is_hfa (ty)) - { - copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context, - stack, &state); - } - else if (ty->size > 16) - { - /* If the argument is a composite type that is larger than 16 - bytes, then the argument has been copied to memory, and - the argument is replaced by a pointer to the copy. */ + { + void *dest; + int elems; - copy_to_register_or_stack (context, stack, &state, - &(ecif->avalue[i]), FFI_TYPE_POINTER); - } - else if (available_x (&state) >= (ty->size + 7) / 8) - { - /* If the argument is a composite type and the size in - double-words is not more than the number of available - X registers, then the argument is copied into consecutive - X registers. */ - int j; - for (j = 0; j < (ty->size + 7) / 8; j++) - { - memcpy (allocate_to_x (context, &state), - &(((UINT64 *) ecif->avalue[i])[j]), - sizeof (UINT64)); + h = is_hfa (ty); + if (h) + { + do_hfa: + elems = h >> 8; + if (state.nsrn + elems <= N_V_ARG_REG) + { + dest = &context->v[state.nsrn]; + state.nsrn += elems; + extend_hfa_type (dest, a, h); + break; + } + state.nsrn = N_V_ARG_REG; + dest = allocate_to_stack (&state, stack, ty->alignment, s); + } + else if (s > 16) + { + /* If the argument is a composite type that is larger than 16 + bytes, then the argument has been copied to memory, and + the argument is replaced by a pointer to the copy. */ + a = &avalue[i]; + t = FFI_TYPE_POINTER; + goto do_pointer; + } + else + { + size_t n = (s + 7) / 8; + if (state.ngrn + n <= N_X_ARG_REG) + { + /* If the argument is a composite type and the size in + double-words is not more than the number of available + X registers, then the argument is copied into + consecutive X registers. */ + dest = &context->x[state.ngrn]; + state.ngrn += n; + } + else + { + /* Otherwise, there are insufficient X registers. Further + X register allocations are prevented, the NSAA is + adjusted and the argument is copied to memory at the + adjusted NSAA. */ + state.ngrn = N_X_ARG_REG; + dest = allocate_to_stack (&state, stack, ty->alignment, s); + } } - } - else - { - /* Otherwise, there are insufficient X registers. Further X - register allocations are prevented, the NSAA is adjusted - (by allocate_to_stack ()) and the argument is copied to - memory at the adjusted NSAA. */ - state.ngrn = N_X_ARG_REG; - - memcpy (allocate_to_stack (&state, stack, ty->alignment, - ty->size), ecif->avalue + i, ty->size); + memcpy (dest, a, s); } break; default: - FFI_ASSERT (0); - break; + abort(); } #if defined (__APPLE__) - if (i + 1 == ecif->cif->aarch64_nfixedargs) + if (i + 1 == cif->aarch64_nfixedargs) { state.ngrn = N_X_ARG_REG; state.nsrn = N_V_ARG_REG; - state.allocating_variadic = 1; } #endif } - return ecif->cif->aarch64_flags; + return cif->aarch64_flags; } ffi_status @@ -846,94 +726,61 @@ void ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { extended_cif ecif; - int h; + struct call_context context; + size_t stack_bytes; + int h, t; ecif.cif = cif; ecif.avalue = avalue; ecif.rvalue = rvalue; - switch (cif->abi) + stack_bytes = cif->bytes; + + memset (&context, 0, sizeof (context)); + if (is_register_candidate (cif->rtype)) { - case FFI_SYSV: - { - struct call_context context; - size_t stack_bytes; + ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); - /* Figure out the total amount of stack space we need, the - above call frame space needs to be 16 bytes aligned to - ensure correct alignment of the first object inserted in - that space hence the ALIGN applied to cif->bytes.*/ - stack_bytes = ALIGN(cif->bytes, 16); + t = cif->rtype->type; + switch (t) + { + case FFI_TYPE_INT: + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_POINTER: + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + *(ffi_arg *)rvalue = extend_integer_type (&context.x[0], t); + break; - memset (&context, 0, sizeof (context)); - if (is_register_candidate (cif->rtype)) - { - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_INT: - case FFI_TYPE_SINT64: - { - void *addr = get_basic_type_addr (cif->rtype->type, - &context, 0); - copy_basic_type (rvalue, addr, cif->rtype->type); - break; - } + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + compress_hfa_type (rvalue, &context.v[0], 0x100 + t); + break; - case FFI_TYPE_STRUCT: - h = is_hfa (cif->rtype); - if (h) - { - int j; - int type = h & 0xff; - int elems = h >> 8; - for (j = 0; j < elems; j++) - { - void *reg = get_basic_type_addr (type, &context, j); - copy_basic_type (rvalue, reg, type); - rvalue += get_basic_type_size (type); - } - } - else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) - { - size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)); - memcpy (rvalue, get_x_addr (&context, 0), size); - } - else - { - FFI_ASSERT (0); - } - break; + case FFI_TYPE_STRUCT: + h = is_hfa (cif->rtype); + if (h) + compress_hfa_type (rvalue, &context.v[0], h); + else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) + memcpy (rvalue, &context.x[0], cif->rtype->size); + else + abort(); + break; - default: - FFI_ASSERT (0); - break; - } - } - else - { - context.x8 = (uintptr_t)rvalue; - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, - stack_bytes, fn); - } - break; - } - - default: - FFI_ASSERT (0); - break; + default: + abort(); + } + } + else + { + context.x8 = (uintptr_t)rvalue; + ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); } } @@ -1000,203 +847,158 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, ffi_cif *cif = closure->cif; void **avalue = (void**) alloca (cif->nargs * sizeof (void*)); void *rvalue = NULL; - int i, h; + int i, h, nargs = cif->nargs; struct arg_state state; + ffi_type *rtype; arg_init (&state, ALIGN(cif->bytes, 16)); - for (i = 0; i < cif->nargs; i++) + for (i = 0; i < nargs; i++) { ffi_type *ty = cif->arg_types[i]; + int t = ty->type; + size_t n, s = ty->size; - switch (ty->type) + switch (t) { case FFI_TYPE_VOID: FFI_ASSERT (0); break; + case FFI_TYPE_INT: case FFI_TYPE_UINT8: case FFI_TYPE_SINT8: case FFI_TYPE_UINT16: case FFI_TYPE_SINT16: case FFI_TYPE_UINT32: case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: case FFI_TYPE_UINT64: case FFI_TYPE_SINT64: + case FFI_TYPE_POINTER: + avalue[i] = allocate_int_to_reg_or_stack (context, &state, stack, s); + break; + case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - avalue[i] = allocate_to_register_or_stack (context, stack, - &state, ty->type); - break; + /* Scalar float is a degenerate case of HFA. */ + h = t + 0x100; + goto do_hfa; case FFI_TYPE_STRUCT: h = is_hfa (ty); if (h) { - unsigned n = h >> 8; - if (available_v (&state) < n) + do_hfa: + n = h >> 8; + if (state.nsrn + n <= N_V_ARG_REG) { - state.nsrn = N_V_ARG_REG; - avalue[i] = allocate_to_stack (&state, stack, ty->alignment, - ty->size); + void *reg = &context->v[state.nsrn]; + state.nsrn += n; + + /* Eeek! We need a pointer to the structure, however the + homogeneous float elements are being passed in individual + registers, therefore for float and double the structure + is not represented as a contiguous sequence of bytes in + our saved register context. We don't need the original + contents of the register storage, so we reformat the + structure into the same memory. */ + avalue[i] = compress_hfa_type (reg, reg, h); } else { - switch (h & 0xff) - { - case FFI_TYPE_FLOAT: - { - /* Eeek! We need a pointer to the structure, - however the homogeneous float elements are - being passed in individual S registers, - therefore the structure is not represented as - a contiguous sequence of bytes in our saved - register context. We need to fake up a copy - of the structure laid out in memory - correctly. The fake can be tossed once the - closure function has returned hence alloca() - is sufficient. */ - unsigned j; - UINT32 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < n; j++) - memcpy (&p[j], - allocate_to_s (context, &state), - sizeof (*p)); - break; - } - - case FFI_TYPE_DOUBLE: - { - /* Eeek! We need a pointer to the structure, - however the homogeneous float elements are - being passed in individual S registers, - therefore the structure is not represented as - a contiguous sequence of bytes in our saved - register context. We need to fake up a copy - of the structure laid out in memory - correctly. The fake can be tossed once the - closure function has returned hence alloca() - is sufficient. */ - unsigned j; - UINT64 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < n; j++) - memcpy (&p[j], - allocate_to_d (context, &state), - sizeof (*p)); - break; - } - - case FFI_TYPE_LONGDOUBLE: - memcpy (&avalue[i], - allocate_to_v (context, &state), - sizeof (*avalue)); - break; - - default: - FFI_ASSERT (0); - break; - } + state.nsrn = N_V_ARG_REG; + avalue[i] = allocate_to_stack (&state, stack, + ty->alignment, s); } } - else if (ty->size > 16) + else if (s > 16) { /* Replace Composite type of size greater than 16 with a pointer. */ - memcpy (&avalue[i], - allocate_to_register_or_stack (context, stack, - &state, FFI_TYPE_POINTER), - sizeof (avalue[i])); - } - else if (available_x (&state) >= (ty->size + 7) / 8) - { - avalue[i] = get_x_addr (context, state.ngrn); - state.ngrn += (ty->size + 7) / 8; + avalue[i] = *(void **) + allocate_int_to_reg_or_stack (context, &state, stack, + sizeof (void *)); } else { - state.ngrn = N_X_ARG_REG; - - avalue[i] = allocate_to_stack (&state, stack, ty->alignment, - ty->size); + n = (s + 7) / 8; + if (state.ngrn + n <= N_X_ARG_REG) + { + avalue[i] = &context->x[state.ngrn]; + state.ngrn += n; + } + else + { + state.ngrn = N_X_ARG_REG; + avalue[i] = allocate_to_stack (&state, stack, + ty->alignment, s); + } } break; default: - FFI_ASSERT (0); - break; + abort(); } } - /* Figure out where the return value will be passed, either in - registers or in a memory block allocated by the caller and passed - in x8. */ - - if (is_register_candidate (cif->rtype)) + /* Figure out where the return value will be passed, either in registers + or in a memory block allocated by the caller and passed in x8. */ + rtype = cif->rtype; + if (is_register_candidate (rtype)) { + size_t s = rtype->size; + int t; + /* Register candidates are *always* returned in registers. */ /* Allocate a scratchpad for the return value, we will let the callee scrible the result into the scratch pad then move the contents into the appropriate return value location for the call convention. */ - rvalue = alloca (cif->rtype->size); + rvalue = alloca (s); (closure->fun) (cif, rvalue, avalue, closure->user_data); /* Copy the return value into the call context so that it is returned as expected to our caller. */ - switch (cif->rtype->type) + t = rtype->type; + switch (t) { case FFI_TYPE_VOID: break; + case FFI_TYPE_INT: case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: case FFI_TYPE_UINT32: - case FFI_TYPE_POINTER: case FFI_TYPE_UINT64: case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: - case FFI_TYPE_INT: case FFI_TYPE_SINT32: case FFI_TYPE_SINT64: + case FFI_TYPE_POINTER: + context->x[0] = extend_integer_type (rvalue, t); + break; + case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - { - void *addr = get_basic_type_addr (cif->rtype->type, context, 0); - copy_basic_type (addr, rvalue, cif->rtype->type); - break; - } + extend_hfa_type (&context->v[0], rvalue, 0x100 + t); + break; + case FFI_TYPE_STRUCT: h = is_hfa (cif->rtype); if (h) - { - int j; - int type = h & 0xff; - int elems = h >> 8; - for (j = 0; j < elems; j++) - { - void *reg = get_basic_type_addr (type, context, j); - copy_basic_type (reg, rvalue, type); - rvalue += get_basic_type_size (type); - } - } - else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) - { - size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)) ; - memcpy (get_x_addr (context, 0), rvalue, size); - } + extend_hfa_type (&context->v[0], rvalue, h); else - { - FFI_ASSERT (0); + { + FFI_ASSERT (s <= 16); + memcpy (&context->x[0], rvalue, s); } break; + default: - FFI_ASSERT (0); - break; + abort(); } } else From 325471ea6a7bf954943485458a1bd391635dfaa8 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 13:58:59 -0400 Subject: [PATCH 28/58] aarch64: Merge prep_args with ffi_call Use the trick to allocate the stack frame for ffi_call_SYSV within ffi_call itself. --- src/aarch64/ffi.c | 195 +++++++++++++++++++++------------------------ src/aarch64/sysv.S | 188 +++++++++++++------------------------------ 2 files changed, 143 insertions(+), 240 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index d19384b..a067303 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -71,14 +71,6 @@ ffi_clear_cache (void *start, void *end) #endif } -extern void -ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *, - extended_cif *), - struct call_context *context, - extended_cif *, - size_t, - void (*fn)(void)); - extern void ffi_closure_SYSV (ffi_closure *); @@ -311,12 +303,11 @@ struct arg_state /* Initialize a procedure call argument marshalling state. */ static void -arg_init (struct arg_state *state, size_t call_frame_size) +arg_init (struct arg_state *state) { state->ngrn = 0; state->nsrn = 0; state->nsaa = 0; - #if defined (__APPLE__) state->allocating_variadic = 0; #endif @@ -529,27 +520,88 @@ allocate_int_to_reg_or_stack (struct call_context *context, return allocate_to_stack (state, stack, size, size); } -/* Marshall the arguments from FFI representation to procedure call - context and stack. */ - -static unsigned -aarch64_prep_args (struct call_context *context, unsigned char *stack, - extended_cif *ecif) +ffi_status +ffi_prep_cif_machdep (ffi_cif *cif) { - ffi_cif *cif = ecif->cif; - void **avalue = ecif->avalue; - int i, nargs = cif->nargs; + /* Round the stack up to a multiple of the stack alignment requirement. */ + cif->bytes = ALIGN(cif->bytes, 16); + + /* Initialize our flags. We are interested if this CIF will touch a + vector register, if so we will enable context save and load to + those registers, otherwise not. This is intended to be friendly + to lazy float context switching in the kernel. */ + cif->aarch64_flags = 0; + + if (is_v_register_candidate (cif->rtype)) + { + cif->aarch64_flags |= AARCH64_FLAG_ARG_V; + } + else + { + int i; + for (i = 0; i < cif->nargs; i++) + if (is_v_register_candidate (cif->arg_types[i])) + { + cif->aarch64_flags |= AARCH64_FLAG_ARG_V; + break; + } + } + +#if defined (__APPLE__) + cif->aarch64_nfixedargs = 0; +#endif + + return FFI_OK; +} + +#if defined (__APPLE__) + +/* Perform Apple-specific cif processing for variadic calls */ +ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, + unsigned int nfixedargs, + unsigned int ntotalargs) +{ + ffi_status status; + + status = ffi_prep_cif_machdep (cif); + + cif->aarch64_nfixedargs = nfixedargs; + + return status; +} + +#endif + +extern void ffi_call_SYSV (void *stack, void *frame, + void (*fn)(void), int flags) FFI_HIDDEN; + +/* Call a function with the provided arguments and capture the return + value. */ +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + struct call_context *context; + void *stack, *frame; struct arg_state state; + size_t stack_bytes; + int i, nargs = cif->nargs; + int h, t; + ffi_type *rtype; - arg_init (&state, cif->bytes); + /* Allocate consectutive stack for everything we'll need. */ + stack_bytes = cif->bytes; + stack = alloca (stack_bytes + 32 + sizeof(struct call_context)); + frame = stack + stack_bytes; + context = frame + 32; + arg_init (&state); for (i = 0; i < nargs; i++) { ffi_type *ty = cif->arg_types[i]; size_t s = ty->size; - int h, t = ty->type; void *a = avalue[i]; + t = ty->type; switch (t) { case FFI_TYPE_VOID: @@ -665,83 +717,12 @@ aarch64_prep_args (struct call_context *context, unsigned char *stack, #endif } - return cif->aarch64_flags; -} - -ffi_status -ffi_prep_cif_machdep (ffi_cif *cif) -{ - /* Round the stack up to a multiple of the stack alignment requirement. */ - cif->bytes = ALIGN(cif->bytes, 16); - - /* Initialize our flags. We are interested if this CIF will touch a - vector register, if so we will enable context save and load to - those registers, otherwise not. This is intended to be friendly - to lazy float context switching in the kernel. */ - cif->aarch64_flags = 0; - - if (is_v_register_candidate (cif->rtype)) + rtype = cif->rtype; + if (is_register_candidate (rtype)) { - cif->aarch64_flags |= AARCH64_FLAG_ARG_V; - } - else - { - int i; - for (i = 0; i < cif->nargs; i++) - if (is_v_register_candidate (cif->arg_types[i])) - { - cif->aarch64_flags |= AARCH64_FLAG_ARG_V; - break; - } - } + ffi_call_SYSV (stack, frame, fn, cif->aarch64_flags); -#if defined (__APPLE__) - cif->aarch64_nfixedargs = 0; -#endif - - return FFI_OK; -} - -#if defined (__APPLE__) - -/* Perform Apple-specific cif processing for variadic calls */ -ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, - unsigned int nfixedargs, - unsigned int ntotalargs) -{ - ffi_status status; - - status = ffi_prep_cif_machdep (cif); - - cif->aarch64_nfixedargs = nfixedargs; - - return status; -} - -#endif - -/* Call a function with the provided arguments and capture the return - value. */ -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) -{ - extended_cif ecif; - struct call_context context; - size_t stack_bytes; - int h, t; - - ecif.cif = cif; - ecif.avalue = avalue; - ecif.rvalue = rvalue; - - stack_bytes = cif->bytes; - - memset (&context, 0, sizeof (context)); - if (is_register_candidate (cif->rtype)) - { - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); - - t = cif->rtype->type; + t = rtype->type; switch (t) { case FFI_TYPE_INT: @@ -754,33 +735,35 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) case FFI_TYPE_POINTER: case FFI_TYPE_UINT64: case FFI_TYPE_SINT64: - *(ffi_arg *)rvalue = extend_integer_type (&context.x[0], t); + *(ffi_arg *)rvalue = extend_integer_type (&context->x[0], t); break; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - compress_hfa_type (rvalue, &context.v[0], 0x100 + t); + compress_hfa_type (rvalue, &context->v[0], 0x100 + t); break; case FFI_TYPE_STRUCT: h = is_hfa (cif->rtype); if (h) - compress_hfa_type (rvalue, &context.v[0], h); - else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) - memcpy (rvalue, &context.x[0], cif->rtype->size); + compress_hfa_type (rvalue, &context->v[0], h); else - abort(); + { + FFI_ASSERT (rtype->size <= 16); + memcpy (rvalue, &context->x[0], rtype->size); + } break; default: - abort(); + FFI_ASSERT (0); + break; } } else { - context.x8 = (uintptr_t)rvalue; - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); + context->x8 = (uintptr_t)rvalue; + ffi_call_SYSV (stack, frame, fn, cif->aarch64_flags); } } @@ -851,7 +834,7 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, struct arg_state state; ffi_type *rtype; - arg_init (&state, ALIGN(cif->bytes, 16)); + arg_init (&state); for (i = 0; i < nargs; i++) { diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index fa7ff5b..a5f636a 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -22,6 +22,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #define LIBFFI_ASM #include #include +#include #include "internal.h" #ifdef HAVE_MACHINE_ASM_H @@ -38,158 +39,77 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #endif #endif -#define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off -#define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off -#define cfi_restore(reg) .cfi_restore reg -#define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg + .text + .align 2 - .text - .align 2 - - .globl CNAME(ffi_call_SYSV) + .globl CNAME(ffi_call_SYSV) #ifdef __ELF__ - .type CNAME(ffi_call_SYSV), #function + .type CNAME(ffi_call_SYSV), #function + .hidden CNAME(ffi_call_SYSV) #endif -/* ffi_call_SYSV() - - Create a stack frame, setup an argument context, call the callee - and extract the result. - - The maximum required argument stack size is provided, - ffi_call_SYSV() allocates that stack space then calls the - prepare_fn to populate register context and stack. The - argument passing registers are loaded from the register - context and the callee called, on return the register passing - register are saved back to the context. Our caller will - extract the return value from the final state of the saved - register context. - - Prototype: - - extern unsigned - ffi_call_SYSV (void (*)(struct call_context *context, unsigned char *, - extended_cif *), - struct call_context *context, - extended_cif *, - size_t required_stack_size, - void (*fn)(void)); +/* ffi_call_SYSV + extern void ffi_call_SYSV (void *stack, void *frame, + void (*fn)(void), int flags); Therefore on entry we have: - x0 prepare_fn - x1 &context - x2 &ecif - x3 bytes - x4 fn + x0 stack + x1 frame + x2 fn + x3 flags +*/ - This function uses the following stack frame layout: - - == - saved x30(lr) - x29(fp)-> saved x29(fp) - saved x24 - saved x23 - saved x22 - sp' -> saved x21 - ... - sp -> (constructed callee stack arguments) - == - - Voila! */ - -#define ffi_call_SYSV_FS (8 * 4) - - .cfi_startproc + cfi_startproc CNAME(ffi_call_SYSV): - stp x29, x30, [sp, #-16]! - cfi_adjust_cfa_offset (16) - cfi_rel_offset (x29, 0) - cfi_rel_offset (x30, 8) + /* Use a stack frame allocated by our caller. */ + cfi_def_cfa(x1, 32); + stp x29, x30, [x1] + mov x29, x1 + mov sp, x0 + cfi_def_cfa_register(x29) + cfi_rel_offset (x29, 0) + cfi_rel_offset (x30, 8) - mov x29, sp - cfi_def_cfa_register (x29) - sub sp, sp, #ffi_call_SYSV_FS + str w3, [x29, #16] /* save flags */ + mov x9, x2 /* save fn */ - stp x21, x22, [sp, #0] - cfi_rel_offset (x21, 0 - ffi_call_SYSV_FS) - cfi_rel_offset (x22, 8 - ffi_call_SYSV_FS) - - stp x23, x24, [sp, #16] - cfi_rel_offset (x23, 16 - ffi_call_SYSV_FS) - cfi_rel_offset (x24, 24 - ffi_call_SYSV_FS) - - mov x21, x1 - mov x22, x2 - mov x24, x4 - - /* Allocate the stack space for the actual arguments, many - arguments will be passed in registers, but we assume - worst case and allocate sufficient stack for ALL of - the arguments. */ - sub sp, sp, x3 - - /* unsigned (*prepare_fn) (struct call_context *context, - unsigned char *stack, extended_cif *ecif); - */ - mov x23, x0 - mov x0, x1 - mov x1, sp - /* x2 already in place */ - blr x23 - - /* Preserve the flags returned. */ - mov x23, x0 - - /* Figure out if we should touch the vector registers. */ - tbz x23, #AARCH64_FLAG_ARG_V_BIT, 1f - - /* Load the vector argument passing registers. */ - ldp q0, q1, [x21, #0] - ldp q2, q3, [x21, #32] - ldp q4, q5, [x21, #64] - ldp q6, q7, [x21, #96] + /* Load the vector argument passing registers, if necessary. */ + tbz w3, #AARCH64_FLAG_ARG_V_BIT, 1f + ldp q0, q1, [x29, #32 + 0] + ldp q2, q3, [x29, #32 + 32] + ldp q4, q5, [x29, #32 + 64] + ldp q6, q7, [x29, #32 + 96] 1: - /* Load the core argument passing registers, including + /* Load the core argument passing registers, including the structure return pointer. */ - ldp x0, x1, [x21, #16*N_V_ARG_REG + 0] - ldp x2, x3, [x21, #16*N_V_ARG_REG + 16] - ldp x4, x5, [x21, #16*N_V_ARG_REG + 32] - ldp x6, x7, [x21, #16*N_V_ARG_REG + 48] - ldr x8, [x21, #16*N_V_ARG_REG + 64] + ldp x0, x1, [x29, #32 + 16*N_V_ARG_REG + 0] + ldp x2, x3, [x29, #32 + 16*N_V_ARG_REG + 16] + ldp x4, x5, [x29, #32 + 16*N_V_ARG_REG + 32] + ldp x6, x7, [x29, #32 + 16*N_V_ARG_REG + 48] + ldr x8, [x29, #32 + 16*N_V_ARG_REG + 64] - blr x24 + blr x9 /* call fn */ - /* Save the core return registers. */ - stp x0, x1, [x21, #16*N_V_ARG_REG] + ldr w3, [x29, #16] /* reload flags */ - /* Figure out if we should touch the vector registers. */ - tbz x23, #AARCH64_FLAG_ARG_V_BIT, 1f - - /* Save the vector return registers. */ - stp q0, q1, [x21, #0] - stp q2, q3, [x21, #32] -1: - /* All done, unwind our stack frame. */ - ldp x21, x22, [x29, # - ffi_call_SYSV_FS] - cfi_restore (x21) - cfi_restore (x22) - - ldp x23, x24, [x29, # - ffi_call_SYSV_FS + 16] - cfi_restore (x23) - cfi_restore (x24) - - mov sp, x29 + /* Partially deconstruct the stack frame. */ + mov sp, x29 cfi_def_cfa_register (sp) + ldp x29, x30, [x29] - ldp x29, x30, [sp], #16 - cfi_adjust_cfa_offset (-16) - cfi_restore (x29) - cfi_restore (x30) + /* Save the core return registers. */ + stp x0, x1, [sp, #32 + 16*N_V_ARG_REG] - ret + /* Save the vector return registers, if necessary. */ + tbz w3, #AARCH64_FLAG_ARG_V_BIT, 1f + stp q0, q1, [sp, #32 + 0] + stp q2, q3, [sp, #32 + 32] +1: + /* All done. */ + ret - .cfi_endproc + cfi_endproc #ifdef __ELF__ .size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV) #endif @@ -237,7 +157,7 @@ CNAME(ffi_call_SYSV): .align 2 .globl CNAME(ffi_closure_SYSV) - .cfi_startproc + cfi_startproc CNAME(ffi_closure_SYSV): stp x29, x30, [sp, #-16]! cfi_adjust_cfa_offset (16) @@ -310,7 +230,7 @@ CNAME(ffi_closure_SYSV): cfi_restore (x30) ret - .cfi_endproc + cfi_endproc #ifdef __ELF__ .size CNAME(ffi_closure_SYSV), .-CNAME(ffi_closure_SYSV) #endif From 4fe1aea1211aaf228c2db867b8ac6f1620de72d4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 17:06:19 -0400 Subject: [PATCH 29/58] aarch64: Move return value handling into ffi_call_SYSV This lets us pass return data directly to the caller of ffi_call in most cases, rather than storing it into temporary storage first. --- src/aarch64/ffi.c | 208 +++++++++++++++++++++++------------------ src/aarch64/internal.h | 43 ++++++++- src/aarch64/sysv.S | 125 +++++++++++++++++++------ 3 files changed, 260 insertions(+), 116 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index a067303..ffa1363 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -523,30 +523,90 @@ allocate_int_to_reg_or_stack (struct call_context *context, ffi_status ffi_prep_cif_machdep (ffi_cif *cif) { + ffi_type *rtype = cif->rtype; + size_t bytes = cif->bytes; + int flags, aarch64_flags, i, n; + + switch (rtype->type) + { + case FFI_TYPE_VOID: + flags = AARCH64_RET_VOID; + break; + case FFI_TYPE_UINT8: + flags = AARCH64_RET_UINT8; + break; + case FFI_TYPE_UINT16: + flags = AARCH64_RET_UINT16; + break; + case FFI_TYPE_UINT32: + flags = AARCH64_RET_UINT32; + break; + case FFI_TYPE_SINT8: + flags = AARCH64_RET_SINT8; + break; + case FFI_TYPE_SINT16: + flags = AARCH64_RET_SINT16; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + flags = AARCH64_RET_SINT32; + break; + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + flags = AARCH64_RET_INT64; + break; + case FFI_TYPE_POINTER: + flags = (sizeof(void *) == 4 ? AARCH64_RET_UINT32 : AARCH64_RET_INT64); + break; + + case FFI_TYPE_FLOAT: + flags = AARCH64_RET_S1; + break; + case FFI_TYPE_DOUBLE: + flags = AARCH64_RET_D1; + break; + case FFI_TYPE_LONGDOUBLE: + flags = AARCH64_RET_Q1; + break; + + case FFI_TYPE_STRUCT: + { + int h = is_hfa (rtype); + size_t s = rtype->size; + + if (h) + flags = (h & 0xff) * 4 + 4 - (h >> 8); + else if (s > 16) + { + flags = AARCH64_RET_VOID | AARCH64_RET_IN_MEM; + bytes += 8; + } + else if (s == 16) + flags = AARCH64_RET_INT128; + else if (s == 8) + flags = AARCH64_RET_INT64; + else + flags = AARCH64_RET_INT128 | AARCH64_RET_NEED_COPY; + } + break; + + default: + abort(); + } + + aarch64_flags = 0; + for (i = 0, n = cif->nargs; i < n; i++) + if (is_v_register_candidate (cif->arg_types[i])) + { + aarch64_flags = AARCH64_FLAG_ARG_V; + flags |= AARCH64_FLAG_ARG_V; + break; + } + /* Round the stack up to a multiple of the stack alignment requirement. */ - cif->bytes = ALIGN(cif->bytes, 16); - - /* Initialize our flags. We are interested if this CIF will touch a - vector register, if so we will enable context save and load to - those registers, otherwise not. This is intended to be friendly - to lazy float context switching in the kernel. */ - cif->aarch64_flags = 0; - - if (is_v_register_candidate (cif->rtype)) - { - cif->aarch64_flags |= AARCH64_FLAG_ARG_V; - } - else - { - int i; - for (i = 0; i < cif->nargs; i++) - if (is_v_register_candidate (cif->arg_types[i])) - { - cif->aarch64_flags |= AARCH64_FLAG_ARG_V; - break; - } - } - + cif->bytes = ALIGN(bytes, 16); + cif->flags = flags; + cif->aarch64_flags = aarch64_flags; #if defined (__APPLE__) cif->aarch64_nfixedargs = 0; #endif @@ -555,51 +615,65 @@ ffi_prep_cif_machdep (ffi_cif *cif) } #if defined (__APPLE__) - /* Perform Apple-specific cif processing for variadic calls */ ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) { - ffi_status status; - - status = ffi_prep_cif_machdep (cif); - + ffi_status status = ffi_prep_cif_machdep (cif); cif->aarch64_nfixedargs = nfixedargs; - return status; } +#endif /* __APPLE__ */ -#endif - -extern void ffi_call_SYSV (void *stack, void *frame, - void (*fn)(void), int flags) FFI_HIDDEN; +extern void ffi_call_SYSV (struct call_context *context, void *frame, + void (*fn)(void), void *rvalue, int flags) + FFI_HIDDEN; /* Call a function with the provided arguments and capture the return value. */ void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) { struct call_context *context; - void *stack, *frame; + void *stack, *frame, *rvalue; struct arg_state state; - size_t stack_bytes; - int i, nargs = cif->nargs; - int h, t; + size_t stack_bytes, rtype_size, rsize; + int i, nargs, flags; ffi_type *rtype; - /* Allocate consectutive stack for everything we'll need. */ + flags = cif->flags; + rtype = cif->rtype; + rtype_size = rtype->size; stack_bytes = cif->bytes; - stack = alloca (stack_bytes + 32 + sizeof(struct call_context)); + + /* If the target function returns a structure via hidden pointer, + then we cannot allow a null rvalue. Otherwise, mash a null + rvalue to void return type. */ + rsize = 0; + if (flags & AARCH64_RET_IN_MEM) + { + if (orig_rvalue == NULL) + rsize = rtype_size; + } + else if (orig_rvalue == NULL) + flags &= AARCH64_FLAG_ARG_V; + else if (flags & AARCH64_RET_NEED_COPY) + rsize = 16; + + /* Allocate consectutive stack for everything we'll need. */ + context = alloca (sizeof(struct call_context) + stack_bytes + 32 + rsize); + stack = context + 1; frame = stack + stack_bytes; - context = frame + 32; + rvalue = (rsize ? frame + 32 : orig_rvalue); arg_init (&state); - for (i = 0; i < nargs; i++) + for (i = 0, nargs = cif->nargs; i < nargs; i++) { ffi_type *ty = cif->arg_types[i]; size_t s = ty->size; void *a = avalue[i]; + int h, t; t = ty->type; switch (t) @@ -717,54 +791,10 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) #endif } - rtype = cif->rtype; - if (is_register_candidate (rtype)) - { - ffi_call_SYSV (stack, frame, fn, cif->aarch64_flags); + ffi_call_SYSV (context, frame, fn, rvalue, flags); - t = rtype->type; - switch (t) - { - case FFI_TYPE_INT: - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - *(ffi_arg *)rvalue = extend_integer_type (&context->x[0], t); - break; - - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - compress_hfa_type (rvalue, &context->v[0], 0x100 + t); - break; - - case FFI_TYPE_STRUCT: - h = is_hfa (cif->rtype); - if (h) - compress_hfa_type (rvalue, &context->v[0], h); - else - { - FFI_ASSERT (rtype->size <= 16); - memcpy (rvalue, &context->x[0], rtype->size); - } - break; - - default: - FFI_ASSERT (0); - break; - } - } - else - { - context->x8 = (uintptr_t)rvalue; - ffi_call_SYSV (stack, frame, fn, cif->aarch64_flags); - } + if (flags & AARCH64_RET_NEED_COPY) + memcpy (orig_rvalue, rvalue, rtype_size); } static unsigned char trampoline [] = diff --git a/src/aarch64/internal.h b/src/aarch64/internal.h index b6b6104..a3070db 100644 --- a/src/aarch64/internal.h +++ b/src/aarch64/internal.h @@ -18,7 +18,48 @@ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#define AARCH64_FLAG_ARG_V_BIT 0 +#define AARCH64_RET_VOID 0 +#define AARCH64_RET_INT64 1 +#define AARCH64_RET_INT128 2 + +#define AARCH64_RET_UNUSED3 3 +#define AARCH64_RET_UNUSED4 4 +#define AARCH64_RET_UNUSED5 5 +#define AARCH64_RET_UNUSED6 6 +#define AARCH64_RET_UNUSED7 7 + +/* Note that FFI_TYPE_FLOAT == 2, _DOUBLE == 3, _LONGDOUBLE == 4, + so _S4 through _Q1 are layed out as (TYPE * 4) + (4 - COUNT). */ +#define AARCH64_RET_S4 8 +#define AARCH64_RET_S3 9 +#define AARCH64_RET_S2 10 +#define AARCH64_RET_S1 11 + +#define AARCH64_RET_D4 12 +#define AARCH64_RET_D3 13 +#define AARCH64_RET_D2 14 +#define AARCH64_RET_D1 15 + +#define AARCH64_RET_Q4 16 +#define AARCH64_RET_Q3 17 +#define AARCH64_RET_Q2 18 +#define AARCH64_RET_Q1 19 + +/* Note that each of the sub-64-bit integers gets two entries. */ +#define AARCH64_RET_UINT8 20 +#define AARCH64_RET_UINT16 22 +#define AARCH64_RET_UINT32 24 + +#define AARCH64_RET_SINT8 26 +#define AARCH64_RET_SINT16 28 +#define AARCH64_RET_SINT32 30 + +#define AARCH64_RET_MASK 31 + +#define AARCH64_RET_IN_MEM (1 << 5) +#define AARCH64_RET_NEED_COPY (1 << 6) + +#define AARCH64_FLAG_ARG_V_BIT 7 #define AARCH64_FLAG_ARG_V (1 << AARCH64_FLAG_ARG_V_BIT) #define N_X_ARG_REG 8 diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index a5f636a..ba15663 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -40,9 +40,9 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #endif .text - .align 2 + .align 4 - .globl CNAME(ffi_call_SYSV) + .globl CNAME(ffi_call_SYSV) #ifdef __ELF__ .type CNAME(ffi_call_SYSV), #function .hidden CNAME(ffi_call_SYSV) @@ -50,14 +50,15 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* ffi_call_SYSV extern void ffi_call_SYSV (void *stack, void *frame, - void (*fn)(void), int flags); + void (*fn)(void), void *rvalue, int flags); Therefore on entry we have: x0 stack x1 frame x2 fn - x3 flags + x3 rvalue + x4 flags */ cfi_startproc @@ -71,43 +72,111 @@ CNAME(ffi_call_SYSV): cfi_rel_offset (x29, 0) cfi_rel_offset (x30, 8) - str w3, [x29, #16] /* save flags */ mov x9, x2 /* save fn */ + mov x8, x3 /* install structure return */ + stp x3, x4, [x29, #16] /* save rvalue and flags */ /* Load the vector argument passing registers, if necessary. */ - tbz w3, #AARCH64_FLAG_ARG_V_BIT, 1f - ldp q0, q1, [x29, #32 + 0] - ldp q2, q3, [x29, #32 + 32] - ldp q4, q5, [x29, #32 + 64] - ldp q6, q7, [x29, #32 + 96] + tbz w4, #AARCH64_FLAG_ARG_V_BIT, 1f + ldp q0, q1, [sp, #0] + ldp q2, q3, [sp, #32] + ldp q4, q5, [sp, #64] + ldp q6, q7, [sp, #96] 1: /* Load the core argument passing registers, including the structure return pointer. */ - ldp x0, x1, [x29, #32 + 16*N_V_ARG_REG + 0] - ldp x2, x3, [x29, #32 + 16*N_V_ARG_REG + 16] - ldp x4, x5, [x29, #32 + 16*N_V_ARG_REG + 32] - ldp x6, x7, [x29, #32 + 16*N_V_ARG_REG + 48] - ldr x8, [x29, #32 + 16*N_V_ARG_REG + 64] + ldp x0, x1, [sp, #16*N_V_ARG_REG + 0] + ldp x2, x3, [sp, #16*N_V_ARG_REG + 16] + ldp x4, x5, [sp, #16*N_V_ARG_REG + 32] + ldp x6, x7, [sp, #16*N_V_ARG_REG + 48] + + /* Deallocate the context, leaving the stacked arguments. */ + add sp, sp, #CALL_CONTEXT_SIZE blr x9 /* call fn */ - ldr w3, [x29, #16] /* reload flags */ + ldp x3, x4, [x29, #16] /* reload rvalue and flags */ /* Partially deconstruct the stack frame. */ mov sp, x29 cfi_def_cfa_register (sp) ldp x29, x30, [x29] - /* Save the core return registers. */ - stp x0, x1, [sp, #32 + 16*N_V_ARG_REG] + /* Save the return value as directed. */ + adr x5, 0f + and w4, w4, #AARCH64_RET_MASK + add x5, x5, x4, lsl #3 + br x5 - /* Save the vector return registers, if necessary. */ - tbz w3, #AARCH64_FLAG_ARG_V_BIT, 1f - stp q0, q1, [sp, #32 + 0] - stp q2, q3, [sp, #32 + 32] -1: - /* All done. */ + /* Note that each table entry is 2 insns, and thus 8 bytes. + For integer data, note that we're storing into ffi_arg + and therefore we want to extend to 64 bits; these types + have two consecutive entries allocated for them. */ + .align 4 +0: ret /* VOID */ + nop +1: str x0, [x3] /* INT64 */ ret +2: stp x0, x1, [x3] /* INT128 */ + ret +3: brk #1000 /* UNUSED */ + ret +4: brk #1000 /* UNUSED */ + ret +5: brk #1000 /* UNUSED */ + ret +6: brk #1000 /* UNUSED */ + ret +7: brk #1000 /* UNUSED */ + ret +8: st4 { v0.s-v3.s }[0], [x3] /* S4 */ + ret +9: st3 { v0.s-v2.s }[0], [x3] /* S3 */ + ret +10: stp s0, s1, [x3] /* S2 */ + ret +11: str s0, [x3] /* S1 */ + ret +12: st4 { v0.d-v3.d }[0], [x3] /* D4 */ + ret +13: st3 { v0.d-v2.d }[0], [x3] /* D3 */ + ret +14: stp d0, d1, [x3] /* D2 */ + ret +15: str d0, [x3] /* D1 */ + ret +16: str q3, [x3, #48] /* Q4 */ + nop +17: str q2, [x3, #32] /* Q3 */ + nop +18: stp q0, q1, [x3] /* Q2 */ + ret +19: str q0, [x3] /* Q1 */ + ret +20: uxtb w0, w0 /* UINT8 */ + str x0, [x3] +21: ret /* reserved */ + nop +22: uxth w0, w0 /* UINT16 */ + str x0, [x3] +23: ret /* reserved */ + nop +24: mov w0, w0 /* UINT32 */ + str x0, [x3] +25: ret /* reserved */ + nop +26: sxtb x0, w0 /* SINT8 */ + str x0, [x3] +27: ret /* reserved */ + nop +28: sxth x0, w0 /* SINT16 */ + str x0, [x3] +29: ret /* reserved */ + nop +30: sxtw x0, w0 /* SINT32 */ + str x0, [x3] +31: ret /* reserved */ + nop cfi_endproc #ifdef __ELF__ @@ -154,9 +223,13 @@ CNAME(ffi_call_SYSV): Voila! */ .text - .align 2 + .align 4 - .globl CNAME(ffi_closure_SYSV) + .globl CNAME(ffi_closure_SYSV) +#ifdef __ELF__ + .type CNAME(ffi_closure_SYSV), #function + .hidden CNAME(ffi_closure_SYSV) +#endif cfi_startproc CNAME(ffi_closure_SYSV): stp x29, x30, [sp, #-16]! From 12cf89ee04dc01650f71bc38ce414c953c83199a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 21:53:30 -0400 Subject: [PATCH 30/58] aarch64: Move return value handling into ffi_closure_SYSV As with the change to ffi_call_SYSV, this avoids copying data into a temporary buffer. --- src/aarch64/ffi.c | 196 ++++++------------------------- src/aarch64/ffitarget.h | 2 +- src/aarch64/sysv.S | 247 ++++++++++++++++++++++------------------ 3 files changed, 175 insertions(+), 270 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index ffa1363..c5a429a 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -71,9 +71,6 @@ ffi_clear_cache (void *start, void *end) #endif } -extern void -ffi_closure_SYSV (ffi_closure *); - /* Test for an FFI floating point representation. */ static unsigned @@ -211,69 +208,6 @@ is_hfa(const ffi_type *ty) return (ele_count << 8) | candidate; } -/* Test if an ffi_type is a candidate for passing in a register. - - This test does not check that sufficient registers of the - appropriate class are actually available, merely that IFF - sufficient registers are available then the argument will be passed - in register(s). - - Note that an ffi_type that is deemed to be a register candidate - will always be returned in registers. - - Returns 1 if a register candidate else 0. */ - -static int -is_register_candidate (ffi_type *ty) -{ - switch (ty->type) - { - case FFI_TYPE_VOID: - return 0; - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_UINT64: - case FFI_TYPE_POINTER: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_SINT64: - return 1; - - case FFI_TYPE_STRUCT: - if (is_hfa (ty)) - { - return 1; - } - else if (ty->size > 16) - { - /* Too large. Will be replaced with a pointer to memory. The - pointer MAY be passed in a register, but the value will - not. This test specifically fails since the argument will - never be passed by value in registers. */ - return 0; - } - else - { - /* Might be passed in registers depending on the number of - registers required. */ - return (ty->size + 7) / 8 < N_X_ARG_REG; - } - break; - - default: - FFI_ASSERT (0); - break; - } - - return 0; -} - /* Test if an ffi_type argument or result is a candidate for a vector register. */ @@ -797,42 +731,42 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) memcpy (orig_rvalue, rvalue, rtype_size); } -static unsigned char trampoline [] = -{ 0x70, 0x00, 0x00, 0x58, /* ldr x16, 1f */ - 0x91, 0x00, 0x00, 0x10, /* adr x17, 2f */ - 0x00, 0x02, 0x1f, 0xd6 /* br x16 */ -}; - /* Build a trampoline. */ -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \ - ({unsigned char *__tramp = (unsigned char*)(TRAMP); \ - UINT64 __fun = (UINT64)(FUN); \ - UINT64 __ctx = (UINT64)(CTX); \ - UINT64 __flags = (UINT64)(FLAGS); \ - memcpy (__tramp, trampoline, sizeof (trampoline)); \ - memcpy (__tramp + 12, &__fun, sizeof (__fun)); \ - memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \ - memcpy (__tramp + 28, &__flags, sizeof (__flags)); \ - ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \ - }) +extern void ffi_closure_SYSV (void) FFI_HIDDEN; +extern void ffi_closure_SYSV_V (void) FFI_HIDDEN; ffi_status -ffi_prep_closure_loc (ffi_closure* closure, +ffi_prep_closure_loc (ffi_closure *closure, ffi_cif* cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc) { + static const unsigned char trampoline[16] = { + 0x90, 0x00, 0x00, 0x58, /* ldr x16, tramp+16 */ + 0xf1, 0xff, 0xff, 0x10, /* adr x17, tramp+0 */ + 0x00, 0x02, 0x1f, 0xd6 /* br x16 */ + }; + char *tramp = closure->tramp; + void (*start)(void); + if (cif->abi != FFI_SYSV) return FFI_BAD_ABI; - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc, - cif->aarch64_flags); - - closure->cif = cif; + closure->cif = cif; + closure->fun = fun; closure->user_data = user_data; - closure->fun = fun; + + memcpy (tramp, trampoline, sizeof(trampoline)); + + if (cif->flags & AARCH64_FLAG_ARG_V) + start = ffi_closure_SYSV_V; + else + start = ffi_closure_SYSV; + *(UINT64 *)(tramp + 16) = (uintptr_t)start; + + ffi_clear_cache(tramp, tramp + FFI_TRAMPOLINE_SIZE); return FFI_OK; } @@ -853,20 +787,20 @@ ffi_prep_closure_loc (ffi_closure* closure, descriptors, invokes the wrapped function, then marshalls the return value back into the call context. */ -void FFI_HIDDEN -ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, - void *stack) +int FFI_HIDDEN +ffi_closure_SYSV_inner (ffi_cif *cif, + void (*fun)(ffi_cif*,void*,void**,void*), + void *user_data, + struct call_context *context, + void *stack, void *rvalue) { - ffi_cif *cif = closure->cif; void **avalue = (void**) alloca (cif->nargs * sizeof (void*)); - void *rvalue = NULL; - int i, h, nargs = cif->nargs; + int i, h, nargs, flags; struct arg_state state; - ffi_type *rtype; arg_init (&state); - for (i = 0; i < nargs; i++) + for (i = 0, nargs = cif->nargs; i < nargs; i++) { ffi_type *ty = cif->arg_types[i]; int t = ty->type; @@ -955,69 +889,11 @@ ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, } } - /* Figure out where the return value will be passed, either in registers - or in a memory block allocated by the caller and passed in x8. */ - rtype = cif->rtype; - if (is_register_candidate (rtype)) - { - size_t s = rtype->size; - int t; + flags = cif->flags; + if (flags & AARCH64_RET_IN_MEM) + rvalue = (void *)(uintptr_t)context->x8; - /* Register candidates are *always* returned in registers. */ + fun (cif, rvalue, avalue, user_data); - /* Allocate a scratchpad for the return value, we will let the - callee scrible the result into the scratch pad then move the - contents into the appropriate return value location for the - call convention. */ - rvalue = alloca (s); - (closure->fun) (cif, rvalue, avalue, closure->user_data); - - /* Copy the return value into the call context so that it is returned - as expected to our caller. */ - t = rtype->type; - switch (t) - { - case FFI_TYPE_VOID: - break; - - case FFI_TYPE_INT: - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_POINTER: - context->x[0] = extend_integer_type (rvalue, t); - break; - - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - extend_hfa_type (&context->v[0], rvalue, 0x100 + t); - break; - - case FFI_TYPE_STRUCT: - h = is_hfa (cif->rtype); - if (h) - extend_hfa_type (&context->v[0], rvalue, h); - else - { - FFI_ASSERT (s <= 16); - memcpy (&context->x[0], rvalue, s); - } - break; - - default: - abort(); - } - } - else - { - rvalue = (void *)(uintptr_t)context->x8; - (closure->fun) (cif, rvalue, avalue, closure->user_data); - } + return flags; } - diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index 336f28a..b488bbe 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -42,7 +42,7 @@ typedef enum ffi_abi /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 -#define FFI_TRAMPOLINE_SIZE 36 +#define FFI_TRAMPOLINE_SIZE 24 #define FFI_NATIVE_RAW_API 0 /* ---- Internal ---- */ diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index ba15663..abd848d 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -37,17 +37,17 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #else #define CNAME(x) x #endif +#endif + +#ifdef __AARCH64EB__ +# define BE(X) X +#else +# define BE(X) 0 #endif .text .align 4 - .globl CNAME(ffi_call_SYSV) -#ifdef __ELF__ - .type CNAME(ffi_call_SYSV), #function - .hidden CNAME(ffi_call_SYSV) -#endif - /* ffi_call_SYSV extern void ffi_call_SYSV (void *stack, void *frame, void (*fn)(void), void *rvalue, int flags); @@ -179,131 +179,160 @@ CNAME(ffi_call_SYSV): nop cfi_endproc -#ifdef __ELF__ - .size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV) -#endif -#define ffi_closure_SYSV_FS (8 * 2 + CALL_CONTEXT_SIZE) + .globl CNAME(ffi_call_SYSV) +#ifdef __ELF__ + .type CNAME(ffi_call_SYSV), #function + .hidden CNAME(ffi_call_SYSV) + .size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV) +#endif /* ffi_closure_SYSV Closure invocation glue. This is the low level code invoked directly by the closure trampoline to setup and call a closure. - On entry x17 points to a struct trampoline_data, x16 has been clobbered + On entry x17 points to a struct ffi_closure, x16 has been clobbered all other registers are preserved. We allocate a call context and save the argument passing registers, then invoked the generic C ffi_closure_SYSV_inner() function to do all the real work, on return we load the result passing registers back from the call context. +*/ - On entry +#define ffi_closure_SYSV_FS (8*2 + CALL_CONTEXT_SIZE + 64) - extern void - ffi_closure_SYSV (struct trampoline_data *); + .align 4 +CNAME(ffi_closure_SYSV_V): + cfi_startproc + stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! + cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) + cfi_rel_offset (x29, 0) + cfi_rel_offset (x30, 8) - struct trampoline_data - { - UINT64 *ffi_closure; - UINT64 flags; - }; + /* Save the argument passing vector registers. */ + stp q0, q1, [sp, #16 + 0] + stp q2, q3, [sp, #16 + 32] + stp q4, q5, [sp, #16 + 64] + stp q6, q7, [sp, #16 + 96] + b 0f + cfi_endproc - This function uses the following stack frame layout: + .globl CNAME(ffi_closure_SYSV_V) +#ifdef __ELF__ + .type CNAME(ffi_closure_SYSV_V), #function + .hidden CNAME(ffi_closure_SYSV_V) + .size CNAME(ffi_closure_SYSV_V), . - CNAME(ffi_closure_SYSV_V) +#endif - == - saved x30(lr) - x29(fp)-> saved x29(fp) - saved x22 - saved x21 - ... - sp -> call_context - == + .align 4 + cfi_startproc +CNAME(ffi_closure_SYSV): + stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! + cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) + cfi_rel_offset (x29, 0) + cfi_rel_offset (x30, 8) +0: + mov x29, sp - Voila! */ + /* Save the argument passing core registers. */ + stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0] + stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16] + stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32] + stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48] + str x8, [sp, #16 + 16*N_V_ARG_REG + 64] - .text - .align 4 + /* Load ffi_closure_inner arguments. */ + ldp x0, x1, [x17, #FFI_TRAMPOLINE_SIZE] /* load cif, fn */ + ldr x2, [x17, #FFI_TRAMPOLINE_SIZE+16] /* load user_data */ + add x3, sp, #16 /* load context */ + add x4, sp, #ffi_closure_SYSV_FS /* load stack */ + add x5, sp, #16+CALL_CONTEXT_SIZE /* load rvalue */ + bl CNAME(ffi_closure_SYSV_inner) - .globl CNAME(ffi_closure_SYSV) + /* Load the return value as directed. */ + adr x1, 0f + and w0, w0, #AARCH64_RET_MASK + add x1, x1, x0, lsl #3 + add x3, sp, #16+CALL_CONTEXT_SIZE + br x1 + + /* Note that each table entry is 2 insns, and thus 8 bytes. */ + .align 4 +0: b 99f /* VOID */ + nop +1: ldr x0, [x3] /* INT64 */ + b 99f +2: ldp x0, x1, [x3] /* INT128 */ + b 99f +3: brk #1000 /* UNUSED */ + nop +4: brk #1000 /* UNUSED */ + nop +5: brk #1000 /* UNUSED */ + nop +6: brk #1000 /* UNUSED */ + nop +7: brk #1000 /* UNUSED */ + nop +8: ldr s3, [x3, #12] /* S4 */ + nop +9: ldr s2, [x2, #8] /* S3 */ + nop +10: ldp s0, s1, [x3] /* S2 */ + b 99f +11: ldr s0, [x3] /* S1 */ + b 99f +12: ldr d3, [x3, #24] /* D4 */ + nop +13: ldr d2, [x3, #16] /* D3 */ + nop +14: ldp d0, d1, [x3] /* D2 */ + b 99f +15: ldr d0, [x3] /* D1 */ + b 99f +16: ldr q3, [x3, #48] /* Q4 */ + nop +17: ldr q2, [x3, #32] /* Q3 */ + nop +18: ldp q0, q1, [x3] /* Q2 */ + b 99f +19: ldr q0, [x3] /* Q1 */ + b 99f +20: ldrb w0, [x3, #BE(7)] /* UINT8 */ + b 99f +21: brk #1000 /* reserved */ + nop +22: ldrh w0, [x3, #BE(6)] /* UINT16 */ + b 99f +23: brk #1000 /* reserved */ + nop +24: ldr w0, [x3, #BE(4)] /* UINT32 */ + b 99f +25: brk #1000 /* reserved */ + nop +26: ldrsb x0, [x3, #BE(7)] /* SINT8 */ + b 99f +27: brk #1000 /* reserved */ + nop +28: ldrsh x0, [x3, #BE(6)] /* SINT16 */ + b 99f +29: brk #1000 /* reserved */ + nop +30: ldrsw x0, [x3, #BE(4)] /* SINT32 */ + nop +31: /* reserved */ +99: ldp x29, x30, [sp], #ffi_closure_SYSV_FS + cfi_adjust_cfa_offset (-ffi_closure_SYSV_FS) + cfi_restore (x29) + cfi_restore (x30) + ret + cfi_endproc + + .globl CNAME(ffi_closure_SYSV) #ifdef __ELF__ .type CNAME(ffi_closure_SYSV), #function .hidden CNAME(ffi_closure_SYSV) -#endif - cfi_startproc -CNAME(ffi_closure_SYSV): - stp x29, x30, [sp, #-16]! - cfi_adjust_cfa_offset (16) - cfi_rel_offset (x29, 0) - cfi_rel_offset (x30, 8) - - mov x29, sp - cfi_def_cfa_register (x29) - - sub sp, sp, #ffi_closure_SYSV_FS - - stp x21, x22, [x29, #-16] - cfi_rel_offset (x21, -16) - cfi_rel_offset (x22, -8) - - /* Load x21 with &call_context. */ - mov x21, sp - /* Preserve our struct trampoline_data * */ - mov x22, x17 - - /* Save the rest of the argument passing registers, including - the structure return pointer. */ - stp x0, x1, [x21, #16*N_V_ARG_REG + 0] - stp x2, x3, [x21, #16*N_V_ARG_REG + 16] - stp x4, x5, [x21, #16*N_V_ARG_REG + 32] - stp x6, x7, [x21, #16*N_V_ARG_REG + 48] - str x8, [x21, #16*N_V_ARG_REG + 64] - - /* Figure out if we should touch the vector registers. */ - ldr x0, [x22, #8] - tbz x0, #AARCH64_FLAG_ARG_V_BIT, 1f - - /* Save the argument passing vector registers. */ - stp q0, q1, [x21, #0] - stp q2, q3, [x21, #32] - stp q4, q5, [x21, #64] - stp q6, q7, [x21, #96] -1: - /* Load &ffi_closure.. */ - ldr x0, [x22, #0] - mov x1, x21 - /* Compute the location of the stack at the point that the - trampoline was called. */ - add x2, x29, #16 - - bl CNAME(ffi_closure_SYSV_inner) - - /* Figure out if we should touch the vector registers. */ - ldr x0, [x22, #8] - tbz x0, #AARCH64_FLAG_ARG_V_BIT, 1f - - /* Load the result passing vector registers. */ - ldp q0, q1, [x21, #0] - ldp q2, q3, [x21, #32] -1: - /* Load the result passing core registers. */ - ldp x0, x1, [x21, #16*N_V_ARG_REG + 0] - - /* We are done, unwind our frame. */ - ldp x21, x22, [x29, #-16] - cfi_restore (x21) - cfi_restore (x22) - - mov sp, x29 - cfi_def_cfa_register (sp) - - ldp x29, x30, [sp], #16 - cfi_adjust_cfa_offset (-16) - cfi_restore (x29) - cfi_restore (x30) - - ret - cfi_endproc -#ifdef __ELF__ - .size CNAME(ffi_closure_SYSV), .-CNAME(ffi_closure_SYSV) + .size CNAME(ffi_closure_SYSV), . - CNAME(ffi_closure_SYSV) #endif From 4a3cbcaa4f1d3834e0b74c9a841c3f51c5e0454c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 22:32:13 -0400 Subject: [PATCH 31/58] aarch64: Unify scalar fp and hfa handling Since an HFA of a single element is exactly the same as scalar, this tidies things up a bit. --- src/aarch64/ffi.c | 225 +++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 134 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index c5a429a..f69c350 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -71,16 +71,7 @@ ffi_clear_cache (void *start, void *end) #endif } -/* Test for an FFI floating point representation. */ - -static unsigned -is_floating_type (unsigned short type) -{ - return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE - || type == FFI_TYPE_LONGDOUBLE); -} - -/* A subroutine of is_hfa. Given a structure type, return the type code +/* A subroutine of is_vfp_type. Given a structure type, return the type code of the first non-structure element. Recurse for structure elements. Return -1 if the structure is in fact empty, i.e. no nested elements. */ @@ -106,7 +97,7 @@ is_hfa0 (const ffi_type *ty) return ret; } -/* A subroutine of is_hfa. Given a structure type, return true if all +/* A subroutine of is_vfp_type. Given a structure type, return true if all of the non-structure elements are the same as CANDIDATE. */ static int @@ -131,23 +122,35 @@ is_hfa1 (const ffi_type *ty, int candidate) return 1; } -/* Determine if TY is an homogenous floating point aggregate (HFA). +/* Determine if TY may be allocated to the FP registers. This is both an + fp scalar type as well as an homogenous floating point aggregate (HFA). That is, a structure consisting of 1 to 4 members of all the same type, - where that type is a floating point scalar. + where that type is an fp scalar. - Returns non-zero iff TY is an HFA. The result is an encoded value where - bits 0-7 contain the type code, and bits 8-10 contain the element count. */ + Returns non-zero iff TY is an HFA. The result is the AARCH64_RET_* + constant for the type. */ static int -is_hfa(const ffi_type *ty) +is_vfp_type (const ffi_type *ty) { ffi_type **elements; int candidate, i; size_t size, ele_count; /* Quickest tests first. */ - if (ty->type != FFI_TYPE_STRUCT) - return 0; + switch (ty->type) + { + default: + return 0; + case FFI_TYPE_FLOAT: + return AARCH64_RET_S1; + case FFI_TYPE_DOUBLE: + return AARCH64_RET_D1; + case FFI_TYPE_LONGDOUBLE: + return AARCH64_RET_Q1; + case FFI_TYPE_STRUCT: + break; + } /* No HFA types are smaller than 4 bytes, or larger than 64 bytes. */ size = ty->size; @@ -205,17 +208,7 @@ is_hfa(const ffi_type *ty) } /* All tests succeeded. Encode the result. */ - return (ele_count << 8) | candidate; -} - -/* Test if an ffi_type argument or result is a candidate for a vector - register. */ - -static int -is_v_register_candidate (ffi_type *ty) -{ - return is_floating_type (ty->type) - || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty)); + return candidate * 4 + (4 - ele_count); } /* Representation of the procedure call argument marshalling @@ -302,9 +295,7 @@ extend_integer_type (void *source, int type) static void extend_hfa_type (void *dest, void *src, int h) { - int n = (h >> 8); - int t = h & 0xff; - int f = (t - FFI_TYPE_FLOAT) * 4 + 4 - n; + int f = h - AARCH64_RET_S4; void *x0; asm volatile ( @@ -358,82 +349,68 @@ extend_hfa_type (void *dest, void *src, int h) static void * compress_hfa_type (void *dest, void *reg, int h) { - int n = h >> 8; - switch (h & 0xff) + switch (h) { - case FFI_TYPE_FLOAT: - switch (n) + case AARCH64_RET_S1: + if (dest == reg) { - default: - if (dest == reg) - { #ifdef __AARCH64EB__ - dest += 12; + dest += 12; #endif - } - else - *(float *)dest = *(float *)reg; - break; - case 2: - asm("ldp q16, q17, [%1]\n\t" - "st2 { v16.s, v17.s }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); - break; - case 3: - asm("ldp q16, q17, [%1]\n\t" - "ldr q18, [%1, #32]\n\t" - "st3 { v16.s, v17.s, v18.s }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); - break; - case 4: - asm("ldp q16, q17, [%1]\n\t" - "ldp q18, q19, [%1, #32]\n\t" - "st4 { v16.s, v17.s, v18.s, v19.s }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); - break; } + else + *(float *)dest = *(float *)reg; + break; + case AARCH64_RET_S2: + asm ("ldp q16, q17, [%1]\n\t" + "st2 { v16.s, v17.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); + break; + case AARCH64_RET_S3: + asm ("ldp q16, q17, [%1]\n\t" + "ldr q18, [%1, #32]\n\t" + "st3 { v16.s, v17.s, v18.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); + break; + case AARCH64_RET_S4: + asm ("ldp q16, q17, [%1]\n\t" + "ldp q18, q19, [%1, #32]\n\t" + "st4 { v16.s, v17.s, v18.s, v19.s }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); break; - case FFI_TYPE_DOUBLE: - switch (n) + case AARCH64_RET_D1: + if (dest == reg) { - default: - if (dest == reg) - { #ifdef __AARCH64EB__ - dest += 8; + dest += 8; #endif - } - else - *(double *)dest = *(double *)reg; - break; - case 2: - asm("ldp q16, q17, [%1]\n\t" - "st2 { v16.d, v17.d }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); - break; - case 3: - asm("ldp q16, q17, [%1]\n\t" - "ldr q18, [%1, #32]\n\t" - "st3 { v16.d, v17.d, v18.d }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); - break; - case 4: - asm("ldp q16, q17, [%1]\n\t" - "ldp q18, q19, [%1, #32]\n\t" - "st4 { v16.d, v17.d, v18.d, v19.d }[0], [%0]" - : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); - break; } + else + *(double *)dest = *(double *)reg; break; - - case FFI_TYPE_LONGDOUBLE: - if (dest != reg) - return memcpy (dest, reg, 16 * n); + case AARCH64_RET_D2: + asm ("ldp q16, q17, [%1]\n\t" + "st2 { v16.d, v17.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17"); + break; + case AARCH64_RET_D3: + asm ("ldp q16, q17, [%1]\n\t" + "ldr q18, [%1, #32]\n\t" + "st3 { v16.d, v17.d, v18.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18"); + break; + case AARCH64_RET_D4: + asm ("ldp q16, q17, [%1]\n\t" + "ldp q18, q19, [%1, #32]\n\t" + "st4 { v16.d, v17.d, v18.d, v19.d }[0], [%0]" + : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19"); break; default: - FFI_ASSERT (0); + if (dest != reg) + return memcpy (dest, reg, 16 * (4 - (h & 3))); + break; } return dest; } @@ -494,34 +471,25 @@ ffi_prep_cif_machdep (ffi_cif *cif) break; case FFI_TYPE_FLOAT: - flags = AARCH64_RET_S1; - break; case FFI_TYPE_DOUBLE: - flags = AARCH64_RET_D1; - break; case FFI_TYPE_LONGDOUBLE: - flags = AARCH64_RET_Q1; - break; - case FFI_TYPE_STRUCT: - { - int h = is_hfa (rtype); - size_t s = rtype->size; - - if (h) - flags = (h & 0xff) * 4 + 4 - (h >> 8); - else if (s > 16) - { - flags = AARCH64_RET_VOID | AARCH64_RET_IN_MEM; - bytes += 8; - } - else if (s == 16) - flags = AARCH64_RET_INT128; - else if (s == 8) - flags = AARCH64_RET_INT64; - else - flags = AARCH64_RET_INT128 | AARCH64_RET_NEED_COPY; - } + flags = is_vfp_type (rtype); + if (flags == 0) + { + size_t s = rtype->size; + if (s > 16) + { + flags = AARCH64_RET_VOID | AARCH64_RET_IN_MEM; + bytes += 8; + } + else if (s == 16) + flags = AARCH64_RET_INT128; + else if (s == 8) + flags = AARCH64_RET_INT64; + else + flags = AARCH64_RET_INT128 | AARCH64_RET_NEED_COPY; + } break; default: @@ -530,7 +498,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) aarch64_flags = 0; for (i = 0, n = cif->nargs; i < n; i++) - if (is_v_register_candidate (cif->arg_types[i])) + if (is_vfp_type (cif->arg_types[i])) { aarch64_flags = AARCH64_FLAG_ARG_V; flags |= AARCH64_FLAG_ARG_V; @@ -652,20 +620,14 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - /* Scalar float is a degenerate case of HFA. */ - h = t + 0x100; - goto do_hfa; - case FFI_TYPE_STRUCT: { void *dest; - int elems; - h = is_hfa (ty); + h = is_vfp_type (ty); if (h) { - do_hfa: - elems = h >> 8; + int elems = 4 - (h & 3); if (state.nsrn + elems <= N_V_ARG_REG) { dest = &context->v[state.nsrn]; @@ -828,16 +790,11 @@ ffi_closure_SYSV_inner (ffi_cif *cif, case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - /* Scalar float is a degenerate case of HFA. */ - h = t + 0x100; - goto do_hfa; - case FFI_TYPE_STRUCT: - h = is_hfa (ty); + h = is_vfp_type (ty); if (h) { - do_hfa: - n = h >> 8; + n = 4 - (h & 3); if (state.nsrn + n <= N_V_ARG_REG) { void *reg = &context->v[state.nsrn]; From 658b2b56001e293848ea22b70068bd7f8daa7205 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 22:36:07 -0400 Subject: [PATCH 32/58] aarch64: Remove aarch64_flags This field was useless from the start, since the normal flags field is available for backend use. --- src/aarch64/ffi.c | 5 +---- src/aarch64/ffitarget.h | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index f69c350..b3e0b16 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -436,7 +436,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) { ffi_type *rtype = cif->rtype; size_t bytes = cif->bytes; - int flags, aarch64_flags, i, n; + int flags, i, n; switch (rtype->type) { @@ -496,11 +496,9 @@ ffi_prep_cif_machdep (ffi_cif *cif) abort(); } - aarch64_flags = 0; for (i = 0, n = cif->nargs; i < n; i++) if (is_vfp_type (cif->arg_types[i])) { - aarch64_flags = AARCH64_FLAG_ARG_V; flags |= AARCH64_FLAG_ARG_V; break; } @@ -508,7 +506,6 @@ ffi_prep_cif_machdep (ffi_cif *cif) /* Round the stack up to a multiple of the stack alignment requirement. */ cif->bytes = ALIGN(bytes, 16); cif->flags = flags; - cif->aarch64_flags = aarch64_flags; #if defined (__APPLE__) cif->aarch64_nfixedargs = 0; #endif diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index b488bbe..6d6d3e6 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -49,9 +49,7 @@ typedef enum ffi_abi #if defined (__APPLE__) #define FFI_TARGET_SPECIFIC_VARIADIC -#define FFI_EXTRA_CIF_FIELDS unsigned aarch64_flags; unsigned aarch64_nfixedargs -#else -#define FFI_EXTRA_CIF_FIELDS unsigned aarch64_flags +#define FFI_EXTRA_CIF_FIELDS unsigned aarch64_nfixedargs #endif #endif From a992f8789b5202ca180e67b16b47ee834160665c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 22:58:09 -0400 Subject: [PATCH 33/58] aarch64: Add support for complex types --- src/aarch64/ffi.c | 34 +++++++++++++++++++++++++--------- src/aarch64/ffitarget.h | 2 ++ testsuite/libffi.call/call.exp | 9 +++++---- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index b3e0b16..4f85140 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -85,7 +85,7 @@ is_hfa0 (const ffi_type *ty) for (i = 0; elements[i]; ++i) { ret = elements[i]->type; - if (ret == FFI_TYPE_STRUCT) + if (ret == FFI_TYPE_STRUCT || ret == FFI_TYPE_COMPLEX) { ret = is_hfa0 (elements[i]); if (ret < 0) @@ -110,7 +110,7 @@ is_hfa1 (const ffi_type *ty, int candidate) for (i = 0; elements[i]; ++i) { int t = elements[i]->type; - if (t == FFI_TYPE_STRUCT) + if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX) { if (!is_hfa1 (elements[i], candidate)) return 0; @@ -138,16 +138,27 @@ is_vfp_type (const ffi_type *ty) size_t size, ele_count; /* Quickest tests first. */ - switch (ty->type) + candidate = ty->type; + switch (candidate) { default: return 0; case FFI_TYPE_FLOAT: - return AARCH64_RET_S1; case FFI_TYPE_DOUBLE: - return AARCH64_RET_D1; case FFI_TYPE_LONGDOUBLE: - return AARCH64_RET_Q1; + ele_count = 1; + goto done; + case FFI_TYPE_COMPLEX: + candidate = ty->elements[0]->type; + switch (candidate) + { + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + ele_count = 2; + goto done; + } + return 0; case FFI_TYPE_STRUCT: break; } @@ -160,7 +171,7 @@ is_vfp_type (const ffi_type *ty) /* Find the type of the first non-structure member. */ elements = ty->elements; candidate = elements[0]->type; - if (candidate == FFI_TYPE_STRUCT) + if (candidate == FFI_TYPE_STRUCT || candidate == FFI_TYPE_COMPLEX) { for (i = 0; ; ++i) { @@ -198,16 +209,18 @@ is_vfp_type (const ffi_type *ty) /* Finally, make sure that all scalar elements are the same type. */ for (i = 0; elements[i]; ++i) { - if (elements[i]->type == FFI_TYPE_STRUCT) + int t = elements[i]->type; + if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX) { if (!is_hfa1 (elements[i], candidate)) return 0; } - else if (elements[i]->type != candidate) + else if (t != candidate) return 0; } /* All tests succeeded. Encode the result. */ + done: return candidate * 4 + (4 - ele_count); } @@ -474,6 +487,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: case FFI_TYPE_STRUCT: + case FFI_TYPE_COMPLEX: flags = is_vfp_type (rtype); if (flags == 0) { @@ -618,6 +632,7 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: case FFI_TYPE_STRUCT: + case FFI_TYPE_COMPLEX: { void *dest; @@ -788,6 +803,7 @@ ffi_closure_SYSV_inner (ffi_cif *cif, case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: case FFI_TYPE_STRUCT: + case FFI_TYPE_COMPLEX: h = is_vfp_type (ty); if (h) { diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index 6d6d3e6..7461386 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -52,4 +52,6 @@ typedef enum ffi_abi #define FFI_EXTRA_CIF_FIELDS unsigned aarch64_nfixedargs #endif +#define FFI_TARGET_HAS_COMPLEX_TYPE + #endif diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index d42dae5..676b323 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -26,10 +26,11 @@ run-many-tests $tlist "" # ??? We really should preprocess ffi.h and grep # for FFI_TARGET_HAS_COMPLEX_TYPE. -if { [istarget s390*] - || [istarget x86_64*] - || [istarget i?86*] } { - run-many-tests $ctlist "" +if { [istarget aarch64*] + || [istarget i?86*] + || [istarget s390*] + || [istarget x86_64*] } { + run-many-tests $ctlist "" } else { foreach test $ctlist { unsupported "$test" From 0e41c73b092064e986d484270a13046479d6bda3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 22 Oct 2014 23:48:12 -0400 Subject: [PATCH 34/58] aarch64: Move x8 out of call_context Reduces stack size. It was only used by the closure, and there are available argument registers. --- src/aarch64/ffi.c | 5 ++--- src/aarch64/internal.h | 2 +- src/aarch64/sysv.S | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index 4f85140..f546ab2 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -52,7 +52,6 @@ struct call_context { struct _v v[N_V_ARG_REG]; UINT64 x[N_X_ARG_REG]; - UINT64 x8; }; #if defined (__clang__) && defined (__APPLE__) @@ -766,7 +765,7 @@ ffi_closure_SYSV_inner (ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, struct call_context *context, - void *stack, void *rvalue) + void *stack, void *rvalue, void *struct_rvalue) { void **avalue = (void**) alloca (cif->nargs * sizeof (void*)); int i, h, nargs, flags; @@ -861,7 +860,7 @@ ffi_closure_SYSV_inner (ffi_cif *cif, flags = cif->flags; if (flags & AARCH64_RET_IN_MEM) - rvalue = (void *)(uintptr_t)context->x8; + rvalue = struct_rvalue; fun (cif, rvalue, avalue, user_data); diff --git a/src/aarch64/internal.h b/src/aarch64/internal.h index a3070db..9c3e077 100644 --- a/src/aarch64/internal.h +++ b/src/aarch64/internal.h @@ -64,4 +64,4 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #define N_X_ARG_REG 8 #define N_V_ARG_REG 8 -#define CALL_CONTEXT_SIZE (N_V_ARG_REG * 16 + N_X_ARG_REG * 8 + 16) +#define CALL_CONTEXT_SIZE (N_V_ARG_REG * 16 + N_X_ARG_REG * 8) diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index abd848d..7f00a3f 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -241,7 +241,6 @@ CNAME(ffi_closure_SYSV): stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16] stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32] stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48] - str x8, [sp, #16 + 16*N_V_ARG_REG + 64] /* Load ffi_closure_inner arguments. */ ldp x0, x1, [x17, #FFI_TRAMPOLINE_SIZE] /* load cif, fn */ @@ -249,6 +248,7 @@ CNAME(ffi_closure_SYSV): add x3, sp, #16 /* load context */ add x4, sp, #ffi_closure_SYSV_FS /* load stack */ add x5, sp, #16+CALL_CONTEXT_SIZE /* load rvalue */ + mov x6, x8 /* load struct_rval */ bl CNAME(ffi_closure_SYSV_inner) /* Load the return value as directed. */ From c6352b664c31835b9527d747f8a29b4754cabb0e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 23 Oct 2014 00:26:14 -0400 Subject: [PATCH 35/58] aarch64: Add support for Go closures --- src/aarch64/ffi.c | 52 ++++++++++++++++++++++++++++++---- src/aarch64/ffitarget.h | 4 +++ src/aarch64/sysv.S | 62 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index f546ab2..0cace9d 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -539,13 +539,14 @@ ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, #endif /* __APPLE__ */ extern void ffi_call_SYSV (struct call_context *context, void *frame, - void (*fn)(void), void *rvalue, int flags) - FFI_HIDDEN; + void (*fn)(void), void *rvalue, int flags, + void *closure) FFI_HIDDEN; /* Call a function with the provided arguments and capture the return value. */ -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, + void **avalue, void *closure) { struct call_context *context; void *stack, *frame, *rvalue; @@ -698,12 +699,27 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *orig_rvalue, void **avalue) #endif } - ffi_call_SYSV (context, frame, fn, rvalue, flags); + ffi_call_SYSV (context, frame, fn, rvalue, flags, closure); if (flags & AARCH64_RET_NEED_COPY) memcpy (orig_rvalue, rvalue, rtype_size); } +void +ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} + +#ifdef FFI_GO_CLOSURES +void +ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); +} +#endif /* FFI_GO_CLOSURES */ + /* Build a trampoline. */ extern void ffi_closure_SYSV (void) FFI_HIDDEN; @@ -744,6 +760,32 @@ ffi_prep_closure_loc (ffi_closure *closure, return FFI_OK; } +#ifdef FFI_GO_CLOSURES +extern void ffi_go_closure_SYSV (void) FFI_HIDDEN; +extern void ffi_go_closure_SYSV_V (void) FFI_HIDDEN; + +ffi_status +ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif* cif, + void (*fun)(ffi_cif*,void*,void**,void*)) +{ + void (*start)(void); + + if (cif->abi != FFI_SYSV) + return FFI_BAD_ABI; + + if (cif->flags & AARCH64_FLAG_ARG_V) + start = ffi_go_closure_SYSV_V; + else + start = ffi_go_closure_SYSV; + + closure->tramp = start; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} +#endif /* FFI_GO_CLOSURES */ + /* Primary handler to setup and invoke a function within a closure. A closure when invoked enters via the assembler wrapper diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index 7461386..80d09af 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -50,6 +50,10 @@ typedef enum ffi_abi #if defined (__APPLE__) #define FFI_TARGET_SPECIFIC_VARIADIC #define FFI_EXTRA_CIF_FIELDS unsigned aarch64_nfixedargs +#else +/* iOS reserves x18 for the system. Disable Go closures until + a new static chain is chosen. */ +#define FFI_GO_CLOSURES 1 #endif #define FFI_TARGET_HAS_COMPLEX_TYPE diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index 7f00a3f..1fb68f2 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -50,7 +50,8 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* ffi_call_SYSV extern void ffi_call_SYSV (void *stack, void *frame, - void (*fn)(void), void *rvalue, int flags); + void (*fn)(void), void *rvalue, + int flags, void *closure); Therefore on entry we have: @@ -59,6 +60,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ x2 fn x3 rvalue x4 flags + x5 closure */ cfi_startproc @@ -74,6 +76,9 @@ CNAME(ffi_call_SYSV): mov x9, x2 /* save fn */ mov x8, x3 /* install structure return */ +#ifdef FFI_GO_CLOSURES + mov x18, x5 /* install static chain */ +#endif stp x3, x4, [x29, #16] /* save rvalue and flags */ /* Load the vector argument passing registers, if necessary. */ @@ -245,6 +250,7 @@ CNAME(ffi_closure_SYSV): /* Load ffi_closure_inner arguments. */ ldp x0, x1, [x17, #FFI_TRAMPOLINE_SIZE] /* load cif, fn */ ldr x2, [x17, #FFI_TRAMPOLINE_SIZE+16] /* load user_data */ +.Ldo_closure: add x3, sp, #16 /* load context */ add x4, sp, #ffi_closure_SYSV_FS /* load stack */ add x5, sp, #16+CALL_CONTEXT_SIZE /* load rvalue */ @@ -336,3 +342,57 @@ CNAME(ffi_closure_SYSV): .hidden CNAME(ffi_closure_SYSV) .size CNAME(ffi_closure_SYSV), . - CNAME(ffi_closure_SYSV) #endif + +#ifdef FFI_GO_CLOSURES + .align 4 +CNAME(ffi_go_closure_SYSV_V): + cfi_startproc + stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! + cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) + cfi_rel_offset (x29, 0) + cfi_rel_offset (x30, 8) + + /* Save the argument passing vector registers. */ + stp q0, q1, [sp, #16 + 0] + stp q2, q3, [sp, #16 + 32] + stp q4, q5, [sp, #16 + 64] + stp q6, q7, [sp, #16 + 96] + b 0f + cfi_endproc + + .globl CNAME(ffi_go_closure_SYSV_V) +#ifdef __ELF__ + .type CNAME(ffi_go_closure_SYSV_V), #function + .hidden CNAME(ffi_go_closure_SYSV_V) + .size CNAME(ffi_go_closure_SYSV_V), . - CNAME(ffi_go_closure_SYSV_V) +#endif + + .align 4 + cfi_startproc +CNAME(ffi_go_closure_SYSV): + stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! + cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) + cfi_rel_offset (x29, 0) + cfi_rel_offset (x30, 8) +0: + mov x29, sp + + /* Save the argument passing core registers. */ + stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0] + stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16] + stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32] + stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48] + + /* Load ffi_closure_inner arguments. */ + ldp x0, x1, [x18, #8] /* load cif, fn */ + mov x2, x18 /* load user_data */ + b .Ldo_closure + cfi_endproc + + .globl CNAME(ffi_go_closure_SYSV) +#ifdef __ELF__ + .type CNAME(ffi_go_closure_SYSV), #function + .hidden CNAME(ffi_go_closure_SYSV) + .size CNAME(ffi_go_closure_SYSV), . - CNAME(ffi_go_closure_SYSV) +#endif +#endif /* FFI_GO_CLOSURES */ From 32a26b75ba92665fa093b539318e83d916e3effb Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 12:33:53 -0700 Subject: [PATCH 36/58] alpha: Reorganize cif flags Unties the backend from changes to FFI_TYPE_* constants, and allows compilation to succeed after the addition of FFI_TYPE_COMPLEX. Delete the hand-written unwind info. --- src/alpha/ffi.c | 63 +++++--- src/alpha/internal.h | 19 +++ src/alpha/osf.S | 343 ++++++++++++------------------------------- 3 files changed, 161 insertions(+), 264 deletions(-) create mode 100644 src/alpha/internal.h diff --git a/src/alpha/ffi.c b/src/alpha/ffi.c index 192f691..519bd2c 100644 --- a/src/alpha/ffi.c +++ b/src/alpha/ffi.c @@ -28,6 +28,7 @@ #include #include #include +#include "internal.h" /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; all further uses in this file will refer to the 128-bit type. */ @@ -48,6 +49,8 @@ extern void ffi_closure_osf(void) FFI_HIDDEN; ffi_status ffi_prep_cif_machdep(ffi_cif *cif) { + int flags; + /* Adjust cif->bytes to represent a minimum 6 words for the temporary register argument loading area. */ if (cif->bytes < 6*FFI_SIZEOF_ARG) @@ -56,21 +59,46 @@ ffi_prep_cif_machdep(ffi_cif *cif) /* Set the return type flag */ switch (cif->rtype->type) { - case FFI_TYPE_STRUCT: + case FFI_TYPE_VOID: + flags = ALPHA_FLAGS(ALPHA_ST_VOID, ALPHA_LD_VOID); + break; + case FFI_TYPE_INT: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_INT32); + break; case FFI_TYPE_FLOAT: + flags = ALPHA_FLAGS(ALPHA_ST_FLOAT, ALPHA_LD_FLOAT); + break; case FFI_TYPE_DOUBLE: - cif->flags = cif->rtype->type; + flags = ALPHA_FLAGS(ALPHA_ST_DOUBLE, ALPHA_LD_DOUBLE); + break; + case FFI_TYPE_UINT8: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_UINT8); + break; + case FFI_TYPE_SINT8: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_SINT8); + break; + case FFI_TYPE_UINT16: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_UINT16); + break; + case FFI_TYPE_SINT16: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_SINT16); + break; + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + case FFI_TYPE_POINTER: + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_INT64); break; - case FFI_TYPE_LONGDOUBLE: - /* 128-bit long double is returned in memory, like a struct. */ - cif->flags = FFI_TYPE_STRUCT; + case FFI_TYPE_STRUCT: + /* Passed in memory, with a hidden pointer. */ + flags = ALPHA_RET_IN_MEM; break; - default: - cif->flags = FFI_TYPE_INT; - break; + abort(); } + cif->flags = flags; return FFI_OK; } @@ -80,20 +108,20 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { unsigned long *stack, *argp; - long i, avn; + long i, avn, flags = cif->flags; ffi_type **arg_types; /* If the return value is a struct and we don't have a return value address then we need to make one. */ - if (rvalue == NULL && cif->flags == FFI_TYPE_STRUCT) + if (rvalue == NULL && flags == ALPHA_RET_IN_MEM) rvalue = alloca(cif->rtype->size); /* Allocate the space for the arguments, plus 4 words of temp space for ffi_call_osf. */ argp = stack = alloca(cif->bytes + 4*FFI_SIZEOF_ARG); - if (cif->flags == FFI_TYPE_STRUCT) - *(void **) argp++ = rvalue; + if (flags == ALPHA_RET_IN_MEM) + *argp++ = (unsigned long)rvalue; i = 0; avn = cif->nargs; @@ -166,7 +194,8 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) i++, arg_types++, avalue++; } - ffi_call_osf(stack, cif->bytes, cif->flags, rvalue, fn); + flags = (flags >> ALPHA_ST_SHIFT) & 0xff; + ffi_call_osf(stack, cif->bytes, flags, rvalue, fn); } @@ -211,16 +240,16 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) ffi_cif *cif; void **avalue; ffi_type **arg_types; - long i, avn, argn; + long i, avn, argn, flags; cif = closure->cif; avalue = alloca(cif->nargs * sizeof(void *)); - + flags = cif->flags; argn = 0; /* Copy the caller's structure return address to that the closure returns the data directly to the caller. */ - if (cif->flags == FFI_TYPE_STRUCT) + if (flags == ALPHA_RET_IN_MEM) { rvalue = (void *) argp[0]; argn = 1; @@ -284,5 +313,5 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) closure->fun (cif, rvalue, avalue, closure->user_data); /* Tell ffi_closure_osf how to perform return type promotions. */ - return cif->rtype->type; + return (flags >> ALPHA_LD_SHIFT) & 0xff; } diff --git a/src/alpha/internal.h b/src/alpha/internal.h new file mode 100644 index 0000000..664a2a6 --- /dev/null +++ b/src/alpha/internal.h @@ -0,0 +1,19 @@ +#define ALPHA_ST_VOID 0 +#define ALPHA_ST_INT 1 +#define ALPHA_ST_FLOAT 2 +#define ALPHA_ST_DOUBLE 3 + +#define ALPHA_LD_VOID 0 +#define ALPHA_LD_INT64 1 +#define ALPHA_LD_INT32 2 +#define ALPHA_LD_UINT16 3 +#define ALPHA_LD_SINT16 4 +#define ALPHA_LD_UINT8 5 +#define ALPHA_LD_SINT8 6 +#define ALPHA_LD_FLOAT 7 +#define ALPHA_LD_DOUBLE 8 + +#define ALPHA_ST_SHIFT 0 +#define ALPHA_LD_SHIFT 8 +#define ALPHA_RET_IN_MEM 0x10000 +#define ALPHA_FLAGS(S, L) (((L) << ALPHA_LD_SHIFT) | (S)) diff --git a/src/alpha/osf.S b/src/alpha/osf.S index 6b9f4df..fb9c595 100644 --- a/src/alpha/osf.S +++ b/src/alpha/osf.S @@ -1,7 +1,7 @@ /* ----------------------------------------------------------------------- - osf.S - Copyright (c) 1998, 2001, 2007, 2008, 2011 Red Hat - - Alpha/OSF Foreign Function Interface + osf.S - Copyright (c) 1998, 2001, 2007, 2008, 2011, 2014 Red Hat + + Alpha/OSF Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -24,13 +24,21 @@ DEALINGS IN THE SOFTWARE. ----------------------------------------------------------------------- */ -#define LIBFFI_ASM +#define LIBFFI_ASM #include #include +#include +#include "internal.h" .arch ev6 .text +/* Aid in building a direct addressed jump table, 4 insns per entry. */ +.macro E index + .align 4 + .org 99b + \index * 16 +.endm + /* ffi_call_osf (void *args, unsigned long bytes, unsigned flags, void *raddr, void (*fnaddr)(void)); @@ -38,7 +46,7 @@ for this function. This has been allocated by ffi_call. We also deallocate some of the stack that has been alloca'd. */ - .align 3 + .align 4 .globl ffi_call_osf .ent ffi_call_osf FFI_HIDDEN(ffi_call_osf) @@ -46,15 +54,17 @@ ffi_call_osf: .frame $15, 32, $26, 0 .mask 0x4008000, -32 -$LFB1: + cfi_startproc addq $16,$17,$1 mov $16, $30 stq $26, 0($1) stq $15, 8($1) stq $18, 16($1) mov $1, $15 -$LCFI1: .prologue 0 + cfi_def_cfa($15, 32) + cfi_rel_offset($26, 0) + cfi_rel_offset($15, 8) stq $19, 24($1) mov $20, $27 @@ -77,71 +87,61 @@ $LCFI1: lda $30, 48($30) jsr $26, ($27), 0 - ldgp $29, 0($26) - - # If the return value pointer is NULL, assume no return value. - ldq $19, 24($15) - ldq $18, 16($15) +0: + ldah $29, 0($26) !gpdisp!1 + ldq $2, 24($15) + lda $29, 0($29) !gpdisp!1 + ldq $3, 16($15) + lda $1, 99f-0b($26) ldq $26, 0($15) -$LCFI2: - beq $19, $noretval - - # Store the return value out in the proper type. - cmpeq $18, FFI_TYPE_INT, $1 - bne $1, $retint - cmpeq $18, FFI_TYPE_FLOAT, $2 - bne $2, $retfloat - cmpeq $18, FFI_TYPE_DOUBLE, $3 - bne $3, $retdouble - - .align 3 -$noretval: ldq $15, 8($15) - ret + cfi_restore($26) + cfi_restore($15) + cfi_def_cfa($sp, 0) + cmoveq $2, ALPHA_ST_VOID, $3 # mash null return to void + addq $3, $3, $3 + s8addq $3, $1, $1 # 99f + stcode * 16 + jmp $31, ($1), $st_int .align 4 -$retint: - stq $0, 0($19) - nop - ldq $15, 8($15) +99: +E ALPHA_ST_VOID + ret +E ALPHA_ST_INT +$st_int: + stq $0, 0($2) + ret +E ALPHA_ST_FLOAT + sts $f0, 0($2) + ret +E ALPHA_ST_DOUBLE + stt $f0, 0($2) ret - .align 4 -$retfloat: - sts $f0, 0($19) - nop - ldq $15, 8($15) - ret - - .align 4 -$retdouble: - stt $f0, 0($19) - nop - ldq $15, 8($15) - ret -$LFE1: - + cfi_endproc .end ffi_call_osf /* ffi_closure_osf(...) Receives the closure argument in $1. */ - .align 3 +#define CLOSURE_FS (16*8) + + .align 4 .globl ffi_closure_osf .ent ffi_closure_osf FFI_HIDDEN(ffi_closure_osf) ffi_closure_osf: - .frame $30, 16*8, $26, 0 - .mask 0x4000000, -16*8 -$LFB2: + .frame $30, CLOSURE_FS, $26, 0 + .mask 0x4000000, -CLOSURE_FS + cfi_startproc ldgp $29, 0($27) - subq $30, 16*8, $30 -$LCFI5: + subq $30, CLOSURE_FS, $30 + cfi_adjust_cfa_offset(CLOSURE_FS) stq $26, 0($30) -$LCFI6: .prologue 1 + cfi_rel_offset($26, 0) # Store all of the potential argument registers in va_list format. stt $f16, 4*8($30) @@ -162,225 +162,74 @@ $LCFI6: lda $17, 2*8($30) lda $18, 10*8($30) jsr $26, ffi_closure_osf_inner - ldgp $29, 0($26) +0: + ldah $29, 0($26) !gpdisp!2 + lda $2, 99f-0b($26) + s4addq $0, 0, $1 # ldcode * 4 + ldq $0, 16($30) # preload return value + s4addq $1, $2, $1 # 99f + ldcode * 16 + lda $29, 0($29) !gpdisp!2 ldq $26, 0($30) - - # Load up the return value in the proper type. - lda $1, $load_table - s4addq $0, $1, $1 - ldl $1, 0($1) - addq $1, $29, $1 + cfi_restore($26) jmp $31, ($1), $load_32 - .align 4 -$load_none: - addq $30, 16*8, $30 +.macro epilogue + addq $30, CLOSURE_FS, $30 + cfi_adjust_cfa_offset(-CLOSURE_FS) ret + .align 4 + cfi_adjust_cfa_offset(CLOSURE_FS) +.endm .align 4 -$load_float: - lds $f0, 16($30) - nop - addq $30, 16*8, $30 - ret +99: +E ALPHA_LD_VOID + epilogue - .align 4 -$load_double: - ldt $f0, 16($30) - nop - addq $30, 16*8, $30 - ret +E ALPHA_LD_INT64 + epilogue - .align 4 -$load_u8: -#ifdef __alpha_bwx__ - ldbu $0, 16($30) - nop -#else - ldq $0, 16($30) - and $0, 255, $0 -#endif - addq $30, 16*8, $30 - ret +E ALPHA_LD_INT32 +$load_32: + sextl $0, $0 + epilogue - .align 4 -$load_s8: -#ifdef __alpha_bwx__ - ldbu $0, 16($30) - sextb $0, $0 -#else - ldq $0, 16($30) - sll $0, 56, $0 - sra $0, 56, $0 -#endif - addq $30, 16*8, $30 - ret - - .align 4 -$load_u16: -#ifdef __alpha_bwx__ - ldwu $0, 16($30) - nop -#else - ldq $0, 16($30) +E ALPHA_LD_UINT16 zapnot $0, 3, $0 -#endif - addq $30, 16*8, $30 - ret + epilogue - .align 4 -$load_s16: +E ALPHA_LD_SINT16 #ifdef __alpha_bwx__ - ldwu $0, 16($30) sextw $0, $0 #else - ldq $0, 16($30) sll $0, 48, $0 sra $0, 48, $0 #endif - addq $30, 16*8, $30 - ret + epilogue - .align 4 -$load_32: - ldl $0, 16($30) - nop - addq $30, 16*8, $30 - ret +E ALPHA_LD_UINT8 + and $0, 0xff, $0 + epilogue - .align 4 -$load_64: - ldq $0, 16($30) - nop - addq $30, 16*8, $30 - ret -$LFE2: - - .end ffi_closure_osf - -#ifdef __ELF__ -.section .rodata +E ALPHA_LD_SINT8 +#ifdef __alpha_bwx__ + sextb $0, $0 #else -.rdata + sll $0, 56, $0 + sra $0, 56, $0 #endif -$load_table: - .gprel32 $load_none # FFI_TYPE_VOID - .gprel32 $load_32 # FFI_TYPE_INT - .gprel32 $load_float # FFI_TYPE_FLOAT - .gprel32 $load_double # FFI_TYPE_DOUBLE - .gprel32 $load_none # FFI_TYPE_LONGDOUBLE - .gprel32 $load_u8 # FFI_TYPE_UINT8 - .gprel32 $load_s8 # FFI_TYPE_SINT8 - .gprel32 $load_u16 # FFI_TYPE_UINT16 - .gprel32 $load_s16 # FFI_TYPE_SINT16 - .gprel32 $load_32 # FFI_TYPE_UINT32 - .gprel32 $load_32 # FFI_TYPE_SINT32 - .gprel32 $load_64 # FFI_TYPE_UINT64 - .gprel32 $load_64 # FFI_TYPE_SINT64 - .gprel32 $load_none # FFI_TYPE_STRUCT - .gprel32 $load_64 # FFI_TYPE_POINTER + epilogue -/* Assert that the table above is in sync with ffi.h. */ +E ALPHA_LD_FLOAT + lds $f0, 16($sp) + epilogue -#if FFI_TYPE_FLOAT != 2 \ - || FFI_TYPE_DOUBLE != 3 \ - || FFI_TYPE_UINT8 != 5 \ - || FFI_TYPE_SINT8 != 6 \ - || FFI_TYPE_UINT16 != 7 \ - || FFI_TYPE_SINT16 != 8 \ - || FFI_TYPE_UINT32 != 9 \ - || FFI_TYPE_SINT32 != 10 \ - || FFI_TYPE_UINT64 != 11 \ - || FFI_TYPE_SINT64 != 12 \ - || FFI_TYPE_STRUCT != 13 \ - || FFI_TYPE_POINTER != 14 \ - || FFI_TYPE_LAST != 14 -#error "osf.S out of sync with ffi.h" -#endif +E ALPHA_LD_DOUBLE + ldt $f0, 16($sp) + epilogue -#ifdef __ELF__ -# define UA_SI .4byte -# define FDE_ENCODING 0x1b /* pcrel sdata4 */ -# define FDE_ENCODE(X) .4byte X-. -# define FDE_ARANGE(X) .4byte X -#elif defined __osf__ -# define UA_SI .align 0; .long -# define FDE_ENCODING 0x50 /* aligned absolute */ -# define FDE_ENCODE(X) .align 3; .quad X -# define FDE_ARANGE(X) .align 0; .quad X -#endif - -#ifdef __ELF__ - .section .eh_frame,EH_FRAME_FLAGS,@progbits -#elif defined __osf__ - .data - .align 3 - .globl _GLOBAL__F_ffi_call_osf -_GLOBAL__F_ffi_call_osf: -#endif -__FRAME_BEGIN__: - UA_SI $LECIE1-$LSCIE1 # Length of Common Information Entry -$LSCIE1: - UA_SI 0x0 # CIE Identifier Tag - .byte 0x1 # CIE Version - .ascii "zR\0" # CIE Augmentation - .byte 0x1 # uleb128 0x1; CIE Code Alignment Factor - .byte 0x78 # sleb128 -8; CIE Data Alignment Factor - .byte 26 # CIE RA Column - .byte 0x1 # uleb128 0x1; Augmentation size - .byte FDE_ENCODING # FDE Encoding - .byte 0xc # DW_CFA_def_cfa - .byte 30 # uleb128 column 30 - .byte 0 # uleb128 offset 0 - .align 3 -$LECIE1: -$LSFDE1: - UA_SI $LEFDE1-$LASFDE1 # FDE Length -$LASFDE1: - UA_SI $LASFDE1-__FRAME_BEGIN__ # FDE CIE offset - FDE_ENCODE($LFB1) # FDE initial location - FDE_ARANGE($LFE1-$LFB1) # FDE address range - .byte 0x0 # uleb128 0x0; Augmentation size - - .byte 0x4 # DW_CFA_advance_loc4 - UA_SI $LCFI1-$LFB1 - .byte 0x9a # DW_CFA_offset, column 26 - .byte 4 # uleb128 4*-8 - .byte 0x8f # DW_CFA_offset, column 15 - .byte 0x3 # uleb128 3*-8 - .byte 0xc # DW_CFA_def_cfa - .byte 15 # uleb128 column 15 - .byte 32 # uleb128 offset 32 - - .byte 0x4 # DW_CFA_advance_loc4 - UA_SI $LCFI2-$LCFI1 - .byte 0xda # DW_CFA_restore, column 26 - .align 3 -$LEFDE1: - -$LSFDE3: - UA_SI $LEFDE3-$LASFDE3 # FDE Length -$LASFDE3: - UA_SI $LASFDE3-__FRAME_BEGIN__ # FDE CIE offset - FDE_ENCODE($LFB2) # FDE initial location - FDE_ARANGE($LFE2-$LFB2) # FDE address range - .byte 0x0 # uleb128 0x0; Augmentation size - - .byte 0x4 # DW_CFA_advance_loc4 - UA_SI $LCFI5-$LFB2 - .byte 0xe # DW_CFA_def_cfa_offset - .byte 0x80,0x1 # uleb128 128 - - .byte 0x4 # DW_CFA_advance_loc4 - UA_SI $LCFI6-$LCFI5 - .byte 0x9a # DW_CFA_offset, column 26 - .byte 16 # uleb128 offset 16*-8 - .align 3 -$LEFDE3: -#if defined __osf__ - .align 0 - .long 0 # End of Table -#endif + cfi_endproc + .end ffi_closure_osf #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits From 5f917371afcb4d3363fd33b547844c47264f6d4e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 13:47:26 -0700 Subject: [PATCH 37/58] alpha: Clean up conversion of float values Don't use "real" conversion to double, lest we raise exceptions when passing signalling nans. --- src/alpha/ffi.c | 95 ++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/src/alpha/ffi.c b/src/alpha/ffi.c index 519bd2c..2f9e7e5 100644 --- a/src/alpha/ffi.c +++ b/src/alpha/ffi.c @@ -45,6 +45,20 @@ extern void ffi_call_osf(void *, unsigned long, unsigned, void *, void (*)(void) FFI_HIDDEN; extern void ffi_closure_osf(void) FFI_HIDDEN; +/* Promote a float value to its in-register double representation. + Unlike actually casting to double, this does not trap on NaN. */ +static inline UINT64 lds(void *ptr) +{ + UINT64 ret; + asm("lds %0,%1" : "=f"(ret) : "m"(*(UINT32 *)ptr)); + return ret; +} + +/* And the reverse. */ +static inline void sts(void *ptr, UINT64 val) +{ + asm("sts %1,%0" : "=m"(*(UINT32 *)ptr) : "f"(val)); +} ffi_status ffi_prep_cif_machdep(ffi_cif *cif) @@ -127,71 +141,67 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) avn = cif->nargs; arg_types = cif->arg_types; - while (i < avn) + for (i = 0, avn = cif->nargs; i < avn; i++) { - size_t size = (*arg_types)->size; + ffi_type *ty = arg_types[i]; + void *valp = avalue[i]; + unsigned long val; + size_t size; - switch ((*arg_types)->type) + switch (ty->type) { case FFI_TYPE_SINT8: - *(SINT64 *) argp = *(SINT8 *)(* avalue); + val = *(SINT8 *)valp; break; case FFI_TYPE_UINT8: - *(SINT64 *) argp = *(UINT8 *)(* avalue); + val = *(UINT8 *)valp; break; case FFI_TYPE_SINT16: - *(SINT64 *) argp = *(SINT16 *)(* avalue); + val = *(SINT16 *)valp; break; case FFI_TYPE_UINT16: - *(SINT64 *) argp = *(UINT16 *)(* avalue); + val = *(UINT16 *)valp; break; case FFI_TYPE_SINT32: case FFI_TYPE_UINT32: /* Note that unsigned 32-bit quantities are sign extended. */ - *(SINT64 *) argp = *(SINT32 *)(* avalue); + val = *(SINT32 *)valp; break; - + case FFI_TYPE_SINT64: case FFI_TYPE_UINT64: case FFI_TYPE_POINTER: - *(UINT64 *) argp = *(UINT64 *)(* avalue); + case FFI_TYPE_DOUBLE: + val = *(UINT64 *)valp; + break; + + case FFI_TYPE_LONGDOUBLE: + /* Note that 128-bit long double is passed by reference. */ + val = (unsigned long)valp; break; case FFI_TYPE_FLOAT: if (argp - stack < 6) - { - /* Note the conversion -- all the fp regs are loaded as - doubles. The in-register format is the same. */ - *(double *) argp = *(float *)(* avalue); - } + val = lds(valp); else - *(float *) argp = *(float *)(* avalue); - break; - - case FFI_TYPE_DOUBLE: - *(double *) argp = *(double *)(* avalue); - break; - - case FFI_TYPE_LONGDOUBLE: - /* 128-bit long double is passed by reference. */ - *(long double **) argp = (long double *)(* avalue); - size = sizeof (long double *); + val = *(UINT32 *)valp; break; case FFI_TYPE_STRUCT: - memcpy(argp, *avalue, (*arg_types)->size); - break; + size = ty->size; + memcpy(argp, valp, size); + argp += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; + continue; default: - FFI_ASSERT(0); + abort(); } - argp += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - i++, arg_types++, avalue++; + *argp++ = val; } flags = (flags >> ALPHA_ST_SHIFT) & 0xff; @@ -255,14 +265,13 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) argn = 1; } - i = 0; - avn = cif->nargs; arg_types = cif->arg_types; /* Grab the addresses of the arguments from the stack frame. */ - while (i < avn) + for (i = 0, avn = cif->nargs; i < avn; i++) { size_t size = arg_types[i]->size; + void *valp = &argp[argn]; switch (arg_types[i]->type) { @@ -276,28 +285,26 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) case FFI_TYPE_UINT64: case FFI_TYPE_POINTER: case FFI_TYPE_STRUCT: - avalue[i] = &argp[argn]; break; case FFI_TYPE_FLOAT: + /* Floats coming from registers need conversion from double + back to float format. */ if (argn < 6) { - /* Floats coming from registers need conversion from double - back to float format. */ - *(float *)&argp[argn - 6] = *(double *)&argp[argn - 6]; - avalue[i] = &argp[argn - 6]; + valp = &argp[argn - 6]; + sts(valp, argp[argn - 6]); } - else - avalue[i] = &argp[argn]; break; case FFI_TYPE_DOUBLE: - avalue[i] = &argp[argn - (argn < 6 ? 6 : 0)]; + if (argn < 6) + valp = &argp[argn - 6]; break; case FFI_TYPE_LONGDOUBLE: /* 128-bit long double is passed by reference. */ - avalue[i] = (long double *) argp[argn]; + valp = (long double *) argp[argn]; size = sizeof (long double *); break; @@ -305,8 +312,8 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) abort (); } + avalue[i] = valp; argn += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - i++; } /* Invoke the closure. */ From f41bec3b576aa5ff8915b1188446c2dc086dfe64 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 20:46:48 -0700 Subject: [PATCH 38/58] alpha: Add support for complex types --- src/alpha/ffi.c | 278 +++++++++++++++---- src/alpha/ffitarget.h | 3 + src/alpha/internal.h | 4 + src/alpha/osf.S | 18 ++ testsuite/libffi.call/call.exp | 1 + testsuite/libffi.call/cls_complex_va_float.c | 6 + 6 files changed, 253 insertions(+), 57 deletions(-) diff --git a/src/alpha/ffi.c b/src/alpha/ffi.c index 2f9e7e5..1e5187e 100644 --- a/src/alpha/ffi.c +++ b/src/alpha/ffi.c @@ -1,8 +1,8 @@ /* ----------------------------------------------------------------------- ffi.c - Copyright (c) 2012 Anthony Green - Copyright (c) 1998, 2001, 2007, 2008 Red Hat, Inc. - - Alpha Foreign Function Interface + Copyright (c) 1998, 2001, 2007, 2008 Red Hat, Inc. + + Alpha Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -60,18 +60,61 @@ static inline void sts(void *ptr, UINT64 val) asm("sts %1,%0" : "=m"(*(UINT32 *)ptr) : "f"(val)); } -ffi_status +ffi_status FFI_HIDDEN ffi_prep_cif_machdep(ffi_cif *cif) { - int flags; + size_t bytes = 0; + int flags, i, avn; + ffi_type *rtype, *itype; - /* Adjust cif->bytes to represent a minimum 6 words for the temporary - register argument loading area. */ - if (cif->bytes < 6*FFI_SIZEOF_ARG) - cif->bytes = 6*FFI_SIZEOF_ARG; + if (cif->abi != FFI_OSF) + return FFI_BAD_ABI; + + /* Compute the size of the argument area. */ + for (i = 0, avn = cif->nargs; i < avn; i++) + { + itype = cif->arg_types[i]; + switch (itype->type) + { + case FFI_TYPE_INT: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + case FFI_TYPE_POINTER: + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + /* All take one 8 byte slot. */ + bytes += 8; + break; + + case FFI_TYPE_VOID: + case FFI_TYPE_STRUCT: + /* Passed by value in N slots. */ + bytes += ALIGN(itype->size, FFI_SIZEOF_ARG); + break; + + case FFI_TYPE_COMPLEX: + /* _Complex long double passed by reference; others in 2 slots. */ + if (itype->elements[0]->type == FFI_TYPE_LONGDOUBLE) + bytes += 8; + else + bytes += 16; + break; + + default: + abort(); + } + } /* Set the return type flag */ - switch (cif->rtype->type) + rtype = cif->rtype; + switch (rtype->type) { case FFI_TYPE_VOID: flags = ALPHA_FLAGS(ALPHA_ST_VOID, ALPHA_LD_VOID); @@ -109,22 +152,83 @@ ffi_prep_cif_machdep(ffi_cif *cif) /* Passed in memory, with a hidden pointer. */ flags = ALPHA_RET_IN_MEM; break; + case FFI_TYPE_COMPLEX: + itype = rtype->elements[0]; + switch (itype->type) + { + case FFI_TYPE_FLOAT: + flags = ALPHA_FLAGS(ALPHA_ST_CPLXF, ALPHA_LD_CPLXF); + break; + case FFI_TYPE_DOUBLE: + flags = ALPHA_FLAGS(ALPHA_ST_CPLXD, ALPHA_LD_CPLXD); + break; + default: + if (rtype->size <= 8) + flags = ALPHA_FLAGS(ALPHA_ST_INT, ALPHA_LD_INT64); + else + flags = ALPHA_RET_IN_MEM; + break; + } + break; default: abort(); } cif->flags = flags; - + + /* Include the hidden structure pointer in args requirement. */ + if (flags == ALPHA_RET_IN_MEM) + bytes += 8; + /* Minimum size is 6 slots, so that ffi_call_osf can pop them. */ + if (bytes < 6*8) + bytes = 6*8; + cif->bytes = bytes; + return FFI_OK; } +static unsigned long +extend_basic_type(void *valp, int type, int argn) +{ + switch (type) + { + case FFI_TYPE_SINT8: + return *(SINT8 *)valp; + case FFI_TYPE_UINT8: + return *(UINT8 *)valp; + case FFI_TYPE_SINT16: + return *(SINT16 *)valp; + case FFI_TYPE_UINT16: + return *(UINT16 *)valp; + + case FFI_TYPE_FLOAT: + if (argn < 6) + return lds(valp); + /* FALLTHRU */ + + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + /* Note that unsigned 32-bit quantities are sign extended. */ + return *(SINT32 *)valp; + + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + case FFI_TYPE_POINTER: + case FFI_TYPE_DOUBLE: + return *(UINT64 *)valp; + + default: + abort(); + } +} void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { - unsigned long *stack, *argp; - long i, avn, flags = cif->flags; + unsigned long *argp; + long i, avn, argn, flags = cif->flags; ffi_type **arg_types; - + /* If the return value is a struct and we don't have a return value address then we need to make one. */ if (rvalue == NULL && flags == ALPHA_RET_IN_MEM) @@ -132,12 +236,12 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) /* Allocate the space for the arguments, plus 4 words of temp space for ffi_call_osf. */ - argp = stack = alloca(cif->bytes + 4*FFI_SIZEOF_ARG); + argp = alloca(cif->bytes + 4*FFI_SIZEOF_ARG); + argn = 0; if (flags == ALPHA_RET_IN_MEM) - *argp++ = (unsigned long)rvalue; + argp[argn++] = (unsigned long)rvalue; - i = 0; avn = cif->nargs; arg_types = cif->arg_types; @@ -145,67 +249,59 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { ffi_type *ty = arg_types[i]; void *valp = avalue[i]; - unsigned long val; + int type = ty->type; size_t size; - switch (ty->type) + switch (type) { + case FFI_TYPE_INT: case FFI_TYPE_SINT8: - val = *(SINT8 *)valp; - break; - case FFI_TYPE_UINT8: - val = *(UINT8 *)valp; - break; - case FFI_TYPE_SINT16: - val = *(SINT16 *)valp; - break; - case FFI_TYPE_UINT16: - val = *(UINT16 *)valp; - break; - case FFI_TYPE_SINT32: case FFI_TYPE_UINT32: - /* Note that unsigned 32-bit quantities are sign extended. */ - val = *(SINT32 *)valp; - break; - case FFI_TYPE_SINT64: case FFI_TYPE_UINT64: case FFI_TYPE_POINTER: + case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: - val = *(UINT64 *)valp; + argp[argn] = extend_basic_type(valp, type, argn); + argn++; break; case FFI_TYPE_LONGDOUBLE: + by_reference: /* Note that 128-bit long double is passed by reference. */ - val = (unsigned long)valp; - break; - - case FFI_TYPE_FLOAT: - if (argp - stack < 6) - val = lds(valp); - else - val = *(UINT32 *)valp; + argp[argn++] = (unsigned long)valp; break; + case FFI_TYPE_VOID: case FFI_TYPE_STRUCT: size = ty->size; - memcpy(argp, valp, size); - argp += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - continue; + memcpy(argp + argn, valp, size); + argn += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; + break; + + case FFI_TYPE_COMPLEX: + type = ty->elements[0]->type; + if (type == FFI_TYPE_LONGDOUBLE) + goto by_reference; + + /* Most complex types passed as two separate arguments. */ + size = ty->elements[0]->size; + argp[argn] = extend_basic_type(valp, type, argn); + argp[argn + 1] = extend_basic_type(valp + size, type, argn + 1); + argn += 2; + break; default: abort(); } - - *argp++ = val; } flags = (flags >> ALPHA_ST_SHIFT) & 0xff; - ffi_call_osf(stack, cif->bytes, flags, rvalue, fn); + ffi_call_osf(argp, cif->bytes, flags, rvalue, fn); } @@ -243,7 +339,6 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } - long FFI_HIDDEN ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) { @@ -266,15 +361,18 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) } arg_types = cif->arg_types; - + /* Grab the addresses of the arguments from the stack frame. */ for (i = 0, avn = cif->nargs; i < avn; i++) { - size_t size = arg_types[i]->size; + ffi_type *ty = arg_types[i]; + int type = ty->type; void *valp = &argp[argn]; + size_t size; - switch (arg_types[i]->type) + switch (type) { + case FFI_TYPE_INT: case FFI_TYPE_SINT8: case FFI_TYPE_UINT8: case FFI_TYPE_SINT16: @@ -284,7 +382,13 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) case FFI_TYPE_SINT64: case FFI_TYPE_UINT64: case FFI_TYPE_POINTER: + argn += 1; + break; + + case FFI_TYPE_VOID: case FFI_TYPE_STRUCT: + size = ty->size; + argn += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; break; case FFI_TYPE_FLOAT: @@ -295,17 +399,78 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) valp = &argp[argn - 6]; sts(valp, argp[argn - 6]); } + argn += 1; break; case FFI_TYPE_DOUBLE: if (argn < 6) valp = &argp[argn - 6]; + argn += 1; break; case FFI_TYPE_LONGDOUBLE: + by_reference: /* 128-bit long double is passed by reference. */ - valp = (long double *) argp[argn]; - size = sizeof (long double *); + valp = (void *)argp[argn]; + argn += 1; + break; + + case FFI_TYPE_COMPLEX: + type = ty->elements[0]->type; + switch (type) + { + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + /* Passed as separate arguments, but they wind up sequential. */ + break; + + case FFI_TYPE_INT: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + /* Passed as separate arguments. Disjoint, but there's room + enough in one slot to hold the pair. */ + size = ty->elements[0]->size; + memcpy(valp + size, valp + 8, size); + break; + + case FFI_TYPE_FLOAT: + /* Passed as separate arguments. Disjoint, and each piece + may need conversion back to float. */ + if (argn < 6) + { + valp = &argp[argn - 6]; + sts(valp, argp[argn - 6]); + } + if (argn + 1 < 6) + sts(valp + 4, argp[argn + 1 - 6]); + else + *(UINT32 *)(valp + 4) = argp[argn + 1]; + break; + + case FFI_TYPE_DOUBLE: + /* Passed as separate arguments. Only disjoint if one part + is in fp regs and the other is on the stack. */ + if (argn < 5) + valp = &argp[argn - 6]; + else if (argn == 5) + { + valp = alloca(16); + ((UINT64 *)valp)[0] = argp[5 - 6]; + ((UINT64 *)valp)[1] = argp[6]; + } + break; + + case FFI_TYPE_LONGDOUBLE: + goto by_reference; + + default: + abort(); + } + argn += 2; break; default: @@ -313,7 +478,6 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) } avalue[i] = valp; - argn += ALIGN(size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; } /* Invoke the closure. */ diff --git a/src/alpha/ffitarget.h b/src/alpha/ffitarget.h index af145bc..60f92fd 100644 --- a/src/alpha/ffitarget.h +++ b/src/alpha/ffitarget.h @@ -44,6 +44,9 @@ typedef enum ffi_abi { } ffi_abi; #endif +#define FFI_TARGET_SPECIFIC_STACK_SPACE_ALLOCATION +#define FFI_TARGET_HAS_COMPLEX_TYPE + /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 diff --git a/src/alpha/internal.h b/src/alpha/internal.h index 664a2a6..44da192 100644 --- a/src/alpha/internal.h +++ b/src/alpha/internal.h @@ -2,6 +2,8 @@ #define ALPHA_ST_INT 1 #define ALPHA_ST_FLOAT 2 #define ALPHA_ST_DOUBLE 3 +#define ALPHA_ST_CPLXF 4 +#define ALPHA_ST_CPLXD 5 #define ALPHA_LD_VOID 0 #define ALPHA_LD_INT64 1 @@ -12,6 +14,8 @@ #define ALPHA_LD_SINT8 6 #define ALPHA_LD_FLOAT 7 #define ALPHA_LD_DOUBLE 8 +#define ALPHA_LD_CPLXF 9 +#define ALPHA_LD_CPLXD 10 #define ALPHA_ST_SHIFT 0 #define ALPHA_LD_SHIFT 8 diff --git a/src/alpha/osf.S b/src/alpha/osf.S index fb9c595..4059f82 100644 --- a/src/alpha/osf.S +++ b/src/alpha/osf.S @@ -117,6 +117,14 @@ E ALPHA_ST_FLOAT E ALPHA_ST_DOUBLE stt $f0, 0($2) ret +E ALPHA_ST_CPLXF + sts $f0, 0($2) + sts $f1, 4($2) + ret +E ALPHA_ST_CPLXD + stt $f0, 0($2) + stt $f1, 8($2) + ret cfi_endproc .end ffi_call_osf @@ -228,6 +236,16 @@ E ALPHA_LD_DOUBLE ldt $f0, 16($sp) epilogue +E ALPHA_LD_CPLXF + lds $f0, 16($sp) + lds $f1, 20($sp) + epilogue + +E ALPHA_LD_CPLXD + ldt $f0, 16($sp) + ldt $f1, 24($sp) + epilogue + cfi_endproc .end ffi_closure_osf diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index 676b323..82e9fe8 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -27,6 +27,7 @@ run-many-tests $tlist "" # ??? We really should preprocess ffi.h and grep # for FFI_TARGET_HAS_COMPLEX_TYPE. if { [istarget aarch64*] + || [istarget alpha*] || [istarget i?86*] || [istarget s390*] || [istarget x86_64*] } { diff --git a/testsuite/libffi.call/cls_complex_va_float.c b/testsuite/libffi.call/cls_complex_va_float.c index 0b79979..2b17826 100644 --- a/testsuite/libffi.call/cls_complex_va_float.c +++ b/testsuite/libffi.call/cls_complex_va_float.c @@ -6,5 +6,11 @@ /* { dg-do run } */ +/* Alpha splits _Complex into two arguments. It's illegal to pass + float through varargs, so _Complex float goes badly. In sort of + gets passed as _Complex double, but the compiler doesn't agree + with itself on this issue. */ +/* { dg-do run { xfail alpha*-*-* } } */ + #include "complex_defs_float.inc" #include "cls_complex_va.inc" From 9761b7bb70b4c47dc11dc74830964a0b3b3176d3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 21:26:52 -0700 Subject: [PATCH 39/58] alpha: Add support for Go closures --- src/alpha/ffi.c | 53 ++++++++++++++++++++++------ src/alpha/ffitarget.h | 1 + src/alpha/osf.S | 82 +++++++++++++++++++++++++++++-------------- 3 files changed, 99 insertions(+), 37 deletions(-) diff --git a/src/alpha/ffi.c b/src/alpha/ffi.c index 1e5187e..efae4cc 100644 --- a/src/alpha/ffi.c +++ b/src/alpha/ffi.c @@ -41,9 +41,11 @@ # define FFI_TYPE_LONGDOUBLE 4 #endif -extern void ffi_call_osf(void *, unsigned long, unsigned, void *, void (*)(void)) - FFI_HIDDEN; +extern void ffi_call_osf(void *stack, void *frame, unsigned flags, + void *raddr, void (*fn)(void), void *closure) + FFI_HIDDEN; extern void ffi_closure_osf(void) FFI_HIDDEN; +extern void ffi_go_closure_osf(void) FFI_HIDDEN; /* Promote a float value to its in-register double representation. Unlike actually casting to double, this does not trap on NaN. */ @@ -222,12 +224,14 @@ extend_basic_type(void *valp, int type, int argn) } } -void -ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) { unsigned long *argp; long i, avn, argn, flags = cif->flags; ffi_type **arg_types; + void *frame; /* If the return value is a struct and we don't have a return value address then we need to make one. */ @@ -236,7 +240,8 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) /* Allocate the space for the arguments, plus 4 words of temp space for ffi_call_osf. */ - argp = alloca(cif->bytes + 4*FFI_SIZEOF_ARG); + argp = frame = alloca(cif->bytes + 4*FFI_SIZEOF_ARG); + frame += cif->bytes; argn = 0; if (flags == ALPHA_RET_IN_MEM) @@ -301,9 +306,21 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) } flags = (flags >> ALPHA_ST_SHIFT) & 0xff; - ffi_call_osf(argp, cif->bytes, flags, rvalue, fn); + ffi_call_osf(argp, frame, flags, rvalue, fn, closure); } +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int(cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int(cif, fn, rvalue, avalue, closure); +} ffi_status ffi_prep_closure_loc (ffi_closure* closure, @@ -339,15 +356,31 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } +ffi_status +ffi_prep_go_closure (ffi_go_closure* closure, + ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*)) +{ + if (cif->abi != FFI_OSF) + return FFI_BAD_ABI; + + closure->tramp = (void *)ffi_go_closure_osf; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + long FFI_HIDDEN -ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) +ffi_closure_osf_inner (ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, + void *rvalue, unsigned long *argp) { - ffi_cif *cif; void **avalue; ffi_type **arg_types; long i, avn, argn, flags; - cif = closure->cif; avalue = alloca(cif->nargs * sizeof(void *)); flags = cif->flags; argn = 0; @@ -481,7 +514,7 @@ ffi_closure_osf_inner(ffi_closure *closure, void *rvalue, unsigned long *argp) } /* Invoke the closure. */ - closure->fun (cif, rvalue, avalue, closure->user_data); + fun (cif, rvalue, avalue, user_data); /* Tell ffi_closure_osf how to perform return type promotions. */ return (flags >> ALPHA_LD_SHIFT) & 0xff; diff --git a/src/alpha/ffitarget.h b/src/alpha/ffitarget.h index 60f92fd..a02dbd0 100644 --- a/src/alpha/ffitarget.h +++ b/src/alpha/ffitarget.h @@ -50,6 +50,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 +#define FFI_GO_CLOSURES 1 #define FFI_TRAMPOLINE_SIZE 24 #define FFI_NATIVE_RAW_API 0 diff --git a/src/alpha/osf.S b/src/alpha/osf.S index 4059f82..b031828 100644 --- a/src/alpha/osf.S +++ b/src/alpha/osf.S @@ -39,10 +39,10 @@ .org 99b + \index * 16 .endm -/* ffi_call_osf (void *args, unsigned long bytes, unsigned flags, - void *raddr, void (*fnaddr)(void)); +/* ffi_call_osf (void *stack, void *frame, unsigned flags, + void *raddr, void (*fnaddr)(void), void *closure) - Bit o trickiness here -- ARGS+BYTES is the base of the stack frame + Bit o trickiness here -- FRAME is the base of the stack frame for this function. This has been allocated by ffi_call. We also deallocate some of the stack that has been alloca'd. */ @@ -52,22 +52,21 @@ FFI_HIDDEN(ffi_call_osf) ffi_call_osf: - .frame $15, 32, $26, 0 - .mask 0x4008000, -32 cfi_startproc - addq $16,$17,$1 + cfi_def_cfa($17, 32) mov $16, $30 - stq $26, 0($1) - stq $15, 8($1) - stq $18, 16($1) - mov $1, $15 + stq $26, 0($17) + stq $15, 8($17) + mov $17, $15 .prologue 0 - cfi_def_cfa($15, 32) + cfi_def_cfa_register($15) cfi_rel_offset($26, 0) cfi_rel_offset($15, 8) - stq $19, 24($1) - mov $20, $27 + stq $18, 16($17) # save flags into frame + stq $19, 24($17) # save rvalue into frame + mov $20, $27 # fn into place for call + mov $21, $1 # closure into static chain # Load up all of the (potential) argument registers. ldq $16, 0($30) @@ -89,16 +88,16 @@ ffi_call_osf: jsr $26, ($27), 0 0: ldah $29, 0($26) !gpdisp!1 - ldq $2, 24($15) + ldq $2, 24($15) # reload rvalue lda $29, 0($29) !gpdisp!1 - ldq $3, 16($15) + ldq $3, 16($15) # reload flags lda $1, 99f-0b($26) ldq $26, 0($15) ldq $15, 8($15) cfi_restore($26) cfi_restore($15) cfi_def_cfa($sp, 0) - cmoveq $2, ALPHA_ST_VOID, $3 # mash null return to void + cmoveq $2, ALPHA_ST_VOID, $3 # mash null rvalue to void addq $3, $3, $3 s8addq $3, $1, $1 # 99f + stcode * 16 jmp $31, ($1), $st_int @@ -135,14 +134,38 @@ E ALPHA_ST_CPLXD #define CLOSURE_FS (16*8) + .align 4 + .globl ffi_go_closure_osf + .ent ffi_go_closure_osf + FFI_HIDDEN(ffi_go_closure_osf) + +ffi_go_closure_osf: + cfi_startproc + ldgp $29, 0($27) + subq $30, CLOSURE_FS, $30 + cfi_adjust_cfa_offset(CLOSURE_FS) + stq $26, 0($30) + .prologue 1 + cfi_rel_offset($26, 0) + + stq $16, 10*8($30) + stq $17, 11*8($30) + stq $18, 12*8($30) + + ldq $16, 8($1) # load cif + ldq $17, 16($1) # load fun + mov $1, $18 # closure is user_data + br $do_closure + + cfi_endproc + .end ffi_go_closure_osf + .align 4 .globl ffi_closure_osf .ent ffi_closure_osf FFI_HIDDEN(ffi_closure_osf) ffi_closure_osf: - .frame $30, CLOSURE_FS, $26, 0 - .mask 0x4000000, -CLOSURE_FS cfi_startproc ldgp $29, 0($27) subq $30, CLOSURE_FS, $30 @@ -152,23 +175,28 @@ ffi_closure_osf: cfi_rel_offset($26, 0) # Store all of the potential argument registers in va_list format. + stq $16, 10*8($30) + stq $17, 11*8($30) + stq $18, 12*8($30) + + ldq $16, 24($1) # load cif + ldq $17, 32($1) # load fun + ldq $18, 40($1) # load user_data + +$do_closure: + stq $19, 13*8($30) + stq $20, 14*8($30) + stq $21, 15*8($30) stt $f16, 4*8($30) stt $f17, 5*8($30) stt $f18, 6*8($30) stt $f19, 7*8($30) stt $f20, 8*8($30) stt $f21, 9*8($30) - stq $16, 10*8($30) - stq $17, 11*8($30) - stq $18, 12*8($30) - stq $19, 13*8($30) - stq $20, 14*8($30) - stq $21, 15*8($30) # Call ffi_closure_osf_inner to do the bulk of the work. - mov $1, $16 - lda $17, 2*8($30) - lda $18, 10*8($30) + lda $19, 2*8($30) + lda $20, 10*8($30) jsr $26, ffi_closure_osf_inner 0: ldah $29, 0($26) !gpdisp!2 From c129bea82a25c9b0f3af3bf48c35c55257092f87 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 15 Oct 2014 17:28:53 -0400 Subject: [PATCH 40/58] arm: Reindent arm/ffi.c --- src/arm/ffi.c | 749 ++++++++++++++++++++++++++------------------------ 1 file changed, 395 insertions(+), 354 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 6691ab5..c5fd831 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -37,122 +37,126 @@ static int vfp_type_p (ffi_type *); static void layout_vfp_args (ffi_cif *); -int ffi_prep_args_SYSV(char *stack, extended_cif *ecif, float *vfp_space); -int ffi_prep_args_VFP(char *stack, extended_cif *ecif, float *vfp_space); +int ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space); +int ffi_prep_args_VFP (char *stack, extended_cif *ecif, float *vfp_space); -static char* ffi_align(ffi_type **p_arg, char *argp) +static char * +ffi_align (ffi_type **p_arg, char *argp) { /* Align if necessary */ register size_t alignment = (*p_arg)->alignment; if (alignment < 4) - { - alignment = 4; - } + { + alignment = 4; + } #ifdef _WIN32_WCE if (alignment > 4) - { - alignment = 4; - } + { + alignment = 4; + } #endif if ((alignment - 1) & (unsigned) argp) - { - argp = (char *) ALIGN(argp, alignment); - } - + { + argp = (char *) ALIGN (argp, alignment); + } if ((*p_arg)->type == FFI_TYPE_STRUCT) - { - argp = (char *) ALIGN(argp, 4); - } + { + argp = (char *) ALIGN (argp, 4); + } return argp; } -static size_t ffi_put_arg(ffi_type **arg_type, void **arg, char *stack) +static size_t +ffi_put_arg (ffi_type **arg_type, void **arg, char *stack) { - register char* argp = stack; - register ffi_type **p_arg = arg_type; - register void **p_argv = arg; - register size_t z = (*p_arg)->size; - if (z < sizeof(int)) - { - z = sizeof(int); - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed int *) argp = (signed int)*(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(unsigned int *) argp = (unsigned int)*(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(signed int *) argp = (signed int)*(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(unsigned int *) argp = (unsigned int)*(UINT16 *)(* p_argv); - break; - - case FFI_TYPE_STRUCT: - memcpy(argp, *p_argv, (*p_arg)->size); - break; + register char *argp = stack; + register ffi_type **p_arg = arg_type; + register void **p_argv = arg; + register size_t z = (*p_arg)->size; - default: - FFI_ASSERT(0); - } - } - else if (z == sizeof(int)) + if (z < sizeof (int)) { - if ((*p_arg)->type == FFI_TYPE_FLOAT) - *(float *) argp = *(float *)(* p_argv); - else - *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); + z = sizeof (int); + switch ((*p_arg)->type) + { + case FFI_TYPE_SINT8: + *(signed int *) argp = (signed int) *(SINT8 *) (*p_argv); + break; + + case FFI_TYPE_UINT8: + *(unsigned int *) argp = (unsigned int) *(UINT8 *) (*p_argv); + break; + + case FFI_TYPE_SINT16: + *(signed int *) argp = (signed int) *(SINT16 *) (*p_argv); + break; + + case FFI_TYPE_UINT16: + *(unsigned int *) argp = (unsigned int) *(UINT16 *) (*p_argv); + break; + + case FFI_TYPE_STRUCT: + memcpy (argp, *p_argv, (*p_arg)->size); + break; + + default: + FFI_ASSERT (0); + } + } + else if (z == sizeof (int)) + { + if ((*p_arg)->type == FFI_TYPE_FLOAT) + *(float *) argp = *(float *) (*p_argv); + else + *(unsigned int *) argp = (unsigned int) *(UINT32 *) (*p_argv); + } + else if (z == sizeof (double) && (*p_arg)->type == FFI_TYPE_DOUBLE) + { + *(double *) argp = *(double *) (*p_argv); } - else if (z == sizeof(double) && (*p_arg)->type == FFI_TYPE_DOUBLE) - { - *(double *) argp = *(double *)(* p_argv); - } else { - memcpy(argp, *p_argv, z); + memcpy (argp, *p_argv, z); } return z; } + /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments - + The vfp_space parameter is the load area for VFP regs, the return value is cif->vfp_used (word bitset of VFP regs used for passing arguments). These are only used for the VFP hard-float ABI. */ -int ffi_prep_args_SYSV(char *stack, extended_cif *ecif, float *vfp_space) +int +ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space) { register unsigned int i; register void **p_argv; register char *argp; register ffi_type **p_arg; argp = stack; - - if ( ecif->cif->flags == FFI_TYPE_STRUCT ) { - *(void **) argp = ecif->rvalue; - argp += 4; - } + if (ecif->cif->flags == FFI_TYPE_STRUCT) + { + *(void **) argp = ecif->rvalue; + argp += 4; + } p_argv = ecif->avalue; for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - (i != 0); - i--, p_arg++, p_argv++) + (i != 0); i--, p_arg++, p_argv++) { - argp = ffi_align(p_arg, argp); - argp += ffi_put_arg(p_arg, p_argv, argp); + argp = ffi_align (p_arg, argp); + argp += ffi_put_arg (p_arg, p_argv, argp); } return 0; } -int ffi_prep_args_VFP(char *stack, extended_cif *ecif, float *vfp_space) +int +ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) { register unsigned int i, vi = 0; register void **p_argv; @@ -162,79 +166,79 @@ int ffi_prep_args_VFP(char *stack, extended_cif *ecif, float *vfp_space) char done_with_regs = 0; char is_vfp_type; - // make sure we are using FFI_VFP - FFI_ASSERT(ecif->cif->abi == FFI_VFP); + /* Make sure we are using FFI_VFP. */ + FFI_ASSERT (ecif->cif->abi == FFI_VFP); - /* the first 4 words on the stack are used for values passed in core - * registers. */ + /* The first 4 words on the stack are used for values + passed in core registers. */ regp = stack; eo_regp = argp = regp + 16; - - /* if the function returns an FFI_TYPE_STRUCT in memory, that address is - * passed in r0 to the function */ - if ( ecif->cif->flags == FFI_TYPE_STRUCT ) { - *(void **) regp = ecif->rvalue; - regp += 4; - } + /* If the function returns an FFI_TYPE_STRUCT in memory, + that address is passed in r0 to the function. */ + if (ecif->cif->flags == FFI_TYPE_STRUCT) + { + *(void **) regp = ecif->rvalue; + regp += 4; + } p_argv = ecif->avalue; for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - (i != 0); - i--, p_arg++, p_argv++) + (i != 0); i--, p_arg++, p_argv++) { is_vfp_type = vfp_type_p (*p_arg); /* Allocated in VFP registers. */ - if(vi < ecif->cif->vfp_nargs && is_vfp_type) - { - char *vfp_slot = (char *)(vfp_space + ecif->cif->vfp_args[vi++]); - ffi_put_arg(p_arg, p_argv, vfp_slot); - continue; - } + if (vi < ecif->cif->vfp_nargs && is_vfp_type) + { + char *vfp_slot = (char *) (vfp_space + ecif->cif->vfp_args[vi++]); + ffi_put_arg (p_arg, p_argv, vfp_slot); + continue; + } /* Try allocating in core registers. */ else if (!done_with_regs && !is_vfp_type) - { - char *tregp = ffi_align(p_arg, regp); - size_t size = (*p_arg)->size; - size = (size < 4)? 4 : size; // pad - /* Check if there is space left in the aligned register area to place - * the argument */ - if(tregp + size <= eo_regp) - { - regp = tregp + ffi_put_arg(p_arg, p_argv, tregp); - done_with_regs = (regp == argp); - // ensure we did not write into the stack area - FFI_ASSERT(regp <= argp); - continue; - } - /* In case there are no arguments in the stack area yet, - the argument is passed in the remaining core registers and on the - stack. */ - else if (!stack_used) - { - stack_used = 1; - done_with_regs = 1; - argp = tregp + ffi_put_arg(p_arg, p_argv, tregp); - FFI_ASSERT(eo_regp < argp); - continue; - } - } + { + char *tregp = ffi_align (p_arg, regp); + size_t size = (*p_arg)->size; + size = (size < 4) ? 4 : size; // pad + /* Check if there is space left in the aligned register + area to place the argument. */ + if (tregp + size <= eo_regp) + { + regp = tregp + ffi_put_arg (p_arg, p_argv, tregp); + done_with_regs = (regp == argp); + // ensure we did not write into the stack area + FFI_ASSERT (regp <= argp); + continue; + } + /* In case there are no arguments in the stack area yet, + the argument is passed in the remaining core registers + and on the stack. */ + else if (!stack_used) + { + stack_used = 1; + done_with_regs = 1; + argp = tregp + ffi_put_arg (p_arg, p_argv, tregp); + FFI_ASSERT (eo_regp < argp); + continue; + } + } /* Base case, arguments are passed on the stack */ stack_used = 1; - argp = ffi_align(p_arg, argp); - argp += ffi_put_arg(p_arg, p_argv, argp); + argp = ffi_align (p_arg, argp); + argp += ffi_put_arg (p_arg, p_argv, argp); } /* Indicate the VFP registers used. */ return ecif->cif->vfp_used; } /* Perform machine dependent cif processing */ -ffi_status ffi_prep_cif_machdep(ffi_cif *cif) +ffi_status +ffi_prep_cif_machdep (ffi_cif * cif) { int type_code; - /* Round the stack up to a multiple of 8 bytes. This isn't needed + /* Round the stack up to a multiple of 8 bytes. This isn't needed everywhere, but it is on some platforms, and it doesn't harm anything when it isn't needed. */ cif->bytes = (cif->bytes + 7) & ~7; @@ -254,21 +258,24 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) break; case FFI_TYPE_STRUCT: - if (cif->abi == FFI_VFP - && (type_code = vfp_type_p (cif->rtype)) != 0) + if (cif->abi == FFI_VFP && (type_code = vfp_type_p (cif->rtype)) != 0) { /* A Composite Type passed in VFP registers, either FFI_TYPE_STRUCT_VFP_FLOAT or FFI_TYPE_STRUCT_VFP_DOUBLE. */ cif->flags = (unsigned) type_code; } else if (cif->rtype->size <= 4) - /* A Composite Type not larger than 4 bytes is returned in r0. */ - cif->flags = (unsigned)FFI_TYPE_INT; + { + /* A Composite Type not larger than 4 bytes is returned in r0. */ + cif->flags = (unsigned) FFI_TYPE_INT; + } else - /* A Composite Type larger than 4 bytes, or whose size cannot - be determined statically ... is stored in memory at an - address passed [in r0]. */ - cif->flags = (unsigned)FFI_TYPE_STRUCT; + { + /* A Composite Type larger than 4 bytes, or whose size cannot + be determined statically ... is stored in memory at an + address passed [in r0]. */ + cif->flags = (unsigned) FFI_TYPE_STRUCT; + } break; default: @@ -276,9 +283,10 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) break; } - /* Map out the register placements of VFP register args. - The VFP hard-float calling conventions are slightly more sophisticated than - the base calling conventions, so we do it here instead of in ffi_prep_args(). */ + /* Map out the register placements of VFP register args. The VFP + hard-float calling conventions are slightly more sophisticated + than the base calling conventions, so we do it here instead of + in ffi_prep_args(). */ if (cif->abi == FFI_VFP) layout_vfp_args (cif); @@ -286,54 +294,56 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) } /* Perform machine dependent cif processing for variadic calls */ -ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, - unsigned int nfixedargs, - unsigned int ntotalargs) +ffi_status +ffi_prep_cif_machdep_var (ffi_cif * cif, + unsigned int nfixedargs, unsigned int ntotalargs) { /* VFP variadic calls actually use the SYSV ABI */ if (cif->abi == FFI_VFP) - cif->abi = FFI_SYSV; + cif->abi = FFI_SYSV; - return ffi_prep_cif_machdep(cif); + return ffi_prep_cif_machdep (cif); } -/* Prototypes for assembly functions, in sysv.S */ -extern void ffi_call_SYSV (void (*fn)(void), extended_cif *, unsigned, unsigned, unsigned *); -extern void ffi_call_VFP (void (*fn)(void), extended_cif *, unsigned, unsigned, unsigned *); +/* Prototypes for assembly functions, in sysv.S. */ +extern void ffi_call_SYSV (void (*fn) (void), extended_cif *, unsigned, + unsigned, unsigned *); +extern void ffi_call_VFP (void (*fn) (void), extended_cif *, unsigned, + unsigned, unsigned *); -void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +void +ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) { extended_cif ecif; - int small_struct = (cif->flags == FFI_TYPE_INT + int small_struct = (cif->flags == FFI_TYPE_INT && cif->rtype->type == FFI_TYPE_STRUCT); int vfp_struct = (cif->flags == FFI_TYPE_STRUCT_VFP_FLOAT || cif->flags == FFI_TYPE_STRUCT_VFP_DOUBLE); unsigned int temp; - + ecif.cif = cif; ecif.avalue = avalue; - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ + /* If the return value is a struct and we don't have a return + value address then we need to make one. */ - if ((rvalue == NULL) && - (cif->flags == FFI_TYPE_STRUCT)) + if ((rvalue == NULL) && (cif->flags == FFI_TYPE_STRUCT)) { - ecif.rvalue = alloca(cif->rtype->size); + ecif.rvalue = alloca (cif->rtype->size); } else if (small_struct) ecif.rvalue = &temp; else if (vfp_struct) { /* Largest case is double x 4. */ - ecif.rvalue = alloca(32); + ecif.rvalue = alloca (32); } else ecif.rvalue = rvalue; - switch (cif->abi) + switch (cif->abi) { case FFI_SYSV: ffi_call_SYSV (fn, &ecif, cif->bytes, cif->flags, ecif.rvalue); @@ -346,30 +356,30 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) #endif default: - FFI_ASSERT(0); + FFI_ASSERT (0); break; } if (small_struct) { - FFI_ASSERT(rvalue != NULL); + FFI_ASSERT (rvalue != NULL); memcpy (rvalue, &temp, cif->rtype->size); } - else if (vfp_struct) { - FFI_ASSERT(rvalue != NULL); + FFI_ASSERT (rvalue != NULL); memcpy (rvalue, ecif.rvalue, cif->rtype->size); } - } /** private members **/ static void ffi_prep_incoming_args_SYSV (char *stack, void **ret, - void** args, ffi_cif* cif, float *vfp_stack); + void **args, ffi_cif *cif, + float *vfp_stack); static void ffi_prep_incoming_args_VFP (char *stack, void **ret, - void** args, ffi_cif* cif, float *vfp_stack); + void **args, ffi_cif *cif, + float *vfp_stack); void ffi_closure_SYSV (ffi_closure *); @@ -378,25 +388,25 @@ void ffi_closure_VFP (ffi_closure *); /* This function is jumped to by the trampoline */ unsigned int FFI_HIDDEN -ffi_closure_inner (ffi_closure *closure, +ffi_closure_inner (ffi_closure *closure, void **respp, void *args, void *vfp_args) { // our various things... - ffi_cif *cif; - void **arg_area; + ffi_cif *cif; + void **arg_area; - cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); + cif = closure->cif; + arg_area = (void **) alloca (cif->nargs * sizeof (void *)); /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding + * element in that array points to the corresponding * value on the stack; and if the function returns * a structure, it will re-set RESP to point to the * structure return address. */ if (cif->abi == FFI_VFP) - ffi_prep_incoming_args_VFP(args, respp, arg_area, cif, vfp_args); + ffi_prep_incoming_args_VFP (args, respp, arg_area, cif, vfp_args); else - ffi_prep_incoming_args_SYSV(args, respp, arg_area, cif, vfp_args); + ffi_prep_incoming_args_SYSV (args, respp, arg_area, cif, vfp_args); (closure->fun) (cif, *respp, arg_area, closure->user_data); @@ -404,11 +414,11 @@ ffi_closure_inner (ffi_closure *closure, } /*@-exportheader@*/ -static void -ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, - void **avalue, ffi_cif *cif, - /* Used only under VFP hard-float ABI. */ - float *vfp_stack) +static void +ffi_prep_incoming_args_SYSV (char *stack, void **rvalue, + void **avalue, ffi_cif *cif, + /* Used only under VFP hard-float ABI. */ + float *vfp_stack) /*@=exportheader@*/ { register unsigned int i; @@ -418,10 +428,11 @@ ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, argp = stack; - if ( cif->flags == FFI_TYPE_STRUCT ) { - *rvalue = *(void **) argp; - argp += 4; - } + if (cif->flags == FFI_TYPE_STRUCT) + { + *rvalue = *(void **) argp; + argp += 4; + } p_argv = avalue; @@ -429,25 +440,25 @@ ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, { size_t z; - argp = ffi_align(p_arg, argp); + argp = ffi_align (p_arg, argp); z = (*p_arg)->size; /* because we're little endian, this is what it turns into. */ - *p_argv = (void*) argp; + *p_argv = (void *) argp; p_argv++; argp += z; } - + return; } /*@-exportheader@*/ -static void -ffi_prep_incoming_args_VFP(char *stack, void **rvalue, - void **avalue, ffi_cif *cif, +static void +ffi_prep_incoming_args_VFP (char *stack, void **rvalue, + void **avalue, ffi_cif * cif, /* Used only under VFP hard-float ABI. */ float *vfp_stack) /*@=exportheader@*/ @@ -460,76 +471,77 @@ ffi_prep_incoming_args_VFP(char *stack, void **rvalue, char stack_used = 0; char is_vfp_type; - FFI_ASSERT(cif->abi == FFI_VFP); + FFI_ASSERT (cif->abi == FFI_VFP); regp = stack; eo_regp = argp = regp + 16; - if ( cif->flags == FFI_TYPE_STRUCT ) { - *rvalue = *(void **) regp; - regp += 4; - } + if (cif->flags == FFI_TYPE_STRUCT) + { + *rvalue = *(void **) regp; + regp += 4; + } p_argv = avalue; for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) { - size_t z; - is_vfp_type = vfp_type_p (*p_arg); + size_t z; + is_vfp_type = vfp_type_p (*p_arg); - if(vi < cif->vfp_nargs && is_vfp_type) - { - *p_argv++ = (void*)(vfp_stack + cif->vfp_args[vi++]); - continue; - } - else if (!done_with_regs && !is_vfp_type) - { - char* tregp = ffi_align(p_arg, regp); + if (vi < cif->vfp_nargs && is_vfp_type) + { + *p_argv++ = (void *) (vfp_stack + cif->vfp_args[vi++]); + continue; + } + else if (!done_with_regs && !is_vfp_type) + { + char *tregp = ffi_align (p_arg, regp); - z = (*p_arg)->size; - z = (z < 4)? 4 : z; // pad - - /* if the arguments either fits into the registers or uses registers - * and stack, while we haven't read other things from the stack */ - if(tregp + z <= eo_regp || !stack_used) - { - /* because we're little endian, this is what it turns into. */ - *p_argv = (void*) tregp; + z = (*p_arg)->size; + z = (z < 4) ? 4 : z; // pad - p_argv++; - regp = tregp + z; - // if we read past the last core register, make sure we have not read - // from the stack before and continue reading after regp - if(regp > eo_regp) - { - if(stack_used) - { - abort(); // we should never read past the end of the register - // are if the stack is already in use - } - argp = regp; - } - if(regp >= eo_regp) - { - done_with_regs = 1; - stack_used = 1; - } - continue; - } - } - stack_used = 1; + /* if the arguments either fits into the registers or uses registers + * and stack, while we haven't read other things from the stack */ + if (tregp + z <= eo_regp || !stack_used) + { + /* because we're little endian, this is what it turns into. */ + *p_argv = (void *) tregp; - argp = ffi_align(p_arg, argp); + p_argv++; + regp = tregp + z; + // if we read past the last core register, make sure we have not read + // from the stack before and continue reading after regp + if (regp > eo_regp) + { + if (stack_used) + { + abort (); // we should never read past the end of the register + // are if the stack is already in use + } + argp = regp; + } + if (regp >= eo_regp) + { + done_with_regs = 1; + stack_used = 1; + } + continue; + } + } + stack_used = 1; - z = (*p_arg)->size; + argp = ffi_align (p_arg, argp); - /* because we're little endian, this is what it turns into. */ + z = (*p_arg)->size; - *p_argv = (void*) argp; + /* because we're little endian, this is what it turns into. */ - p_argv++; - argp += z; + *p_argv = (void *) argp; + + p_argv++; + argp += z; } - + return; } @@ -549,7 +561,8 @@ extern void *ffi_closure_trampoline_table_page; typedef struct ffi_trampoline_table ffi_trampoline_table; typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry; -struct ffi_trampoline_table { +struct ffi_trampoline_table +{ /* contiguous writable and executable pages */ vm_address_t config_page; vm_address_t trampoline_page; @@ -563,8 +576,9 @@ struct ffi_trampoline_table { ffi_trampoline_table *next; }; -struct ffi_trampoline_table_entry { - void *(*trampoline)(); +struct ffi_trampoline_table_entry +{ + void *(*trampoline) (); ffi_trampoline_table_entry *next; }; @@ -591,63 +605,80 @@ ffi_trampoline_table_alloc () ffi_trampoline_table *table = NULL; /* Loop until we can allocate two contiguous pages */ - while (table == NULL) { - vm_address_t config_page = 0x0; - kern_return_t kt; + while (table == NULL) + { + vm_address_t config_page = 0x0; + kern_return_t kt; - /* Try to allocate two pages */ - kt = vm_allocate (mach_task_self (), &config_page, PAGE_SIZE*2, VM_FLAGS_ANYWHERE); - if (kt != KERN_SUCCESS) { - fprintf(stderr, "vm_allocate() failure: %d at %s:%d\n", kt, __FILE__, __LINE__); - break; + /* Try to allocate two pages */ + kt = + vm_allocate (mach_task_self (), &config_page, PAGE_SIZE * 2, + VM_FLAGS_ANYWHERE); + if (kt != KERN_SUCCESS) + { + fprintf (stderr, "vm_allocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + break; + } + + /* Now drop the second half of the allocation to make room for the trampoline table */ + vm_address_t trampoline_page = config_page + PAGE_SIZE; + kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_SIZE); + if (kt != KERN_SUCCESS) + { + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + break; + } + + /* Remap the trampoline table to directly follow the config page */ + vm_prot_t cur_prot; + vm_prot_t max_prot; + + kt = + vm_remap (mach_task_self (), &trampoline_page, PAGE_SIZE, 0x0, FALSE, + mach_task_self (), + (vm_address_t) & ffi_closure_trampoline_table_page, FALSE, + &cur_prot, &max_prot, VM_INHERIT_SHARE); + + /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */ + if (kt != KERN_SUCCESS) + { + /* Log unexpected failures */ + if (kt != KERN_NO_SPACE) + { + fprintf (stderr, "vm_remap() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + } + + vm_deallocate (mach_task_self (), config_page, PAGE_SIZE); + continue; + } + + /* We have valid trampoline and config pages */ + table = calloc (1, sizeof (ffi_trampoline_table)); + table->free_count = FFI_TRAMPOLINE_COUNT; + table->config_page = config_page; + table->trampoline_page = trampoline_page; + + /* Create and initialize the free list */ + table->free_list_pool = + calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry)); + + uint16_t i; + for (i = 0; i < table->free_count; i++) + { + ffi_trampoline_table_entry *entry = &table->free_list_pool[i]; + entry->trampoline = + (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE)); + + if (i < table->free_count - 1) + entry->next = &table->free_list_pool[i + 1]; + } + + table->free_list = table->free_list_pool; } - /* Now drop the second half of the allocation to make room for the trampoline table */ - vm_address_t trampoline_page = config_page+PAGE_SIZE; - kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) { - fprintf(stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, __FILE__, __LINE__); - break; - } - - /* Remap the trampoline table to directly follow the config page */ - vm_prot_t cur_prot; - vm_prot_t max_prot; - - kt = vm_remap (mach_task_self (), &trampoline_page, PAGE_SIZE, 0x0, FALSE, mach_task_self (), (vm_address_t) &ffi_closure_trampoline_table_page, FALSE, &cur_prot, &max_prot, VM_INHERIT_SHARE); - - /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */ - if (kt != KERN_SUCCESS) { - /* Log unexpected failures */ - if (kt != KERN_NO_SPACE) { - fprintf(stderr, "vm_remap() failure: %d at %s:%d\n", kt, __FILE__, __LINE__); - } - - vm_deallocate (mach_task_self (), config_page, PAGE_SIZE); - continue; - } - - /* We have valid trampoline and config pages */ - table = calloc (1, sizeof(ffi_trampoline_table)); - table->free_count = FFI_TRAMPOLINE_COUNT; - table->config_page = config_page; - table->trampoline_page = trampoline_page; - - /* Create and initialize the free list */ - table->free_list_pool = calloc(FFI_TRAMPOLINE_COUNT, sizeof(ffi_trampoline_table_entry)); - - uint16_t i; - for (i = 0; i < table->free_count; i++) { - ffi_trampoline_table_entry *entry = &table->free_list_pool[i]; - entry->trampoline = (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE)); - - if (i < table->free_count - 1) - entry->next = &table->free_list_pool[i+1]; - } - - table->free_list = table->free_list_pool; - } - return table; } @@ -655,36 +686,38 @@ void * ffi_closure_alloc (size_t size, void **code) { /* Create the closure */ - ffi_closure *closure = malloc(size); + ffi_closure *closure = malloc (size); if (closure == NULL) return NULL; - pthread_mutex_lock(&ffi_trampoline_lock); + pthread_mutex_lock (&ffi_trampoline_lock); /* Check for an active trampoline table with available entries. */ ffi_trampoline_table *table = ffi_trampoline_tables; - if (table == NULL || table->free_list == NULL) { - table = ffi_trampoline_table_alloc (); - if (table == NULL) { - free(closure); - return NULL; + if (table == NULL || table->free_list == NULL) + { + table = ffi_trampoline_table_alloc (); + if (table == NULL) + { + free (closure); + return NULL; + } + + /* Insert the new table at the top of the list */ + table->next = ffi_trampoline_tables; + if (table->next != NULL) + table->next->prev = table; + + ffi_trampoline_tables = table; } - /* Insert the new table at the top of the list */ - table->next = ffi_trampoline_tables; - if (table->next != NULL) - table->next->prev = table; - - ffi_trampoline_tables = table; - } - /* Claim the free entry */ ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list; ffi_trampoline_tables->free_list = entry->next; ffi_trampoline_tables->free_count--; entry->next = NULL; - pthread_mutex_unlock(&ffi_trampoline_lock); + pthread_mutex_unlock (&ffi_trampoline_lock); /* Initialize the return values */ *code = entry->trampoline; @@ -699,7 +732,7 @@ ffi_closure_free (void *ptr) { ffi_closure *closure = ptr; - pthread_mutex_lock(&ffi_trampoline_lock); + pthread_mutex_lock (&ffi_trampoline_lock); /* Fetch the table and entry references */ ffi_trampoline_table *table = closure->trampoline_table; @@ -712,36 +745,43 @@ ffi_closure_free (void *ptr) /* If all trampolines within this table are free, and at least one other table exists, deallocate * the table */ - if (table->free_count == FFI_TRAMPOLINE_COUNT && ffi_trampoline_tables != table) { - /* Remove from the list */ - if (table->prev != NULL) - table->prev->next = table->next; + if (table->free_count == FFI_TRAMPOLINE_COUNT + && ffi_trampoline_tables != table) + { + /* Remove from the list */ + if (table->prev != NULL) + table->prev->next = table->next; - if (table->next != NULL) - table->next->prev = table->prev; + if (table->next != NULL) + table->next->prev = table->prev; - /* Deallocate pages */ - kern_return_t kt; - kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf(stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, __FILE__, __LINE__); + /* Deallocate pages */ + kern_return_t kt; + kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE); + if (kt != KERN_SUCCESS) + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); - kt = vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf(stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, __FILE__, __LINE__); + kt = + vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE); + if (kt != KERN_SUCCESS) + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); - /* Deallocate free list */ - free (table->free_list_pool); - free (table); - } else if (ffi_trampoline_tables != table) { - /* Otherwise, bump this table to the top of the list */ - table->prev = NULL; - table->next = ffi_trampoline_tables; - if (ffi_trampoline_tables != NULL) - ffi_trampoline_tables->prev = table; + /* Deallocate free list */ + free (table->free_list_pool); + free (table); + } + else if (ffi_trampoline_tables != table) + { + /* Otherwise, bump this table to the top of the list */ + table->prev = NULL; + table->next = ffi_trampoline_tables; + if (ffi_trampoline_tables != NULL) + ffi_trampoline_tables->prev = table; - ffi_trampoline_tables = table; - } + ffi_trampoline_tables = table; + } pthread_mutex_unlock (&ffi_trampoline_lock); @@ -770,13 +810,12 @@ ffi_closure_free (void *ptr) /* the cif must already be prep'ed */ ffi_status -ffi_prep_closure_loc (ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void *user_data, - void *codeloc) +ffi_prep_closure_loc (ffi_closure * closure, + ffi_cif * cif, + void (*fun) (ffi_cif *, void *, void **, void *), + void *user_data, void *codeloc) { - void (*closure_func)(ffi_closure*) = NULL; + void (*closure_func) (ffi_closure *) = NULL; if (cif->abi == FFI_SYSV) closure_func = &ffi_closure_SYSV; @@ -788,25 +827,24 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_BAD_ABI; #if FFI_EXEC_TRAMPOLINE_TABLE - void **config = FFI_TRAMPOLINE_CODELOC_CONFIG(codeloc); + void **config = FFI_TRAMPOLINE_CODELOC_CONFIG (codeloc); config[0] = closure; config[1] = closure_func; #else - FFI_INIT_TRAMPOLINE (&closure->tramp[0], \ - closure_func, \ - codeloc); + FFI_INIT_TRAMPOLINE (&closure->tramp[0], closure_func, codeloc); #endif - closure->cif = cif; + closure->cif = cif; closure->user_data = user_data; - closure->fun = fun; + closure->fun = fun; return FFI_OK; } /* Below are routines for VFP hard-float support. */ -static int rec_vfp_type_p (ffi_type *t, int *elt, int *elnum) +static int +rec_vfp_type_p (ffi_type * t, int *elt, int *elnum) { switch (t->type) { @@ -833,7 +871,7 @@ static int rec_vfp_type_p (ffi_type *t, int *elt, int *elnum) while (*el) { int el_elt = 0, el_elnum = 0; - if (! rec_vfp_type_p (*el, &el_elt, &el_elnum) + if (!rec_vfp_type_p (*el, &el_elt, &el_elnum) || (base_elt && base_elt != el_elt) || total_elnum + el_elnum > 4) return 0; @@ -845,12 +883,13 @@ static int rec_vfp_type_p (ffi_type *t, int *elt, int *elnum) *elt = base_elt; return 1; } - default: ; + default:; } return 0; } -static int vfp_type_p (ffi_type *t) +static int +vfp_type_p (ffi_type * t) { int elt, elnum; if (rec_vfp_type_p (t, &elt, &elnum)) @@ -869,7 +908,8 @@ static int vfp_type_p (ffi_type *t) return 0; } -static int place_vfp_arg (ffi_cif *cif, ffi_type *t) +static int +place_vfp_arg (ffi_cif * cif, ffi_type * t) { short reg = cif->vfp_reg_free; int nregs = t->size / sizeof (float); @@ -903,7 +943,7 @@ static int place_vfp_arg (ffi_cif *cif, ffi_type *t) cif->vfp_reg_free = reg; } return 0; - next_reg: ; + next_reg:; } // done, mark all regs as used cif->vfp_reg_free = 16; @@ -911,21 +951,22 @@ static int place_vfp_arg (ffi_cif *cif, ffi_type *t) return 1; } -static void layout_vfp_args (ffi_cif *cif) +static void +layout_vfp_args (ffi_cif * cif) { int i; /* Init VFP fields */ cif->vfp_used = 0; cif->vfp_nargs = 0; cif->vfp_reg_free = 0; - memset (cif->vfp_args, -1, 16); /* Init to -1. */ + memset (cif->vfp_args, -1, 16); /* Init to -1. */ for (i = 0; i < cif->nargs; i++) { ffi_type *t = cif->arg_types[i]; if (vfp_type_p (t) && place_vfp_arg (cif, t) == 1) - { - break; - } + { + break; + } } } From 57b24fb3f5471262ce57e1a912774d6e7de5ada7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 00:53:21 -0400 Subject: [PATCH 41/58] arm: Deref ffi_align argument --- src/arm/ffi.c | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index c5fd831..7d86e94 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -40,30 +40,19 @@ static void layout_vfp_args (ffi_cif *); int ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space); int ffi_prep_args_VFP (char *stack, extended_cif *ecif, float *vfp_space); -static char * -ffi_align (ffi_type **p_arg, char *argp) +static void * +ffi_align (ffi_type *ty, void *p) { /* Align if necessary */ - register size_t alignment = (*p_arg)->alignment; - if (alignment < 4) - { - alignment = 4; - } + size_t alignment; #ifdef _WIN32_WCE - if (alignment > 4) - { - alignment = 4; - } + alignment = 4; +#else + alignment = ty->alignment; + if (alignment < 4) + alignment = 4; #endif - if ((alignment - 1) & (unsigned) argp) - { - argp = (char *) ALIGN (argp, alignment); - } - if ((*p_arg)->type == FFI_TYPE_STRUCT) - { - argp = (char *) ALIGN (argp, 4); - } - return argp; + return (void *) ALIGN (p, alignment); } static size_t @@ -148,7 +137,7 @@ ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space) for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; (i != 0); i--, p_arg++, p_argv++) { - argp = ffi_align (p_arg, argp); + argp = ffi_align (*p_arg, argp); argp += ffi_put_arg (p_arg, p_argv, argp); } @@ -199,7 +188,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) /* Try allocating in core registers. */ else if (!done_with_regs && !is_vfp_type) { - char *tregp = ffi_align (p_arg, regp); + char *tregp = ffi_align (*p_arg, regp); size_t size = (*p_arg)->size; size = (size < 4) ? 4 : size; // pad /* Check if there is space left in the aligned register @@ -226,7 +215,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) } /* Base case, arguments are passed on the stack */ stack_used = 1; - argp = ffi_align (p_arg, argp); + argp = ffi_align (*p_arg, argp); argp += ffi_put_arg (p_arg, p_argv, argp); } /* Indicate the VFP registers used. */ @@ -440,7 +429,7 @@ ffi_prep_incoming_args_SYSV (char *stack, void **rvalue, { size_t z; - argp = ffi_align (p_arg, argp); + argp = ffi_align (*p_arg, argp); z = (*p_arg)->size; @@ -495,7 +484,7 @@ ffi_prep_incoming_args_VFP (char *stack, void **rvalue, } else if (!done_with_regs && !is_vfp_type) { - char *tregp = ffi_align (p_arg, regp); + char *tregp = ffi_align (*p_arg, regp); z = (*p_arg)->size; z = (z < 4) ? 4 : z; // pad @@ -530,7 +519,7 @@ ffi_prep_incoming_args_VFP (char *stack, void **rvalue, } stack_used = 1; - argp = ffi_align (p_arg, argp); + argp = ffi_align (*p_arg, argp); z = (*p_arg)->size; From 0d39b4bb692b1a8e4f3354badcd9e123f8276bd4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 01:02:52 -0400 Subject: [PATCH 42/58] arm: Deref ffi_put_arg arguments --- src/arm/ffi.c | 88 ++++++++++++++++++++++----------------------------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 7d86e94..5e5ad0a 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -56,58 +56,46 @@ ffi_align (ffi_type *ty, void *p) } static size_t -ffi_put_arg (ffi_type **arg_type, void **arg, char *stack) +ffi_put_arg (ffi_type *ty, void *src, void *dst) { - register char *argp = stack; - register ffi_type **p_arg = arg_type; - register void **p_argv = arg; - register size_t z = (*p_arg)->size; + size_t z = ty->size; - if (z < sizeof (int)) + switch (ty->type) { - z = sizeof (int); - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed int *) argp = (signed int) *(SINT8 *) (*p_argv); - break; + case FFI_TYPE_SINT8: + *(UINT32 *)dst = *(SINT8 *)src; + break; + case FFI_TYPE_UINT8: + *(UINT32 *)dst = *(UINT8 *)src; + break; + case FFI_TYPE_SINT16: + *(UINT32 *)dst = *(SINT16 *)src; + break; + case FFI_TYPE_UINT16: + *(UINT32 *)dst = *(UINT16 *)src; + break; - case FFI_TYPE_UINT8: - *(unsigned int *) argp = (unsigned int) *(UINT8 *) (*p_argv); - break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + case FFI_TYPE_POINTER: + case FFI_TYPE_FLOAT: + *(UINT32 *)dst = *(UINT32 *)src; + break; - case FFI_TYPE_SINT16: - *(signed int *) argp = (signed int) *(SINT16 *) (*p_argv); - break; + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + case FFI_TYPE_DOUBLE: + *(UINT64 *)dst = *(UINT64 *)src; + break; - case FFI_TYPE_UINT16: - *(unsigned int *) argp = (unsigned int) *(UINT16 *) (*p_argv); - break; - - case FFI_TYPE_STRUCT: - memcpy (argp, *p_argv, (*p_arg)->size); - break; - - default: - FFI_ASSERT (0); - } + case FFI_TYPE_STRUCT: + default: + memcpy (dst, src, z); + break; } - else if (z == sizeof (int)) - { - if ((*p_arg)->type == FFI_TYPE_FLOAT) - *(float *) argp = *(float *) (*p_argv); - else - *(unsigned int *) argp = (unsigned int) *(UINT32 *) (*p_argv); - } - else if (z == sizeof (double) && (*p_arg)->type == FFI_TYPE_DOUBLE) - { - *(double *) argp = *(double *) (*p_argv); - } - else - { - memcpy (argp, *p_argv, z); - } - return z; + + return ALIGN (z, 4); } /* ffi_prep_args is called by the assembly routine once stack space @@ -138,7 +126,7 @@ ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space) (i != 0); i--, p_arg++, p_argv++) { argp = ffi_align (*p_arg, argp); - argp += ffi_put_arg (p_arg, p_argv, argp); + argp += ffi_put_arg (*p_arg, *p_argv, argp); } return 0; @@ -182,7 +170,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) if (vi < ecif->cif->vfp_nargs && is_vfp_type) { char *vfp_slot = (char *) (vfp_space + ecif->cif->vfp_args[vi++]); - ffi_put_arg (p_arg, p_argv, vfp_slot); + ffi_put_arg (*p_arg, *p_argv, vfp_slot); continue; } /* Try allocating in core registers. */ @@ -195,7 +183,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) area to place the argument. */ if (tregp + size <= eo_regp) { - regp = tregp + ffi_put_arg (p_arg, p_argv, tregp); + regp = tregp + ffi_put_arg (*p_arg, *p_argv, tregp); done_with_regs = (regp == argp); // ensure we did not write into the stack area FFI_ASSERT (regp <= argp); @@ -208,7 +196,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) { stack_used = 1; done_with_regs = 1; - argp = tregp + ffi_put_arg (p_arg, p_argv, tregp); + argp = tregp + ffi_put_arg (*p_arg, *p_argv, tregp); FFI_ASSERT (eo_regp < argp); continue; } @@ -216,7 +204,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) /* Base case, arguments are passed on the stack */ stack_used = 1; argp = ffi_align (*p_arg, argp); - argp += ffi_put_arg (p_arg, p_argv, argp); + argp += ffi_put_arg (*p_arg, *p_argv, argp); } /* Indicate the VFP registers used. */ return ecif->cif->vfp_used; From a74a3aaddbcc730d9aa0a4787f136c9cc19b82b3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 01:21:22 -0400 Subject: [PATCH 43/58] arm: Rewrite vfp_type_p Do not modify the ffi_type. Rearrange the tests so that we quickly eliminate structures that cannot match. Return an encoded value of element count and base type. --- src/arm/ffi.c | 226 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 151 insertions(+), 75 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 5e5ad0a..d00ed89 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -34,7 +34,7 @@ #include /* Forward declares. */ -static int vfp_type_p (ffi_type *); +static int vfp_type_p (const ffi_type *); static void layout_vfp_args (ffi_cif *); int ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space); @@ -141,7 +141,6 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) register ffi_type **p_arg; char stack_used = 0; char done_with_regs = 0; - char is_vfp_type; /* Make sure we are using FFI_VFP. */ FFI_ASSERT (ecif->cif->abi == FFI_VFP); @@ -164,7 +163,7 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; (i != 0); i--, p_arg++, p_argv++) { - is_vfp_type = vfp_type_p (*p_arg); + int is_vfp_type = vfp_type_p (*p_arg); /* Allocated in VFP registers. */ if (vi < ecif->cif->vfp_nargs && is_vfp_type) @@ -214,7 +213,6 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) ffi_status ffi_prep_cif_machdep (ffi_cif * cif) { - int type_code; /* Round the stack up to a multiple of 8 bytes. This isn't needed everywhere, but it is on some platforms, and it doesn't harm anything when it isn't needed. */ @@ -235,13 +233,25 @@ ffi_prep_cif_machdep (ffi_cif * cif) break; case FFI_TYPE_STRUCT: - if (cif->abi == FFI_VFP && (type_code = vfp_type_p (cif->rtype)) != 0) + if (cif->abi == FFI_VFP) { - /* A Composite Type passed in VFP registers, either - FFI_TYPE_STRUCT_VFP_FLOAT or FFI_TYPE_STRUCT_VFP_DOUBLE. */ - cif->flags = (unsigned) type_code; + int h = vfp_type_p (cif->rtype); + if (h) + { + int ele_count = h >> 8; + int type_code = h & 0xff; + if (ele_count > 1) + { + if (type_code == FFI_TYPE_FLOAT) + type_code = FFI_TYPE_STRUCT_VFP_FLOAT; + else + type_code = FFI_TYPE_STRUCT_VFP_DOUBLE; + } + cif->flags = type_code; + break; + } } - else if (cif->rtype->size <= 4) + if (cif->rtype->size <= 4) { /* A Composite Type not larger than 4 bytes is returned in r0. */ cif->flags = (unsigned) FFI_TYPE_INT; @@ -446,7 +456,6 @@ ffi_prep_incoming_args_VFP (char *stack, void **rvalue, register ffi_type **p_arg; char done_with_regs = 0; char stack_used = 0; - char is_vfp_type; FFI_ASSERT (cif->abi == FFI_VFP); regp = stack; @@ -462,8 +471,8 @@ ffi_prep_incoming_args_VFP (char *stack, void **rvalue, for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) { + int is_vfp_type = vfp_type_p (*p_arg); size_t z; - is_vfp_type = vfp_type_p (*p_arg); if (vi < cif->vfp_nargs && is_vfp_type) { @@ -820,81 +829,150 @@ ffi_prep_closure_loc (ffi_closure * closure, /* Below are routines for VFP hard-float support. */ +/* A subroutine of vfp_type_p. Given a structure type, return the type code + of the first non-structure element. Recurse for structure elements. + Return -1 if the structure is in fact empty, i.e. no nested elements. */ + static int -rec_vfp_type_p (ffi_type * t, int *elt, int *elnum) +is_hfa0 (const ffi_type *ty) { - switch (t->type) + ffi_type **elements = ty->elements; + int i, ret = -1; + + if (elements != NULL) + for (i = 0; elements[i]; ++i) + { + ret = elements[i]->type; + if (ret == FFI_TYPE_STRUCT) + { + ret = is_hfa0 (elements[i]); + if (ret < 0) + continue; + } + break; + } + + return ret; +} + +/* A subroutine of vfp_type_p. Given a structure type, return true if all + of the non-structure elements are the same as CANDIDATE. */ + +static int +is_hfa1 (const ffi_type *ty, int candidate) +{ + ffi_type **elements = ty->elements; + int i; + + if (elements != NULL) + for (i = 0; elements[i]; ++i) + { + int t = elements[i]->type; + if (t == FFI_TYPE_STRUCT) + { + if (!is_hfa1 (elements[i], candidate)) + return 0; + } + else if (t != candidate) + return 0; + } + + return 1; +} + +/* Determine if TY is an homogenous floating point aggregate (HFA). + That is, a structure consisting of 1 to 4 members of all the same type, + where that type is a floating point scalar. + + Returns non-zero iff TY is an HFA. The result is an encoded value where + bits 0-7 contain the type code, and bits 8-10 contain the element count. */ + +static int +vfp_type_p (const ffi_type *ty) +{ + ffi_type **elements; + int candidate, i; + size_t size, ele_count; + + /* Quickest tests first. */ + switch (ty->type) { + default: + return 0; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: - *elt = (int) t->type; - *elnum = 1; - return 1; - - case FFI_TYPE_STRUCT_VFP_FLOAT: - *elt = FFI_TYPE_FLOAT; - *elnum = t->size / sizeof (float); - return 1; - - case FFI_TYPE_STRUCT_VFP_DOUBLE: - *elt = FFI_TYPE_DOUBLE; - *elnum = t->size / sizeof (double); - return 1; - - case FFI_TYPE_STRUCT:; - { - int base_elt = 0, total_elnum = 0; - ffi_type **el = t->elements; - while (*el) - { - int el_elt = 0, el_elnum = 0; - if (!rec_vfp_type_p (*el, &el_elt, &el_elnum) - || (base_elt && base_elt != el_elt) - || total_elnum + el_elnum > 4) - return 0; - base_elt = el_elt; - total_elnum += el_elnum; - el++; - } - *elnum = total_elnum; - *elt = base_elt; - return 1; - } - default:; + return 0x100 + ty->type; + case FFI_TYPE_STRUCT: + break; } - return 0; -} -static int -vfp_type_p (ffi_type * t) -{ - int elt, elnum; - if (rec_vfp_type_p (t, &elt, &elnum)) + /* No HFA types are smaller than 4 bytes, or larger than 32 bytes. */ + size = ty->size; + if (size < 4 || size > 32) + return 0; + + /* Find the type of the first non-structure member. */ + elements = ty->elements; + candidate = elements[0]->type; + if (candidate == FFI_TYPE_STRUCT) { - if (t->type == FFI_TYPE_STRUCT) - { - if (elnum == 1) - t->type = elt; - else - t->type = (elt == FFI_TYPE_FLOAT - ? FFI_TYPE_STRUCT_VFP_FLOAT - : FFI_TYPE_STRUCT_VFP_DOUBLE); - } - return (int) t->type; + for (i = 0; ; ++i) + { + candidate = is_hfa0 (elements[i]); + if (candidate >= 0) + break; + } } - return 0; + + /* If the first member is not a floating point type, it's not an HFA. + Also quickly re-check the size of the structure. */ + switch (candidate) + { + case FFI_TYPE_FLOAT: + ele_count = size / sizeof(float); + if (size != ele_count * sizeof(float)) + return 0; + break; + case FFI_TYPE_DOUBLE: + ele_count = size / sizeof(double); + if (size != ele_count * sizeof(double)) + return 0; + break; + default: + return 0; + } + if (ele_count > 4) + return 0; + + /* Finally, make sure that all scalar elements are the same type. */ + for (i = 0; elements[i]; ++i) + { + if (elements[i]->type == FFI_TYPE_STRUCT) + { + if (!is_hfa1 (elements[i], candidate)) + return 0; + } + else if (elements[i]->type != candidate) + return 0; + } + + /* All tests succeeded. Encode the result. */ + return (ele_count << 8) | candidate; } static int -place_vfp_arg (ffi_cif * cif, ffi_type * t) +place_vfp_arg (ffi_cif *cif, int h) { - short reg = cif->vfp_reg_free; - int nregs = t->size / sizeof (float); - int align = ((t->type == FFI_TYPE_STRUCT_VFP_FLOAT - || t->type == FFI_TYPE_FLOAT) ? 1 : 2); + unsigned short reg = cif->vfp_reg_free; + int align = 1, nregs = h >> 8; + + if ((h & 0xff) == FFI_TYPE_DOUBLE) + align = 2, nregs *= 2; + /* Align register number. */ if ((reg & 1) && align == 2) reg++; + while (reg + nregs <= 16) { int s, new_used = 0; @@ -940,10 +1018,8 @@ layout_vfp_args (ffi_cif * cif) for (i = 0; i < cif->nargs; i++) { - ffi_type *t = cif->arg_types[i]; - if (vfp_type_p (t) && place_vfp_arg (cif, t) == 1) - { - break; - } + int h = vfp_type_p (cif->arg_types[i]); + if (h && place_vfp_arg (cif, h) == 1) + break; } } From e7f15f60e86ad8931b2c4a516945d0b76b4bda20 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 01:27:16 -0400 Subject: [PATCH 44/58] arm: Rewrite ffi_call Use the trick to allocate the stack frame for ffi_call_SYSV within ffi_call itself. --- src/arm/ffi.c | 319 +++++++++++++++++---------------- src/arm/ffitarget.h | 2 +- src/arm/internal.h | 7 + src/arm/sysv.S | 424 ++++++++++++++------------------------------ 4 files changed, 303 insertions(+), 449 deletions(-) create mode 100644 src/arm/internal.h diff --git a/src/arm/ffi.c b/src/arm/ffi.c index d00ed89..c91b869 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -30,16 +30,13 @@ #include #include - #include +#include "internal.h" /* Forward declares. */ static int vfp_type_p (const ffi_type *); static void layout_vfp_args (ffi_cif *); -int ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space); -int ffi_prep_args_VFP (char *stack, extended_cif *ecif, float *vfp_space); - static void * ffi_align (ffi_type *ty, void *p) { @@ -98,53 +95,44 @@ ffi_put_arg (ffi_type *ty, void *src, void *dst) return ALIGN (z, 4); } -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments +/* ffi_prep_args is called once stack space has been allocated + for the function's arguments. The vfp_space parameter is the load area for VFP regs, the return value is cif->vfp_used (word bitset of VFP regs used for passing arguments). These are only used for the VFP hard-float ABI. */ -int -ffi_prep_args_SYSV (char *stack, extended_cif *ecif, float *vfp_space) +static void +ffi_prep_args_SYSV (ffi_cif *cif, int flags, void *rvalue, + void **avalue, char *argp) { - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - argp = stack; + ffi_type **arg_types = cif->arg_types; + int i, n; - if (ecif->cif->flags == FFI_TYPE_STRUCT) + if (flags == ARM_TYPE_STRUCT) { - *(void **) argp = ecif->rvalue; + *(void **) argp = rvalue; argp += 4; } - p_argv = ecif->avalue; - - for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - (i != 0); i--, p_arg++, p_argv++) + for (i = 0, n = cif->nargs; i < n; i++) { - argp = ffi_align (*p_arg, argp); - argp += ffi_put_arg (*p_arg, *p_argv, argp); + ffi_type *ty = arg_types[i]; + argp = ffi_align (ty, argp); + argp += ffi_put_arg (ty, avalue[i], argp); } - - return 0; } -int -ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) +static void +ffi_prep_args_VFP (ffi_cif *cif, int flags, void *rvalue, + void **avalue, char *stack, char *vfp_space) { - register unsigned int i, vi = 0; - register void **p_argv; - register char *argp, *regp, *eo_regp; - register ffi_type **p_arg; + ffi_type **arg_types = cif->arg_types; + int i, n, vi = 0; + char *argp, *regp, *eo_regp; char stack_used = 0; char done_with_regs = 0; - /* Make sure we are using FFI_VFP. */ - FFI_ASSERT (ecif->cif->abi == FFI_VFP); - /* The first 4 words on the stack are used for values passed in core registers. */ regp = stack; @@ -152,37 +140,36 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) /* If the function returns an FFI_TYPE_STRUCT in memory, that address is passed in r0 to the function. */ - if (ecif->cif->flags == FFI_TYPE_STRUCT) + if (flags == ARM_TYPE_STRUCT) { - *(void **) regp = ecif->rvalue; + *(void **) regp = rvalue; regp += 4; } - p_argv = ecif->avalue; - - for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; - (i != 0); i--, p_arg++, p_argv++) + for (i = 0, n = cif->nargs; i < n; i++) { - int is_vfp_type = vfp_type_p (*p_arg); + ffi_type *ty = arg_types[i]; + void *a = avalue[i]; + int is_vfp_type = vfp_type_p (ty); /* Allocated in VFP registers. */ - if (vi < ecif->cif->vfp_nargs && is_vfp_type) + if (vi < cif->vfp_nargs && is_vfp_type) { - char *vfp_slot = (char *) (vfp_space + ecif->cif->vfp_args[vi++]); - ffi_put_arg (*p_arg, *p_argv, vfp_slot); + char *vfp_slot = vfp_space + cif->vfp_args[vi++] * 4; + ffi_put_arg (ty, a, vfp_slot); continue; } /* Try allocating in core registers. */ else if (!done_with_regs && !is_vfp_type) { - char *tregp = ffi_align (*p_arg, regp); - size_t size = (*p_arg)->size; + char *tregp = ffi_align (ty, regp); + size_t size = ty->size; size = (size < 4) ? 4 : size; // pad /* Check if there is space left in the aligned register area to place the argument. */ if (tregp + size <= eo_regp) { - regp = tregp + ffi_put_arg (*p_arg, *p_argv, tregp); + regp = tregp + ffi_put_arg (ty, a, tregp); done_with_regs = (regp == argp); // ensure we did not write into the stack area FFI_ASSERT (regp <= argp); @@ -195,88 +182,98 @@ ffi_prep_args_VFP (char *stack, extended_cif * ecif, float *vfp_space) { stack_used = 1; done_with_regs = 1; - argp = tregp + ffi_put_arg (*p_arg, *p_argv, tregp); + argp = tregp + ffi_put_arg (ty, a, tregp); FFI_ASSERT (eo_regp < argp); continue; } } /* Base case, arguments are passed on the stack */ stack_used = 1; - argp = ffi_align (*p_arg, argp); - argp += ffi_put_arg (*p_arg, *p_argv, argp); + argp = ffi_align (ty, argp); + argp += ffi_put_arg (ty, a, argp); } - /* Indicate the VFP registers used. */ - return ecif->cif->vfp_used; } /* Perform machine dependent cif processing */ ffi_status -ffi_prep_cif_machdep (ffi_cif * cif) +ffi_prep_cif_machdep (ffi_cif *cif) { + int flags = 0, cabi = cif->abi; + size_t bytes; + /* Round the stack up to a multiple of 8 bytes. This isn't needed everywhere, but it is on some platforms, and it doesn't harm anything when it isn't needed. */ - cif->bytes = (cif->bytes + 7) & ~7; + bytes = ALIGN (cif->bytes, 8); - /* Set the return type flag */ - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - cif->flags = (unsigned) cif->rtype->type; - break; - - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - cif->flags = (unsigned) FFI_TYPE_SINT64; - break; - - case FFI_TYPE_STRUCT: - if (cif->abi == FFI_VFP) - { - int h = vfp_type_p (cif->rtype); - if (h) - { - int ele_count = h >> 8; - int type_code = h & 0xff; - if (ele_count > 1) - { - if (type_code == FFI_TYPE_FLOAT) - type_code = FFI_TYPE_STRUCT_VFP_FLOAT; - else - type_code = FFI_TYPE_STRUCT_VFP_DOUBLE; - } - cif->flags = type_code; - break; - } - } - if (cif->rtype->size <= 4) - { - /* A Composite Type not larger than 4 bytes is returned in r0. */ - cif->flags = (unsigned) FFI_TYPE_INT; - } - else - { - /* A Composite Type larger than 4 bytes, or whose size cannot - be determined statically ... is stored in memory at an - address passed [in r0]. */ - cif->flags = (unsigned) FFI_TYPE_STRUCT; - } - break; - - default: - cif->flags = FFI_TYPE_INT; - break; - } + /* Minimum stack space is the 4 register arguments that we pop. */ + if (bytes < 4*4) + bytes = 4*4; + cif->bytes = bytes; /* Map out the register placements of VFP register args. The VFP hard-float calling conventions are slightly more sophisticated than the base calling conventions, so we do it here instead of in ffi_prep_args(). */ - if (cif->abi == FFI_VFP) + if (cabi == FFI_VFP) layout_vfp_args (cif); + /* Set the return type flag */ + switch (cif->rtype->type) + { + case FFI_TYPE_VOID: + flags = ARM_TYPE_VOID; + break; + + case FFI_TYPE_INT: + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_POINTER: + flags = ARM_TYPE_INT; + break; + + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + flags = ARM_TYPE_INT64; + break; + + case FFI_TYPE_FLOAT: + flags = (cabi == FFI_VFP ? ARM_TYPE_VFP_S : ARM_TYPE_INT); + break; + case FFI_TYPE_DOUBLE: + flags = (cabi == FFI_VFP ? ARM_TYPE_VFP_D : ARM_TYPE_INT64); + break; + + case FFI_TYPE_STRUCT: + if (cabi == FFI_VFP) + { + int h = vfp_type_p (cif->rtype); + + flags = ARM_TYPE_VFP_N; + if (h == 0x100 + FFI_TYPE_FLOAT) + flags = ARM_TYPE_VFP_S; + if (h == 0x100 + FFI_TYPE_DOUBLE) + flags = ARM_TYPE_VFP_D; + if (h != 0) + break; + } + + /* A Composite Type not larger than 4 bytes is returned in r0. + A Composite Type larger than 4 bytes, or whose size cannot + be determined statically ... is stored in memory at an + address passed [in r0]. */ + flags = (cif->rtype->size <= 4 ? ARM_TYPE_INT : ARM_TYPE_STRUCT); + break; + + default: + abort(); + } + cif->flags = flags; + return FFI_OK; } @@ -293,69 +290,83 @@ ffi_prep_cif_machdep_var (ffi_cif * cif, } /* Prototypes for assembly functions, in sysv.S. */ -extern void ffi_call_SYSV (void (*fn) (void), extended_cif *, unsigned, - unsigned, unsigned *); -extern void ffi_call_VFP (void (*fn) (void), extended_cif *, unsigned, - unsigned, unsigned *); + +struct call_frame +{ + void *fp; + void *lr; + void *rvalue; + int flags; +}; + +extern void ffi_call_SYSV (void *stack, struct call_frame *, + void (*fn) (void)) FFI_HIDDEN; +extern void ffi_call_VFP (void *vfp_space, struct call_frame *, + void (*fn) (void), unsigned vfp_used) FFI_HIDDEN; void ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) { - extended_cif ecif; + int flags = cif->flags; + ffi_type *rtype = cif->rtype; + size_t bytes, rsize, vfp_size; + char *stack, *vfp_space, *new_rvalue; + struct call_frame *frame; - int small_struct = (cif->flags == FFI_TYPE_INT - && cif->rtype->type == FFI_TYPE_STRUCT); - int vfp_struct = (cif->flags == FFI_TYPE_STRUCT_VFP_FLOAT - || cif->flags == FFI_TYPE_STRUCT_VFP_DOUBLE); - - unsigned int temp; - - ecif.cif = cif; - ecif.avalue = avalue; - - /* If the return value is a struct and we don't have a return - value address then we need to make one. */ - - if ((rvalue == NULL) && (cif->flags == FFI_TYPE_STRUCT)) + rsize = 0; + if (rvalue == NULL) { - ecif.rvalue = alloca (cif->rtype->size); + /* If the return value is a struct and we don't have a return + value address then we need to make one. Otherwise the return + value is in registers and we can ignore them. */ + if (flags == ARM_TYPE_STRUCT) + rsize = rtype->size; + else + flags = ARM_TYPE_VOID; } - else if (small_struct) - ecif.rvalue = &temp; - else if (vfp_struct) + else if (flags == ARM_TYPE_VFP_N) { /* Largest case is double x 4. */ - ecif.rvalue = alloca (32); + rsize = 32; + } + else if (flags == ARM_TYPE_INT && rtype->type == FFI_TYPE_STRUCT) + rsize = 4; + + /* Largest case. */ + vfp_size = (cif->abi == FFI_VFP && cif->vfp_used ? 8*8: 0); + + bytes = cif->bytes; + stack = alloca (vfp_size + bytes + sizeof(struct call_frame) + rsize); + + vfp_space = NULL; + if (vfp_size) + { + vfp_space = stack; + stack += vfp_size; + } + + frame = (struct call_frame *)(stack + bytes); + + new_rvalue = rvalue; + if (rsize) + new_rvalue = (void *)(frame + 1); + + frame->rvalue = new_rvalue; + frame->flags = flags; + + if (vfp_space) + { + ffi_prep_args_VFP (cif, flags, new_rvalue, avalue, stack, vfp_space); + ffi_call_VFP (vfp_space, frame, fn, cif->vfp_used); } else - ecif.rvalue = rvalue; - - switch (cif->abi) { - case FFI_SYSV: - ffi_call_SYSV (fn, &ecif, cif->bytes, cif->flags, ecif.rvalue); - break; - - case FFI_VFP: -#ifdef __ARM_EABI__ - ffi_call_VFP (fn, &ecif, cif->bytes, cif->flags, ecif.rvalue); - break; -#endif - - default: - FFI_ASSERT (0); - break; - } - if (small_struct) - { - FFI_ASSERT (rvalue != NULL); - memcpy (rvalue, &temp, cif->rtype->size); - } - else if (vfp_struct) - { - FFI_ASSERT (rvalue != NULL); - memcpy (rvalue, ecif.rvalue, cif->rtype->size); + ffi_prep_args_SYSV (cif, flags, new_rvalue, avalue, stack); + ffi_call_SYSV (stack, frame, fn); } + + if (rvalue && rvalue != new_rvalue) + memcpy (rvalue, new_rvalue, rtype->size); } /** private members **/ diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index 26d494d..6355904 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -53,7 +53,7 @@ typedef enum ffi_abi { #define FFI_EXTRA_CIF_FIELDS \ int vfp_used; \ - short vfp_reg_free, vfp_nargs; \ + unsigned short vfp_reg_free, vfp_nargs; \ signed char vfp_args[16] \ /* Internally used. */ diff --git a/src/arm/internal.h b/src/arm/internal.h new file mode 100644 index 0000000..6cf0b2a --- /dev/null +++ b/src/arm/internal.h @@ -0,0 +1,7 @@ +#define ARM_TYPE_VFP_S 0 +#define ARM_TYPE_VFP_D 1 +#define ARM_TYPE_VFP_N 2 +#define ARM_TYPE_INT64 3 +#define ARM_TYPE_INT 4 +#define ARM_TYPE_VOID 5 +#define ARM_TYPE_STRUCT 6 diff --git a/src/arm/sysv.S b/src/arm/sysv.S index 541bbe9..b967d97 100644 --- a/src/arm/sysv.S +++ b/src/arm/sysv.S @@ -1,8 +1,8 @@ /* ----------------------------------------------------------------------- sysv.S - Copyright (c) 1998, 2008, 2011 Red Hat, Inc. Copyright (c) 2011 Plausible Labs Cooperative, Inc. - - ARM Foreign Function Interface + + ARM Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -28,219 +28,155 @@ #define LIBFFI_ASM #include #include -#ifdef HAVE_MACHINE_ASM_H -#include -#else -#ifdef __USER_LABEL_PREFIX__ -#define CONCAT1(a, b) CONCAT2(a, b) -#define CONCAT2(a, b) a ## b +#include +#include "internal.h" -/* Use the right prefix for global labels. */ -#define CNAME(x) CONCAT1 (__USER_LABEL_PREFIX__, x) -#else -#define CNAME(x) x -#endif -#ifdef __APPLE__ -#define ENTRY(x) .globl _##x; _##x: -#else -#define ENTRY(x) .globl CNAME(x); .type CNAME(x),%function; CNAME(x): -#endif /* __APPLE__ */ -#endif - -#ifdef __ELF__ -#define LSYM(x) .x -#else -#define LSYM(x) x -#endif - -/* Use the SOFTFP return value ABI on Mac OS X, as per the iOS ABI - Function Call Guide */ -#ifdef __APPLE__ -#define __SOFTFP__ -#endif - -/* We need a better way of testing for this, but for now, this is all - we can do. */ -@ This selects the minimum architecture level required. -#define __ARM_ARCH__ 3 - -#if defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) -# undef __ARM_ARCH__ -# define __ARM_ARCH__ 4 -#endif - -#if defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) \ - || defined(__ARM_ARCH_5E__) || defined(__ARM_ARCH_5TE__) \ - || defined(__ARM_ARCH_5TEJ__) -# undef __ARM_ARCH__ -# define __ARM_ARCH__ 5 -#endif - -#if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ +/* GCC 4.8 provides __ARM_ARCH; construct it otherwise. */ +#ifndef __ARM_ARCH +# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7EM__) +# define __ARM_ARCH 7 +# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) \ || defined(__ARM_ARCH_6M__) -# undef __ARM_ARCH__ -# define __ARM_ARCH__ 6 -#endif - -#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ - || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ - || defined(__ARM_ARCH_7EM__) -# undef __ARM_ARCH__ -# define __ARM_ARCH__ 7 -#endif - -#if __ARM_ARCH__ >= 5 -# define call_reg(x) blx x -#elif defined (__ARM_ARCH_4T__) -# define call_reg(x) mov lr, pc ; bx x -# if defined(__thumb__) || defined(__THUMB_INTERWORK__) -# define __INTERWORKING__ +# define __ARM_ARCH 6 +# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) \ + || defined(__ARM_ARCH_5E__) || defined(__ARM_ARCH_5TE__) \ + || defined(__ARM_ARCH_5TEJ__) +# define __ARM_ARCH 5 +# else +# define __ARM_ARCH 4 # endif -#else -# define call_reg(x) mov lr, pc ; mov pc, x #endif /* Conditionally compile unwinder directives. */ +.macro UNWIND text:vararg #ifdef __ARM_EABI__ -#define UNWIND -#else -#define UNWIND @ + \text #endif - -.syntax unified - -#if defined(__thumb__) && !defined(__THUMB_INTERWORK__) -#define ARM_FUNC_START(name) \ - .text; \ - .align 2; \ - .thumb; \ - .thumb_func; \ - ENTRY(name); \ - bx pc; \ - nop; \ - .arm; \ - UNWIND .fnstart; \ -_L__##name: -#else -#define ARM_FUNC_START(name) \ - .text; \ - .align 2; \ - .arm; \ - ENTRY(name); \ - UNWIND .fnstart +.endm +#if defined(HAVE_AS_CFI_PSEUDO_OP) && defined(__ARM_EABI__) + .cfi_sections .debug_frame #endif -.macro RETLDM regs=, cond=, dirn=ia -#if defined (__INTERWORKING__) - .ifc "\regs","" - ldr\cond lr, [sp], #4 - .else - ldm\cond\dirn sp!, {\regs, lr} - .endif - bx\cond lr +#define CONCAT(a, b) CONCAT2(a, b) +#define CONCAT2(a, b) a ## b + +#ifdef __USER_LABEL_PREFIX__ +# define CNAME(X) CONCAT (__USER_LABEL_PREFIX__, X) #else - .ifc "\regs","" - ldr\cond pc, [sp], #4 - .else - ldm\cond\dirn sp!, {\regs, pc} - .endif +# define CNAME(X) X #endif +#ifdef __ELF__ +# define SIZE(X) .size CNAME(X), . - CNAME(X) +# define TYPE(X, Y) .type CNAME(X), Y +#else +# define SIZE(X) +# define TYPE(X, Y) +#endif + +#define ARM_FUNC_START(name, gl) \ + .align 3; \ + .ifne gl; .globl CNAME(name); FFI_HIDDEN(CNAME(name)); .endif; \ + TYPE(name, %function); \ + CNAME(name): + +#define ARM_FUNC_END(name) \ + SIZE(name) + +/* Aid in defining a jump table with 8 bytes between entries. */ +.macro E index + .if . - 0b - 8*\index + .error "type table out of sync" + .endif .endm - @ r0: ffi_prep_args - @ r1: &ecif - @ r2: cif->bytes - @ r3: fig->flags - @ sp+0: ecif.rvalue + .text + .syntax unified + .arm - @ This assumes we are using gas. -ARM_FUNC_START(ffi_call_SYSV) - @ Save registers - stmfd sp!, {r0-r3, fp, lr} - UNWIND .save {r0-r3, fp, lr} - mov fp, sp + /* We require interworking on LDM, which implies ARMv5T, + which implies the existance of BLX. */ + .arch armv5t - UNWIND .setfp fp, sp + /* Note that we use STC and LDC to encode VFP instructions, + so that we do not need ".fpu vfp", nor get that added to + the object file attributes. These will not be executed + unless the FFI_VFP abi is used. */ - @ Make room for all of the new args. - sub sp, fp, r2 + @ r0: stack + @ r1: frame + @ r2: fn + @ r3: vfp_used - @ Place all of the ffi_prep_args in position - mov r0, sp - @ r1 already set +ARM_FUNC_START(ffi_call_VFP, 1) + UNWIND .fnstart + cfi_startproc - @ Call ffi_prep_args(stack, &ecif) - bl CNAME(ffi_prep_args_SYSV) + cmp r3, #3 @ load only d0 if possible + ldcle p11, cr0, [r0] @ vldrle d0, [sp] + ldcgt p11, cr0, [r0], {16} @ vldmgt sp, {d0-d7} + add r0, r0, #64 @ discard the vfp register args + /* FALLTHRU */ +ARM_FUNC_END(ffi_call_VFP) - @ move first 4 parameters in registers - ldmia sp, {r0-r3} +ARM_FUNC_START(ffi_call_SYSV, 1) + stm r1, {fp, lr} + mov fp, r1 - @ and adjust stack - sub lr, fp, sp @ cif->bytes == fp - sp - ldr ip, [fp] @ load fn() in advance - cmp lr, #16 - movhs lr, #16 - add sp, sp, lr + @ This is a bit of a lie wrt the origin of the unwind info, but + @ now we've got the usual frame pointer and two saved registers. + UNWIND .save {fp,lr} + UNWIND .setfp fp, sp + cfi_def_cfa(fp, 8) + cfi_rel_offset(fp, 0) + cfi_rel_offset(lr, 4) - @ call (fn) (...) - call_reg(ip) - - @ Remove the space we pushed for the args - mov sp, fp + mov sp, r0 @ install the stack pointer + mov lr, r2 @ move the fn pointer out of the way + ldmia sp!, {r0-r3} @ move first 4 parameters in registers. + blx lr @ call fn @ Load r2 with the pointer to storage for the return value - ldr r2, [sp, #24] + @ Load r3 with the return type code + ldr r2, [fp, #8] + ldr r3, [fp, #12] - @ Load r3 with the return type code - ldr r3, [sp, #12] + @ Deallocate the stack with the arguments. + mov sp, fp + cfi_def_cfa_register(sp) - @ If the return value pointer is NULL, assume no return value. - cmp r2, #0 - beq LSYM(Lepilogue) + @ Store values stored in registers. + .align 3 + add pc, pc, r3, lsl #3 + nop +0: +E ARM_TYPE_VFP_S + stc p10, cr0, [r2] @ vstr s0, [r2] + pop {fp,pc} +E ARM_TYPE_VFP_D + stc p11, cr0, [r2] @ vstr d0, [r2] + pop {fp,pc} +E ARM_TYPE_VFP_N + stc p11, cr0, [r2], {8} @ vstm r2, {d0-d3} + pop {fp,pc} +E ARM_TYPE_INT64 + str r1, [r2, #4] + nop +E ARM_TYPE_INT + str r0, [r2] + pop {fp,pc} +E ARM_TYPE_VOID + pop {fp,pc} + nop +E ARM_TYPE_STRUCT + pop {fp,pc} -@ return INT - cmp r3, #FFI_TYPE_INT -#if defined(__SOFTFP__) || defined(__ARM_EABI__) - cmpne r3, #FFI_TYPE_FLOAT -#endif - streq r0, [r2] - beq LSYM(Lepilogue) - - @ return INT64 - cmp r3, #FFI_TYPE_SINT64 -#if defined(__SOFTFP__) || defined(__ARM_EABI__) - cmpne r3, #FFI_TYPE_DOUBLE -#endif - stmiaeq r2, {r0, r1} - -#if !defined(__SOFTFP__) && !defined(__ARM_EABI__) - beq LSYM(Lepilogue) - -@ return FLOAT - cmp r3, #FFI_TYPE_FLOAT - stfeqs f0, [r2] - beq LSYM(Lepilogue) - -@ return DOUBLE or LONGDOUBLE - cmp r3, #FFI_TYPE_DOUBLE - stfeqd f0, [r2] -#endif - -LSYM(Lepilogue): -#if defined (__INTERWORKING__) - ldmia sp!, {r0-r3,fp, lr} - bx lr -#else - ldmia sp!, {r0-r3,fp, pc} -#endif - -.ffi_call_SYSV_end: - UNWIND .fnend -#ifdef __ELF__ - .size CNAME(ffi_call_SYSV),.ffi_call_SYSV_end-CNAME(ffi_call_SYSV) -#endif + cfi_endproc + UNWIND .fnend +ARM_FUNC_END(ffi_call_SYSV) /* @@ -251,7 +187,8 @@ LSYM(Lepilogue): void *args; */ -ARM_FUNC_START(ffi_closure_SYSV) +ARM_FUNC_START(ffi_closure_SYSV, 1) + UNWIND .fnstart UNWIND .pad #16 add ip, sp, #16 stmfd sp!, {ip, lr} @@ -310,116 +247,16 @@ ARM_FUNC_START(ffi_closure_SYSV) ldfd f0, [sp] b .Lclosure_epilogue #endif - -.ffi_closure_SYSV_end: UNWIND .fnend -#ifdef __ELF__ - .size CNAME(ffi_closure_SYSV),.ffi_closure_SYSV_end-CNAME(ffi_closure_SYSV) -#endif +ARM_FUNC_END(ffi_closure_SYSV) /* Below are VFP hard-float ABI call and closure implementations. Add VFP FPU directive here. This is only compiled into the library under EABI. */ #ifdef __ARM_EABI__ - .fpu vfp - - @ r0: fn - @ r1: &ecif - @ r2: cif->bytes - @ r3: fig->flags - @ sp+0: ecif.rvalue - -ARM_FUNC_START(ffi_call_VFP) - @ Save registers - stmfd sp!, {r0-r3, fp, lr} - UNWIND .save {r0-r3, fp, lr} - mov fp, sp - UNWIND .setfp fp, sp - - @ Make room for all of the new args. - sub sp, sp, r2 - - @ Make room for loading VFP args - sub sp, sp, #64 - - @ Place all of the ffi_prep_args in position - mov r0, sp - @ r1 already set - sub r2, fp, #64 @ VFP scratch space - - @ Call ffi_prep_args(stack, &ecif, vfp_space) - bl CNAME(ffi_prep_args_VFP) - - @ Load VFP register args if needed - cmp r0, #0 - mov ip, fp - beq LSYM(Lbase_args) - - @ Load only d0 if possible - cmp r0, #3 - sub ip, fp, #64 - flddle d0, [ip] - fldmiadgt ip, {d0-d7} - -LSYM(Lbase_args): - @ move first 4 parameters in registers - ldmia sp, {r0-r3} - - @ and adjust stack - sub lr, ip, sp @ cif->bytes == (fp - 64) - sp - ldr ip, [fp] @ load fn() in advance - cmp lr, #16 - movhs lr, #16 - add sp, sp, lr - - @ call (fn) (...) - call_reg(ip) - - @ Remove the space we pushed for the args - mov sp, fp - - @ Load r2 with the pointer to storage for - @ the return value - ldr r2, [sp, #24] - - @ Load r3 with the return type code - ldr r3, [sp, #12] - - @ If the return value pointer is NULL, - @ assume no return value. - cmp r2, #0 - beq LSYM(Lepilogue_vfp) - - cmp r3, #FFI_TYPE_INT - streq r0, [r2] - beq LSYM(Lepilogue_vfp) - - cmp r3, #FFI_TYPE_SINT64 - stmeqia r2, {r0, r1} - beq LSYM(Lepilogue_vfp) - - cmp r3, #FFI_TYPE_FLOAT - fstseq s0, [r2] - beq LSYM(Lepilogue_vfp) - - cmp r3, #FFI_TYPE_DOUBLE - fstdeq d0, [r2] - beq LSYM(Lepilogue_vfp) - - cmp r3, #FFI_TYPE_STRUCT_VFP_FLOAT - cmpne r3, #FFI_TYPE_STRUCT_VFP_DOUBLE - fstmiadeq r2, {d0-d3} - -LSYM(Lepilogue_vfp): - RETLDM "r0-r3,fp" - -.ffi_call_VFP_end: - UNWIND .fnend - .size CNAME(ffi_call_VFP),.ffi_call_VFP_end-CNAME(ffi_call_VFP) - - -ARM_FUNC_START(ffi_closure_VFP) +ARM_FUNC_START(ffi_closure_VFP, 1) + UNWIND .fnstart fstmfdd sp!, {d0-d7} @ r0-r3, then d0-d7 UNWIND .pad #80 @@ -475,16 +312,15 @@ ARM_FUNC_START(ffi_closure_VFP) .Lretdouble_struct_vfp: fldmiad sp, {d0-d3} b .Lclosure_epilogue_vfp - -.ffi_closure_VFP_end: UNWIND .fnend - .size CNAME(ffi_closure_VFP),.ffi_closure_VFP_end-CNAME(ffi_closure_VFP) +ARM_FUNC_END(ffi_closure_VFP) #endif -ENTRY(ffi_arm_trampoline) +ARM_FUNC_START(ffi_arm_trampoline, 1) stmfd sp!, {r0-r3} ldr r0, [pc] ldr pc, [pc] +ARM_FUNC_END(ffi_arm_trampoline) #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",%progbits From a4b785ea695e929b91d64e1e8b514411ae8bafe8 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 17 Oct 2014 02:07:32 -0400 Subject: [PATCH 45/58] arm: Rewrite ffi_closure Move the push of the argument registers into ffi_closure_SYSV, reducing the size of the trampoline. --- Makefile.am | 7 +- src/arm/ffi.c | 222 +-- src/arm/ffitarget.h | 2 +- src/arm/gentramp.sh | 118 -- src/arm/sysv.S | 221 +-- src/arm/trampoline.S | 4450 ------------------------------------------ 6 files changed, 187 insertions(+), 4833 deletions(-) delete mode 100755 src/arm/gentramp.sh delete mode 100644 src/arm/trampoline.S diff --git a/Makefile.am b/Makefile.am index 97a7bd3..95c082a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,9 +48,9 @@ EXTRA_DIST = LICENSE ChangeLog.v1 ChangeLog.libgcj \ src/xtensa/ffitarget.h src/xtensa/ffi.c src/xtensa/sysv.S \ ChangeLog.libffi m4/libtool.m4 m4/lt~obsolete.m4 \ m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 \ - m4/ltversion.m4 src/arm/gentramp.sh src/debug.c msvcc.sh \ + m4/ltversion.m4 src/debug.c msvcc.sh \ generate-darwin-source-and-headers.py \ - libffi.xcodeproj/project.pbxproj src/arm/trampoline.S \ + libffi.xcodeproj/project.pbxproj \ libtool-ldflags ChangeLog.libffi-3.1 info_TEXINFOS = doc/libffi.texi @@ -190,9 +190,6 @@ nodist_libffi_la_SOURCES += src/arc/arcompact.S src/arc/ffi.c endif if ARM nodist_libffi_la_SOURCES += src/arm/sysv.S src/arm/ffi.c -if FFI_EXEC_TRAMPOLINE_TABLE -nodist_libffi_la_SOURCES += src/arm/trampoline.S -endif endif if AVR32 nodist_libffi_la_SOURCES += src/avr32/sysv.S src/avr32/ffi.c diff --git a/src/arm/ffi.c b/src/arm/ffi.c index c91b869..395af97 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -364,157 +364,87 @@ ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) ffi_prep_args_SYSV (cif, flags, new_rvalue, avalue, stack); ffi_call_SYSV (stack, frame, fn); } - + if (rvalue && rvalue != new_rvalue) memcpy (rvalue, new_rvalue, rtype->size); } -/** private members **/ - -static void ffi_prep_incoming_args_SYSV (char *stack, void **ret, - void **args, ffi_cif *cif, - float *vfp_stack); - -static void ffi_prep_incoming_args_VFP (char *stack, void **ret, - void **args, ffi_cif *cif, - float *vfp_stack); - -void ffi_closure_SYSV (ffi_closure *); - -void ffi_closure_VFP (ffi_closure *); - -/* This function is jumped to by the trampoline */ - -unsigned int FFI_HIDDEN -ffi_closure_inner (ffi_closure *closure, - void **respp, void *args, void *vfp_args) +static void * +ffi_prep_incoming_args_SYSV (ffi_cif *cif, void *rvalue, + char *argp, void **avalue) { - // our various things... - ffi_cif *cif; - void **arg_area; + ffi_type **arg_types = cif->arg_types; + int i, n; - cif = closure->cif; - arg_area = (void **) alloca (cif->nargs * sizeof (void *)); - - /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding - * value on the stack; and if the function returns - * a structure, it will re-set RESP to point to the - * structure return address. */ - if (cif->abi == FFI_VFP) - ffi_prep_incoming_args_VFP (args, respp, arg_area, cif, vfp_args); - else - ffi_prep_incoming_args_SYSV (args, respp, arg_area, cif, vfp_args); - - (closure->fun) (cif, *respp, arg_area, closure->user_data); - - return cif->flags; -} - -/*@-exportheader@*/ -static void -ffi_prep_incoming_args_SYSV (char *stack, void **rvalue, - void **avalue, ffi_cif *cif, - /* Used only under VFP hard-float ABI. */ - float *vfp_stack) -/*@=exportheader@*/ -{ - register unsigned int i; - register void **p_argv; - register char *argp; - register ffi_type **p_arg; - - argp = stack; - - if (cif->flags == FFI_TYPE_STRUCT) + if (cif->flags == ARM_TYPE_STRUCT) { - *rvalue = *(void **) argp; + rvalue = *(void **) argp; argp += 4; } - p_argv = avalue; - - for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) + for (i = 0, n = cif->nargs; i < n; i++) { - size_t z; + ffi_type *ty = arg_types[i]; + size_t z = ty->size; - argp = ffi_align (*p_arg, argp); - - z = (*p_arg)->size; - - /* because we're little endian, this is what it turns into. */ - - *p_argv = (void *) argp; - - p_argv++; + argp = ffi_align (ty, argp); + avalue[i] = (void *) argp; argp += z; } - return; + return rvalue; } -/*@-exportheader@*/ -static void -ffi_prep_incoming_args_VFP (char *stack, void **rvalue, - void **avalue, ffi_cif * cif, - /* Used only under VFP hard-float ABI. */ - float *vfp_stack) -/*@=exportheader@*/ +static void * +ffi_prep_incoming_args_VFP (ffi_cif *cif, void *rvalue, char *stack, + char *vfp_space, void **avalue) { - register unsigned int i, vi = 0; - register void **p_argv; - register char *argp, *regp, *eo_regp; - register ffi_type **p_arg; + ffi_type **arg_types = cif->arg_types; + int i, n, vi = 0; + char *argp, *regp, *eo_regp; char done_with_regs = 0; char stack_used = 0; - FFI_ASSERT (cif->abi == FFI_VFP); regp = stack; eo_regp = argp = regp + 16; - if (cif->flags == FFI_TYPE_STRUCT) + if (cif->flags == ARM_TYPE_STRUCT) { - *rvalue = *(void **) regp; + rvalue = *(void **) regp; regp += 4; } - p_argv = avalue; - - for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) + for (i = 0, n = cif->nargs; i < n; i++) { - int is_vfp_type = vfp_type_p (*p_arg); - size_t z; + ffi_type *ty = arg_types[i]; + int is_vfp_type = vfp_type_p (ty); + size_t z = ty->size; if (vi < cif->vfp_nargs && is_vfp_type) { - *p_argv++ = (void *) (vfp_stack + cif->vfp_args[vi++]); + avalue[i] = vfp_space + cif->vfp_args[vi++] * 4; continue; } else if (!done_with_regs && !is_vfp_type) { - char *tregp = ffi_align (*p_arg, regp); + char *tregp = ffi_align (ty, regp); - z = (*p_arg)->size; z = (z < 4) ? 4 : z; // pad - /* if the arguments either fits into the registers or uses registers - * and stack, while we haven't read other things from the stack */ + /* If the arguments either fits into the registers or uses registers + and stack, while we haven't read other things from the stack */ if (tregp + z <= eo_regp || !stack_used) { - /* because we're little endian, this is what it turns into. */ - *p_argv = (void *) tregp; - - p_argv++; + /* Because we're little endian, this is what it turns into. */ + avalue[i] = (void *) tregp; regp = tregp + z; - // if we read past the last core register, make sure we have not read - // from the stack before and continue reading after regp + + /* If we read past the last core register, make sure we + have not read from the stack before and continue + reading after regp. */ if (regp > eo_regp) { - if (stack_used) - { - abort (); // we should never read past the end of the register - // are if the stack is already in use - } + FFI_ASSERT (!stack_used); argp = regp; } if (regp >= eo_regp) @@ -525,26 +455,41 @@ ffi_prep_incoming_args_VFP (char *stack, void **rvalue, continue; } } + stack_used = 1; - - argp = ffi_align (*p_arg, argp); - - z = (*p_arg)->size; - - /* because we're little endian, this is what it turns into. */ - - *p_argv = (void *) argp; - - p_argv++; + argp = ffi_align (ty, argp); + avalue[i] = (void *) argp; argp += z; } - return; + return rvalue; } -/* How to make a trampoline. */ +int FFI_HIDDEN +ffi_closure_inner_SYSV (ffi_closure *closure, void *rvalue, char *argp) +{ + ffi_cif *cif = closure->cif; + void **avalue = (void **) alloca (cif->nargs * sizeof (void *)); -extern unsigned int ffi_arm_trampoline[3]; + rvalue = ffi_prep_incoming_args_SYSV (cif, rvalue, argp, avalue); + closure->fun (cif, rvalue, avalue, closure->user_data); + return cif->flags; +} + +int FFI_HIDDEN +ffi_closure_inner_VFP (ffi_closure *closure, void *rvalue, + char *argp, char *vfp_space) +{ + ffi_cif *cif = closure->cif; + void **avalue = (void **) alloca (cif->nargs * sizeof (void *)); + + rvalue = ffi_prep_incoming_args_VFP (cif, rvalue, argp, vfp_space, avalue); + closure->fun (cif, rvalue, avalue, closure->user_data); + return cif->flags; +} + +void ffi_closure_SYSV (void) FFI_HIDDEN; +void ffi_closure_VFP (void) FFI_HIDDEN; #if FFI_EXEC_TRAMPOLINE_TABLE @@ -788,19 +733,7 @@ ffi_closure_free (void *ptr) #else -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX) \ -({ unsigned char *__tramp = (unsigned char*)(TRAMP); \ - unsigned int __fun = (unsigned int)(FUN); \ - unsigned int __ctx = (unsigned int)(CTX); \ - unsigned char *insns = (unsigned char *)(CTX); \ - memcpy (__tramp, ffi_arm_trampoline, sizeof ffi_arm_trampoline); \ - *(unsigned int*) &__tramp[12] = __ctx; \ - *(unsigned int*) &__tramp[16] = __fun; \ - __clear_cache((&__tramp[0]), (&__tramp[19])); /* Clear data mapping. */ \ - __clear_cache(insns, insns + 3 * sizeof (unsigned int)); \ - /* Clear instruction \ - mapping. */ \ - }) +extern unsigned int ffi_arm_trampoline[2] FFI_HIDDEN; #endif @@ -812,15 +745,15 @@ ffi_prep_closure_loc (ffi_closure * closure, void (*fun) (ffi_cif *, void *, void **, void *), void *user_data, void *codeloc) { - void (*closure_func) (ffi_closure *) = NULL; + void (*closure_func) (void) = ffi_closure_SYSV; - if (cif->abi == FFI_SYSV) - closure_func = &ffi_closure_SYSV; -#ifdef __ARM_EABI__ - else if (cif->abi == FFI_VFP) - closure_func = &ffi_closure_VFP; -#endif - else + if (cif->abi == FFI_VFP) + { + /* We only need take the vfp path if there are vfp arguments. */ + if (cif->vfp_used) + closure_func = ffi_closure_VFP; + } + else if (cif->abi != FFI_SYSV) return FFI_BAD_ABI; #if FFI_EXEC_TRAMPOLINE_TABLE @@ -828,12 +761,15 @@ ffi_prep_closure_loc (ffi_closure * closure, config[0] = closure; config[1] = closure_func; #else - FFI_INIT_TRAMPOLINE (&closure->tramp[0], closure_func, codeloc); + memcpy (closure->tramp, ffi_arm_trampoline, 8); + __clear_cache(closure->tramp, closure->tramp + 8); /* clear data map */ + __clear_cache(codeloc, codeloc + 8); /* clear insn map */ + *(void (**)(void))(closure->tramp + 8) = closure_func; #endif closure->cif = cif; - closure->user_data = user_data; closure->fun = fun; + closure->user_data = user_data; return FFI_OK; } diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index 6355904..421bacf 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -65,7 +65,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 -#define FFI_TRAMPOLINE_SIZE 20 +#define FFI_TRAMPOLINE_SIZE 12 #define FFI_NATIVE_RAW_API 0 #endif diff --git a/src/arm/gentramp.sh b/src/arm/gentramp.sh deleted file mode 100755 index 05c43a3..0000000 --- a/src/arm/gentramp.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/sh - -# ----------------------------------------------------------------------- -# gentramp.sh - Copyright (c) 2010, Plausible Labs Cooperative, Inc. -# -# ARM Trampoline Page Generator -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# ``Software''), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - -PROGNAME=$0 - -# Each trampoline is exactly 3 instructions, or 12 bytes. If any of these values change, -# the entire arm trampoline implementation must be updated to match, too. - -# Size of an individual trampoline, in bytes -TRAMPOLINE_SIZE=12 - -# Page size, in bytes -PAGE_SIZE=4096 - -# Compute the size of the reachable config page; The first 16 bytes of the config page -# are unreachable due to our maximum pc-relative ldr offset. -PAGE_AVAIL=`expr $PAGE_SIZE - 16` - -# Compute the number of of available trampolines. -TRAMPOLINE_COUNT=`expr $PAGE_AVAIL / $TRAMPOLINE_SIZE` - -header () { - echo "# GENERATED CODE - DO NOT EDIT" - echo "# This file was generated by $PROGNAME" - echo "" - - # Write out the license header -cat << EOF -# Copyright (c) 2010, Plausible Labs Cooperative, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# ``Software''), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - -EOF - - # Write out the trampoline table, aligned to the page boundary - echo ".text" - echo ".align 12" - echo ".globl _ffi_closure_trampoline_table_page" - echo "_ffi_closure_trampoline_table_page:" -} - - -# WARNING - Don't modify the trampoline code size without also updating the relevant libffi code -trampoline () { - cat << END - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - -END -} - -main () { - # Write out the header - header - - # Write out the trampolines - local i=0 - while [ $i -lt ${TRAMPOLINE_COUNT} ]; do - trampoline - local i=`expr $i + 1` - done -} - -main diff --git a/src/arm/sysv.S b/src/arm/sysv.S index b967d97..ce5450d 100644 --- a/src/arm/sysv.S +++ b/src/arm/sysv.S @@ -189,139 +189,128 @@ ARM_FUNC_END(ffi_call_SYSV) ARM_FUNC_START(ffi_closure_SYSV, 1) UNWIND .fnstart - UNWIND .pad #16 - add ip, sp, #16 - stmfd sp!, {ip, lr} - UNWIND .save {r0, lr} - add r2, sp, #8 - UNWIND .pad #16 - sub sp, sp, #16 - str sp, [sp, #8] - add r1, sp, #8 - bl CNAME(ffi_closure_inner) - cmp r0, #FFI_TYPE_INT - beq .Lretint + cfi_startproc + stmdb sp!, {r0-r3} @ save argument regs + cfi_adjust_cfa_offset(16) + mov r0, ip @ load closure + add ip, sp, #16 @ compute entry sp + sub sp, sp, #32 @ allocate rvalue space + stmdb sp!, {sp,lr} - cmp r0, #FFI_TYPE_FLOAT -#if defined(__SOFTFP__) || defined(__ARM_EABI__) - beq .Lretint -#else - beq .Lretfloat -#endif + /* Remember that EABI unwind info only applies at call sites. + We need do nothing except note the save of the stack pointer + and the link registers. */ + UNWIND .save {sp,lr} + cfi_adjust_cfa_offset(8) + cfi_rel_offset(lr, 4) - cmp r0, #FFI_TYPE_DOUBLE -#if defined(__SOFTFP__) || defined(__ARM_EABI__) - beq .Lretlonglong -#else - beq .Lretdouble -#endif + add r1, sp, #8 @ load respp + add r2, sp, #8+32 @ load args + mov r3, #0 @ load vfp_args - cmp r0, #FFI_TYPE_LONGDOUBLE -#if defined(__SOFTFP__) || defined(__ARM_EABI__) - beq .Lretlonglong -#else - beq .Lretlongdouble -#endif + bl CNAME(ffi_closure_inner_SYSV) - cmp r0, #FFI_TYPE_SINT64 - beq .Lretlonglong -.Lclosure_epilogue: - add sp, sp, #16 - ldmfd sp, {sp, pc} -.Lretint: - ldr r0, [sp] - b .Lclosure_epilogue -.Lretlonglong: - ldr r0, [sp] - ldr r1, [sp, #4] - b .Lclosure_epilogue - -#if !defined(__SOFTFP__) && !defined(__ARM_EABI__) -.Lretfloat: - ldfs f0, [sp] - b .Lclosure_epilogue -.Lretdouble: - ldfd f0, [sp] - b .Lclosure_epilogue -.Lretlongdouble: - ldfd f0, [sp] - b .Lclosure_epilogue -#endif - UNWIND .fnend + @ Load values returned in registers. + add r2, sp, #8 @ load respp + adr r3, CNAME(ffi_closure_ret) + add pc, r3, r0, lsl #3 + cfi_endproc + UNWIND .fnend ARM_FUNC_END(ffi_closure_SYSV) - -/* Below are VFP hard-float ABI call and closure implementations. - Add VFP FPU directive here. This is only compiled into the library - under EABI. */ -#ifdef __ARM_EABI__ ARM_FUNC_START(ffi_closure_VFP, 1) UNWIND .fnstart - fstmfdd sp!, {d0-d7} - @ r0-r3, then d0-d7 - UNWIND .pad #80 - add ip, sp, #80 - stmfd sp!, {ip, lr} - UNWIND .save {r0, lr} - add r2, sp, #72 - add r3, sp, #8 - UNWIND .pad #72 - sub sp, sp, #72 - str sp, [sp, #64] - add r1, sp, #64 - bl CNAME(ffi_closure_inner) + cfi_startproc + stmdb sp!, {r0-r3} @ save argument regs + cfi_adjust_cfa_offset(16) + sub sp, sp, #64+32 @ allocate vfp+rvalue space + cfi_adjust_cfa_offset(64+32) + stc p11, cr0, [sp], {16} @ vstm sp, {d0-d7} + mov r0, ip @ load closure + add ip, sp, #16+64+32 @ compute entry sp + stmdb sp!, {ip,lr} - cmp r0, #FFI_TYPE_INT - beq .Lretint_vfp + /* See above. */ + UNWIND .save {sp,lr} + cfi_adjust_cfa_offset(8) + cfi_rel_offset(sp, 0) + cfi_rel_offset(lr, 4) - cmp r0, #FFI_TYPE_FLOAT - beq .Lretfloat_vfp + add r1, sp, #8+64 @ load respp + add r2, sp, #8+64+32 @ load args + add r3, sp, #8 @ load vfp_args - cmp r0, #FFI_TYPE_DOUBLE - cmpne r0, #FFI_TYPE_LONGDOUBLE - beq .Lretdouble_vfp + bl CNAME(ffi_closure_inner_VFP) - cmp r0, #FFI_TYPE_SINT64 - beq .Lretlonglong_vfp - - cmp r0, #FFI_TYPE_STRUCT_VFP_FLOAT - beq .Lretfloat_struct_vfp - - cmp r0, #FFI_TYPE_STRUCT_VFP_DOUBLE - beq .Lretdouble_struct_vfp - -.Lclosure_epilogue_vfp: - add sp, sp, #72 - ldmfd sp, {sp, pc} - -.Lretfloat_vfp: - flds s0, [sp] - b .Lclosure_epilogue_vfp -.Lretdouble_vfp: - fldd d0, [sp] - b .Lclosure_epilogue_vfp -.Lretint_vfp: - ldr r0, [sp] - b .Lclosure_epilogue_vfp -.Lretlonglong_vfp: - ldmia sp, {r0, r1} - b .Lclosure_epilogue_vfp -.Lretfloat_struct_vfp: - fldmiad sp, {d0-d1} - b .Lclosure_epilogue_vfp -.Lretdouble_struct_vfp: - fldmiad sp, {d0-d3} - b .Lclosure_epilogue_vfp - UNWIND .fnend + @ Load values returned in registers. + add r2, sp, #8+64 @ load respp + adr r3, CNAME(ffi_closure_ret) + add pc, r3, r0, lsl #3 + cfi_endproc + UNWIND .fnend ARM_FUNC_END(ffi_closure_VFP) -#endif + +/* Load values returned in registers for both closure entry points. + Note that we use LDM with SP in the register set. This is deprecated + by ARM, but not yet unpredictable. */ + +ARM_FUNC_START(ffi_closure_ret, 0) + cfi_startproc + cfi_rel_offset(sp, 0) + cfi_rel_offset(lr, 4) +0: +E ARM_TYPE_VFP_S + ldc p10, cr0, [r2] @ vldr s0, [r2] + ldm sp, {sp,pc} +E ARM_TYPE_VFP_D + ldc p11, cr0, [r2] @ vldr d0, [r2] + ldm sp, {sp,pc} +E ARM_TYPE_VFP_N + ldc p11, cr0, [r2], {8} @ vldm r2, {d0-d3} + ldm sp, {sp,pc} +E ARM_TYPE_INT64 + ldr r1, [r2, #4] + nop +E ARM_TYPE_INT + ldr r0, [r2] + ldm sp, {sp,pc} +E ARM_TYPE_VOID + ldm sp, {sp,pc} + nop +E ARM_TYPE_STRUCT + ldm sp, {sp,pc} + cfi_endproc +ARM_FUNC_END(ffi_closure_ret) + +#if FFI_EXEC_TRAMPOLINE_TABLE + +/* ??? The iOS support should be updated. The first insn used to + be STMFD, but that's been moved into ffi_closure_SYSV. If the + writable page is put after this one we can make use of the + pc+8 feature of the architecture. We can also reduce the size + of the thunk to 8 and pack more of these into the page. + + In the meantime, simply replace the STMFD with a NOP so as to + keep all the magic numbers the same within ffi.c. */ + + .align 12 +ARM_FUNC_START(ffi_closure_trampoline_table_page) +.rept 4096 / 12 + nop + ldr ip, [pc, #-4092] + ldr pc, [pc, #-4092] +.endr + +#else ARM_FUNC_START(ffi_arm_trampoline, 1) - stmfd sp!, {r0-r3} - ldr r0, [pc] - ldr pc, [pc] +0: adr ip, 0b + ldr pc, 1f +1: .long 0 ARM_FUNC_END(ffi_arm_trampoline) +#endif /* FFI_EXEC_TRAMPOLINE_TABLE */ + #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",%progbits #endif diff --git a/src/arm/trampoline.S b/src/arm/trampoline.S deleted file mode 100644 index 935e8de..0000000 --- a/src/arm/trampoline.S +++ /dev/null @@ -1,4450 +0,0 @@ -# GENERATED CODE - DO NOT EDIT -# This file was generated by src/arm/gentramp.sh - -# Copyright (c) 2010, Plausible Labs Cooperative, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# Software''), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED AS IS'', WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -# ----------------------------------------------------------------------- - -.text -.align 12 -.globl _ffi_closure_trampoline_table_page -_ffi_closure_trampoline_table_page: - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - - - // trampoline - // Save to stack - stmfd sp!, {r0-r3} - - // Load the context argument from the config page. - // This places the first usable config value at _ffi_closure_trampoline_table-4080 - // This accounts for the above 4-byte stmfd instruction, plus 8 bytes constant when loading from pc. - ldr r0, [pc, #-4092] - - // Load the jump address from the config page. - ldr pc, [pc, #-4092] - From 5e88ebe62717b8e1e2f8d96c10f78a7134caa765 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 20 Oct 2014 15:10:43 -0400 Subject: [PATCH 46/58] arm: Remove internal FFI_TYPE constants These have been replaced by the contents of internal.h. --- src/arm/ffitarget.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index 421bacf..bf36750 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -56,10 +56,6 @@ typedef enum ffi_abi { unsigned short vfp_reg_free, vfp_nargs; \ signed char vfp_args[16] \ -/* Internally used. */ -#define FFI_TYPE_STRUCT_VFP_FLOAT (FFI_TYPE_LAST + 1) -#define FFI_TYPE_STRUCT_VFP_DOUBLE (FFI_TYPE_LAST + 2) - #define FFI_TARGET_SPECIFIC_VARIADIC /* ---- Definitions for closures ----------------------------------------- */ From a529bec27b64ab846ab2643ecbeb22e01f35c7dc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 11:26:59 -0400 Subject: [PATCH 47/58] arm: Add support for complex types --- src/arm/ffi.c | 28 +++++++++++++++++++++------- src/arm/ffitarget.h | 1 + testsuite/libffi.call/call.exp | 1 + 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 395af97..9a11e8f 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -87,9 +87,12 @@ ffi_put_arg (ffi_type *ty, void *src, void *dst) break; case FFI_TYPE_STRUCT: - default: + case FFI_TYPE_COMPLEX: memcpy (dst, src, z); break; + + default: + abort(); } return ALIGN (z, 4); @@ -249,6 +252,7 @@ ffi_prep_cif_machdep (ffi_cif *cif) break; case FFI_TYPE_STRUCT: + case FFI_TYPE_COMPLEX: if (cabi == FFI_VFP) { int h = vfp_type_p (cif->rtype); @@ -790,7 +794,7 @@ is_hfa0 (const ffi_type *ty) for (i = 0; elements[i]; ++i) { ret = elements[i]->type; - if (ret == FFI_TYPE_STRUCT) + if (ret == FFI_TYPE_STRUCT || ret == FFI_TYPE_COMPLEX) { ret = is_hfa0 (elements[i]); if (ret < 0) @@ -815,7 +819,7 @@ is_hfa1 (const ffi_type *ty, int candidate) for (i = 0; elements[i]; ++i) { int t = elements[i]->type; - if (t == FFI_TYPE_STRUCT) + if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX) { if (!is_hfa1 (elements[i], candidate)) return 0; @@ -842,13 +846,21 @@ vfp_type_p (const ffi_type *ty) size_t size, ele_count; /* Quickest tests first. */ + candidate = ty->type; switch (ty->type) { default: return 0; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: - return 0x100 + ty->type; + ele_count = 1; + goto done; + case FFI_TYPE_COMPLEX: + candidate = ty->elements[0]->type; + if (candidate != FFI_TYPE_FLOAT && candidate != FFI_TYPE_DOUBLE) + return 0; + ele_count = 2; + goto done; case FFI_TYPE_STRUCT: break; } @@ -861,7 +873,7 @@ vfp_type_p (const ffi_type *ty) /* Find the type of the first non-structure member. */ elements = ty->elements; candidate = elements[0]->type; - if (candidate == FFI_TYPE_STRUCT) + if (candidate == FFI_TYPE_STRUCT || candidate == FFI_TYPE_COMPLEX) { for (i = 0; ; ++i) { @@ -894,16 +906,18 @@ vfp_type_p (const ffi_type *ty) /* Finally, make sure that all scalar elements are the same type. */ for (i = 0; elements[i]; ++i) { - if (elements[i]->type == FFI_TYPE_STRUCT) + int t = elements[i]->type; + if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX) { if (!is_hfa1 (elements[i], candidate)) return 0; } - else if (elements[i]->type != candidate) + else if (t != candidate) return 0; } /* All tests succeeded. Encode the result. */ + done: return (ele_count << 8) | candidate; } diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index bf36750..744a1e1 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -57,6 +57,7 @@ typedef enum ffi_abi { signed char vfp_args[16] \ #define FFI_TARGET_SPECIFIC_VARIADIC +#define FFI_TARGET_HAS_COMPLEX_TYPE /* ---- Definitions for closures ----------------------------------------- */ diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index 82e9fe8..7a5c44f 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -28,6 +28,7 @@ run-many-tests $tlist "" # for FFI_TARGET_HAS_COMPLEX_TYPE. if { [istarget aarch64*] || [istarget alpha*] + || [istarget arm*] || [istarget i?86*] || [istarget s390*] || [istarget x86_64*] } { From 6fa617dabc3d99e19d47ac9bc4127c56e3163008 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 21 Oct 2014 11:27:11 -0400 Subject: [PATCH 48/58] arm: Add argument space for the hidden struct return pointer This should have been failing all along, but it's only exposed by the complex_int test case. --- src/arm/ffi.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 9a11e8f..eabab47 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -202,17 +202,7 @@ ffi_status ffi_prep_cif_machdep (ffi_cif *cif) { int flags = 0, cabi = cif->abi; - size_t bytes; - - /* Round the stack up to a multiple of 8 bytes. This isn't needed - everywhere, but it is on some platforms, and it doesn't harm anything - when it isn't needed. */ - bytes = ALIGN (cif->bytes, 8); - - /* Minimum stack space is the 4 register arguments that we pop. */ - if (bytes < 4*4) - bytes = 4*4; - cif->bytes = bytes; + size_t bytes = cif->bytes; /* Map out the register placements of VFP register args. The VFP hard-float calling conventions are slightly more sophisticated @@ -270,12 +260,29 @@ ffi_prep_cif_machdep (ffi_cif *cif) A Composite Type larger than 4 bytes, or whose size cannot be determined statically ... is stored in memory at an address passed [in r0]. */ - flags = (cif->rtype->size <= 4 ? ARM_TYPE_INT : ARM_TYPE_STRUCT); + if (cif->rtype->size <= 4) + flags = ARM_TYPE_INT; + else + { + flags = ARM_TYPE_STRUCT; + bytes += 4; + } break; default: abort(); } + + /* Round the stack up to a multiple of 8 bytes. This isn't needed + everywhere, but it is on some platforms, and it doesn't harm anything + when it isn't needed. */ + bytes = ALIGN (bytes, 8); + + /* Minimum stack space is the 4 register arguments that we pop. */ + if (bytes < 4*4) + bytes = 4*4; + + cif->bytes = bytes; cif->flags = flags; return FFI_OK; From ab83cbb90b280195e636976098d3674f5ffc1d0a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 29 Oct 2014 14:38:42 -0400 Subject: [PATCH 49/58] arm: Add support for Go closures --- src/arm/ffi.c | 78 +++++++++++++++++++++++++++++++++++++-------- src/arm/ffitarget.h | 1 + src/arm/sysv.S | 63 +++++++++++++++++++++++------------- 3 files changed, 106 insertions(+), 36 deletions(-) diff --git a/src/arm/ffi.c b/src/arm/ffi.c index eabab47..9c8732d 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -308,6 +308,7 @@ struct call_frame void *lr; void *rvalue; int flags; + void *closure; }; extern void ffi_call_SYSV (void *stack, struct call_frame *, @@ -315,8 +316,9 @@ extern void ffi_call_SYSV (void *stack, struct call_frame *, extern void ffi_call_VFP (void *vfp_space, struct call_frame *, void (*fn) (void), unsigned vfp_used) FFI_HIDDEN; -void -ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) +static void +ffi_call_int (ffi_cif * cif, void (*fn) (void), void *rvalue, + void **avalue, void *closure) { int flags = cif->flags; ffi_type *rtype = cif->rtype; @@ -364,6 +366,7 @@ ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) frame->rvalue = new_rvalue; frame->flags = flags; + frame->closure = closure; if (vfp_space) { @@ -380,6 +383,19 @@ ffi_call (ffi_cif * cif, void (*fn) (void), void *rvalue, void **avalue) memcpy (rvalue, new_rvalue, rtype->size); } +void +ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); +} + static void * ffi_prep_incoming_args_SYSV (ffi_cif *cif, void *rvalue, char *argp, void **avalue) @@ -476,31 +492,43 @@ ffi_prep_incoming_args_VFP (ffi_cif *cif, void *rvalue, char *stack, return rvalue; } -int FFI_HIDDEN -ffi_closure_inner_SYSV (ffi_closure *closure, void *rvalue, char *argp) +struct closure_frame { - ffi_cif *cif = closure->cif; - void **avalue = (void **) alloca (cif->nargs * sizeof (void *)); + char vfp_space[8*8] __attribute__((aligned(8))); + char result[8*4]; + char argp[]; +}; - rvalue = ffi_prep_incoming_args_SYSV (cif, rvalue, argp, avalue); - closure->fun (cif, rvalue, avalue, closure->user_data); +int FFI_HIDDEN +ffi_closure_inner_SYSV (ffi_cif *cif, + void (*fun) (ffi_cif *, void *, void **, void *), + void *user_data, + struct closure_frame *frame) +{ + void **avalue = (void **) alloca (cif->nargs * sizeof (void *)); + void *rvalue = ffi_prep_incoming_args_SYSV (cif, frame->result, + frame->argp, avalue); + fun (cif, rvalue, avalue, user_data); return cif->flags; } int FFI_HIDDEN -ffi_closure_inner_VFP (ffi_closure *closure, void *rvalue, - char *argp, char *vfp_space) +ffi_closure_inner_VFP (ffi_cif *cif, + void (*fun) (ffi_cif *, void *, void **, void *), + void *user_data, + struct closure_frame *frame) { - ffi_cif *cif = closure->cif; void **avalue = (void **) alloca (cif->nargs * sizeof (void *)); - - rvalue = ffi_prep_incoming_args_VFP (cif, rvalue, argp, vfp_space, avalue); - closure->fun (cif, rvalue, avalue, closure->user_data); + void *rvalue = ffi_prep_incoming_args_VFP (cif, frame->result, frame->argp, + frame->vfp_space, avalue); + fun (cif, rvalue, avalue, user_data); return cif->flags; } void ffi_closure_SYSV (void) FFI_HIDDEN; void ffi_closure_VFP (void) FFI_HIDDEN; +void ffi_go_closure_SYSV (void) FFI_HIDDEN; +void ffi_go_closure_VFP (void) FFI_HIDDEN; #if FFI_EXEC_TRAMPOLINE_TABLE @@ -785,6 +813,28 @@ ffi_prep_closure_loc (ffi_closure * closure, return FFI_OK; } +ffi_status +ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif, + void (*fun) (ffi_cif *, void *, void **, void *)) +{ + void (*closure_func) (void) = ffi_go_closure_SYSV; + + if (cif->abi == FFI_VFP) + { + /* We only need take the vfp path if there are vfp arguments. */ + if (cif->vfp_used) + closure_func = ffi_go_closure_VFP; + } + else if (cif->abi != FFI_SYSV) + return FFI_BAD_ABI; + + closure->tramp = closure_func; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + /* Below are routines for VFP hard-float support. */ /* A subroutine of vfp_type_p. Given a structure type, return the type code diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index 744a1e1..4f473f9 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -62,6 +62,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 +#define FFI_GO_CLOSURES 1 #define FFI_TRAMPOLINE_SIZE 12 #define FFI_NATIVE_RAW_API 0 diff --git a/src/arm/sysv.S b/src/arm/sysv.S index ce5450d..fd16589 100644 --- a/src/arm/sysv.S +++ b/src/arm/sysv.S @@ -136,6 +136,7 @@ ARM_FUNC_START(ffi_call_SYSV, 1) mov sp, r0 @ install the stack pointer mov lr, r2 @ move the fn pointer out of the way + ldr ip, [fp, #16] @ install the static chain ldmia sp!, {r0-r3} @ move first 4 parameters in registers. blx lr @ call fn @@ -180,22 +181,33 @@ ARM_FUNC_END(ffi_call_SYSV) /* - unsigned int FFI_HIDDEN - ffi_closure_inner (closure, respp, args) - ffi_closure *closure; - void **respp; - void *args; + int ffi_closure_inner_* (cif, fun, user_data, frame) */ +ARM_FUNC_START(ffi_go_closure_SYSV, 1) + cfi_startproc + stmdb sp!, {r0-r3} @ save argument regs + cfi_adjust_cfa_offset(16) + ldr r0, [ip, #4] @ load cif + ldr r1, [ip, #8] @ load fun + mov r2, ip @ load user_data + b 0f + cfi_endproc +ARM_FUNC_END(ffi_go_closure_SYSV) + ARM_FUNC_START(ffi_closure_SYSV, 1) UNWIND .fnstart cfi_startproc stmdb sp!, {r0-r3} @ save argument regs cfi_adjust_cfa_offset(16) - mov r0, ip @ load closure + ldr r0, [ip, #FFI_TRAMPOLINE_SIZE] @ load cif + ldr r1, [ip, #FFI_TRAMPOLINE_SIZE+4] @ load fun + ldr r2, [ip, #FFI_TRAMPOLINE_SIZE+8] @ load user_data +0: add ip, sp, #16 @ compute entry sp - sub sp, sp, #32 @ allocate rvalue space - stmdb sp!, {sp,lr} + sub sp, sp, #64+32 @ allocate frame + cfi_adjust_cfa_offset(64+32) + stmdb sp!, {ip,lr} /* Remember that EABI unwind info only applies at call sites. We need do nothing except note the save of the stack pointer @@ -204,46 +216,53 @@ ARM_FUNC_START(ffi_closure_SYSV, 1) cfi_adjust_cfa_offset(8) cfi_rel_offset(lr, 4) - add r1, sp, #8 @ load respp - add r2, sp, #8+32 @ load args - mov r3, #0 @ load vfp_args - + add r3, sp, #8 @ load frame bl CNAME(ffi_closure_inner_SYSV) @ Load values returned in registers. - add r2, sp, #8 @ load respp + add r2, sp, #8+64 @ load result adr r3, CNAME(ffi_closure_ret) add pc, r3, r0, lsl #3 cfi_endproc UNWIND .fnend ARM_FUNC_END(ffi_closure_SYSV) +ARM_FUNC_START(ffi_go_closure_VFP, 1) + cfi_startproc + stmdb sp!, {r0-r3} @ save argument regs + cfi_adjust_cfa_offset(16) + ldr r0, [ip, #4] @ load cif + ldr r1, [ip, #8] @ load fun + mov r2, ip @ load user_data + b 0f + cfi_endproc +ARM_FUNC_END(ffi_go_closure_VFP) + ARM_FUNC_START(ffi_closure_VFP, 1) UNWIND .fnstart cfi_startproc stmdb sp!, {r0-r3} @ save argument regs cfi_adjust_cfa_offset(16) - sub sp, sp, #64+32 @ allocate vfp+rvalue space + ldr r0, [ip, #FFI_TRAMPOLINE_SIZE] @ load cif + ldr r1, [ip, #FFI_TRAMPOLINE_SIZE+4] @ load fun + ldr r2, [ip, #FFI_TRAMPOLINE_SIZE+8] @ load user_data +0: + add ip, sp, #16 + sub sp, sp, #64+32 @ allocate frame cfi_adjust_cfa_offset(64+32) stc p11, cr0, [sp], {16} @ vstm sp, {d0-d7} - mov r0, ip @ load closure - add ip, sp, #16+64+32 @ compute entry sp stmdb sp!, {ip,lr} /* See above. */ UNWIND .save {sp,lr} cfi_adjust_cfa_offset(8) - cfi_rel_offset(sp, 0) cfi_rel_offset(lr, 4) - add r1, sp, #8+64 @ load respp - add r2, sp, #8+64+32 @ load args - add r3, sp, #8 @ load vfp_args - + add r3, sp, #8 @ load frame bl CNAME(ffi_closure_inner_VFP) @ Load values returned in registers. - add r2, sp, #8+64 @ load respp + add r2, sp, #8+64 @ load result adr r3, CNAME(ffi_closure_ret) add pc, r3, r0, lsl #3 cfi_endproc From fd4f7cc0cabae66a91e40f8645791740b9416bc1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 24 Oct 2014 11:11:53 -0700 Subject: [PATCH 50/58] sparc: Eliminate long double ifdefs --- src/sparc/ffi.c | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index 9f0fded..c2a0d14 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -27,9 +27,19 @@ #include #include - #include +/* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; + all further uses in this file will refer to the 128-bit type. */ +#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE +# if FFI_TYPE_LONGDOUBLE != 4 +# error FFI_TYPE_LONGDOUBLE out of date +# endif +#else +# undef FFI_TYPE_LONGDOUBLE +# define FFI_TYPE_LONGDOUBLE 4 +#endif + /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments */ @@ -72,10 +82,7 @@ void ffi_prep_args_v8(char *stack, extended_cif *ecif) size_t z; if ((*p_arg)->type == FFI_TYPE_STRUCT -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - || (*p_arg)->type == FFI_TYPE_LONGDOUBLE -#endif - ) + || (*p_arg)->type == FFI_TYPE_LONGDOUBLE) { *(unsigned int *) argp = (unsigned long)(* p_argv); z = sizeof(int); @@ -176,9 +183,7 @@ int ffi_prep_args_v9(char *stack, extended_cif *ecif) /* FALLTHROUGH */ case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE case FFI_TYPE_LONGDOUBLE: -#endif ret = 1; /* We should promote into FP regs as well as integer. */ break; } @@ -296,9 +301,7 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_VOID: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE case FFI_TYPE_LONGDOUBLE: -#endif cif->flags = cif->rtype->type; break; @@ -351,9 +354,7 @@ int ffi_v9_layout_struct(ffi_type *arg, int off, char *ret, char *intg, char *fl break; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE case FFI_TYPE_LONGDOUBLE: -#endif memmove(ret + off, flt + off, (*ptr)->size); off += (*ptr)->size; break; @@ -412,10 +413,7 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) FFI_ASSERT(0); #else if (rvalue && (cif->rtype->type == FFI_TYPE_STRUCT -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - || cif->flags == FFI_TYPE_LONGDOUBLE -#endif - )) + || cif->flags == FFI_TYPE_LONGDOUBLE)) { /* For v8, we need an "unimp" with size of returning struct */ /* behind "call", so we alloc some executable space for it. */ @@ -551,11 +549,7 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, /* Copy the caller's structure return address so that the closure returns the data directly to the caller. */ - if (cif->flags == FFI_TYPE_STRUCT -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - || cif->flags == FFI_TYPE_LONGDOUBLE -#endif - ) + if (cif->flags == FFI_TYPE_STRUCT || cif->flags == FFI_TYPE_LONGDOUBLE) rvalue = (void *) gpr[0]; /* Always skip the structure return address. */ @@ -565,10 +559,7 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, for (i = 0; i < cif->nargs; i++) { if (arg_types[i]->type == FFI_TYPE_STRUCT -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - || arg_types[i]->type == FFI_TYPE_LONGDOUBLE -#endif - ) + || arg_types[i]->type == FFI_TYPE_LONGDOUBLE) { /* Straight copy of invisible reference. */ avalue[i] = (void *)gpr[argn++]; @@ -656,17 +647,12 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, argn += ALIGN(arg_types[i]->size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; /* Align on a 16-byte boundary. */ -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE if (arg_types[i]->type == FFI_TYPE_LONGDOUBLE && (argn % 2) != 0) argn++; -#endif if (i < fp_slot_max && (arg_types[i]->type == FFI_TYPE_FLOAT || arg_types[i]->type == FFI_TYPE_DOUBLE -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - || arg_types[i]->type == FFI_TYPE_LONGDOUBLE -#endif - )) + || arg_types[i]->type == FFI_TYPE_LONGDOUBLE)) avalue[i] = ((char *) &fpr[argn]) - arg_types[i]->size; else avalue[i] = ((char *) &gpr[argn]) - arg_types[i]->size; From 5d7b54450582f31d0d5d3ecca7f330b70446f195 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 24 Oct 2014 11:45:53 -0700 Subject: [PATCH 51/58] sparc: Tidy up symbols Assembly to use local labels, .type annotation, hidden annotation. I do retain the _prefix for the symbols, but given that it wasn't done consistently across all symbols, I doubt it's actually needed. --- src/sparc/ffi.c | 34 ++++---- src/sparc/v8.S | 202 ++++++++++++++++-------------------------------- src/sparc/v9.S | 156 ++++++++++++------------------------- 3 files changed, 135 insertions(+), 257 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index c2a0d14..1f38f54 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -44,7 +44,8 @@ /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments */ -void ffi_prep_args_v8(char *stack, extended_cif *ecif) +void FFI_HIDDEN +ffi_prep_args_v8(char *stack, extended_cif *ecif) { int i; void **p_argv; @@ -127,7 +128,8 @@ void ffi_prep_args_v8(char *stack, extended_cif *ecif) return; } -int ffi_prep_args_v9(char *stack, extended_cif *ecif) +int FFI_HIDDEN +ffi_prep_args_v9(char *stack, extended_cif *ecif) { int i, ret = 0; int tmp; @@ -254,7 +256,8 @@ int ffi_prep_args_v9(char *stack, extended_cif *ecif) } /* Perform machine dependent cif processing */ -ffi_status ffi_prep_cif_machdep(ffi_cif *cif) +ffi_status FFI_HIDDEN +ffi_prep_cif_machdep(ffi_cif *cif) { int wordsize; @@ -337,7 +340,8 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) return FFI_OK; } -int ffi_v9_layout_struct(ffi_type *arg, int off, char *ret, char *intg, char *flt) +static int +ffi_v9_layout_struct(ffi_type *arg, int off, char *ret, char *intg, char *flt) { ffi_type **ptr = &arg->elements[0]; @@ -371,14 +375,14 @@ int ffi_v9_layout_struct(ffi_type *arg, int off, char *ret, char *intg, char *fl #ifdef SPARC64 extern int ffi_call_v9(void *, extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)(void)); + unsigned, unsigned *, void (*fn)(void)) FFI_HIDDEN; #else extern int ffi_call_v8(void *, extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)(void)); + unsigned, unsigned *, void (*fn)(void)) FFI_HIDDEN; #endif #ifndef __GNUC__ -void ffi_flush_icache (void *, size_t); +void ffi_flush_icache (void *, size_t) FFI_HIDDEN; #endif void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) @@ -478,9 +482,9 @@ void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) #ifdef SPARC64 -extern void ffi_closure_v9(void); +extern void ffi_closure_v9(void) FFI_HIDDEN; #else -extern void ffi_closure_v8(void); +extern void ffi_closure_v8(void) FFI_HIDDEN; #endif ffi_status @@ -534,9 +538,9 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } -int -ffi_closure_sparc_inner_v8(ffi_closure *closure, - void *rvalue, unsigned long *gpr, unsigned long *scratch) +int FFI_HIDDEN +ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, + unsigned long *gpr, unsigned long *scratch) { ffi_cif *cif; ffi_type **arg_types; @@ -592,9 +596,9 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, return cif->rtype->type; } -int -ffi_closure_sparc_inner_v9(ffi_closure *closure, - void *rvalue, unsigned long *gpr, double *fpr) +int FFI_HIDDEN +ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, + unsigned long *gpr, double *fpr) { ffi_cif *cif; ffi_type **arg_types; diff --git a/src/sparc/v8.S b/src/sparc/v8.S index 6bf7ac0..10c66ba 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -28,18 +28,31 @@ #define LIBFFI_ASM #include #include +#include #define STACKFRAME 96 /* Minimum stack framesize for SPARC */ #define ARGS (64+4) /* Offset of register area in frame */ -#ifndef __GNUC__ - .text - .align 8 -.globl ffi_flush_icache -.globl _ffi_flush_icache +#define C2(X, Y) X ## Y +#define C1(X, Y) C2(X, Y) -ffi_flush_icache: -_ffi_flush_icache: +#ifdef __USER_LABEL_PREFIX__ +# define C(Y) C1(__USER_LABEL_PREFIX__, Y) +#else +# define C(Y) Y +#endif +#define L(Y) C1(.L, Y) + + .text + +#ifndef __GNUC__ + .align 8 + .globl C(ffi_flush_icache) + .type C(ffi_flush_icache),@function + FFI_HIDDEN(C(ffi_flush_icache)) + +C(ffi_flush_icache): + cfi_startproc add %o0, %o1, %o2 #ifdef SPARC64 1: flush %o0 @@ -56,20 +69,20 @@ _ffi_flush_icache: nop retl nop -.ffi_flush_icache_end: - .size ffi_flush_icache,.ffi_flush_icache_end-ffi_flush_icache + cfi_endproc + .size C(ffi_flush_icache), . - C(ffi_flush_icache) #endif - .text .align 8 -.globl ffi_call_v8 -.globl _ffi_call_v8 + .globl C(ffi_call_v8) + .type C(ffi_call_v8),@function + FFI_HIDDEN(C(ffi_call_v8)) -ffi_call_v8: -_ffi_call_v8: -.LLFB1: +C(ffi_call_v8): + cfi_startproc save %sp, -STACKFRAME, %sp -.LLCFI0: + cfi_def_cfa_register(%fp) + cfi_window_save sub %sp, %i2, %sp ! alloca() space in stack for frame to set up add %sp, STACKFRAME, %l0 ! %l0 has start of @@ -91,81 +104,81 @@ _ffi_call_v8: ! If the return value pointer is NULL, assume no return value. tst %i4 - bz done + bz L(done) nop cmp %i3, FFI_TYPE_INT - be,a done + be,a L(done) st %o0, [%i4] ! (delay) cmp %i3, FFI_TYPE_FLOAT - be,a done + be,a L(done) st %f0, [%i4+0] ! (delay) cmp %i3, FFI_TYPE_DOUBLE - be,a double + be,a L(double) st %f0, [%i4+0] ! (delay) cmp %i3, FFI_TYPE_SINT8 - be,a sint8 + be,a L(sint8) sll %o0, 24, %o0 ! (delay) cmp %i3, FFI_TYPE_UINT8 - be,a uint8 + be,a L(uint8) sll %o0, 24, %o0 ! (delay) cmp %i3, FFI_TYPE_SINT16 - be,a sint16 + be,a L(sint16) sll %o0, 16, %o0 ! (delay) cmp %i3, FFI_TYPE_UINT16 - be,a uint16 + be,a L(uint16) sll %o0, 16, %o0 ! (delay) cmp %i3, FFI_TYPE_SINT64 - be,a longlong + be,a L(longlong) st %o0, [%i4+0] ! (delay) -done: + +L(done): ret restore -double: +L(double): st %f1, [%i4+4] ret restore -sint8: +L(sint8): sra %o0, 24, %o0 st %o0, [%i4+0] ret restore -uint8: +L(uint8): srl %o0, 24, %o0 st %o0, [%i4+0] ret restore -sint16: +L(sint16): sra %o0, 16, %o0 st %o0, [%i4+0] ret restore -uint16: +L(uint16): srl %o0, 16, %o0 st %o0, [%i4+0] ret restore -longlong: +L(longlong): st %o1, [%i4+4] ret restore -.LLFE1: + cfi_endproc -.ffi_call_v8_end: - .size ffi_call_v8,.ffi_call_v8_end-ffi_call_v8 + .size C(ffi_call_v8),. - C(ffi_call_v8) #undef STACKFRAME @@ -178,15 +191,16 @@ longlong: Receives the closure argument in %g2. */ - .text .align 8 - .globl ffi_closure_v8 + .globl C(ffi_closure_v8) + .type C(ffi_closure_v8),@function + FFI_HIDDEN(C(ffi_closure_v8)) -ffi_closure_v8: +C(ffi_closure_v8): #ifdef HAVE_AS_REGISTER_PSEUDO_OP .register %g2, #scratch #endif -.LLFB2: + cfi_startproc ! Reserve frame space for all arguments in case ! we need to align them on a 8-byte boundary. ld [%g2+FFI_TRAMPOLINE_SIZE], %g1 @@ -196,7 +210,8 @@ ffi_closure_v8: ! %g1 == STACKFRAME + 8*nargs neg %g1 save %sp, %g1, %sp -.LLCFI1: + cfi_def_cfa_register(%fp) + cfi_window_save ! Store all of the potential argument registers in va_list format. st %i0, [%fp+68+0] @@ -216,130 +231,47 @@ ffi_closure_v8: ! Load up the return value in the proper type. ! See ffi_prep_cif_machdep for the list of cases. cmp %o0, FFI_TYPE_VOID - be done1 + be L(done1) cmp %o0, FFI_TYPE_INT - be done1 + be L(done1) ld [%fp-8], %i0 cmp %o0, FFI_TYPE_FLOAT - be,a done1 + be,a L(done1) ld [%fp-8], %f0 cmp %o0, FFI_TYPE_DOUBLE - be,a done1 + be,a L(done1) ldd [%fp-8], %f0 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE cmp %o0, FFI_TYPE_LONGDOUBLE - be done2 + be L(done2) #endif cmp %o0, FFI_TYPE_STRUCT - be done2 + be L(done2) cmp %o0, FFI_TYPE_SINT64 - be,a done1 + be,a L(done1) ldd [%fp-8], %i0 cmp %o0, FFI_TYPE_UINT64 - be,a done1 + be,a L(done1) ldd [%fp-8], %i0 ld [%fp-8], %i0 -done1: +L(done1): jmp %i7+8 restore -done2: +L(done2): ! Skip 'unimp'. jmp %i7+12 restore -.LLFE2: -.ffi_closure_v8_end: - .size ffi_closure_v8,.ffi_closure_v8_end-ffi_closure_v8 - -#ifdef SPARC64 -#define WS 8 -#define nword xword -#define uanword uaxword -#else -#define WS 4 -#define nword long -#define uanword uaword -#endif - -#ifdef HAVE_RO_EH_FRAME - .section ".eh_frame",#alloc -#else - .section ".eh_frame",#alloc,#write -#endif -.LLframe1: - .uaword .LLECIE1-.LLSCIE1 ! Length of Common Information Entry -.LLSCIE1: - .uaword 0x0 ! CIE Identifier Tag - .byte 0x1 ! CIE Version - .ascii "zR\0" ! CIE Augmentation - .byte 0x1 ! uleb128 0x1; CIE Code Alignment Factor - .byte 0x80-WS ! sleb128 -WS; CIE Data Alignment Factor - .byte 0xf ! CIE RA Column - .byte 0x1 ! uleb128 0x1; Augmentation size -#ifdef HAVE_AS_SPARC_UA_PCREL - .byte 0x1b ! FDE Encoding (pcrel sdata4) -#else - .byte 0x50 ! FDE Encoding (aligned absolute) -#endif - .byte 0xc ! DW_CFA_def_cfa - .byte 0xe ! uleb128 0xe - .byte 0x0 ! uleb128 0x0 - .align WS -.LLECIE1: -.LLSFDE1: - .uaword .LLEFDE1-.LLASFDE1 ! FDE Length -.LLASFDE1: - .uaword .LLASFDE1-.LLframe1 ! FDE CIE offset -#ifdef HAVE_AS_SPARC_UA_PCREL - .uaword %r_disp32(.LLFB1) - .uaword .LLFE1-.LLFB1 ! FDE address range -#else - .align WS - .nword .LLFB1 - .uanword .LLFE1-.LLFB1 ! FDE address range -#endif - .byte 0x0 ! uleb128 0x0; Augmentation size - .byte 0x4 ! DW_CFA_advance_loc4 - .uaword .LLCFI0-.LLFB1 - .byte 0xd ! DW_CFA_def_cfa_register - .byte 0x1e ! uleb128 0x1e - .byte 0x2d ! DW_CFA_GNU_window_save - .byte 0x9 ! DW_CFA_register - .byte 0xf ! uleb128 0xf - .byte 0x1f ! uleb128 0x1f - .align WS -.LLEFDE1: -.LLSFDE2: - .uaword .LLEFDE2-.LLASFDE2 ! FDE Length -.LLASFDE2: - .uaword .LLASFDE2-.LLframe1 ! FDE CIE offset -#ifdef HAVE_AS_SPARC_UA_PCREL - .uaword %r_disp32(.LLFB2) - .uaword .LLFE2-.LLFB2 ! FDE address range -#else - .align WS - .nword .LLFB2 - .uanword .LLFE2-.LLFB2 ! FDE address range -#endif - .byte 0x0 ! uleb128 0x0; Augmentation size - .byte 0x4 ! DW_CFA_advance_loc4 - .uaword .LLCFI1-.LLFB2 - .byte 0xd ! DW_CFA_def_cfa_register - .byte 0x1e ! uleb128 0x1e - .byte 0x2d ! DW_CFA_GNU_window_save - .byte 0x9 ! DW_CFA_register - .byte 0xf ! uleb128 0xf - .byte 0x1f ! uleb128 0x1f - .align WS -.LLEFDE2: + cfi_endproc + .size C(ffi_closure_v8), . - C(ffi_closure_v8) #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits diff --git a/src/sparc/v9.S b/src/sparc/v9.S index bf31a2b..aba468e 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -27,25 +27,38 @@ #define LIBFFI_ASM #include #include +#include #ifdef SPARC64 /* Only compile this in for 64bit builds, because otherwise the object file will have inproper architecture due to used instructions. */ +#define C2(X, Y) X ## Y +#define C1(X, Y) C2(X, Y) + +#ifdef __USER_LABEL_PREFIX__ +# define C(Y) C1(__USER_LABEL_PREFIX__, Y) +#else +# define C(Y) Y +#endif +#define L(Y) C1(.L, Y) + + #define STACKFRAME 176 /* Minimum stack framesize for SPARC 64-bit */ #define STACK_BIAS 2047 #define ARGS (128) /* Offset of register area in frame */ .text .align 8 -.globl ffi_call_v9 -.globl _ffi_call_v9 + .globl C(ffi_call_v9) + .type C(ffi_call_v9),@function + FFI_HIDDEN(C(ffi_call_v9)) -ffi_call_v9: -_ffi_call_v9: -.LLFB1: +C(ffi_call_v9): + cfi_startproc save %sp, -STACKFRAME, %sp -.LLCFI0: + cfi_def_cfa_register(%fp) + cfi_window_save sub %sp, %i2, %sp ! alloca() space in stack for frame to set up add %sp, STACKFRAME+STACK_BIAS, %l0 ! %l0 has start of @@ -83,34 +96,35 @@ _ffi_call_v9: sub %l0, STACK_BIAS, %sp ! (delay) switch to frame ! If the return value pointer is NULL, assume no return value. - brz,pn %i4, done + brz,pn %i4, L(done) nop cmp %i3, FFI_TYPE_INT - be,a,pt %icc, done + be,a,pt %icc, L(done) stx %o0, [%i4+0] ! (delay) cmp %i3, FFI_TYPE_FLOAT - be,a,pn %icc, done + be,a,pn %icc, L(done) st %f0, [%i4+0] ! (delay) cmp %i3, FFI_TYPE_DOUBLE - be,a,pn %icc, done + be,a,pn %icc, L(done) std %f0, [%i4+0] ! (delay) cmp %i3, FFI_TYPE_STRUCT - be,pn %icc, dostruct + be,pn %icc, L(dostruct) cmp %i3, FFI_TYPE_LONGDOUBLE - bne,pt %icc, done + bne,pt %icc, L(done) nop std %f0, [%i4+0] std %f2, [%i4+8] -done: ret +L(done): + ret restore -dostruct: +L(dostruct): /* This will not work correctly for unions. */ stx %o0, [%i4+0] stx %o1, [%i4+8] @@ -122,10 +136,9 @@ dostruct: std %f6, [%i4+56] ret restore -.LLFE1: -.ffi_call_v9_end: - .size ffi_call_v9,.ffi_call_v9_end-ffi_call_v9 + cfi_endproc + .size C(ffi_call_v9), . - C(ffi_call_v9) #undef STACKFRAME @@ -138,14 +151,16 @@ dostruct: Receives the closure argument in %g1. */ - .text .align 8 - .globl ffi_closure_v9 + .globl C(ffi_closure_v9) + .type C(ffi_closure_v9),@function + FFI_HIDDEN(C(ffi_closure_v9)) -ffi_closure_v9: -.LLFB2: +C(ffi_closure_v9): + cfi_startproc save %sp, -STACKFRAME, %sp -.LLCFI1: + cfi_def_cfa_register(%fp) + cfi_window_save ! Store all of the potential argument registers in va_list format. stx %i0, [FP+128+0] @@ -177,28 +192,28 @@ ffi_closure_v9: mov %g1, %o0 add %fp, STACK_BIAS-160, %o1 add %fp, STACK_BIAS+128, %o2 - call ffi_closure_sparc_inner_v9 + call C(ffi_closure_sparc_inner_v9) add %fp, STACK_BIAS-128, %o3 ! Load up the return value in the proper type. ! See ffi_prep_cif_machdep for the list of cases. cmp %o0, FFI_TYPE_VOID - be,pn %icc, done1 + be,pn %icc, L(done1) cmp %o0, FFI_TYPE_INT - be,pn %icc, integer + be,pn %icc, L(integer) cmp %o0, FFI_TYPE_FLOAT - be,a,pn %icc, done1 + be,a,pn %icc, L(done1) ld [FP-160], %f0 cmp %o0, FFI_TYPE_DOUBLE - be,a,pn %icc, done1 + be,a,pn %icc, L(done1) ldd [FP-160], %f0 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE cmp %o0, FFI_TYPE_LONGDOUBLE - be,a,pn %icc, longdouble1 + be,a,pn %icc, L(longdouble1) ldd [FP-160], %f0 #endif @@ -211,97 +226,24 @@ ffi_closure_v9: ldd [FP-144], %f4 ldd [FP-136], %f6 -integer: +L(integer): ldx [FP-160], %i0 -done1: +L(done1): ret restore #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE -longdouble1: +L(longdouble1): ldd [FP-152], %f2 ret restore #endif -.LLFE2: -.ffi_closure_v9_end: - .size ffi_closure_v9,.ffi_closure_v9_end-ffi_closure_v9 - -#ifdef HAVE_RO_EH_FRAME - .section ".eh_frame",#alloc -#else - .section ".eh_frame",#alloc,#write -#endif -.LLframe1: - .uaword .LLECIE1-.LLSCIE1 ! Length of Common Information Entry -.LLSCIE1: - .uaword 0x0 ! CIE Identifier Tag - .byte 0x1 ! CIE Version - .ascii "zR\0" ! CIE Augmentation - .byte 0x1 ! uleb128 0x1; CIE Code Alignment Factor - .byte 0x78 ! sleb128 -8; CIE Data Alignment Factor - .byte 0xf ! CIE RA Column - .byte 0x1 ! uleb128 0x1; Augmentation size -#ifdef HAVE_AS_SPARC_UA_PCREL - .byte 0x1b ! FDE Encoding (pcrel sdata4) -#else - .byte 0x50 ! FDE Encoding (aligned absolute) -#endif - .byte 0xc ! DW_CFA_def_cfa - .byte 0xe ! uleb128 0xe - .byte 0xff,0xf ! uleb128 0x7ff - .align 8 -.LLECIE1: -.LLSFDE1: - .uaword .LLEFDE1-.LLASFDE1 ! FDE Length -.LLASFDE1: - .uaword .LLASFDE1-.LLframe1 ! FDE CIE offset -#ifdef HAVE_AS_SPARC_UA_PCREL - .uaword %r_disp32(.LLFB1) - .uaword .LLFE1-.LLFB1 ! FDE address range -#else - .align 8 - .xword .LLFB1 - .uaxword .LLFE1-.LLFB1 ! FDE address range -#endif - .byte 0x0 ! uleb128 0x0; Augmentation size - .byte 0x4 ! DW_CFA_advance_loc4 - .uaword .LLCFI0-.LLFB1 - .byte 0xd ! DW_CFA_def_cfa_register - .byte 0x1e ! uleb128 0x1e - .byte 0x2d ! DW_CFA_GNU_window_save - .byte 0x9 ! DW_CFA_register - .byte 0xf ! uleb128 0xf - .byte 0x1f ! uleb128 0x1f - .align 8 -.LLEFDE1: -.LLSFDE2: - .uaword .LLEFDE2-.LLASFDE2 ! FDE Length -.LLASFDE2: - .uaword .LLASFDE2-.LLframe1 ! FDE CIE offset -#ifdef HAVE_AS_SPARC_UA_PCREL - .uaword %r_disp32(.LLFB2) - .uaword .LLFE2-.LLFB2 ! FDE address range -#else - .align 8 - .xword .LLFB2 - .uaxword .LLFE2-.LLFB2 ! FDE address range -#endif - .byte 0x0 ! uleb128 0x0; Augmentation size - .byte 0x4 ! DW_CFA_advance_loc4 - .uaword .LLCFI1-.LLFB2 - .byte 0xd ! DW_CFA_def_cfa_register - .byte 0x1e ! uleb128 0x1e - .byte 0x2d ! DW_CFA_GNU_window_save - .byte 0x9 ! DW_CFA_register - .byte 0xf ! uleb128 0xf - .byte 0x1f ! uleb128 0x1f - .align 8 -.LLEFDE2: -#endif + cfi_endproc + .size C(ffi_closure_v9), . - C(ffi_closure_v9) +#endif /* SPARC64 */ #ifdef __linux__ .section .note.GNU-stack,"",@progbits #endif From 2b27890ba77db6a23d37fc70897109e4b2803c2d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 24 Oct 2014 16:10:48 -0700 Subject: [PATCH 52/58] sparc: Rewrite everything It's impossible to call between v8 and v9 ABIs, because of the stack bias in the v9 ABI. So let's not pretend it's just not implemented yet. Split the v9 code out to a separate file. The register windows prevent ffi_call from setting up the entire stack frame the assembly, but we needn't make an indirect call back to prep_args. --- Makefile.am | 4 +- src/prep_cif.c | 11 - src/sparc/ffi.c | 747 ++++++++++++------------------------------ src/sparc/ffi64.c | 433 ++++++++++++++++++++++++ src/sparc/ffitarget.h | 13 +- src/sparc/internal.h | 18 + src/sparc/v8.S | 301 +++++++++-------- src/sparc/v9.S | 314 +++++++++++------- 8 files changed, 1030 insertions(+), 811 deletions(-) create mode 100644 src/sparc/ffi64.c create mode 100644 src/sparc/internal.h diff --git a/Makefile.am b/Makefile.am index 95c082a..7766e90 100644 --- a/Makefile.am +++ b/Makefile.am @@ -36,7 +36,7 @@ EXTRA_DIST = LICENSE ChangeLog.v1 ChangeLog.libgcj \ src/sh/ffi.c src/sh/sysv.S src/sh/ffitarget.h src/sh64/ffi.c \ src/sh64/sysv.S src/sh64/ffitarget.h src/sparc/v8.S \ src/sparc/v9.S src/sparc/ffitarget.h src/sparc/ffi.c \ - src/x86/darwin64.S src/x86/ffi.c src/x86/sysv.S \ + src/sparc/ffi64.c src/x86/darwin64.S src/x86/ffi.c src/x86/sysv.S \ src/x86/darwin.S src/x86/ffiw64.c src/x86/win64.S \ src/x86/ffi64.c src/x86/unix64.S \ src/x86/ffitarget.h src/pa/ffitarget.h src/pa/ffi.c \ @@ -141,7 +141,7 @@ if X86_DARWIN nodist_libffi_la_SOURCES += src/x86/ffi.c src/x86/darwin.S src/x86/ffi64.c src/x86/darwin64.S endif if SPARC -nodist_libffi_la_SOURCES += src/sparc/ffi.c src/sparc/v8.S src/sparc/v9.S +nodist_libffi_la_SOURCES += src/sparc/ffi.c src/sparc/ffi64.c src/sparc/v8.S src/sparc/v9.S endif if ALPHA nodist_libffi_la_SOURCES += src/alpha/ffi.c src/alpha/osf.S diff --git a/src/prep_cif.c b/src/prep_cif.c index be5eae3..5881ceb 100644 --- a/src/prep_cif.c +++ b/src/prep_cif.c @@ -147,9 +147,6 @@ ffi_status FFI_HIDDEN ffi_prep_cif_core(ffi_cif *cif, ffi_abi abi, #if !defined FFI_TARGET_SPECIFIC_STACK_SPACE_ALLOCATION /* Make space for the return structure pointer */ if (cif->rtype->type == FFI_TYPE_STRUCT -#ifdef SPARC - && (cif->abi != FFI_V9 || cif->rtype->size > 32) -#endif #ifdef TILE && (cif->rtype->size > 10 * FFI_SIZEOF_ARG) #endif @@ -179,14 +176,6 @@ ffi_status FFI_HIDDEN ffi_prep_cif_core(ffi_cif *cif, ffi_abi abi, FFI_ASSERT_VALID_TYPE(*ptr); #if !defined FFI_TARGET_SPECIFIC_STACK_SPACE_ALLOCATION -#ifdef SPARC - if (((*ptr)->type == FFI_TYPE_STRUCT - && ((*ptr)->size > 16 || cif->abi != FFI_V9)) - || ((*ptr)->type == FFI_TYPE_LONGDOUBLE - && cif->abi != FFI_V9)) - bytes += sizeof(void*); - else -#endif { /* Add any padding if necessary */ if (((*ptr)->alignment - 1) & bytes) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index 1f38f54..7542847 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -28,6 +28,9 @@ #include #include #include +#include "internal.h" + +#ifndef SPARC64 /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; all further uses in this file will refer to the 128-bit type. */ @@ -40,632 +43,310 @@ # define FFI_TYPE_LONGDOUBLE 4 #endif - -/* ffi_prep_args is called by the assembly routine once stack space - has been allocated for the function's arguments */ - -void FFI_HIDDEN -ffi_prep_args_v8(char *stack, extended_cif *ecif) -{ - int i; - void **p_argv; - char *argp; - ffi_type **p_arg; - - /* Skip 16 words for the window save area */ - argp = stack + 16*sizeof(int); - - /* This should only really be done when we are returning a structure, - however, it's faster just to do it all the time... - - if ( ecif->cif->rtype->type == FFI_TYPE_STRUCT ) */ - *(int *) argp = (long)ecif->rvalue; - - /* And 1 word for the structure return value. */ - argp += sizeof(int); - -#ifdef USING_PURIFY - /* Purify will probably complain in our assembly routine, unless we - zero out this memory. */ - - ((int*)argp)[0] = 0; - ((int*)argp)[1] = 0; - ((int*)argp)[2] = 0; - ((int*)argp)[3] = 0; - ((int*)argp)[4] = 0; - ((int*)argp)[5] = 0; -#endif - - p_argv = ecif->avalue; - - for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; i; i--, p_arg++) - { - size_t z; - - if ((*p_arg)->type == FFI_TYPE_STRUCT - || (*p_arg)->type == FFI_TYPE_LONGDOUBLE) - { - *(unsigned int *) argp = (unsigned long)(* p_argv); - z = sizeof(int); - } - else - { - z = (*p_arg)->size; - if (z < sizeof(int)) - { - z = sizeof(int); - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed int *) argp = *(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(unsigned int *) argp = *(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(signed int *) argp = *(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(unsigned int *) argp = *(UINT16 *)(* p_argv); - break; - - default: - FFI_ASSERT(0); - } - } - else - { - memcpy(argp, *p_argv, z); - } - } - p_argv++; - argp += z; - } - - return; -} - -int FFI_HIDDEN -ffi_prep_args_v9(char *stack, extended_cif *ecif) -{ - int i, ret = 0; - int tmp; - void **p_argv; - char *argp; - ffi_type **p_arg; - - tmp = 0; - - /* Skip 16 words for the window save area */ - argp = stack + 16*sizeof(long long); - -#ifdef USING_PURIFY - /* Purify will probably complain in our assembly routine, unless we - zero out this memory. */ - - ((long long*)argp)[0] = 0; - ((long long*)argp)[1] = 0; - ((long long*)argp)[2] = 0; - ((long long*)argp)[3] = 0; - ((long long*)argp)[4] = 0; - ((long long*)argp)[5] = 0; -#endif - - p_argv = ecif->avalue; - - if (ecif->cif->rtype->type == FFI_TYPE_STRUCT && - ecif->cif->rtype->size > 32) - { - *(unsigned long long *) argp = (unsigned long)ecif->rvalue; - argp += sizeof(long long); - tmp = 1; - } - - for (i = 0, p_arg = ecif->cif->arg_types; i < ecif->cif->nargs; - i++, p_arg++) - { - size_t z; - - z = (*p_arg)->size; - switch ((*p_arg)->type) - { - case FFI_TYPE_STRUCT: - if (z > 16) - { - /* For structures larger than 16 bytes we pass reference. */ - *(unsigned long long *) argp = (unsigned long)* p_argv; - argp += sizeof(long long); - tmp++; - p_argv++; - continue; - } - /* FALLTHROUGH */ - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - case FFI_TYPE_LONGDOUBLE: - ret = 1; /* We should promote into FP regs as well as integer. */ - break; - } - if (z < sizeof(long long)) - { - switch ((*p_arg)->type) - { - case FFI_TYPE_SINT8: - *(signed long long *) argp = *(SINT8 *)(* p_argv); - break; - - case FFI_TYPE_UINT8: - *(unsigned long long *) argp = *(UINT8 *)(* p_argv); - break; - - case FFI_TYPE_SINT16: - *(signed long long *) argp = *(SINT16 *)(* p_argv); - break; - - case FFI_TYPE_UINT16: - *(unsigned long long *) argp = *(UINT16 *)(* p_argv); - break; - - case FFI_TYPE_SINT32: - *(signed long long *) argp = *(SINT32 *)(* p_argv); - break; - - case FFI_TYPE_UINT32: - *(unsigned long long *) argp = *(UINT32 *)(* p_argv); - break; - - case FFI_TYPE_FLOAT: - *(float *) (argp + 4) = *(FLOAT32 *)(* p_argv); /* Right justify */ - break; - - case FFI_TYPE_STRUCT: - memcpy(argp, *p_argv, z); - break; - - default: - FFI_ASSERT(0); - } - z = sizeof(long long); - tmp++; - } - else if (z == sizeof(long long)) - { - memcpy(argp, *p_argv, z); - z = sizeof(long long); - tmp++; - } - else - { - if ((tmp & 1) && (*p_arg)->alignment > 8) - { - tmp++; - argp += sizeof(long long); - } - memcpy(argp, *p_argv, z); - z = 2 * sizeof(long long); - tmp += 2; - } - p_argv++; - argp += z; - } - - return ret; -} - /* Perform machine dependent cif processing */ ffi_status FFI_HIDDEN ffi_prep_cif_machdep(ffi_cif *cif) { - int wordsize; - - if (cif->abi != FFI_V9) - { - wordsize = 4; - - /* If we are returning a struct, this will already have been added. - Otherwise we need to add it because it's always got to be there! */ - - if (cif->rtype->type != FFI_TYPE_STRUCT) - cif->bytes += wordsize; - - /* sparc call frames require that space is allocated for 6 args, - even if they aren't used. Make that space if necessary. */ - - if (cif->bytes < 4*6+4) - cif->bytes = 4*6+4; - } - else - { - wordsize = 8; - - /* sparc call frames require that space is allocated for 6 args, - even if they aren't used. Make that space if necessary. */ - - if (cif->bytes < 8*6) - cif->bytes = 8*6; - } - - /* Adjust cif->bytes. to include 16 words for the window save area, - and maybe the struct/union return pointer area, */ - - cif->bytes += 16 * wordsize; - - /* The stack must be 2 word aligned, so round bytes up - appropriately. */ - - cif->bytes = ALIGN(cif->bytes, 2 * wordsize); + ffi_type *rtype = cif->rtype; + int rtt = rtype->type; + size_t bytes; + int i, n, flags; /* Set the return type flag */ - switch (cif->rtype->type) + switch (rtt) { case FFI_TYPE_VOID: + flags = SPARC_RET_VOID; + break; case FFI_TYPE_FLOAT: + flags = SPARC_RET_FLOAT; + break; case FFI_TYPE_DOUBLE: + flags = SPARC_RET_DOUBLE; + break; case FFI_TYPE_LONGDOUBLE: - cif->flags = cif->rtype->type; - break; - case FFI_TYPE_STRUCT: - if (cif->abi == FFI_V9 && cif->rtype->size > 32) - cif->flags = FFI_TYPE_VOID; - else - cif->flags = FFI_TYPE_STRUCT; + flags = SPARC_RET_STRUCT; break; - case FFI_TYPE_SINT8: + flags = SPARC_RET_SINT8; + break; case FFI_TYPE_UINT8: + flags = SPARC_RET_UINT8; + break; case FFI_TYPE_SINT16: + flags = SPARC_RET_SINT16; + break; case FFI_TYPE_UINT16: - if (cif->abi == FFI_V9) - cif->flags = FFI_TYPE_INT; - else - cif->flags = cif->rtype->type; + flags = SPARC_RET_UINT16; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + case FFI_TYPE_POINTER: + flags = SPARC_RET_UINT32; break; - case FFI_TYPE_SINT64: case FFI_TYPE_UINT64: - if (cif->abi == FFI_V9) - cif->flags = FFI_TYPE_INT; - else - cif->flags = FFI_TYPE_SINT64; + flags = SPARC_RET_INT64; break; - default: - cif->flags = FFI_TYPE_INT; - break; + abort(); } + cif->flags = flags; + + bytes = 0; + for (i = 0, n = cif->nargs; i < n; ++i) + { + ffi_type *ty = cif->arg_types[i]; + size_t z = ty->size; + int tt = ty->type; + + if (tt == FFI_TYPE_STRUCT || tt == FFI_TYPE_LONGDOUBLE) + /* Passed by reference. */ + z = 4; + else + z = ALIGN(z, 4); + bytes += z; + } + + /* Sparc call frames require that space is allocated for 6 args, + even if they aren't used. Make that space if necessary. */ + if (bytes < 6 * 4) + bytes = 6 * 4; + + /* The ABI always requires space for the struct return pointer. */ + bytes += 4; + + /* The stack must be 2 word aligned, so round bytes up appropriately. */ + bytes = ALIGN(bytes, 2 * 4); + + /* Include the call frame to prep_args. */ + bytes += 4*16 + 4*8; + cif->bytes = bytes; + return FFI_OK; } -static int -ffi_v9_layout_struct(ffi_type *arg, int off, char *ret, char *intg, char *flt) +extern void ffi_call_v8(ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, size_t bytes) FFI_HIDDEN; + +int FFI_HIDDEN +ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) { - ffi_type **ptr = &arg->elements[0]; + ffi_type **p_arg; + int flags = cif->flags; + int i, nargs; - while (*ptr != NULL) + if (rvalue == NULL) { - if (off & ((*ptr)->alignment - 1)) - off = ALIGN(off, (*ptr)->alignment); + if (flags == SPARC_RET_STRUCT) + { + /* Since we pass the pointer to the callee, we need a value. + We allowed for this space in ffi_call, before ffi_call_v8 + alloca'd the space. */ + rvalue = (char *)argp + cif->bytes; + } + else + { + /* Otherwise, we can ignore the return value. */ + flags = SPARC_RET_VOID; + } + } - switch ((*ptr)->type) + /* This could only really be done when we are returning a structure. + However, the space is reserved so we can do it unconditionally. */ + *argp++ = (unsigned long)rvalue; + +#ifdef USING_PURIFY + /* Purify will probably complain in our assembly routine, + unless we zero out this memory. */ + memset(argp, 0, 6*4); +#endif + + p_arg = cif->arg_types; + for (i = 0, nargs = cif->nargs; i < nargs; i++) + { + ffi_type *ty = p_arg[i]; + void *a = avalue[i]; + + switch (ty->type) { case FFI_TYPE_STRUCT: - off = ffi_v9_layout_struct(*ptr, off, ret, intg, flt); - off = ALIGN(off, FFI_SIZEOF_ARG); - break; - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - memmove(ret + off, flt + off, (*ptr)->size); - off += (*ptr)->size; + *argp++ = (unsigned long)a; break; + + case FFI_TYPE_DOUBLE: + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + memcpy(argp, a, 8); + argp += 2; + break; + + case FFI_TYPE_INT: + case FFI_TYPE_FLOAT: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_POINTER: + *argp++ = *(unsigned *)a; + break; + + case FFI_TYPE_UINT8: + *argp++ = *(UINT8 *)a; + break; + case FFI_TYPE_SINT8: + *argp++ = *(SINT8 *)a; + break; + case FFI_TYPE_UINT16: + *argp++ = *(UINT16 *)a; + break; + case FFI_TYPE_SINT16: + *argp++ = *(SINT16 *)a; + break; + default: - memmove(ret + off, intg + off, (*ptr)->size); - off += (*ptr)->size; - break; + abort(); } - ptr++; } - return off; + + return flags; } - -#ifdef SPARC64 -extern int ffi_call_v9(void *, extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)(void)) FFI_HIDDEN; -#else -extern int ffi_call_v8(void *, extended_cif *, unsigned, - unsigned, unsigned *, void (*fn)(void)) FFI_HIDDEN; -#endif - -#ifndef __GNUC__ -void ffi_flush_icache (void *, size_t) FFI_HIDDEN; -#endif - -void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { - extended_cif ecif; - void *rval = rvalue; + size_t bytes = cif->bytes; - ecif.cif = cif; - ecif.avalue = avalue; + FFI_ASSERT (cif->abi == FFI_V8); - /* If the return value is a struct and we don't have a return */ - /* value address then we need to make one */ + /* If we've not got a return value, we need to create one if we've + got to pass the return value to the callee. Otherwise ignore it. */ + if (rvalue == NULL && cif->flags == SPARC_RET_STRUCT) + bytes += ALIGN (cif->rtype->size, 8); - ecif.rvalue = rvalue; - if (cif->rtype->type == FFI_TYPE_STRUCT) - { - if (cif->rtype->size <= 32) - rval = alloca(64); - else - { - rval = NULL; - if (rvalue == NULL) - ecif.rvalue = alloca(cif->rtype->size); - } - } - - switch (cif->abi) - { - case FFI_V8: -#ifdef SPARC64 - /* We don't yet support calling 32bit code from 64bit */ - FFI_ASSERT(0); -#else - if (rvalue && (cif->rtype->type == FFI_TYPE_STRUCT - || cif->flags == FFI_TYPE_LONGDOUBLE)) - { - /* For v8, we need an "unimp" with size of returning struct */ - /* behind "call", so we alloc some executable space for it. */ - /* l7 is used, we need to make sure v8.S doesn't use %l7. */ - unsigned int *call_struct = NULL; - ffi_closure_alloc(32, (void **)&call_struct); - if (call_struct) - { - unsigned long f = (unsigned long)fn; - call_struct[0] = 0xae10001f; /* mov %i7, %l7 */ - call_struct[1] = 0xbe10000f; /* mov %o7, %i7 */ - call_struct[2] = 0x03000000 | f >> 10; /* sethi %hi(fn), %g1 */ - call_struct[3] = 0x9fc06000 | (f & 0x3ff); /* jmp %g1+%lo(fn), %o7 */ - call_struct[4] = 0x01000000; /* nop */ - if (cif->rtype->size < 0x7f) - call_struct[5] = cif->rtype->size; /* unimp */ - else - call_struct[5] = 0x01000000; /* nop */ - call_struct[6] = 0x81c7e008; /* ret */ - call_struct[7] = 0xbe100017; /* mov %l7, %i7 */ -#ifdef __GNUC__ - asm volatile ("iflush %0; iflush %0+8; iflush %0+16; iflush %0+24" : : - "r" (call_struct) : "memory"); - /* SPARC v8 requires 5 instructions for flush to be visible */ - asm volatile ("nop; nop; nop; nop; nop"); -#else - ffi_flush_icache (call_struct, 32); -#endif - ffi_call_v8(ffi_prep_args_v8, &ecif, cif->bytes, - cif->flags, rvalue, call_struct); - ffi_closure_free(call_struct); - } - else - { - ffi_call_v8(ffi_prep_args_v8, &ecif, cif->bytes, - cif->flags, rvalue, fn); - } - } - else - { - ffi_call_v8(ffi_prep_args_v8, &ecif, cif->bytes, - cif->flags, rvalue, fn); - } -#endif - break; - case FFI_V9: -#ifdef SPARC64 - ffi_call_v9(ffi_prep_args_v9, &ecif, cif->bytes, - cif->flags, rval, fn); - if (rvalue && rval && cif->rtype->type == FFI_TYPE_STRUCT) - ffi_v9_layout_struct(cif->rtype, 0, (char *)rvalue, (char *)rval, ((char *)rval)+32); -#else - /* And vice versa */ - FFI_ASSERT(0); -#endif - break; - default: - FFI_ASSERT(0); - break; - } + ffi_call_v8(cif, fn, rvalue, avalue, -bytes); } - -#ifdef SPARC64 -extern void ffi_closure_v9(void) FFI_HIDDEN; +#ifdef __GNUC__ +static inline void +ffi_flush_icache (void *p) +{ + /* SPARC v8 requires 5 instructions for flush to be visible */ + asm volatile ("iflush %0; iflush %0+8; nop; nop; nop; nop; nop" + : : "r" (p) : "memory"); +} #else -extern void ffi_closure_v8(void) FFI_HIDDEN; +extern void ffi_flush_icache (void *) FFI_HIDDEN; #endif +extern void ffi_closure_v8(void) FFI_HIDDEN; + ffi_status -ffi_prep_closure_loc (ffi_closure* closure, - ffi_cif* cif, +ffi_prep_closure_loc (ffi_closure *closure, + ffi_cif *cif, void (*fun)(ffi_cif*, void*, void**, void*), void *user_data, void *codeloc) { unsigned int *tramp = (unsigned int *) &closure->tramp[0]; - unsigned long fn; -#ifdef SPARC64 - /* Trampoline address is equal to the closure address. We take advantage - of that to reduce the trampoline size by 8 bytes. */ - if (cif->abi != FFI_V9) - return FFI_BAD_ABI; - fn = (unsigned long) ffi_closure_v9; - tramp[0] = 0x83414000; /* rd %pc, %g1 */ - tramp[1] = 0xca586010; /* ldx [%g1+16], %g5 */ - tramp[2] = 0x81c14000; /* jmp %g5 */ - tramp[3] = 0x01000000; /* nop */ - *((unsigned long *) &tramp[4]) = fn; -#else - unsigned long ctx = (unsigned long) codeloc; + unsigned long ctx = (unsigned long) closure; + unsigned long fn = (unsigned long) ffi_closure_v8; + if (cif->abi != FFI_V8) return FFI_BAD_ABI; - fn = (unsigned long) ffi_closure_v8; + tramp[0] = 0x03000000 | fn >> 10; /* sethi %hi(fn), %g1 */ tramp[1] = 0x05000000 | ctx >> 10; /* sethi %hi(ctx), %g2 */ tramp[2] = 0x81c06000 | (fn & 0x3ff); /* jmp %g1+%lo(fn) */ tramp[3] = 0x8410a000 | (ctx & 0x3ff);/* or %g2, %lo(ctx) */ -#endif closure->cif = cif; closure->fun = fun; closure->user_data = user_data; - /* Flush the Icache. closure is 8 bytes aligned. */ -#ifdef __GNUC__ -#ifdef SPARC64 - asm volatile ("flush %0; flush %0+8" : : "r" (closure) : "memory"); -#else - asm volatile ("iflush %0; iflush %0+8" : : "r" (closure) : "memory"); - /* SPARC v8 requires 5 instructions for flush to be visible */ - asm volatile ("nop; nop; nop; nop; nop"); -#endif -#else - ffi_flush_icache (closure, 16); -#endif + ffi_flush_icache (closure); return FFI_OK; } int FFI_HIDDEN ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, - unsigned long *gpr, unsigned long *scratch) + unsigned long *argp) { ffi_cif *cif; ffi_type **arg_types; void **avalue; - int i, argn; + int i, nargs, flags; cif = closure->cif; arg_types = cif->arg_types; - avalue = alloca(cif->nargs * sizeof(void *)); + nargs = cif->nargs; + flags = cif->flags; + avalue = alloca(nargs * sizeof(void *)); /* Copy the caller's structure return address so that the closure returns the data directly to the caller. */ - if (cif->flags == FFI_TYPE_STRUCT || cif->flags == FFI_TYPE_LONGDOUBLE) - rvalue = (void *) gpr[0]; + if (flags == SPARC_RET_STRUCT) + rvalue = (void *)*argp; /* Always skip the structure return address. */ - argn = 1; + argp++; /* Grab the addresses of the arguments from the stack frame. */ - for (i = 0; i < cif->nargs; i++) + for (i = 0; i < nargs; i++) { - if (arg_types[i]->type == FFI_TYPE_STRUCT - || arg_types[i]->type == FFI_TYPE_LONGDOUBLE) + ffi_type *ty = arg_types[i]; + int tt = ty->type; + void *a = argp; + + switch (tt) { + case FFI_TYPE_STRUCT: + case FFI_TYPE_LONGDOUBLE: /* Straight copy of invisible reference. */ - avalue[i] = (void *)gpr[argn++]; - } - else if ((arg_types[i]->type == FFI_TYPE_DOUBLE - || arg_types[i]->type == FFI_TYPE_SINT64 - || arg_types[i]->type == FFI_TYPE_UINT64) - /* gpr is 8-byte aligned. */ - && (argn % 2) != 0) - { - /* Align on a 8-byte boundary. */ - scratch[0] = gpr[argn]; - scratch[1] = gpr[argn+1]; - avalue[i] = scratch; - scratch -= 2; - argn += 2; - } - else - { - /* Always right-justify. */ - argn += ALIGN(arg_types[i]->size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - avalue[i] = ((char *) &gpr[argn]) - arg_types[i]->size; + a = (void *)*argp; + break; + + case FFI_TYPE_DOUBLE: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + if ((unsigned long)a & 7) + { + /* Align on a 8-byte boundary. */ + UINT64 *tmp = alloca(8); + *tmp = ((UINT64)argp[0] << 32) | argp[1]; + a = tmp; + } + argp++; + break; + + case FFI_TYPE_INT: + case FFI_TYPE_FLOAT: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + case FFI_TYPE_POINTER: + break; + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + a += 2; + break; + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + a += 3; + break; + + default: + abort(); } + argp++; + avalue[i] = a; } /* Invoke the closure. */ (closure->fun) (cif, rvalue, avalue, closure->user_data); /* Tell ffi_closure_sparc how to perform return type promotions. */ - return cif->rtype->type; -} - -int FFI_HIDDEN -ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, - unsigned long *gpr, double *fpr) -{ - ffi_cif *cif; - ffi_type **arg_types; - void **avalue; - int i, argn, fp_slot_max; - - cif = closure->cif; - arg_types = cif->arg_types; - avalue = alloca(cif->nargs * sizeof(void *)); - - /* Copy the caller's structure return address so that the closure - returns the data directly to the caller. */ - if (cif->flags == FFI_TYPE_VOID - && cif->rtype->type == FFI_TYPE_STRUCT) - { - rvalue = (void *) gpr[0]; - /* Skip the structure return address. */ - argn = 1; - } - else - argn = 0; - - fp_slot_max = 16 - argn; - - /* Grab the addresses of the arguments from the stack frame. */ - for (i = 0; i < cif->nargs; i++) - { - if (arg_types[i]->type == FFI_TYPE_STRUCT) - { - if (arg_types[i]->size > 16) - { - /* Straight copy of invisible reference. */ - avalue[i] = (void *)gpr[argn++]; - } - else - { - /* Left-justify. */ - ffi_v9_layout_struct(arg_types[i], - 0, - (char *) &gpr[argn], - (char *) &gpr[argn], - (char *) &fpr[argn]); - avalue[i] = &gpr[argn]; - argn += ALIGN(arg_types[i]->size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - } - } - else - { - /* Right-justify. */ - argn += ALIGN(arg_types[i]->size, FFI_SIZEOF_ARG) / FFI_SIZEOF_ARG; - - /* Align on a 16-byte boundary. */ - if (arg_types[i]->type == FFI_TYPE_LONGDOUBLE && (argn % 2) != 0) - argn++; - if (i < fp_slot_max - && (arg_types[i]->type == FFI_TYPE_FLOAT - || arg_types[i]->type == FFI_TYPE_DOUBLE - || arg_types[i]->type == FFI_TYPE_LONGDOUBLE)) - avalue[i] = ((char *) &fpr[argn]) - arg_types[i]->size; - else - avalue[i] = ((char *) &gpr[argn]) - arg_types[i]->size; - } - } - - /* Invoke the closure. */ - (closure->fun) (cif, rvalue, avalue, closure->user_data); - - /* Tell ffi_closure_sparc how to perform return type promotions. */ - return cif->rtype->type; + return flags; } +#endif /* !SPARC64 */ diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c new file mode 100644 index 0000000..7ed928d --- /dev/null +++ b/src/sparc/ffi64.c @@ -0,0 +1,433 @@ +/* ----------------------------------------------------------------------- + ffi.c - Copyright (c) 2011, 2013 Anthony Green + Copyright (c) 1996, 2003-2004, 2007-2008 Red Hat, Inc. + + SPARC Foreign Function Interface + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ----------------------------------------------------------------------- */ + +#include +#include +#include +#include "internal.h" + +/* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE; + all further uses in this file will refer to the 128-bit type. */ +#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE +# if FFI_TYPE_LONGDOUBLE != 4 +# error FFI_TYPE_LONGDOUBLE out of date +# endif +#else +# undef FFI_TYPE_LONGDOUBLE +# define FFI_TYPE_LONGDOUBLE 4 +#endif + +#ifdef SPARC64 +/* Perform machine dependent cif processing */ + +int FFI_HIDDEN +ffi_v9_layout_struct (ffi_type *arg, int off, void *d, void *si, void *sf) +{ + ffi_type **elts, *t; + + for (elts = arg->elements; (t = *elts) != NULL; elts++) + { + size_t z = t->size; + void *src = si; + + off = ALIGN(off, t->alignment); + switch (t->type) + { + case FFI_TYPE_STRUCT: + off = ffi_v9_layout_struct(t, off, d, si, sf); + off = ALIGN(off, FFI_SIZEOF_ARG); + continue; + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + /* Note that closures start with the argument offset, + so that we know when to stop looking at fp regs. */ + if (off < 128) + src = sf; + break; + } + memcpy(d + off, src + off, z); + off += z; + } + + return off; +} + +ffi_status FFI_HIDDEN +ffi_prep_cif_machdep(ffi_cif *cif) +{ + ffi_type *rtype = cif->rtype; + int rtt = rtype->type; + size_t bytes = 0; + int i, n, flags; + + /* Set the return type flag */ + switch (rtt) + { + case FFI_TYPE_VOID: + flags = SPARC_RET_VOID; + break; + case FFI_TYPE_FLOAT: + flags = SPARC_RET_FLOAT; + break; + case FFI_TYPE_DOUBLE: + flags = SPARC_RET_DOUBLE; + break; + case FFI_TYPE_LONGDOUBLE: + flags = SPARC_RET_LDOUBLE; + break; + + case FFI_TYPE_STRUCT: + if (rtype->size > 32) + { + flags = SPARC_RET_VOID | SPARC_FLAG_RET_IN_MEM; + bytes = 8; + } + else + flags = SPARC_RET_STRUCT; + break; + + case FFI_TYPE_SINT8: + flags = SPARC_RET_SINT8; + break; + case FFI_TYPE_UINT8: + flags = SPARC_RET_UINT8; + break; + case FFI_TYPE_SINT16: + flags = SPARC_RET_SINT16; + break; + case FFI_TYPE_UINT16: + flags = SPARC_RET_UINT16; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + flags = SPARC_RET_SINT32; + break; + case FFI_TYPE_UINT32: + flags = SPARC_RET_UINT32; + break; + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + case FFI_TYPE_POINTER: + flags = SPARC_RET_INT64; + break; + + default: + abort(); + } + + bytes = 0; + for (i = 0, n = cif->nargs; i < n; ++i) + { + ffi_type *ty = cif->arg_types[i]; + size_t z = ty->size; + size_t a = ty->alignment; + + switch (ty->type) + { + case FFI_TYPE_STRUCT: + /* Large structs passed by reference. */ + if (z > 16) + { + a = z = 8; + break; + } + /* ??? FALLTHRU -- check for fp members in the struct. */ + case FFI_TYPE_FLOAT: + case FFI_TYPE_DOUBLE: + case FFI_TYPE_LONGDOUBLE: + flags |= SPARC_FLAG_FP_ARGS; + break; + } + bytes = ALIGN(bytes, a); + bytes += ALIGN(z, 8); + } + + /* Sparc call frames require that space is allocated for 6 args, + even if they aren't used. Make that space if necessary. */ + if (bytes < 6 * 8) + bytes = 6 * 8; + + /* The stack must be 2 word aligned, so round bytes up appropriately. */ + bytes = ALIGN(bytes, 16); + + /* Include the call frame to prep_args. */ + bytes += 8*16 + 8*8; + + cif->bytes = bytes; + cif->flags = flags; + return FFI_OK; +} + +extern void ffi_call_v9(ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, size_t bytes) FFI_HIDDEN; + +/* ffi_prep_args is called by the assembly routine once stack space + has been allocated for the function's arguments */ + +int FFI_HIDDEN +ffi_prep_args_v9(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) +{ + ffi_type **p_arg; + int flags = cif->flags; + int i, nargs; + + if (rvalue == NULL) + { + if (flags & SPARC_FLAG_RET_IN_MEM) + { + /* Since we pass the pointer to the callee, we need a value. + We allowed for this space in ffi_call, before ffi_call_v8 + alloca'd the space. */ + rvalue = (char *)argp + cif->bytes; + } + else + { + /* Otherwise, we can ignore the return value. */ + flags = SPARC_RET_VOID; + } + } + +#ifdef USING_PURIFY + /* Purify will probably complain in our assembly routine, + unless we zero out this memory. */ + memset(argp, 0, 6*8); +#endif + + if (flags & SPARC_FLAG_RET_IN_MEM) + *argp++ = (unsigned long)rvalue; + + p_arg = cif->arg_types; + for (i = 0, nargs = cif->nargs; i < nargs; i++) + { + ffi_type *ty = p_arg[i]; + void *a = avalue[i]; + size_t z; + + switch (ty->type) + { + case FFI_TYPE_SINT8: + *argp++ = *(SINT8 *)a; + break; + case FFI_TYPE_UINT8: + *argp++ = *(UINT8 *)a; + break; + case FFI_TYPE_SINT16: + *argp++ = *(SINT16 *)a; + break; + case FFI_TYPE_UINT16: + *argp++ = *(UINT16 *)a; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + *argp++ = *(SINT32 *)a; + break; + case FFI_TYPE_UINT32: + case FFI_TYPE_FLOAT: + *argp++ = *(UINT32 *)a; + break; + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + case FFI_TYPE_POINTER: + case FFI_TYPE_DOUBLE: + *argp++ = *(UINT64 *)a; + break; + + case FFI_TYPE_LONGDOUBLE: + case FFI_TYPE_STRUCT: + z = ty->size; + if (z > 16) + { + /* For structures larger than 16 bytes we pass reference. */ + *argp++ = (unsigned long)a; + break; + } + if (((unsigned long)argp & 15) && ty->alignment > 8) + argp++; + memcpy(argp, a, z); + argp += ALIGN(z, 8) / 8; + break; + + default: + abort(); + } + } + + return flags; +} + +void +ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + size_t bytes = cif->bytes; + + FFI_ASSERT (cif->abi == FFI_V9); + + if (rvalue == NULL && (cif->flags & SPARC_FLAG_RET_IN_MEM)) + bytes += ALIGN (cif->rtype->size, 16); + + ffi_call_v9(cif, fn, rvalue, avalue, -bytes); +} + +#ifdef __GNUC__ +static inline void +ffi_flush_icache (void *p) +{ + asm volatile ("flush %0; flush %0+8" : : "r" (p) : "memory"); +} +#else +extern void ffi_flush_icache (void *) FFI_HIDDEN; +#endif + +extern void ffi_closure_v9(void) FFI_HIDDEN; + +ffi_status +ffi_prep_closure_loc (ffi_closure* closure, + ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, + void *codeloc) +{ + unsigned int *tramp = (unsigned int *) &closure->tramp[0]; + unsigned long fn; + + if (cif->abi != FFI_V9) + return FFI_BAD_ABI; + + /* Trampoline address is equal to the closure address. We take advantage + of that to reduce the trampoline size by 8 bytes. */ + fn = (unsigned long) ffi_closure_v9; + tramp[0] = 0x83414000; /* rd %pc, %g1 */ + tramp[1] = 0xca586010; /* ldx [%g1+16], %g5 */ + tramp[2] = 0x81c14000; /* jmp %g5 */ + tramp[3] = 0x01000000; /* nop */ + *((unsigned long *) &tramp[4]) = fn; + + closure->cif = cif; + closure->fun = fun; + closure->user_data = user_data; + + ffi_flush_icache (closure); + + return FFI_OK; +} + +int FFI_HIDDEN +ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, + unsigned long *gpr, unsigned long *fpr) +{ + ffi_cif *cif; + ffi_type **arg_types; + void **avalue; + int i, argn, nargs, flags; + + cif = closure->cif; + arg_types = cif->arg_types; + nargs = cif->nargs; + flags = cif->flags; + + avalue = alloca(nargs * sizeof(void *)); + + /* Copy the caller's structure return address so that the closure + returns the data directly to the caller. */ + if (flags & SPARC_FLAG_RET_IN_MEM) + { + rvalue = (void *) gpr[0]; + /* Skip the structure return address. */ + argn = 1; + } + else + argn = 0; + + /* Grab the addresses of the arguments from the stack frame. */ + for (i = 0; i < nargs; i++) + { + ffi_type *ty = arg_types[i]; + void *a = &gpr[argn++]; + size_t z; + + switch (ty->type) + { + case FFI_TYPE_STRUCT: + z = ty->size; + if (z > 16) + a = *(void **)a; + else + { + if (--argn < 16) + ffi_v9_layout_struct(arg_types[i], 8*argn, gpr, gpr, fpr); + argn += ALIGN (z, 8) / 8; + } + break; + + case FFI_TYPE_LONGDOUBLE: + if (--argn & 1) + argn++; + a = (argn < 16 ? fpr : gpr) + argn; + argn += 2; + break; + case FFI_TYPE_DOUBLE: + if (argn <= 16) + a = fpr + argn - 1; + break; + case FFI_TYPE_FLOAT: + if (argn <= 16) + a = fpr + argn - 1; + a += 4; + break; + + case FFI_TYPE_UINT64: + case FFI_TYPE_SINT64: + case FFI_TYPE_POINTER: + break; + case FFI_TYPE_INT: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT32: + a += 4; + break; + case FFI_TYPE_UINT16: + case FFI_TYPE_SINT16: + a += 6; + break; + case FFI_TYPE_UINT8: + case FFI_TYPE_SINT8: + a += 7; + break; + + default: + abort(); + } + avalue[i] = a; + } + + /* Invoke the closure. */ + (closure->fun) (cif, rvalue, avalue, closure->user_data); + + /* Tell ffi_closure_sparc how to perform return type promotions. */ + return flags; +} +#endif /* SPARC64 */ diff --git a/src/sparc/ffitarget.h b/src/sparc/ffitarget.h index d89f787..ff4dc0b 100644 --- a/src/sparc/ffitarget.h +++ b/src/sparc/ffitarget.h @@ -46,18 +46,19 @@ typedef signed long ffi_sarg; typedef enum ffi_abi { FFI_FIRST_ABI = 0, - FFI_V8, - FFI_V8PLUS, - FFI_V9, - FFI_LAST_ABI, #ifdef SPARC64 - FFI_DEFAULT_ABI = FFI_V9 + FFI_V9, + FFI_DEFAULT_ABI = FFI_V9, #else - FFI_DEFAULT_ABI = FFI_V8 + FFI_V8, + FFI_DEFAULT_ABI = FFI_V8, #endif + FFI_LAST_ABI } ffi_abi; #endif +#define FFI_TARGET_SPECIFIC_STACK_SPACE_ALLOCATION + /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 diff --git a/src/sparc/internal.h b/src/sparc/internal.h new file mode 100644 index 0000000..df7c305 --- /dev/null +++ b/src/sparc/internal.h @@ -0,0 +1,18 @@ +#define SPARC_RET_VOID 0 +#define SPARC_RET_STRUCT 1 +#define SPARC_RET_FLOAT 2 +#define SPARC_RET_DOUBLE 3 +#define SPARC_RET_UINT8 4 +#define SPARC_RET_SINT8 5 +#define SPARC_RET_UINT16 6 +#define SPARC_RET_SINT16 7 +#define SPARC_RET_INT64 8 +#define SPARC_RET_UINT32 9 + +/* These two are only used for V9. */ +#define SPARC_RET_SINT32 10 +#define SPARC_RET_LDOUBLE 11 + +#define SPARC_FLAG_RET_MASK 15 +#define SPARC_FLAG_RET_IN_MEM 32 +#define SPARC_FLAG_FP_ARGS 64 diff --git a/src/sparc/v8.S b/src/sparc/v8.S index 10c66ba..b0d50a3 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -29,9 +29,9 @@ #include #include #include +#include "internal.h" -#define STACKFRAME 96 /* Minimum stack framesize for SPARC */ -#define ARGS (64+4) /* Offset of register area in frame */ +#ifndef SPARC64 #define C2(X, Y) X ## Y #define C1(X, Y) C2(X, Y) @@ -53,26 +53,24 @@ C(ffi_flush_icache): cfi_startproc - add %o0, %o1, %o2 -#ifdef SPARC64 -1: flush %o0 -#else 1: iflush %o0 -#endif - add %o0, 8, %o0 - cmp %o0, %o2 - blt 1b + iflush %o+8 nop nop nop nop nop retl - nop + nop cfi_endproc .size C(ffi_flush_icache), . - C(ffi_flush_icache) #endif +.macro E index + .align 16 + .org 2b + \index * 16 +.endm + .align 8 .globl C(ffi_call_v8) .type C(ffi_call_v8),@function @@ -80,104 +78,104 @@ C(ffi_flush_icache): C(ffi_call_v8): cfi_startproc - save %sp, -STACKFRAME, %sp + ! Allocate a stack frame sized by ffi_call. + save %sp, %o4, %sp cfi_def_cfa_register(%fp) cfi_window_save - - sub %sp, %i2, %sp ! alloca() space in stack for frame to set up - add %sp, STACKFRAME, %l0 ! %l0 has start of - ! frame to set up - mov %l0, %o0 ! call routine to set up frame - call %i0 - mov %i1, %o1 ! (delay) + mov %i0, %o0 ! copy cif + add %sp, 64+32, %o1 ! load args area + mov %i2, %o2 ! copy rvalue + call C(ffi_prep_args_v8) + mov %i3, %o3 ! copy avalue - ld [%l0+ARGS], %o0 ! call foreign function - ld [%l0+ARGS+4], %o1 - ld [%l0+ARGS+8], %o2 - ld [%l0+ARGS+12], %o3 - ld [%l0+ARGS+16], %o4 - ld [%l0+ARGS+20], %o5 - call %i5 - mov %l0, %sp ! (delay) switch to frame - nop ! STRUCT returning functions skip 12 instead of 8 bytes + add %sp, 32, %sp ! deallocate prep frame + and %o0, SPARC_FLAG_RET_MASK, %l0 ! save return type + ld [%sp+64+4], %o0 ! load all argument registers + ld [%sp+64+8], %o1 + ld [%sp+64+12], %o2 + ld [%sp+64+16], %o3 + cmp %l0, SPARC_RET_STRUCT ! struct return needs an unimp 4 + ld [%sp+64+20], %o4 + be 8f + ld [%sp+64+24], %o5 - ! If the return value pointer is NULL, assume no return value. - tst %i4 - bz L(done) + ! Call foreign function + call %i1 + nop + +0: call 1f ! load pc in %o7 + sll %l0, 4, %l0 +1: add %o7, %l0, %o7 ! o7 = 0b + ret_type*16 + jmp %o7+(2f-0b) + nop + + ! Note that each entry is 4 insns, enforced by the E macro. + .align 16 +2: +E SPARC_RET_VOID + ret + restore +E SPARC_RET_STRUCT + unimp +E SPARC_RET_FLOAT + st %f0, [%i2] + ret + restore +E SPARC_RET_DOUBLE + std %f0, [%i2] + ret + restore nop - - cmp %i3, FFI_TYPE_INT - be,a L(done) - st %o0, [%i4] ! (delay) - - cmp %i3, FFI_TYPE_FLOAT - be,a L(done) - st %f0, [%i4+0] ! (delay) - - cmp %i3, FFI_TYPE_DOUBLE - be,a L(double) - st %f0, [%i4+0] ! (delay) - - cmp %i3, FFI_TYPE_SINT8 - be,a L(sint8) - sll %o0, 24, %o0 ! (delay) - - cmp %i3, FFI_TYPE_UINT8 - be,a L(uint8) - sll %o0, 24, %o0 ! (delay) - - cmp %i3, FFI_TYPE_SINT16 - be,a L(sint16) - sll %o0, 16, %o0 ! (delay) - - cmp %i3, FFI_TYPE_UINT16 - be,a L(uint16) - sll %o0, 16, %o0 ! (delay) - - cmp %i3, FFI_TYPE_SINT64 - be,a L(longlong) - st %o0, [%i4+0] ! (delay) - -L(done): +E SPARC_RET_UINT8 + and %o0, 0xff, %o0 + st %o0, [%i2] ret - restore - -L(double): - st %f1, [%i4+4] + restore +E SPARC_RET_SINT8 + sll %o0, 24, %o0 + b 7f + sra %o0, 24, %o0 +E SPARC_RET_UINT16 + sll %o0, 16, %o0 + b 7f + srl %o0, 16, %o0 +E SPARC_RET_SINT16 + sll %o0, 16, %o0 + b 7f + sra %o0, 16, %o0 +E SPARC_RET_INT64 + std %o0, [%i2] ret - restore - -L(sint8): - sra %o0, 24, %o0 - st %o0, [%i4+0] + restore +E SPARC_RET_UINT32 +7: st %o0, [%i2] ret - restore + restore -L(uint8): - srl %o0, 24, %o0 - st %o0, [%i4+0] - ret - restore + ! Unused entries. Don't allow bad data to do worse things. +E 10 + unimp +E 11 + unimp +E 12 + unimp +E 13 + unimp +E 14 + unimp +E 15 + unimp -L(sint16): - sra %o0, 16, %o0 - st %o0, [%i4+0] + ! Struct returning functions expect and skip the unimp here. + .align 8 +8: call %i1 + nop + unimp 4 ret - restore + restore -L(uint16): - srl %o0, 16, %o0 - st %o0, [%i4+0] - ret - restore - -L(longlong): - st %o1, [%i4+4] - ret - restore cfi_endproc - .size C(ffi_call_v8),. - C(ffi_call_v8) @@ -185,7 +183,8 @@ L(longlong): #define STACKFRAME 104 /* 16*4 register window + 1*4 struct return + 6*4 args backing store + - 3*4 locals */ + 2*4 return storage + + 1*4 alignment */ /* ffi_closure_v8(...) @@ -201,15 +200,7 @@ C(ffi_closure_v8): .register %g2, #scratch #endif cfi_startproc - ! Reserve frame space for all arguments in case - ! we need to align them on a 8-byte boundary. - ld [%g2+FFI_TRAMPOLINE_SIZE], %g1 - ld [%g1+4], %g1 - sll %g1, 3, %g1 - add %g1, STACKFRAME, %g1 - ! %g1 == STACKFRAME + 8*nargs - neg %g1 - save %sp, %g1, %sp + save %sp, -STACKFRAME, %sp cfi_def_cfa_register(%fp) cfi_window_save @@ -224,55 +215,75 @@ C(ffi_closure_v8): ! Call ffi_closure_sparc_inner to do the bulk of the work. mov %g2, %o0 add %fp, -8, %o1 - add %fp, 64, %o2 call ffi_closure_sparc_inner_v8 - add %fp, -16, %o3 + add %fp, 64, %o2 - ! Load up the return value in the proper type. - ! See ffi_prep_cif_machdep for the list of cases. - cmp %o0, FFI_TYPE_VOID - be L(done1) +0: call 1f + and %o0, SPARC_FLAG_RET_MASK, %o0 +1: sll %o0, 4, %o0 ! o0 = o0 * 16 + add %o7, %o0, %o7 ! o7 = 0b + o0*16 + jmp %o7+(2f-0b) + nop - cmp %o0, FFI_TYPE_INT - be L(done1) - ld [%fp-8], %i0 - - cmp %o0, FFI_TYPE_FLOAT - be,a L(done1) - ld [%fp-8], %f0 - - cmp %o0, FFI_TYPE_DOUBLE - be,a L(done1) - ldd [%fp-8], %f0 - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - cmp %o0, FFI_TYPE_LONGDOUBLE - be L(done2) -#endif - - cmp %o0, FFI_TYPE_STRUCT - be L(done2) - - cmp %o0, FFI_TYPE_SINT64 - be,a L(done1) - ldd [%fp-8], %i0 - - cmp %o0, FFI_TYPE_UINT64 - be,a L(done1) - ldd [%fp-8], %i0 - - ld [%fp-8], %i0 -L(done1): - jmp %i7+8 + ! Note that each entry is 4 insns, enforced by the E macro. + .align 16 +2: +E SPARC_RET_VOID + ret restore -L(done2): - ! Skip 'unimp'. +E SPARC_RET_STRUCT jmp %i7+12 restore +E SPARC_RET_FLOAT + ld [%fp-8], %f0 + ret + restore +E SPARC_RET_DOUBLE + ldd [%fp-8], %f0 + ret + restore +E SPARC_RET_UINT8 + ldub [%fp-8+3], %i0 + ret + restore +E SPARC_RET_SINT8 + ldsb [%fp-8+3], %i0 + ret + restore +E SPARC_RET_UINT16 + lduh [%fp-8+2], %i0 + ret + restore +E SPARC_RET_SINT16 + ldsh [%fp-8+2], %i0 + ret + restore +E SPARC_RET_INT64 + ldd [%fp-8], %i0 + ret + restore +E SPARC_RET_UINT32 + ld [%fp-8], %i0 + ret + restore + + ! Unused entries. Don't allow bad data to do worse things. +E 10 + unimp +E 11 + unimp +E 12 + unimp +E 13 + unimp +E 14 + unimp +E 15 + unimp cfi_endproc .size C(ffi_closure_v8), . - C(ffi_closure_v8) - +#endif /* !SPARC64 */ #if defined __ELF__ && defined __linux__ .section .note.GNU-stack,"",@progbits #endif diff --git a/src/sparc/v9.S b/src/sparc/v9.S index aba468e..e2fe036 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -28,10 +28,9 @@ #include #include #include +#include "internal.h" #ifdef SPARC64 -/* Only compile this in for 64bit builds, because otherwise the object file - will have inproper architecture due to used instructions. */ #define C2(X, Y) X ## Y #define C1(X, Y) C2(X, Y) @@ -43,12 +42,14 @@ #endif #define L(Y) C1(.L, Y) +.macro E index + .align 16 + .org 2b + \index * 16 +.endm -#define STACKFRAME 176 /* Minimum stack framesize for SPARC 64-bit */ #define STACK_BIAS 2047 -#define ARGS (128) /* Offset of register area in frame */ -.text + .text .align 8 .globl C(ffi_call_v9) .type C(ffi_call_v9),@function @@ -56,86 +57,135 @@ C(ffi_call_v9): cfi_startproc - save %sp, -STACKFRAME, %sp + save %sp, %o4, %sp cfi_def_cfa_register(%fp) cfi_window_save - sub %sp, %i2, %sp ! alloca() space in stack for frame to set up - add %sp, STACKFRAME+STACK_BIAS, %l0 ! %l0 has start of - ! frame to set up + mov %i0, %o0 ! copy cif + add %sp, STACK_BIAS+128+48, %o1 ! load args area + mov %i2, %o2 ! copy rvalue + call C(ffi_prep_args_v9) + mov %i3, %o3 ! copy avalue - mov %l0, %o0 ! call routine to set up frame - call %i0 - mov %i1, %o1 ! (delay) - brz,pt %o0, 1f - ldx [%l0+ARGS], %o0 ! call foreign function + andcc %o0, SPARC_FLAG_FP_ARGS, %g0 ! need fp regs? + add %sp, 48, %sp ! deallocate prep frame + be,pt %xcc, 1f + mov %o0, %l0 ! save flags - ldd [%l0+ARGS], %f0 - ldd [%l0+ARGS+8], %f2 - ldd [%l0+ARGS+16], %f4 - ldd [%l0+ARGS+24], %f6 - ldd [%l0+ARGS+32], %f8 - ldd [%l0+ARGS+40], %f10 - ldd [%l0+ARGS+48], %f12 - ldd [%l0+ARGS+56], %f14 - ldd [%l0+ARGS+64], %f16 - ldd [%l0+ARGS+72], %f18 - ldd [%l0+ARGS+80], %f20 - ldd [%l0+ARGS+88], %f22 - ldd [%l0+ARGS+96], %f24 - ldd [%l0+ARGS+104], %f26 - ldd [%l0+ARGS+112], %f28 - ldd [%l0+ARGS+120], %f30 + ldd [%sp+STACK_BIAS+128], %f0 ! load all fp arg regs + ldd [%sp+STACK_BIAS+128+8], %f2 + ldd [%sp+STACK_BIAS+128+16], %f4 + ldd [%sp+STACK_BIAS+128+24], %f6 + ldd [%sp+STACK_BIAS+128+32], %f8 + ldd [%sp+STACK_BIAS+128+40], %f10 + ldd [%sp+STACK_BIAS+128+48], %f12 + ldd [%sp+STACK_BIAS+128+56], %f14 + ldd [%sp+STACK_BIAS+128+64], %f16 + ldd [%sp+STACK_BIAS+128+72], %f18 + ldd [%sp+STACK_BIAS+128+80], %f20 + ldd [%sp+STACK_BIAS+128+88], %f22 + ldd [%sp+STACK_BIAS+128+96], %f24 + ldd [%sp+STACK_BIAS+128+104], %f26 + ldd [%sp+STACK_BIAS+128+112], %f28 + ldd [%sp+STACK_BIAS+128+120], %f30 -1: ldx [%l0+ARGS+8], %o1 - ldx [%l0+ARGS+16], %o2 - ldx [%l0+ARGS+24], %o3 - ldx [%l0+ARGS+32], %o4 - ldx [%l0+ARGS+40], %o5 - call %i5 - sub %l0, STACK_BIAS, %sp ! (delay) switch to frame +1: ldx [%sp+STACK_BIAS+128], %o0 ! load all int arg regs + ldx [%sp+STACK_BIAS+128+8], %o1 + ldx [%sp+STACK_BIAS+128+16], %o2 + ldx [%sp+STACK_BIAS+128+24], %o3 + ldx [%sp+STACK_BIAS+128+32], %o4 + call %i1 + ldx [%sp+STACK_BIAS+128+40], %o5 - ! If the return value pointer is NULL, assume no return value. - brz,pn %i4, L(done) +0: call 1f ! load pc in %o7 + and %l0, SPARC_FLAG_RET_MASK, %l1 +1: sll %l1, 4, %l1 + add %o7, %l1, %o7 ! o7 = 0b + ret_type*16 + jmp %o7+(2f-0b) nop - cmp %i3, FFI_TYPE_INT - be,a,pt %icc, L(done) - stx %o0, [%i4+0] ! (delay) - - cmp %i3, FFI_TYPE_FLOAT - be,a,pn %icc, L(done) - st %f0, [%i4+0] ! (delay) - - cmp %i3, FFI_TYPE_DOUBLE - be,a,pn %icc, L(done) - std %f0, [%i4+0] ! (delay) - - cmp %i3, FFI_TYPE_STRUCT - be,pn %icc, L(dostruct) - - cmp %i3, FFI_TYPE_LONGDOUBLE - bne,pt %icc, L(done) + .align 16 +2: +E SPARC_RET_VOID + return %i7+8 nop - std %f0, [%i4+0] - std %f2, [%i4+8] +E SPARC_RET_STRUCT + add %sp, STACK_BIAS-64+128+48, %l2 + sub %sp, 64, %sp + b 8f + stx %o0, [%l2] +E SPARC_RET_FLOAT + return %i7+8 + st %f0, [%o2] +E SPARC_RET_DOUBLE + return %i7+8 + std %f0, [%o2] +E SPARC_RET_UINT8 + and %o0, 0xff, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_SINT8 + sll %o0, 24, %o0 + sra %o0, 24, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_UINT16 + sll %o0, 16, %o0 + srl %o0, 16, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_SINT16 + sll %o0, 16, %o0 + sra %o0, 16, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_INT64 + stx %o0, [%i2] + return %i7+8 + nop +E SPARC_RET_UINT32 + srl %o0, 0, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_SINT32 + sra %o0, 0, %i0 + return %i7+8 + stx %o0, [%o2] +E SPARC_RET_LDOUBLE + std %f0, [%i2] + return %i7+8 + std %f2, [%o2+8] -L(done): - ret - restore + ! Unused entries. Don't allow bad data to do worse things. +E 12 + unimp +E 13 + unimp +E 14 + unimp +E 15 + unimp -L(dostruct): - /* This will not work correctly for unions. */ - stx %o0, [%i4+0] - stx %o1, [%i4+8] - stx %o2, [%i4+16] - stx %o3, [%i4+24] - std %f0, [%i4+32] - std %f2, [%i4+40] - std %f4, [%i4+48] - std %f6, [%i4+56] - ret - restore + ! Finish the SPARC_RET_STRUCT sequence. + .align 8 +8: stx %o1, [%l2+8] + stx %o2, [%l2+16] + stx %o3, [%l2+24] + std %f0, [%l2+32] + std %f2, [%l2+40] + std %f4, [%l2+48] + std %f6, [%l2+56] + + ! Copy the structure into place. + ldx [%i0+16], %o0 ! load rtype from cif + mov 0, %o1 ! load off + mov %i2, %o2 ! load dst + mov %l2, %o3 ! load src_int + call C(ffi_v9_layout_struct) + add %l2, 32, %o4 ! load src_fp + + return %i7+8 + nop cfi_endproc .size C(ffi_call_v9), . - C(ffi_call_v9) @@ -195,54 +245,90 @@ C(ffi_closure_v9): call C(ffi_closure_sparc_inner_v9) add %fp, STACK_BIAS-128, %o3 - ! Load up the return value in the proper type. - ! See ffi_prep_cif_machdep for the list of cases. - cmp %o0, FFI_TYPE_VOID - be,pn %icc, L(done1) +0: call 1f ! load pc in %o7 + and %o0, SPARC_FLAG_RET_MASK, %o0 +1: sll %o0, 4, %o0 ! o2 = i2 * 16 + add %o7, %o0, %o7 ! o7 = 0b + i2*16 + jmp %o7+(2f-0b) + nop - cmp %o0, FFI_TYPE_INT - be,pn %icc, L(integer) - - cmp %o0, FFI_TYPE_FLOAT - be,a,pn %icc, L(done1) - ld [FP-160], %f0 - - cmp %o0, FFI_TYPE_DOUBLE - be,a,pn %icc, L(done1) - ldd [FP-160], %f0 - -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE - cmp %o0, FFI_TYPE_LONGDOUBLE - be,a,pn %icc, L(longdouble1) - ldd [FP-160], %f0 -#endif - - ! FFI_TYPE_STRUCT - ldx [FP-152], %i1 - ldx [FP-144], %i2 - ldx [FP-136], %i3 - ldd [FP-160], %f0 - ldd [FP-152], %f2 - ldd [FP-144], %f4 - ldd [FP-136], %f6 - -L(integer): + ! Note that we cannot load the data in the delay slot of + ! the return insn because the data is in the stack frame + ! that is deallocated by the return. + .align 16 +2: +E SPARC_RET_VOID + return %i7+8 + nop +E SPARC_RET_STRUCT ldx [FP-160], %i0 + ldd [FP-160], %f0 + b 8f + ldx [FP-152], %i1 +E SPARC_RET_FLOAT + ld [FP-160], %f0 + return %i7+8 + nop +E SPARC_RET_DOUBLE + ldd [FP-160], %f0 + return %i7+8 + nop +E SPARC_RET_UINT8 + ldub [FP-160+7], %i0 + return %i7+8 + nop +E SPARC_RET_SINT8 + ldsb [FP-160+7], %i0 + return %i7+8 + nop +E SPARC_RET_UINT16 + lduh [FP-160+6], %i0 + return %i7+8 + nop +E SPARC_RET_SINT16 + ldsh [FP-160+6], %i0 + return %i7+8 + nop +E SPARC_RET_INT64 + ldx [FP-160], %i0 + return %i7+8 + nop +E SPARC_RET_UINT32 + lduw [FP-160+4], %i0 + return %i7+8 + nop +E SPARC_RET_SINT32 + ldsw [FP-160+4], %i0 + return %i7+8 + nop +E SPARC_RET_LDOUBLE + ldd [FP-160], %f0 + ldd [FP-160+8], %f2 + return %i7+8 + nop -L(done1): - ret - restore + ! Unused entries. Don't allow bad data to do worse things. +E 12 + unimp +E 13 + unimp +E 14 + unimp +E 15 + unimp -#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE -L(longdouble1): - ldd [FP-152], %f2 - ret - restore -#endif + ! Finish the SPARC_RET_STRUCT sequence. + .align 8 +8: ldd [FP-152], %f2 + ldx [FP-144], %i2 + ldd [FP-144], %f4 + ldx [FP-136], %i3 + ldd [FP-136], %f6 + return %i7+8 + nop cfi_endproc .size C(ffi_closure_v9), . - C(ffi_closure_v9) - #endif /* SPARC64 */ #ifdef __linux__ .section .note.GNU-stack,"",@progbits From 0686c2e71c8e094cf003f199e3f578bcdede43ba Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 25 Oct 2014 14:52:45 -0700 Subject: [PATCH 53/58] sparc: Preprocess float point struct return We can eliminate recursion and speed structure return by flattening a nested structure tree into a bitmask. --- src/sparc/ffi64.c | 126 +++++++++++++++++++++++++++++++++---------- src/sparc/internal.h | 2 + src/sparc/v9.S | 11 ++-- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c index 7ed928d..65ae438 100644 --- a/src/sparc/ffi64.c +++ b/src/sparc/ffi64.c @@ -42,41 +42,103 @@ #endif #ifdef SPARC64 -/* Perform machine dependent cif processing */ -int FFI_HIDDEN -ffi_v9_layout_struct (ffi_type *arg, int off, void *d, void *si, void *sf) +/* Flatten the contents of a structure to the parts that are passed in + floating point registers. The return is a bit mask wherein bit N + set means bytes [4*n, 4*n+3] are passed in %fN. + + We encode both the (running) size (maximum 32) and mask (maxumum 255) + into one integer. The size is placed in the low byte, so that align + and addition work correctly. The mask is placed in the second byte. */ + +static int +ffi_struct_float_mask (ffi_type *struct_type, int size_mask) { ffi_type **elts, *t; - for (elts = arg->elements; (t = *elts) != NULL; elts++) + for (elts = struct_type->elements; (t = *elts) != NULL; elts++) { size_t z = t->size; - void *src = si; + int o, m; - off = ALIGN(off, t->alignment); + size_mask = ALIGN(size_mask, t->alignment); switch (t->type) { case FFI_TYPE_STRUCT: - off = ffi_v9_layout_struct(t, off, d, si, sf); - off = ALIGN(off, FFI_SIZEOF_ARG); + size_mask = ffi_struct_float_mask (t, size_mask); + size_mask = ALIGN(size_mask, FFI_SIZEOF_ARG); continue; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: - /* Note that closures start with the argument offset, - so that we know when to stop looking at fp regs. */ - if (off < 128) - src = sf; + m = (1 << (z / 4)) - 1; /* compute mask for type */ + o = (size_mask >> 2) & 0x3f; /* extract word offset */ + size_mask |= m << (o + 8); /* insert mask into place */ break; } - memcpy(d + off, src + off, z); - off += z; + size_mask += z; } - return off; + size_mask = ALIGN(size_mask, struct_type->alignment); + FFI_ASSERT ((size_mask & 0xff) == struct_type->size); + + return size_mask; } +/* Merge floating point data into integer data. If the structure is + entirely floating point, simply return a pointer to the fp data. */ + +static void * +ffi_struct_float_merge (int size_mask, void *vi, void *vf) +{ + int size = size_mask & 0xff; + int mask = size_mask >> 8; + int n = size >> 2; + + if (mask == 0) + return vi; + else if (mask == (1 << n) - 1) + return vf; + else + { + unsigned int *wi = vi, *wf = vf; + int i; + + for (i = 0; i < n; ++i) + if ((mask >> i) & 1) + wi[i] = wf[i]; + + return vi; + } +} + +/* Similar, but place the data into VD in the end. */ + +void FFI_HIDDEN +ffi_struct_float_copy (int size_mask, void *vd, void *vi, void *vf) +{ + int size = size_mask & 0xff; + int mask = size_mask >> 8; + int n = size >> 2; + + if (mask == 0) + ; + else if (mask == (1 << n) - 1) + vi = vf; + else + { + unsigned int *wd = vd, *wi = vi, *wf = vf; + int i; + + for (i = 0; i < n; ++i) + wd[i] = ((mask >> i) & 1 ? wf : wi)[i]; + return; + } + memcpy (vd, vi, size); +} + +/* Perform machine dependent cif processing */ + ffi_status FFI_HIDDEN ffi_prep_cif_machdep(ffi_cif *cif) { @@ -108,7 +170,10 @@ ffi_prep_cif_machdep(ffi_cif *cif) bytes = 8; } else - flags = SPARC_RET_STRUCT; + { + flags = ffi_struct_float_mask (rtype, 0) << SPARC_FLTMASK_SHIFT; + flags |= SPARC_RET_STRUCT; + } break; case FFI_TYPE_SINT8: @@ -343,7 +408,7 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, ffi_cif *cif; ffi_type **arg_types; void **avalue; - int i, argn, nargs, flags; + int i, argn, argx, nargs, flags; cif = closure->cif; arg_types = cif->arg_types; @@ -364,12 +429,13 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, argn = 0; /* Grab the addresses of the arguments from the stack frame. */ - for (i = 0; i < nargs; i++) + for (i = 0; i < nargs; i++, argn = argx) { ffi_type *ty = arg_types[i]; - void *a = &gpr[argn++]; + void *a = &gpr[argn]; size_t z; + argx = argn + 1; switch (ty->type) { case FFI_TYPE_STRUCT: @@ -378,25 +444,31 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, a = *(void **)a; else { - if (--argn < 16) - ffi_v9_layout_struct(arg_types[i], 8*argn, gpr, gpr, fpr); - argn += ALIGN (z, 8) / 8; + argx = argn + ALIGN (z, 8) / 8; + if (argn < 16) + { + int size_mask = ffi_struct_float_mask (ty, 0); + int argn_mask = (0xffff00 >> argn) & 0xff00; + + /* Eliminate fp registers off the end. */ + size_mask = (size_mask & 0xff) | (size_mask & argn_mask); + a = ffi_struct_float_merge (size_mask, gpr+argn, fpr+argn); + } } break; case FFI_TYPE_LONGDOUBLE: - if (--argn & 1) - argn++; + argn = ALIGN (argn, 2); a = (argn < 16 ? fpr : gpr) + argn; - argn += 2; + argx = argn + 2; break; case FFI_TYPE_DOUBLE: if (argn <= 16) - a = fpr + argn - 1; + a = fpr + argn; break; case FFI_TYPE_FLOAT: if (argn <= 16) - a = fpr + argn - 1; + a = fpr + argn; a += 4; break; diff --git a/src/sparc/internal.h b/src/sparc/internal.h index df7c305..3018928 100644 --- a/src/sparc/internal.h +++ b/src/sparc/internal.h @@ -16,3 +16,5 @@ #define SPARC_FLAG_RET_MASK 15 #define SPARC_FLAG_RET_IN_MEM 32 #define SPARC_FLAG_FP_ARGS 64 + +#define SPARC_FLTMASK_SHIFT 8 diff --git a/src/sparc/v9.S b/src/sparc/v9.S index e2fe036..3d91f2d 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -177,12 +177,11 @@ E 15 std %f6, [%l2+56] ! Copy the structure into place. - ldx [%i0+16], %o0 ! load rtype from cif - mov 0, %o1 ! load off - mov %i2, %o2 ! load dst - mov %l2, %o3 ! load src_int - call C(ffi_v9_layout_struct) - add %l2, 32, %o4 ! load src_fp + srl %l0, SPARC_FLTMASK_SHIFT, %o0 ! load size_mask + mov %i2, %o1 ! load dst + mov %l2, %o2 ! load src_gp + call C(ffi_struct_float_copy) + add %l2, 32, %o3 ! load src_fp return %i7+8 nop From 20da5b41324de524ea3ee1f84f402828b7c332dc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 25 Oct 2014 16:24:41 -0700 Subject: [PATCH 54/58] sparc: Handle more cases of structure return directly --- src/sparc/ffi.c | 4 +- src/sparc/ffi64.c | 40 +++++++++++++--- src/sparc/internal.h | 24 ++++++---- src/sparc/v8.S | 77 ++++++++++++++--------------- src/sparc/v9.S | 112 +++++++++++++++++++++++++------------------ 5 files changed, 152 insertions(+), 105 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index 7542847..1b8f48e 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -59,10 +59,10 @@ ffi_prep_cif_machdep(ffi_cif *cif) flags = SPARC_RET_VOID; break; case FFI_TYPE_FLOAT: - flags = SPARC_RET_FLOAT; + flags = SPARC_RET_F_1; break; case FFI_TYPE_DOUBLE: - flags = SPARC_RET_DOUBLE; + flags = SPARC_RET_F_2; break; case FFI_TYPE_LONGDOUBLE: case FFI_TYPE_STRUCT: diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c index 65ae438..ab3ed09 100644 --- a/src/sparc/ffi64.c +++ b/src/sparc/ffi64.c @@ -66,7 +66,6 @@ ffi_struct_float_mask (ffi_type *struct_type, int size_mask) { case FFI_TYPE_STRUCT: size_mask = ffi_struct_float_mask (t, size_mask); - size_mask = ALIGN(size_mask, FFI_SIZEOF_ARG); continue; case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: @@ -154,13 +153,13 @@ ffi_prep_cif_machdep(ffi_cif *cif) flags = SPARC_RET_VOID; break; case FFI_TYPE_FLOAT: - flags = SPARC_RET_FLOAT; + flags = SPARC_RET_F_1; break; case FFI_TYPE_DOUBLE: - flags = SPARC_RET_DOUBLE; + flags = SPARC_RET_F_2; break; case FFI_TYPE_LONGDOUBLE: - flags = SPARC_RET_LDOUBLE; + flags = SPARC_RET_F_4; break; case FFI_TYPE_STRUCT: @@ -171,8 +170,37 @@ ffi_prep_cif_machdep(ffi_cif *cif) } else { - flags = ffi_struct_float_mask (rtype, 0) << SPARC_FLTMASK_SHIFT; - flags |= SPARC_RET_STRUCT; + int size_mask = ffi_struct_float_mask (rtype, 0); + int word_size = (size_mask >> 2) & 0x3f; + int all_mask = (1 << word_size) - 1; + int fp_mask = size_mask >> 8; + + flags = (size_mask << SPARC_FLTMASK_SHIFT) | SPARC_RET_STRUCT; + + /* For special cases of all-int or all-fp, we can return + the value directly without popping through a struct copy. */ + if (fp_mask == 0) + { + if (rtype->alignment >= 8) + { + if (rtype->size == 8) + flags = SPARC_RET_INT64; + else if (rtype->size == 16) + flags = SPARC_RET_INT128; + } + } + else if (fp_mask == all_mask) + switch (word_size) + { + case 1: flags = SPARC_RET_F_1; break; + case 2: flags = SPARC_RET_F_2; break; + case 3: flags = SPARC_RET_F_3; break; + case 4: flags = SPARC_RET_F_4; break; + /* 5 word structures skipped; handled via RET_STRUCT. */ + case 6: flags = SPARC_RET_F_6; break; + /* 7 word structures skipped; handled via RET_STRUCT. */ + case 8: flags = SPARC_RET_F_8; break; + } } break; diff --git a/src/sparc/internal.h b/src/sparc/internal.h index 3018928..b4494d9 100644 --- a/src/sparc/internal.h +++ b/src/sparc/internal.h @@ -1,17 +1,21 @@ #define SPARC_RET_VOID 0 #define SPARC_RET_STRUCT 1 -#define SPARC_RET_FLOAT 2 -#define SPARC_RET_DOUBLE 3 -#define SPARC_RET_UINT8 4 -#define SPARC_RET_SINT8 5 -#define SPARC_RET_UINT16 6 -#define SPARC_RET_SINT16 7 +#define SPARC_RET_UINT8 2 +#define SPARC_RET_SINT8 3 +#define SPARC_RET_UINT16 4 +#define SPARC_RET_SINT16 5 +#define SPARC_RET_UINT32 6 +#define SPARC_RET_SINT32 7 /* v9 only */ #define SPARC_RET_INT64 8 -#define SPARC_RET_UINT32 9 +#define SPARC_RET_INT128 9 /* v9 only */ -/* These two are only used for V9. */ -#define SPARC_RET_SINT32 10 -#define SPARC_RET_LDOUBLE 11 +/* Note that F_7 is missing, and is handled by SPARC_RET_STRUCT. */ +#define SPARC_RET_F_8 10 +#define SPARC_RET_F_6 11 /* v9 only */ +#define SPARC_RET_F_4 12 +#define SPARC_RET_F_2 13 +#define SPARC_RET_F_3 14 /* v9 only */ +#define SPARC_RET_F_1 15 #define SPARC_FLAG_RET_MASK 15 #define SPARC_FLAG_RET_IN_MEM 32 diff --git a/src/sparc/v8.S b/src/sparc/v8.S index b0d50a3..4adcf6d 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -118,15 +118,6 @@ E SPARC_RET_VOID restore E SPARC_RET_STRUCT unimp -E SPARC_RET_FLOAT - st %f0, [%i2] - ret - restore -E SPARC_RET_DOUBLE - std %f0, [%i2] - ret - restore - nop E SPARC_RET_UINT8 and %o0, 0xff, %o0 st %o0, [%i2] @@ -144,28 +135,34 @@ E SPARC_RET_SINT16 sll %o0, 16, %o0 b 7f sra %o0, 16, %o0 -E SPARC_RET_INT64 - std %o0, [%i2] - ret - restore E SPARC_RET_UINT32 7: st %o0, [%i2] ret restore - - ! Unused entries. Don't allow bad data to do worse things. -E 10 +E SPARC_RET_SINT32 unimp -E 11 +E SPARC_RET_INT64 + std %o0, [%i2] + ret + restore +E SPARC_RET_INT128 unimp -E 12 +E SPARC_RET_F_8 unimp -E 13 +E SPARC_RET_F_6 unimp -E 14 +E SPARC_RET_F_4 unimp -E 15 +E SPARC_RET_F_2 + std %f0, [%i2] + ret + restore +E SPARC_RET_F_3 unimp +E SPARC_RET_F_1 + st %f0, [%i2] + ret + restore ! Struct returning functions expect and skip the unimp here. .align 8 @@ -234,14 +231,6 @@ E SPARC_RET_VOID E SPARC_RET_STRUCT jmp %i7+12 restore -E SPARC_RET_FLOAT - ld [%fp-8], %f0 - ret - restore -E SPARC_RET_DOUBLE - ldd [%fp-8], %f0 - ret - restore E SPARC_RET_UINT8 ldub [%fp-8+3], %i0 ret @@ -258,28 +247,34 @@ E SPARC_RET_SINT16 ldsh [%fp-8+2], %i0 ret restore -E SPARC_RET_INT64 - ldd [%fp-8], %i0 - ret - restore E SPARC_RET_UINT32 ld [%fp-8], %i0 ret restore - - ! Unused entries. Don't allow bad data to do worse things. -E 10 +E SPARC_RET_SINT32 unimp -E 11 +E SPARC_RET_INT64 + ldd [%fp-8], %i0 + ret + restore +E SPARC_RET_INT128 unimp -E 12 +E SPARC_RET_F_8 unimp -E 13 +E SPARC_RET_F_6 unimp -E 14 +E SPARC_RET_F_4 unimp -E 15 +E SPARC_RET_F_2 + ldd [%fp-8], %f0 + ret + restore +E SPARC_RET_F_3 unimp +E SPARC_RET_F_1 + ld [%fp-8], %f0 + ret + restore cfi_endproc .size C(ffi_closure_v8), . - C(ffi_closure_v8) diff --git a/src/sparc/v9.S b/src/sparc/v9.S index 3d91f2d..d893d2f 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -114,12 +114,6 @@ E SPARC_RET_STRUCT sub %sp, 64, %sp b 8f stx %o0, [%l2] -E SPARC_RET_FLOAT - return %i7+8 - st %f0, [%o2] -E SPARC_RET_DOUBLE - return %i7+8 - std %f0, [%o2] E SPARC_RET_UINT8 and %o0, 0xff, %i0 return %i7+8 @@ -139,10 +133,6 @@ E SPARC_RET_SINT16 sra %o0, 16, %i0 return %i7+8 stx %o0, [%o2] -E SPARC_RET_INT64 - stx %o0, [%i2] - return %i7+8 - nop E SPARC_RET_UINT32 srl %o0, 0, %i0 return %i7+8 @@ -151,20 +141,40 @@ E SPARC_RET_SINT32 sra %o0, 0, %i0 return %i7+8 stx %o0, [%o2] -E SPARC_RET_LDOUBLE - std %f0, [%i2] +E SPARC_RET_INT64 + stx %o0, [%i2] return %i7+8 - std %f2, [%o2+8] - - ! Unused entries. Don't allow bad data to do worse things. -E 12 - unimp -E 13 - unimp -E 14 - unimp -E 15 - unimp + nop +E SPARC_RET_INT128 + stx %o0, [%i2] + stx %o1, [%i2+8] + return %i7+8 + nop +E SPARC_RET_F_8 + st %f7, [%i2+7*4] + nop + st %f6, [%i2+6*4] + nop +E SPARC_RET_F_6 + st %f5, [%i2+5*4] + nop + st %f4, [%i2+4*4] + nop +E SPARC_RET_F_4 + std %f2, [%i2+2*4] + return %i7+8 + std %f0, [%o2] +E SPARC_RET_F_2 + return %i7+8 + std %f0, [%o2] +E SPARC_RET_F_3 + st %f2, [%i2+2*4] + nop + st %f1, [%i2+1*4] + nop +E SPARC_RET_F_1 + return %i7+8 + st %f0, [%o2] ! Finish the SPARC_RET_STRUCT sequence. .align 8 @@ -264,14 +274,6 @@ E SPARC_RET_STRUCT ldd [FP-160], %f0 b 8f ldx [FP-152], %i1 -E SPARC_RET_FLOAT - ld [FP-160], %f0 - return %i7+8 - nop -E SPARC_RET_DOUBLE - ldd [FP-160], %f0 - return %i7+8 - nop E SPARC_RET_UINT8 ldub [FP-160+7], %i0 return %i7+8 @@ -288,10 +290,6 @@ E SPARC_RET_SINT16 ldsh [FP-160+6], %i0 return %i7+8 nop -E SPARC_RET_INT64 - ldx [FP-160], %i0 - return %i7+8 - nop E SPARC_RET_UINT32 lduw [FP-160+4], %i0 return %i7+8 @@ -300,21 +298,43 @@ E SPARC_RET_SINT32 ldsw [FP-160+4], %i0 return %i7+8 nop -E SPARC_RET_LDOUBLE +E SPARC_RET_INT64 + ldx [FP-160], %i0 + return %i7+8 + nop +E SPARC_RET_INT128 + ldx [FP-160], %i0 + ldx [FP-160+8], %i1 + return %i7+8 + nop +E SPARC_RET_F_8 + ld [FP-160+7*4], %f7 + nop + ld [FP-160+6*4], %f6 + nop +E SPARC_RET_F_6 + ld [FP-160+5*4], %f5 + nop + ld [FP-160+4*4], %f4 + nop +E SPARC_RET_F_4 ldd [FP-160], %f0 ldd [FP-160+8], %f2 return %i7+8 nop - - ! Unused entries. Don't allow bad data to do worse things. -E 12 - unimp -E 13 - unimp -E 14 - unimp -E 15 - unimp +E SPARC_RET_F_2 + ldd [FP-160], %f0 + return %i7+8 + nop +E SPARC_RET_F_3 + ld [FP-160+2*4], %f2 + nop + ld [FP-160+1*4], %f1 + nop +E SPARC_RET_F_1 + ld [FP-160], %f0 + return %i7+8 + nop ! Finish the SPARC_RET_STRUCT sequence. .align 8 From ad89c2d9e93d5a713ae4113567b4ca6c4fbbf17c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 26 Oct 2014 13:16:03 -0700 Subject: [PATCH 55/58] sparc: Add support for complex types --- src/sparc/ffi.c | 92 ++++++++++++++++++-- src/sparc/ffi64.c | 48 +++++++++-- src/sparc/ffitarget.h | 1 + src/sparc/internal.h | 10 ++- src/sparc/v8.S | 128 ++++++++++++++++++---------- src/sparc/v9.S | 8 +- testsuite/libffi.call/call.exp | 1 + testsuite/libffi.call/complex_int.c | 4 +- 8 files changed, 223 insertions(+), 69 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index 1b8f48e..d319c03 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -90,6 +90,40 @@ ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_UINT64: flags = SPARC_RET_INT64; break; + case FFI_TYPE_COMPLEX: + rtt = rtype->elements[0]->type; + switch (rtt) + { + case FFI_TYPE_FLOAT: + flags = SPARC_RET_F_2; + break; + case FFI_TYPE_DOUBLE: + flags = SPARC_RET_F_4; + break; + case FFI_TYPE_LONGDOUBLE: + flags = SPARC_RET_F_8; + break; + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + flags = SPARC_RET_INT128; + break; + case FFI_TYPE_INT: + case FFI_TYPE_SINT32: + case FFI_TYPE_UINT32: + flags = SPARC_RET_INT64; + break; + case FFI_TYPE_SINT16: + case FFI_TYPE_UINT16: + flags = SP_V8_RET_CPLX16; + break; + case FFI_TYPE_SINT8: + case FFI_TYPE_UINT8: + flags = SP_V8_RET_CPLX8; + break; + default: + abort(); + } + break; default: abort(); } @@ -102,11 +136,24 @@ ffi_prep_cif_machdep(ffi_cif *cif) size_t z = ty->size; int tt = ty->type; - if (tt == FFI_TYPE_STRUCT || tt == FFI_TYPE_LONGDOUBLE) - /* Passed by reference. */ - z = 4; - else - z = ALIGN(z, 4); + switch (tt) + { + case FFI_TYPE_STRUCT: + case FFI_TYPE_LONGDOUBLE: + by_reference: + /* Passed by reference. */ + z = 4; + break; + + case FFI_TYPE_COMPLEX: + tt = ty->elements[0]->type; + if (tt == FFI_TYPE_FLOAT || z > 8) + goto by_reference; + /* FALLTHRU */ + + default: + z = ALIGN(z, 4); + } bytes += z; } @@ -169,11 +216,14 @@ ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) { ffi_type *ty = p_arg[i]; void *a = avalue[i]; + int tt = ty->type; + size_t z; - switch (ty->type) + switch (tt) { case FFI_TYPE_STRUCT: case FFI_TYPE_LONGDOUBLE: + by_reference: *argp++ = (unsigned long)a; break; @@ -205,6 +255,23 @@ ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) *argp++ = *(SINT16 *)a; break; + case FFI_TYPE_COMPLEX: + tt = ty->elements[0]->type; + z = ty->size; + if (tt == FFI_TYPE_FLOAT || z > 8) + goto by_reference; + if (z < 4) + { + memcpy((char *)argp + 4 - z, a, z); + argp++; + } + else + { + memcpy(argp, a, z); + argp += z / 4; + } + break; + default: abort(); } @@ -299,11 +366,13 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, ffi_type *ty = arg_types[i]; int tt = ty->type; void *a = argp; + size_t z; switch (tt) { case FFI_TYPE_STRUCT: case FFI_TYPE_LONGDOUBLE: + by_reference: /* Straight copy of invisible reference. */ a = (void *)*argp; break; @@ -336,6 +405,17 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, a += 3; break; + case FFI_TYPE_COMPLEX: + tt = ty->elements[0]->type; + z = ty->size; + if (tt == FFI_TYPE_FLOAT || z > 8) + goto by_reference; + if (z < 4) + a += 4 - z; + else if (z > 4) + argp++; + break; + default: abort(); } diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c index ab3ed09..1e2d3f4 100644 --- a/src/sparc/ffi64.c +++ b/src/sparc/ffi64.c @@ -52,14 +52,28 @@ and addition work correctly. The mask is placed in the second byte. */ static int -ffi_struct_float_mask (ffi_type *struct_type, int size_mask) +ffi_struct_float_mask (ffi_type *outer_type, int size_mask) { - ffi_type **elts, *t; + ffi_type **elts; + ffi_type *t; - for (elts = struct_type->elements; (t = *elts) != NULL; elts++) + if (outer_type->type == FFI_TYPE_COMPLEX) + { + int m = 0, tt = outer_type->elements[0]->type; + size_t z = outer_type->size; + + if (tt == FFI_TYPE_FLOAT + || tt == FFI_TYPE_DOUBLE + || tt == FFI_TYPE_LONGDOUBLE) + m = (1 << (z / 4)) - 1; + return (m << 8) | z; + } + FFI_ASSERT (outer_type->type == FFI_TYPE_STRUCT); + + for (elts = outer_type->elements; (t = *elts) != NULL; elts++) { size_t z = t->size; - int o, m; + int o, m, tt; size_mask = ALIGN(size_mask, t->alignment); switch (t->type) @@ -67,6 +81,13 @@ ffi_struct_float_mask (ffi_type *struct_type, int size_mask) case FFI_TYPE_STRUCT: size_mask = ffi_struct_float_mask (t, size_mask); continue; + case FFI_TYPE_COMPLEX: + tt = t->elements[0]->type; + if (tt != FFI_TYPE_FLOAT + && tt != FFI_TYPE_DOUBLE + && tt != FFI_TYPE_LONGDOUBLE) + break; + /* FALLTHRU */ case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: @@ -78,8 +99,8 @@ ffi_struct_float_mask (ffi_type *struct_type, int size_mask) size_mask += z; } - size_mask = ALIGN(size_mask, struct_type->alignment); - FFI_ASSERT ((size_mask & 0xff) == struct_type->size); + size_mask = ALIGN(size_mask, outer_type->alignment); + FFI_ASSERT ((size_mask & 0xff) == outer_type->size); return size_mask; } @@ -162,6 +183,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) flags = SPARC_RET_F_4; break; + case FFI_TYPE_COMPLEX: case FFI_TYPE_STRUCT: if (rtype->size > 32) { @@ -194,7 +216,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) { case 1: flags = SPARC_RET_F_1; break; case 2: flags = SPARC_RET_F_2; break; - case 3: flags = SPARC_RET_F_3; break; + case 3: flags = SP_V9_RET_F_3; break; case 4: flags = SPARC_RET_F_4; break; /* 5 word structures skipped; handled via RET_STRUCT. */ case 6: flags = SPARC_RET_F_6; break; @@ -218,7 +240,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) break; case FFI_TYPE_INT: case FFI_TYPE_SINT32: - flags = SPARC_RET_SINT32; + flags = SP_V9_RET_SINT32; break; case FFI_TYPE_UINT32: flags = SPARC_RET_UINT32; @@ -242,6 +264,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) switch (ty->type) { + case FFI_TYPE_COMPLEX: case FFI_TYPE_STRUCT: /* Large structs passed by reference. */ if (z > 16) @@ -249,7 +272,12 @@ ffi_prep_cif_machdep(ffi_cif *cif) a = z = 8; break; } - /* ??? FALLTHRU -- check for fp members in the struct. */ + /* Small structs may be passed in integer or fp regs or both. */ + if (bytes >= 16*8) + break; + if ((ffi_struct_float_mask (ty, 0) & 0xff00) == 0) + break; + /* FALLTHRU */ case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: @@ -351,6 +379,7 @@ ffi_prep_args_v9(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) break; case FFI_TYPE_LONGDOUBLE: + case FFI_TYPE_COMPLEX: case FFI_TYPE_STRUCT: z = ty->size; if (z > 16) @@ -466,6 +495,7 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, argx = argn + 1; switch (ty->type) { + case FFI_TYPE_COMPLEX: case FFI_TYPE_STRUCT: z = ty->size; if (z > 16) diff --git a/src/sparc/ffitarget.h b/src/sparc/ffitarget.h index ff4dc0b..f70c937 100644 --- a/src/sparc/ffitarget.h +++ b/src/sparc/ffitarget.h @@ -58,6 +58,7 @@ typedef enum ffi_abi { #endif #define FFI_TARGET_SPECIFIC_STACK_SPACE_ALLOCATION +#define FFI_TARGET_HAS_COMPLEX_TYPE /* ---- Definitions for closures ----------------------------------------- */ diff --git a/src/sparc/internal.h b/src/sparc/internal.h index b4494d9..f9387d4 100644 --- a/src/sparc/internal.h +++ b/src/sparc/internal.h @@ -5,16 +5,18 @@ #define SPARC_RET_UINT16 4 #define SPARC_RET_SINT16 5 #define SPARC_RET_UINT32 6 -#define SPARC_RET_SINT32 7 /* v9 only */ +#define SP_V9_RET_SINT32 7 /* v9 only */ +#define SP_V8_RET_CPLX16 7 /* v8 only */ #define SPARC_RET_INT64 8 -#define SPARC_RET_INT128 9 /* v9 only */ +#define SPARC_RET_INT128 9 /* Note that F_7 is missing, and is handled by SPARC_RET_STRUCT. */ #define SPARC_RET_F_8 10 -#define SPARC_RET_F_6 11 /* v9 only */ +#define SPARC_RET_F_6 11 #define SPARC_RET_F_4 12 #define SPARC_RET_F_2 13 -#define SPARC_RET_F_3 14 /* v9 only */ +#define SP_V9_RET_F_3 14 /* v9 only */ +#define SP_V8_RET_CPLX8 14 /* v8 only */ #define SPARC_RET_F_1 15 #define SPARC_FLAG_RET_MASK 15 diff --git a/src/sparc/v8.S b/src/sparc/v8.S index 4adcf6d..e76d813 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -1,8 +1,8 @@ /* ----------------------------------------------------------------------- v8.S - Copyright (c) 2013 The Written Word, Inc. Copyright (c) 1996, 1997, 2003, 2004, 2008 Red Hat, Inc. - - SPARC Foreign Function Interface + + SPARC Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -25,7 +25,7 @@ DEALINGS IN THE SOFTWARE. ----------------------------------------------------------------------- */ -#define LIBFFI_ASM +#define LIBFFI_ASM #include #include #include @@ -45,7 +45,7 @@ .text -#ifndef __GNUC__ +#ifndef __GNUC__ .align 8 .globl C(ffi_flush_icache) .type C(ffi_flush_icache),@function @@ -75,7 +75,7 @@ C(ffi_flush_icache): .globl C(ffi_call_v8) .type C(ffi_call_v8),@function FFI_HIDDEN(C(ffi_call_v8)) - + C(ffi_call_v8): cfi_startproc ! Allocate a stack frame sized by ffi_call. @@ -139,26 +139,44 @@ E SPARC_RET_UINT32 7: st %o0, [%i2] ret restore -E SPARC_RET_SINT32 - unimp +E SP_V8_RET_CPLX16 + sth %o0, [%i2+2] + b 9f + srl %o0, 16, %o0 E SPARC_RET_INT64 - std %o0, [%i2] + st %o0, [%i2] + st %o1, [%i2+4] ret restore E SPARC_RET_INT128 - unimp -E SPARC_RET_F_8 - unimp -E SPARC_RET_F_6 - unimp -E SPARC_RET_F_4 - unimp -E SPARC_RET_F_2 - std %f0, [%i2] + std %o0, [%i2] + std %o2, [%i2+8] ret restore -E SPARC_RET_F_3 - unimp +E SPARC_RET_F_8 + st %f7, [%i2+7*4] + nop + st %f6, [%i2+6*4] + nop +E SPARC_RET_F_6 + st %f5, [%i2+5*4] + nop + st %f4, [%i2+4*4] + nop +E SPARC_RET_F_4 + st %f3, [%i2+3*4] + nop + st %f2, [%i2+2*4] + nop +E SPARC_RET_F_2 + st %f1, [%i2+4] + st %f0, [%i2] + ret + restore +E SP_V8_RET_CPLX8 + stb %o0, [%i2+1] + b 10f + srl %o0, 8, %o0 E SPARC_RET_F_1 st %f0, [%i2] ret @@ -172,16 +190,22 @@ E SPARC_RET_F_1 ret restore + .align 8 +9: sth %o0, [%i2] + ret + restore + .align 8 +10: stb %o0, [%i2] + ret + restore + cfi_endproc .size C(ffi_call_v8),. - C(ffi_call_v8) -#undef STACKFRAME -#define STACKFRAME 104 /* 16*4 register window + - 1*4 struct return + - 6*4 args backing store + - 2*4 return storage + - 1*4 alignment */ +/* 16*4 register window + 1*4 struct return + 6*4 args backing store + + 8*4 return storage + 1*4 alignment. */ +#define STACKFRAME (16*4 + 4 + 6*4 + 8*4 + 4) /* ffi_closure_v8(...) @@ -211,7 +235,7 @@ C(ffi_closure_v8): ! Call ffi_closure_sparc_inner to do the bulk of the work. mov %g2, %o0 - add %fp, -8, %o1 + add %fp, -8*4, %o1 call ffi_closure_sparc_inner_v8 add %fp, 64, %o2 @@ -220,8 +244,8 @@ C(ffi_closure_v8): 1: sll %o0, 4, %o0 ! o0 = o0 * 16 add %o7, %o0, %o7 ! o7 = 0b + o0*16 jmp %o7+(2f-0b) - nop - + add %fp, -8*4, %i2 + ! Note that each entry is 4 insns, enforced by the E macro. .align 16 2: @@ -232,47 +256,63 @@ E SPARC_RET_STRUCT jmp %i7+12 restore E SPARC_RET_UINT8 - ldub [%fp-8+3], %i0 + ldub [%i2+3], %i0 ret restore E SPARC_RET_SINT8 - ldsb [%fp-8+3], %i0 + ldsb [%i2+3], %i0 ret restore E SPARC_RET_UINT16 - lduh [%fp-8+2], %i0 + lduh [%i2+2], %i0 ret restore E SPARC_RET_SINT16 - ldsh [%fp-8+2], %i0 + ldsh [%i2+2], %i0 ret restore E SPARC_RET_UINT32 - ld [%fp-8], %i0 + ld [%i2], %i0 + ret + restore +E SP_V8_RET_CPLX16 + ld [%i2], %i0 ret restore -E SPARC_RET_SINT32 - unimp E SPARC_RET_INT64 - ldd [%fp-8], %i0 + ldd [%i2], %i0 ret restore E SPARC_RET_INT128 - unimp + ldd [%i2], %i0 + ldd [%i2+8], %i2 + ret + restore E SPARC_RET_F_8 - unimp + ld [%i2+7*4], %f7 + nop + ld [%i2+6*4], %f6 + nop E SPARC_RET_F_6 - unimp + ld [%i2+5*4], %f5 + nop + ld [%i2+4*4], %f4 + nop E SPARC_RET_F_4 - unimp + ld [%i2+3*4], %f3 + nop + ld [%i2+2*4], %f2 + nop E SPARC_RET_F_2 - ldd [%fp-8], %f0 + ldd [%i2], %f0 + ret + restore +E SP_V8_RET_CPLX8 + lduh [%i2], %i0 ret restore -E SPARC_RET_F_3 - unimp E SPARC_RET_F_1 - ld [%fp-8], %f0 + ld [%i2], %f0 ret restore diff --git a/src/sparc/v9.S b/src/sparc/v9.S index d893d2f..5c3f27b 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -137,7 +137,7 @@ E SPARC_RET_UINT32 srl %o0, 0, %i0 return %i7+8 stx %o0, [%o2] -E SPARC_RET_SINT32 +E SP_V9_RET_SINT32 sra %o0, 0, %i0 return %i7+8 stx %o0, [%o2] @@ -167,7 +167,7 @@ E SPARC_RET_F_4 E SPARC_RET_F_2 return %i7+8 std %f0, [%o2] -E SPARC_RET_F_3 +E SP_V9_RET_F_3 st %f2, [%i2+2*4] nop st %f1, [%i2+1*4] @@ -294,7 +294,7 @@ E SPARC_RET_UINT32 lduw [FP-160+4], %i0 return %i7+8 nop -E SPARC_RET_SINT32 +E SP_V9_RET_SINT32 ldsw [FP-160+4], %i0 return %i7+8 nop @@ -326,7 +326,7 @@ E SPARC_RET_F_2 ldd [FP-160], %f0 return %i7+8 nop -E SPARC_RET_F_3 +E SP_V9_RET_F_3 ld [FP-160+2*4], %f2 nop ld [FP-160+1*4], %f1 diff --git a/testsuite/libffi.call/call.exp b/testsuite/libffi.call/call.exp index 7a5c44f..982c03e 100644 --- a/testsuite/libffi.call/call.exp +++ b/testsuite/libffi.call/call.exp @@ -31,6 +31,7 @@ if { [istarget aarch64*] || [istarget arm*] || [istarget i?86*] || [istarget s390*] + || [istarget sparc*] || [istarget x86_64*] } { run-many-tests $ctlist "" } else { diff --git a/testsuite/libffi.call/complex_int.c b/testsuite/libffi.call/complex_int.c index 4c8e864..bac3190 100644 --- a/testsuite/libffi.call/complex_int.c +++ b/testsuite/libffi.call/complex_int.c @@ -12,9 +12,9 @@ _Complex int f_complex(_Complex int c, int x, int *py) { - c = -(2 * creal (c)) + (cimag (c) + 1)* I; + __real__ c = -2 * __real__ c; + __imag__ c = __imag__ c + 1; *py += x; - return c; } From 92022496ef7a9439f48a2ef11e460c300ac863d7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 26 Oct 2014 14:48:28 -0700 Subject: [PATCH 56/58] sparc: Add support for Go closures --- src/sparc/ffi.c | 45 +++++++++++++++++++++++++++++++++++-------- src/sparc/ffi64.c | 45 +++++++++++++++++++++++++++++++++++-------- src/sparc/ffitarget.h | 1 + src/sparc/v8.S | 38 ++++++++++++++++++++++++++++-------- src/sparc/v9.S | 33 ++++++++++++++++++++++++++----- 5 files changed, 133 insertions(+), 29 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index d319c03..19c3586 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -176,7 +176,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) } extern void ffi_call_v8(ffi_cif *cif, void (*fn)(void), void *rvalue, - void **avalue, size_t bytes) FFI_HIDDEN; + void **avalue, size_t bytes, void *closure) FFI_HIDDEN; int FFI_HIDDEN ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) @@ -280,8 +280,9 @@ ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) return flags; } -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static void +ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) { size_t bytes = cif->bytes; @@ -292,7 +293,20 @@ ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) if (rvalue == NULL && cif->flags == SPARC_RET_STRUCT) bytes += ALIGN (cif->rtype->size, 8); - ffi_call_v8(cif, fn, rvalue, avalue, -bytes); + ffi_call_v8(cif, fn, rvalue, avalue, -bytes, closure); +} + +void +ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int (cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int (cif, fn, rvalue, avalue, closure); } #ifdef __GNUC__ @@ -308,6 +322,7 @@ extern void ffi_flush_icache (void *) FFI_HIDDEN; #endif extern void ffi_closure_v8(void) FFI_HIDDEN; +extern void ffi_go_closure_v8(void) FFI_HIDDEN; ffi_status ffi_prep_closure_loc (ffi_closure *closure, @@ -337,16 +352,30 @@ ffi_prep_closure_loc (ffi_closure *closure, return FFI_OK; } +ffi_status +ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*)) +{ + if (cif->abi != FFI_V8) + return FFI_BAD_ABI; + + closure->tramp = ffi_go_closure_v8; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + int FFI_HIDDEN -ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, +ffi_closure_sparc_inner_v8(ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, void *rvalue, unsigned long *argp) { - ffi_cif *cif; ffi_type **arg_types; void **avalue; int i, nargs, flags; - cif = closure->cif; arg_types = cif->arg_types; nargs = cif->nargs; flags = cif->flags; @@ -424,7 +453,7 @@ ffi_closure_sparc_inner_v8(ffi_closure *closure, void *rvalue, } /* Invoke the closure. */ - (closure->fun) (cif, rvalue, avalue, closure->user_data); + fun (cif, rvalue, avalue, user_data); /* Tell ffi_closure_sparc how to perform return type promotions. */ return flags; diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c index 1e2d3f4..02f3d75 100644 --- a/src/sparc/ffi64.c +++ b/src/sparc/ffi64.c @@ -305,7 +305,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) } extern void ffi_call_v9(ffi_cif *cif, void (*fn)(void), void *rvalue, - void **avalue, size_t bytes) FFI_HIDDEN; + void **avalue, size_t bytes, void *closure) FFI_HIDDEN; /* ffi_prep_args is called by the assembly routine once stack space has been allocated for the function's arguments */ @@ -402,8 +402,9 @@ ffi_prep_args_v9(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) return flags; } -void -ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +static void +ffi_call_int(ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) { size_t bytes = cif->bytes; @@ -412,7 +413,20 @@ ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) if (rvalue == NULL && (cif->flags & SPARC_FLAG_RET_IN_MEM)) bytes += ALIGN (cif->rtype->size, 16); - ffi_call_v9(cif, fn, rvalue, avalue, -bytes); + ffi_call_v9(cif, fn, rvalue, avalue, -bytes, closure); +} + +void +ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) +{ + ffi_call_int(cif, fn, rvalue, avalue, NULL); +} + +void +ffi_call_go(ffi_cif *cif, void (*fn)(void), void *rvalue, + void **avalue, void *closure) +{ + ffi_call_int(cif, fn, rvalue, avalue, closure); } #ifdef __GNUC__ @@ -426,6 +440,7 @@ extern void ffi_flush_icache (void *) FFI_HIDDEN; #endif extern void ffi_closure_v9(void) FFI_HIDDEN; +extern void ffi_go_closure_v9(void) FFI_HIDDEN; ffi_status ffi_prep_closure_loc (ffi_closure* closure, @@ -458,16 +473,30 @@ ffi_prep_closure_loc (ffi_closure* closure, return FFI_OK; } +ffi_status +ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif, + void (*fun)(ffi_cif*, void*, void**, void*)) +{ + if (cif->abi != FFI_V9) + return FFI_BAD_ABI; + + closure->tramp = ffi_go_closure_v9; + closure->cif = cif; + closure->fun = fun; + + return FFI_OK; +} + int FFI_HIDDEN -ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, +ffi_closure_sparc_inner_v9(ffi_cif *cif, + void (*fun)(ffi_cif*, void*, void**, void*), + void *user_data, void *rvalue, unsigned long *gpr, unsigned long *fpr) { - ffi_cif *cif; ffi_type **arg_types; void **avalue; int i, argn, argx, nargs, flags; - cif = closure->cif; arg_types = cif->arg_types; nargs = cif->nargs; flags = cif->flags; @@ -555,7 +584,7 @@ ffi_closure_sparc_inner_v9(ffi_closure *closure, void *rvalue, } /* Invoke the closure. */ - (closure->fun) (cif, rvalue, avalue, closure->user_data); + fun (cif, rvalue, avalue, user_data); /* Tell ffi_closure_sparc how to perform return type promotions. */ return flags; diff --git a/src/sparc/ffitarget.h b/src/sparc/ffitarget.h index f70c937..6982903 100644 --- a/src/sparc/ffitarget.h +++ b/src/sparc/ffitarget.h @@ -63,6 +63,7 @@ typedef enum ffi_abi { /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 +#define FFI_GO_CLOSURES 1 #define FFI_NATIVE_RAW_API 0 #ifdef SPARC64 diff --git a/src/sparc/v8.S b/src/sparc/v8.S index e76d813..66cf76f 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -102,7 +102,7 @@ C(ffi_call_v8): ! Call foreign function call %i1 - nop + mov %i5, %g2 ! load static chain 0: call 1f ! load pc in %o7 sll %l0, 4, %l0 @@ -185,7 +185,7 @@ E SPARC_RET_F_1 ! Struct returning functions expect and skip the unimp here. .align 8 8: call %i1 - nop + mov %i5, %g2 ! load static chain unimp 4 ret restore @@ -211,20 +211,43 @@ E SPARC_RET_F_1 Receives the closure argument in %g2. */ +#ifdef HAVE_AS_REGISTER_PSEUDO_OP + .register %g2, #scratch +#endif + + .align 8 + .globl C(ffi_go_closure_v8) + .type C(ffi_go_closure_v8),@function + FFI_HIDDEN(C(ffi_go_closure_v8)) + +C(ffi_go_closure_v8): + cfi_startproc + save %sp, -STACKFRAME, %sp + cfi_def_cfa_register(%fp) + cfi_window_save + + ld [%g2+4], %o0 ! load cif + ld [%g2+8], %o1 ! load fun + b 0f + mov %g2, %o2 ! load user_data + cfi_endproc + .size C(ffi_go_closure_v8), . - C(ffi_go_closure_v8) + .align 8 .globl C(ffi_closure_v8) .type C(ffi_closure_v8),@function FFI_HIDDEN(C(ffi_closure_v8)) C(ffi_closure_v8): -#ifdef HAVE_AS_REGISTER_PSEUDO_OP - .register %g2, #scratch -#endif cfi_startproc save %sp, -STACKFRAME, %sp cfi_def_cfa_register(%fp) cfi_window_save + ld [%g2+FFI_TRAMPOLINE_SIZE], %o0 ! load cif + ld [%g2+FFI_TRAMPOLINE_SIZE+4], %o1 ! load fun + ld [%g2+FFI_TRAMPOLINE_SIZE+8], %o2 ! load user_data +0: ! Store all of the potential argument registers in va_list format. st %i0, [%fp+68+0] st %i1, [%fp+68+4] @@ -234,10 +257,9 @@ C(ffi_closure_v8): st %i5, [%fp+68+20] ! Call ffi_closure_sparc_inner to do the bulk of the work. - mov %g2, %o0 - add %fp, -8*4, %o1 + add %fp, -8*4, %o3 call ffi_closure_sparc_inner_v8 - add %fp, 64, %o2 + add %fp, 64, %o4 0: call 1f and %o0, SPARC_FLAG_RET_MASK, %o0 diff --git a/src/sparc/v9.S b/src/sparc/v9.S index 5c3f27b..d848f9a 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -94,8 +94,9 @@ C(ffi_call_v9): ldx [%sp+STACK_BIAS+128+16], %o2 ldx [%sp+STACK_BIAS+128+24], %o3 ldx [%sp+STACK_BIAS+128+32], %o4 + ldx [%sp+STACK_BIAS+128+40], %o5 call %i1 - ldx [%sp+STACK_BIAS+128+40], %o5 + mov %i5, %g5 ! load static chain 0: call 1f ! load pc in %o7 and %l0, SPARC_FLAG_RET_MASK, %l1 @@ -210,6 +211,25 @@ E SPARC_RET_F_1 Receives the closure argument in %g1. */ + .align 8 + .globl C(ffi_go_closure_v9) + .type C(ffi_go_closure_v9),@function + FFI_HIDDEN(C(ffi_go_closure_v9)) + +C(ffi_go_closure_v9): + cfi_startproc + save %sp, -STACKFRAME, %sp + cfi_def_cfa_register(%fp) + cfi_window_save + + ldx [%g5+8], %o0 + ldx [%g5+16], %o1 + b 0f + mov %g5, %o2 + + cfi_endproc + .size C(ffi_go_closure_v9), . - C(ffi_go_closure_v9) + .align 8 .globl C(ffi_closure_v9) .type C(ffi_closure_v9),@function @@ -221,6 +241,10 @@ C(ffi_closure_v9): cfi_def_cfa_register(%fp) cfi_window_save + ldx [%g1+FFI_TRAMPOLINE_SIZE], %o0 + ldx [%g1+FFI_TRAMPOLINE_SIZE+8], %o1 + ldx [%g1+FFI_TRAMPOLINE_SIZE+16], %o2 +0: ! Store all of the potential argument registers in va_list format. stx %i0, [FP+128+0] stx %i1, [FP+128+8] @@ -248,11 +272,10 @@ C(ffi_closure_v9): std %f30, [FP-8] ! Call ffi_closure_sparc_inner to do the bulk of the work. - mov %g1, %o0 - add %fp, STACK_BIAS-160, %o1 - add %fp, STACK_BIAS+128, %o2 + add %fp, STACK_BIAS-160, %o3 + add %fp, STACK_BIAS+128, %o4 call C(ffi_closure_sparc_inner_v9) - add %fp, STACK_BIAS-128, %o3 + add %fp, STACK_BIAS-128, %o5 0: call 1f ! load pc in %o7 and %o0, SPARC_FLAG_RET_MASK, %o0 From 6eec410f1512d0584276db5f4b83711330e830d7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 26 Oct 2014 15:29:04 -0700 Subject: [PATCH 57/58] sparc: Re-add abi compliant structure support The original code, removed in the "rewrite" patch, was incorrect for large structures, and required dynamic allocation of a trampoline on every ffi_call. Instead, allocate a 4k entry table of all possible structure returns. The table is 80k, but is read-only and dynamically paged, which ought to be better than allocating the trampoline. This is difficult to test with gcc. One can only use -O0 at present. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63668. --- src/sparc/ffi.c | 19 +++++++++++++------ src/sparc/ffi64.c | 2 +- src/sparc/internal.h | 2 +- src/sparc/v8.S | 31 +++++++++++++++++++++++-------- src/sparc/v9.S | 2 +- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/sparc/ffi.c b/src/sparc/ffi.c index 19c3586..d5212d8 100644 --- a/src/sparc/ffi.c +++ b/src/sparc/ffi.c @@ -66,7 +66,8 @@ ffi_prep_cif_machdep(ffi_cif *cif) break; case FFI_TYPE_LONGDOUBLE: case FFI_TYPE_STRUCT: - flags = SPARC_RET_STRUCT; + flags = (rtype->size & 0xfff) << SPARC_SIZEMASK_SHIFT; + flags |= SPARC_RET_STRUCT; break; case FFI_TYPE_SINT8: flags = SPARC_RET_SINT8; @@ -187,7 +188,7 @@ ffi_prep_args_v8(ffi_cif *cif, unsigned long *argp, void *rvalue, void **avalue) if (rvalue == NULL) { - if (flags == SPARC_RET_STRUCT) + if ((flags & SPARC_FLAG_RET_MASK) == SPARC_RET_STRUCT) { /* Since we pass the pointer to the callee, we need a value. We allowed for this space in ffi_call, before ffi_call_v8 @@ -290,7 +291,8 @@ ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue, /* If we've not got a return value, we need to create one if we've got to pass the return value to the callee. Otherwise ignore it. */ - if (rvalue == NULL && cif->flags == SPARC_RET_STRUCT) + if (rvalue == NULL + && (cif->flags & SPARC_FLAG_RET_MASK) == SPARC_RET_STRUCT) bytes += ALIGN (cif->rtype->size, 8); ffi_call_v8(cif, fn, rvalue, avalue, -bytes, closure); @@ -382,9 +384,14 @@ ffi_closure_sparc_inner_v8(ffi_cif *cif, avalue = alloca(nargs * sizeof(void *)); /* Copy the caller's structure return address so that the closure - returns the data directly to the caller. */ - if (flags == SPARC_RET_STRUCT) - rvalue = (void *)*argp; + returns the data directly to the caller. Also install it so we + can return the address in %o0. */ + if ((flags & SPARC_FLAG_RET_MASK) == SPARC_RET_STRUCT) + { + void *new_rvalue = (void *)*argp; + *(void **)rvalue = new_rvalue; + rvalue = new_rvalue; + } /* Always skip the structure return address. */ argp++; diff --git a/src/sparc/ffi64.c b/src/sparc/ffi64.c index 02f3d75..a4e41d2 100644 --- a/src/sparc/ffi64.c +++ b/src/sparc/ffi64.c @@ -197,7 +197,7 @@ ffi_prep_cif_machdep(ffi_cif *cif) int all_mask = (1 << word_size) - 1; int fp_mask = size_mask >> 8; - flags = (size_mask << SPARC_FLTMASK_SHIFT) | SPARC_RET_STRUCT; + flags = (size_mask << SPARC_SIZEMASK_SHIFT) | SPARC_RET_STRUCT; /* For special cases of all-int or all-fp, we can return the value directly without popping through a struct copy. */ diff --git a/src/sparc/internal.h b/src/sparc/internal.h index f9387d4..0a66472 100644 --- a/src/sparc/internal.h +++ b/src/sparc/internal.h @@ -23,4 +23,4 @@ #define SPARC_FLAG_RET_IN_MEM 32 #define SPARC_FLAG_FP_ARGS 64 -#define SPARC_FLTMASK_SHIFT 8 +#define SPARC_SIZEMASK_SHIFT 8 diff --git a/src/sparc/v8.S b/src/sparc/v8.S index 66cf76f..3a811ef 100644 --- a/src/sparc/v8.S +++ b/src/sparc/v8.S @@ -91,6 +91,7 @@ C(ffi_call_v8): add %sp, 32, %sp ! deallocate prep frame and %o0, SPARC_FLAG_RET_MASK, %l0 ! save return type + srl %o0, SPARC_SIZEMASK_SHIFT, %l1 ! save return size ld [%sp+64+4], %o0 ! load all argument registers ld [%sp+64+8], %o1 ld [%sp+64+12], %o2 @@ -182,14 +183,6 @@ E SPARC_RET_F_1 ret restore - ! Struct returning functions expect and skip the unimp here. - .align 8 -8: call %i1 - mov %i5, %g2 ! load static chain - unimp 4 - ret - restore - .align 8 9: sth %o0, [%i2] ret @@ -199,6 +192,27 @@ E SPARC_RET_F_1 ret restore + ! Struct returning functions expect and skip the unimp here. + ! To make it worse, conforming callees examine the unimp and + ! make sure the low 12 bits of the unimp match the size of + ! the struct being returned. + .align 8 +8: call 1f ! load pc in %o7 + sll %l1, 2, %l0 ! size * 4 +1: sll %l1, 4, %l1 ! size * 16 + add %l0, %l1, %l0 ! size * 20 + add %o7, %l0, %o7 ! o7 = 0b + size*20 + jmp %o7+(2f-8b) + mov %i5, %g2 ! load static chain +2: +.rept 0x1000 + call %i1 + nop + unimp (. - 2b) / 20 + ret + restore +.endr + cfi_endproc .size C(ffi_call_v8),. - C(ffi_call_v8) @@ -275,6 +289,7 @@ E SPARC_RET_VOID ret restore E SPARC_RET_STRUCT + ld [%i2], %i0 jmp %i7+12 restore E SPARC_RET_UINT8 diff --git a/src/sparc/v9.S b/src/sparc/v9.S index d848f9a..52732d3 100644 --- a/src/sparc/v9.S +++ b/src/sparc/v9.S @@ -188,7 +188,7 @@ E SPARC_RET_F_1 std %f6, [%l2+56] ! Copy the structure into place. - srl %l0, SPARC_FLTMASK_SHIFT, %o0 ! load size_mask + srl %l0, SPARC_SIZEMASK_SHIFT, %o0 ! load size_mask mov %i2, %o1 ! load dst mov %l2, %o2 ! load src_gp call C(ffi_struct_float_copy) From 0e303c065779afb42cfdb2ea20c0e1a557dc16f0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 12 Nov 2014 03:58:58 -0800 Subject: [PATCH 58/58] x86: Work around clang bugs http://llvm.org/bugs/show_bug.cgi?21500 http://llvm.org/bugs/show_bug.cgi?21501 http://llvm.org/bugs/show_bug.cgi?21515 --- src/x86/sysv.S | 139 ++++++++++++++++++++++++++--------------------- src/x86/unix64.S | 18 ++++-- 2 files changed, 91 insertions(+), 66 deletions(-) diff --git a/src/x86/sysv.S b/src/x86/sysv.S index e6a8c1e..72cba6c 100644 --- a/src/x86/sysv.S +++ b/src/x86/sysv.S @@ -59,7 +59,12 @@ /* This macro allows the safe creation of jump tables without an actual table. The entry points into the table are all 8 bytes. The use of ORG asserts that we're at the correct location. */ -#define E(X) .align 8; .org 0b + X * 8 +/* ??? The clang assembler doesn't handle .org with symbolic expressions. */ +#ifdef __clang__ +# define E(X) .align 8 +#else +# define E(X) .align 8; .org 0b + X * 8 +#endif .text .align 16 @@ -194,70 +199,75 @@ ENDF(ffi_call_i386) #define closure_FS (16 + 3*4 + 3*4 + 4) -.macro FFI_CLOSURE_SAVE_REGS - movl %eax, 16+R_EAX*4(%esp) - movl %edx, 16+R_EDX*4(%esp) +#define FFI_CLOSURE_SAVE_REGS \ + movl %eax, 16+R_EAX*4(%esp); \ + movl %edx, 16+R_EDX*4(%esp); \ movl %ecx, 16+R_ECX*4(%esp) -.endm -.macro FFI_CLOSURE_COPY_TRAMP_DATA chain - movl FFI_TRAMPOLINE_SIZE(%eax), %edx /* copy cif */ - movl FFI_TRAMPOLINE_SIZE+4(%eax), %ecx /* copy fun */ - movl FFI_TRAMPOLINE_SIZE+8(%eax), %eax /* copy user_data */ - movl %edx, 28(%esp) - movl %ecx, 32(%esp) +#define FFI_CLOSURE_COPY_TRAMP_DATA \ + movl FFI_TRAMPOLINE_SIZE(%eax), %edx; /* copy cif */ \ + movl FFI_TRAMPOLINE_SIZE+4(%eax), %ecx; /* copy fun */ \ + movl FFI_TRAMPOLINE_SIZE+8(%eax), %eax; /* copy user_data */ \ + movl %edx, 28(%esp); \ + movl %ecx, 32(%esp); \ movl %eax, 36(%esp) -.endm -.macro FFI_CLOSURE_CALL_INNER - movl %esp, %ecx /* load closure_data */ - leal closure_FS+4(%esp), %edx /* load incoming stack */ -#ifdef __PIC__ - movl %ebx, 40(%esp) /* save ebx */ - cfi_rel_offset(%ebx, 40) - call __x86.get_pc_thunk.bx /* load got register */ - addl $C(_GLOBAL_OFFSET_TABLE_), %ebx -#endif -#if defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE || !defined __PIC__ - call ffi_closure_inner -#else - call ffi_closure_inner@PLT -#endif -.endm -.macro FFI_CLOSURE_MASK_AND_JUMP - andl $X86_RET_TYPE_MASK, %eax #ifdef __PIC__ - leal 0f@GOTOFF(%ebx, %eax, 8), %eax - movl 40(%esp), %ebx /* restore ebx */ - cfi_restore(%ebx) -#else - leal 0f(, %eax, 8), %eax -#endif +/* We're going to always load the got register here, even if .hidden says + we're going to avoid the PLT call. We'll use the got register in + FFI_CLOSURE_MASK_AND_JUMP. */ +# if defined HAVE_HIDDEN_VISIBILITY_ATTRIBUTE +# define PLT(X) X +# else +# define PLT(X) X@PLT +# endif +# define FFI_CLOSURE_CALL_INNER \ + movl %esp, %ecx; /* load closure_data */ \ + leal closure_FS+4(%esp), %edx; /* load incoming stack */ \ + movl %ebx, 40(%esp); /* save ebx */ \ + cfi_rel_offset(%ebx, 40); \ + call __x86.get_pc_thunk.bx; /* load got register */ \ + addl $C(_GLOBAL_OFFSET_TABLE_), %ebx; \ + call PLT(ffi_closure_inner) +#define FFI_CLOSURE_MASK_AND_JUMP \ + andl $X86_RET_TYPE_MASK, %eax; \ + leal 0f@GOTOFF(%ebx, %eax, 8), %eax; \ + movl 40(%esp), %ebx; /* restore ebx */ \ + cfi_restore(%ebx); \ jmp *%eax -.endm +#else +# define FFI_CLOSURE_CALL_INNER \ + movl %esp, %ecx; /* load closure_data */ \ + leal closure_FS+4(%esp), %edx; /* load incoming stack */ \ + call ffi_closure_inner +#define FFI_CLOSURE_MASK_AND_JUMP \ + andl $X86_RET_TYPE_MASK, %eax; \ + leal 0f(, %eax, 8), %eax; \ + jmp *%eax +#endif /* __PIC__ */ -.macro FFI_GO_CLOSURE suffix, chain, t1, t2 - .align 16 - .globl C(ffi_go_closure_\suffix) - FFI_HIDDEN(C(ffi_go_closure_\suffix)) -C(ffi_go_closure_\suffix): - cfi_startproc - subl $closure_FS, %esp - cfi_adjust_cfa_offset(closure_FS) - FFI_CLOSURE_SAVE_REGS - movl 4(\chain), \t1 /* copy cif */ - movl 8(\chain), \t2 /* copy fun */ - movl \t1, 28(%esp) - movl \t2, 32(%esp) - movl \chain, 36(%esp) /* closure is user_data */ - jmp 88f - cfi_endproc -ENDF(C(ffi_go_closure_\suffix)) -.endm +#define FFI_GO_CLOSURE(suffix, chain, t1, t2) \ + .align 16; \ + .globl C(C1(ffi_go_closure_,suffix)); \ + FFI_HIDDEN(C(C1(ffi_go_closure_,suffix))); \ +C(C1(ffi_go_closure_,suffix)): \ + cfi_startproc; \ + subl $closure_FS, %esp; \ + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ \ + cfi_def_cfa_offset(closure_FS + 4); \ + FFI_CLOSURE_SAVE_REGS; \ + movl 4(chain), t1; /* copy cif */ \ + movl 8(chain), t2; /* copy fun */ \ + movl t1, 28(%esp); \ + movl t2, 32(%esp); \ + movl chain, 36(%esp); /* closure is user_data */ \ + jmp 88f; \ + cfi_endproc; \ +ENDF(C(C1(ffi_go_closure_,suffix))) -FFI_GO_CLOSURE EAX, %eax, %edx, %ecx -FFI_GO_CLOSURE ECX, %ecx, %edx, %eax +FFI_GO_CLOSURE(EAX, %eax, %edx, %ecx) +FFI_GO_CLOSURE(ECX, %ecx, %edx, %eax) /* The closure entry points are reached from the ffi_closure trampoline. On entry, %eax contains the address of the ffi_closure. */ @@ -269,7 +279,8 @@ FFI_GO_CLOSURE ECX, %ecx, %edx, %eax C(ffi_closure_i386): cfi_startproc subl $closure_FS, %esp - cfi_adjust_cfa_offset(closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(closure_FS + 4) FFI_CLOSURE_SAVE_REGS FFI_CLOSURE_COPY_TRAMP_DATA @@ -337,7 +348,7 @@ E(X86_RET_UNUSED15) cfi_endproc ENDF(C(ffi_closure_i386)) -FFI_GO_CLOSURE STDCALL, %ecx, %edx, %eax +FFI_GO_CLOSURE(STDCALL, %ecx, %edx, %eax) /* For REGISTER, we have no available parameter registers, and so we enter here having pushed the closure onto the stack. */ @@ -350,7 +361,8 @@ C(ffi_closure_REGISTER): cfi_def_cfa(%esp, 8) cfi_offset(%eip, -8) subl $closure_FS-4, %esp - cfi_adjust_cfa_offset(closure_FS-4) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(closure_FS + 4) FFI_CLOSURE_SAVE_REGS @@ -372,7 +384,8 @@ ENDF(C(ffi_closure_REGISTER)) C(ffi_closure_STDCALL): cfi_startproc subl $closure_FS, %esp - cfi_adjust_cfa_offset(closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(closure_FS + 4) FFI_CLOSURE_SAVE_REGS @@ -480,7 +493,8 @@ ENDF(C(ffi_closure_STDCALL)) C(ffi_closure_raw_SYSV): cfi_startproc subl $raw_closure_S_FS, %esp - cfi_adjust_cfa_offset(raw_closure_S_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(raw_closure_S_FS + 4) movl %ebx, raw_closure_S_FS-4(%esp) cfi_rel_offset(%ebx, raw_closure_S_FS-4) @@ -575,7 +589,8 @@ C(ffi_closure_raw_THISCALL): /* Rearrange the stack such that %ecx is the first argument. This means moving the return address. */ popl %edx - cfi_adjust_cfa_offset(-4) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(0) cfi_register(%eip, %edx) pushl %ecx cfi_adjust_cfa_offset(4) diff --git a/src/x86/unix64.S b/src/x86/unix64.S index 6066bbf..58cb153 100644 --- a/src/x86/unix64.S +++ b/src/x86/unix64.S @@ -35,9 +35,15 @@ .text +/* This macro allows the safe creation of jump tables without an + actual table. The entry points into the table are all 8 bytes. + The use of ORG asserts that we're at the correct location. */ +/* ??? The clang assembler doesn't handle .org with symbolic expressions. */ .macro E index .align 8 +#ifndef __clang__ .org 0b + \index * 8, 0x90 +#endif .endm /* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags, @@ -220,7 +226,8 @@ E UNIX64_RET_ST_RAX_RDX ffi_closure_unix64_sse: cfi_startproc subq $ffi_closure_FS, %rsp - cfi_adjust_cfa_offset(ffi_closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(ffi_closure_FS + 8) movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp) movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp) @@ -243,7 +250,8 @@ ffi_closure_unix64_sse: ffi_closure_unix64: cfi_startproc subq $ffi_closure_FS, %rsp - cfi_adjust_cfa_offset(ffi_closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(ffi_closure_FS + 8) 0: movq %rdi, ffi_closure_OFS_G+0x00(%rsp) movq %rsi, ffi_closure_OFS_G+0x08(%rsp) @@ -348,7 +356,8 @@ E UNIX64_RET_ST_RAX_RDX ffi_go_closure_unix64_sse: cfi_startproc subq $ffi_closure_FS, %rsp - cfi_adjust_cfa_offset(ffi_closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(ffi_closure_FS + 8) movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp) movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp) @@ -371,7 +380,8 @@ ffi_go_closure_unix64_sse: ffi_go_closure_unix64: cfi_startproc subq $ffi_closure_FS, %rsp - cfi_adjust_cfa_offset(ffi_closure_FS) + /* Note clang bug 21515: adjust_cfa_offset error across endproc. */ + cfi_def_cfa_offset(ffi_closure_FS + 8) 0: movq %rdi, ffi_closure_OFS_G+0x00(%rsp) movq %rsi, ffi_closure_OFS_G+0x08(%rsp)