1515#include "ceval_macros.h"
1616#include "jit_stencils.h"
1717
18- #ifdef MS_WINDOWS
19- #include <psapi.h>
20- #include <windows.h>
21- #else
18+ #ifndef MS_WINDOWS
2219 #include <sys/mman.h>
2320#endif
2421
@@ -46,121 +43,84 @@ alloc(size_t size)
4643static int initialized = 0 ;
4744
4845static void
49- patch_one (unsigned char * location , HoleKind kind , uint64_t value , uint64_t addend )
46+ patch_one (unsigned char * location , HoleKind kind , uint64_t patch )
5047{
48+ uint32_t * addr = (uint32_t * )location ;
5149 switch (kind ) {
5250 case R_386_32 : {
53- uint32_t * addr = (uint32_t * )location ;
54- uint32_t instruction = * addr ;
55- instruction = (uint32_t )(value + addend );
56- * addr = instruction ;
51+ * addr = (uint32_t )patch ;
5752 return ;
5853 }
5954 case R_386_PC32 :
6055 case R_X86_64_PC32 :
6156 case R_X86_64_PLT32 : {
62- uint32_t * addr = (uint32_t * )location ;
63- uint32_t instruction = * addr ;
64- instruction = (uint32_t )(value + addend - (uintptr_t )location );
65- * addr = instruction ;
57+ patch -= (uintptr_t )location ;
58+ * addr = (uint32_t )patch ;
6659 return ;
6760 }
6861 case R_AARCH64_ABS64 :
6962 case R_X86_64_64 : {
70- uint64_t * addr = (uint64_t * )location ;
71- uint64_t instruction = * addr ;
72- instruction = value + addend ;
73- * addr = instruction ;
63+ * (uint64_t * )addr = patch ;
7464 return ;
7565 }
7666 case R_AARCH64_ADR_GOT_PAGE : {
77- uint32_t * addr = (uint32_t * )location ;
78- uint32_t instruction = * addr ;
79- assert ((instruction & 0x9F000000 ) == 0x90000000 );
80- value = (((value + addend ) >> 12 ) << 12 ) - (((uintptr_t )location >> 12 ) << 12 );
81- assert ((value & 0xFFF ) == 0 );
82- // assert((value & ((1ULL << 33) - 1)) == value); // XXX: This should be signed.
83- uint32_t lo = ((uint64_t )value << 17 ) & 0x60000000 ;
84- uint32_t hi = ((uint64_t )value >> 9 ) & 0x00FFFFE0 ;
85- instruction = (instruction & 0x9F00001F ) | hi | lo ;
86- assert ((instruction & 0x9F000000 ) == 0x90000000 );
87- * addr = instruction ;
67+ patch = ((patch >> 12 ) << 12 ) - (((uintptr_t )location >> 12 ) << 12 );
68+ assert ((* addr & 0x9F000000 ) == 0x90000000 );
69+ assert ((patch & 0xFFF ) == 0 );
70+ // assert((patch & ((1ULL << 33) - 1)) == patch); // XXX: This should be signed.
71+ uint32_t lo = (patch << 17 ) & 0x60000000 ;
72+ uint32_t hi = (patch >> 9 ) & 0x00FFFFE0 ;
73+ * addr = (* addr & 0x9F00001F ) | hi | lo ;
8874 return ;
8975 }
9076 case R_AARCH64_CALL26 :
9177 case R_AARCH64_JUMP26 : {
92- uint32_t * addr = (uint32_t * )location ;
93- uint32_t instruction = * addr ;
94- assert (((instruction & 0xFC000000 ) == 0x14000000 ) ||
95- ((instruction & 0xFC000000 ) == 0x94000000 ));
96- value = value + addend - (uintptr_t )location ;
97- assert ((value & 0x3 ) == 0 );
98- // assert((value & ((1ULL << 29) - 1)) == value); // XXX: This should be signed.
99- instruction = (instruction & 0xFC000000 ) | ((uint32_t )(value >> 2 ) & 0x03FFFFFF );
100- assert (((instruction & 0xFC000000 ) == 0x14000000 ) ||
101- ((instruction & 0xFC000000 ) == 0x94000000 ));
102- * addr = instruction ;
78+ patch -= (uintptr_t )location ;
79+ assert (((* addr & 0xFC000000 ) == 0x14000000 ) ||
80+ ((* addr & 0xFC000000 ) == 0x94000000 ));
81+ assert ((patch & 0x3 ) == 0 );
82+ // assert((patch & ((1ULL << 29) - 1)) == patch); // XXX: This should be signed.
83+ * addr = (* addr & 0xFC000000 ) | ((uint32_t )(patch >> 2 ) & 0x03FFFFFF );
10384 return ;
10485 }
10586 case R_AARCH64_LD64_GOT_LO12_NC : {
106- uint32_t * addr = (uint32_t * )location ;
107- uint32_t instruction = * addr ;
108- assert (((instruction & 0x3B000000 ) == 0x39000000 ) ||
109- ((instruction & 0x11C00000 ) == 0x11000000 ));
110- value = (value + addend ) & ((1 << 12 ) - 1 );
111- int implicit_shift = 0 ;
112- if ((instruction & 0x3B000000 ) == 0x39000000 ) {
113- implicit_shift = ((instruction >> 30 ) & 0x3 );
114- if (implicit_shift == 0 && (instruction & 0x04800000 ) == 0x04800000 ) {
115- implicit_shift = 4 ;
87+ patch &= (1 << 12 ) - 1 ;
88+ assert (((* addr & 0x3B000000 ) == 0x39000000 ) ||
89+ ((* addr & 0x11C00000 ) == 0x11000000 ));
90+ int shift = 0 ;
91+ if ((* addr & 0x3B000000 ) == 0x39000000 ) {
92+ shift = ((* addr >> 30 ) & 0x3 );
93+ if (shift == 0 && (* addr & 0x04800000 ) == 0x04800000 ) {
94+ shift = 4 ;
11695 }
11796 }
118- assert (((value & ((1 << implicit_shift ) - 1 )) == 0 ));
119- value >>= implicit_shift ;
120- assert ((value & ((1 << 12 ) - 1 )) == value );
121- instruction = (instruction & 0xFFC003FF ) | ((uint32_t )(value << 10 ) & 0x003FFC00 );
122- assert (((instruction & 0x3B000000 ) == 0x39000000 ) ||
123- ((instruction & 0x11C00000 ) == 0x11000000 ));
124- * addr = instruction ;
97+ assert (((patch & ((1 << shift ) - 1 )) == 0 ));
98+ * addr = (* addr & 0xFFC003FF ) | ((uint32_t )((patch >> shift ) << 10 ) & 0x003FFC00 );
12599 return ;
126100 }
127101 case R_AARCH64_MOVW_UABS_G0_NC : {
128- uint32_t * addr = (uint32_t * )location ;
129- uint32_t instruction = * addr ;
130- assert (((instruction >> 21 ) & 0x3 ) == 0 );
131- instruction = (instruction & 0xFFE0001F ) | ((((value + addend ) >> 0 ) & 0xFFFF ) << 5 );
132- * addr = instruction ;
102+ assert (((* addr >> 21 ) & 0x3 ) == 0 );
103+ * addr = (* addr & 0xFFE0001F ) | (((patch >>
10000
0 ) & 0xFFFF ) << 5 );
133104 return ;
134105 }
135106 case R_AARCH64_MOVW_UABS_G1_NC : {
136- uint32_t * addr = (uint32_t * )location ;
137- uint32_t instruction = * addr ;
138- assert (((instruction >> 21 ) & 0x3 ) == 1 );
139- instruction = (instruction & 0xFFE0001F ) | ((((value + addend ) >> 16 ) & 0xFFFF ) << 5 );
140- * addr = instruction ;
107+ assert (((* addr >> 21 ) & 0x3 ) == 1 );
108+ * addr = (* addr & 0xFFE0001F ) | (((patch >> 16 ) & 0xFFFF ) << 5 );
141109 return ;
142110 }
143111 case R_AARCH64_MOVW_UABS_G2_NC : {
144- uint32_t * addr = (uint32_t * )location ;
145- uint32_t instruction = * addr ;
146- assert (((instruction >> 21 ) & 0x3 ) == 2 );
147- instruction = (instruction & 0xFFE0001F ) | ((((value + addend ) >> 32 ) & 0xFFFF ) << 5 );
148- * addr = instruction ;
112+ assert (((* addr >> 21 ) & 0x3 ) == 2 );
113+ * addr = (* addr & 0xFFE0001F ) | (((patch >> 32 ) & 0xFFFF ) << 5 );
149114 return ;
150115 }
151116 case R_AARCH64_MOVW_UABS_G3 : {
152- uint32_t * addr = (uint32_t * )location ;
153- uint32_t instruction = * addr ;
154- assert (((instruction >> 21 ) & 0x3 ) == 3 );
155- instruction = (instruction & 0xFFE0001F ) | ((((value + addend ) >> 48 ) & 0xFFFF ) << 5 );
156- * addr = instruction ;
117+ assert (((* addr >> 21 ) & 0x3 ) == 3 );
118+ * addr = (* addr & 0xFFE0001F ) | (((patch >> 48 ) & 0xFFFF ) << 5 );
157119 return ;
158120 }
159121 case R_X86_64_GOTOFF64 : {
160- uint64_t * addr = (uint64_t * )location ;
161- uint64_t instruction = * addr ;
162- instruction = value + addend - (uintptr_t )location ;
163- * addr = instruction ;
122+ patch -= (uintptr_t )location ;
123+ * addr = patch ;
164124 return ;
165125 }
166126 case R_X86_64_GOTPC32 :
@@ -178,12 +138,8 @@ copy_and_patch(unsigned char *memory, const Stencil *stencil, uint64_t patches[]
178138 memcpy (memory , stencil -> bytes , stencil -> nbytes );
179139 for (size_t i = 0 ; i < stencil -> nholes ; i ++ ) {
180140 const Hole * hole = & stencil -> holes [i ];
181- patch_one (memory + hole -> offset , hole -> kind , patches [hole -> value ], hole -> addend );
182- }
183- for (size_t i = 0 ; i < stencil -> nloads ; i ++ ) {
184- const SymbolLoad * load = & stencil -> loads [i ];
185- uint64_t value = load -> symbol ;
186- patch_one (memory + load -> offset , load -> kind , value , load -> addend );
141+ uint64_t patch = patches [hole -> value ] + hole -> addend ;
142+ patch_one (memory + hole -> offset , hole -> kind , patch );
187143 }
188144}
189145
@@ -232,13 +188,8 @@ _PyJIT_CompileTrace(_PyUOpInstruction *trace, int size)
232188 offsets [i ] = nbytes ;
233189 _PyUOpInstruction * instruction = & trace [i ];
234190 const Stencil * stencil = & stencils [instruction -> opcode ];
235- // XXX: Assert this once we support everything, and move initialization
236- // to interpreter startup. Then we can only fail due to memory stuff:
237- if (stencil -> nbytes == 0 ) {
238- PyMem_Free (offsets );
239- return NULL ;
240- }
241191 nbytes += stencil -> nbytes ;
192+ assert (stencil -> nbytes );
242193 };
243194 unsigned char * memory = alloc (nbytes );
244195 if (memory == NULL ) {
@@ -270,6 +221,7 @@ _PyJIT_CompileTrace(_PyUOpInstruction *trace, int size)
270221 patches [_JIT_CONTINUE ] = (uintptr_t )head + offsets [0 ];
271222 patches [_JIT_CONTINUE_OPARG ] = instruction_continue -> oparg ;
272223 patches [_JIT_CONTINUE_OPERAND ] = instruction_continue -> operand ;
224+ patches [_JIT_ZERO ] = 0 ;
273225 copy_and_patch (head , stencil , patches );
274226 head += stencil -> nbytes ;
275227 // Then, all of the stencils:
@@ -286,6 +238,7 @@ _PyJIT_CompileTrace(_PyUOpInstruction *trace, int size)
286238 patches [_JIT_JUMP ] = (uintptr_t )memory + offsets [instruction -> oparg % size ];
287239 patches [_JIT_JUMP_OPARG ] = instruction_jump -> oparg ;
288240 patches [_JIT_JUMP_OPERAND ] = instruction_jump -> operand ;
241+ patches [_JIT_ZERO ] = 0 ;
289242 copy_and_patch (head , stencil , patches );
290243 head += stencil -> nbytes ;
291244 };
@@ -307,4 +260,4 @@ _PyJIT_CompileTrace(_PyUOpInstruction *trace, int size)
307260 // Wow, done already?
308261 assert (memory + nbytes == head );
309262 return (_PyJITFunction )memory ;
310- }
263+ }
0 commit comments