8000 Change strategy for unlocking pages in the JIT buffer · ruby/ruby@47114c2 · GitHub
[go: up one dir, main page]
Skip to content

Commit 47114c2

Browse files
tenderlovejhawthorn
tenderlove
and
jhawthorn
committed
Change strategy for unlocking pages in the JIT buffer
This commit changes the strategy we use for unlocking pages in the JIT buffer. Rather than unlock the whole JIT buffer, it unlocks just the page that we're trying to write to. It caches the "currently writing" pages in the codeblock so that we can amortize the cost of mprotect across multiple writes to the same code block. The "currently writing" index only stores the last thing we wrote to. So it's possible for multiple OS pages to be unlocked at the same time. We don't want to keep track of every page that has been unlocked, so when we're done writing, the entire JIT buffer is marked as executable. Co-authored-by: John Hawthorn <john@hawthorn.email>
1 parent 88deb89 commit 47114c2

File tree

7 files changed

+47
-50
lines changed

7 files changed

+47
-50
lines changed

‎misc/yjit_asm_tests.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ void run_assembler_tests(void)
8484
uint8_t* mem_block = alloc_exec_mem(4096);
8585
cb_init(cb, mem_block, 4096);
8686

87-
cb_mark_writeable(cb);
8887
// add
8988
cb_set_pos(cb, 0); add(cb, CL, imm_opnd(3)); check_bytes(cb, "80C103");
9089
cb_set_pos(cb, 0); add(cb, CL, BL); check_bytes(cb, "00D9");
@@ -402,7 +401,7 @@ void run_runtime_tests(void)
402401
int (*function)(void);
403402
function = (int (*)(void))mem_block;
404403

405-
#define TEST(BODY) cb_mark_writeable(cb); cb_set_pos(cb, 0); BODY ret(cb); cb_mark_executable(cb); assert_equal(7, function());
404+
#define TEST(BODY) cb_set_pos(cb, 0); BODY ret(cb); cb_mark_all_executable(cb); assert_equal(7, function());
406405

407406
// add
408407
TEST({ mov(cb, RAX, imm_opnd(0)); add(cb, RAX, imm_opnd(7)); })

‎yjit_asm.c

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,15 +211,16 @@ static uint8_t *alloc_exec_mem(uint32_t mem_size)
211211
}
212212

213213
codeblock_t block;
214+
block.current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
214215
block.mem_block = mem_block;
215216
block.mem_size = mem_size;
216217

217218
codeblock_t * cb = &block;
218219
// Fill the executable memory with INT3 (0xCC) so that
219220
// executing uninitialized memory will fault
220-
cb_mark_writeable(cb);
221+
cb_mark_all_writeable(cb);
221222
memset(mem_block, 0xCC, mem_size);
222-
cb_mark_executable(cb);
223+
cb_mark_all_executable(cb);
223224

224225
return mem_block;
225226
#else
@@ -237,6 +238,7 @@ void cb_init(codeblock_t *cb, uint8_t *mem_block, uint32_t mem_size)
237238
cb->write_pos = 0;
238239
cb->num_labels = 0;
239240
cb->num_refs = 0;
241+
cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
240242
}
241243

242244
// Align the current write position to a multiple of bytes
@@ -284,6 +286,7 @@ void cb_write_byte(codeblock_t *cb, uint8_t byte)
284286
{
285287
assert (cb->mem_block);
286288
assert (cb->write_pos + 1 <= cb->mem_size);
289+
cb_mark_position_writeable(cb);
287290
cb->mem_block[cb->write_pos++] = byte;
288291
}
289292

@@ -1779,16 +1782,31 @@ void cb_write_lock_prefix(codeblock_t *cb)
17791782
cb_write_byte(cb, 0xF0);
17801783
}
17811784

1782-
void cb_mark_writeable(codeblock_t * cb)
1785+
void cb_mark_all_writeable(codeblock_t * cb)
17831786
{
17841787
if (mprotect(cb->mem_block, cb->mem_size, PROT_READ | PROT_WRITE)) {
17851788
fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", (void *)cb->mem_block, strerror(errno));
17861789
abort();
17871790
}
17881791
}
17891792

1790-
void cb_mark_executable(codeblock_t * cb)
1793+
void cb_mark_position_writeable(codeblock_t * cb)
17911794
{
1795+
uint32_t pagesize = (uint32_t)sysconf(_SC_PAGESIZE);
1796+
uint32_t aligned_position = (cb->write_pos / pagesize) * pagesize;
1797+
1798+
if (cb->current_aligned_write_pos != aligned_position) {
1799+
cb->current_aligned_write_pos = aligned_position;
1800+
if (mprotect(cb->mem_block + aligned_position, pagesize, PROT_READ | PROT_WRITE)) {
1801+
fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", (void *)(cb->mem_block + aligned_position), strerror(errno));
1802+
abort();
1803+
}
1804+
}
1805+
}
1806+
1807+
void cb_mark_all_executable(codeblock_t * cb)
1808+
{
1809+
cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
17921810
if (mprotect(cb->mem_block, cb->mem_size, PROT_READ | PROT_EXEC)) {
17931811
fprintf(stderr, "Couldn't make JIT page (%p) executable, errno: %s", (void *)cb->mem_block, strerror(errno));
17941812
abort();

‎yjit_asm.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,14 @@ typedef struct CodeBlock
5555
// Flag to enable or disable comments
5656
bool has_asm;
5757

58+
// Keep track of the current aligned write position.
59+
// Used for changing protection when writing to the JIT buffer
60+
uint32_t current_aligned_write_pos;
5861
} codeblock_t;
5962

63+
// 1 is not aligned so this won't match any pages
64+
#define ALIGNED_WRITE_POSITION_NONE 1
65+
6066
enum OpndType
6167
{
6268
OPND_NONE,
@@ -261,8 +267,9 @@ static inline uint32_t cb_new_label(codeblock_t *cb, const char *name);
261267
static inline void cb_write_label(codeblock_t *cb, uint32_t label_idx);
262268
static inline void cb_label_ref(codeblock_t *cb, uint32_t label_idx);
263269
static inline void cb_link_labels(codeblock_t *cb);
264-
static inline void cb_mark_writeable(codeblock_t *cb);
265-
static inline void cb_mark_executable(codeblock_t *cb);
270+
static inline void cb_mark_all_writeable(codeblock_t *cb);
271+
static inline void cb_mark_position_writeable(codeblock_t *cb);
272+
static inline void cb_mark_all_executable(codeblock_t *cb);
266273

267274
// Encode individual instructions into a code block
268275
static inline void add(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1);

‎yjit_codegen.c

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -475,8 +475,6 @@ gen_full_cfunc_return(void)
475475
codeblock_t *cb = ocb;
476476
outline_full_cfunc_return_pos = ocb->write_pos;
477477

478-
rb_yjit_cb_mark_writeable(cb);
479-
480478
// This chunk of code expect REG_EC to be filled properly and
481479
// RAX to contain the return value of the C method.
482480

@@ -495,7 +493,6 @@ gen_full_cfunc_return(void)
495493

496494
mov(cb, RAX, imm_opnd(Qundef));
497495
ret(cb);
498-
cb_mark_executable(cb);
499496
}
500497

501498
/*
@@ -4656,9 +4653,6 @@ rb_yjit_tracing_invalidate_all(void)
46564653
RB_VM_LOCK_ENTER();
46574654
rb_vm_barrier();
46584655

4659-
rb_yjit_cb_mark_writeable(cb);
4660-
rb_yjit_cb_mark_writeable(ocb);
4661-
46624656
// Make it so all live block versions are no longer valid branch targets
46634657
rb_objspace_each_objects(tracing_invalidate_all_i, NULL);
46644658

@@ -4680,8 +4674,8 @@ rb_yjit_tracing_invalidate_all(void)
46804674
RUBY_ASSERT_ALWAYS(yjit_codepage_frozen_bytes <= old_pos && "frozen bytes should increase monotonically");
46814675
yjit_codepage_frozen_bytes = old_pos;
46824676

4683-
cb_mark_executable(ocb);
4684-
cb_mark_executable(cb);
4677+
cb_mark_all_executable(ocb);
4678+
cb_mark_all_executable(cb);
46854679
RB_VM_LOCK_LEAVE();
46864680
}
46874681

@@ -4753,22 +4747,17 @@ yjit_init_codegen(void)
47534747
uint8_t *mem_block = alloc_exec_mem(mem_size);
47544748

47554749
cb = &block;
4756-
cb_mark_writeable(cb);
47574750
cb_init(cb, mem_block, mem_size/2);
4758-
cb_mark_executable(cb);
47594751

47604752
ocb = &outline_block;
4761-
cb_mark_writeable(ocb);
47624753
cb_init(ocb, mem_block + mem_size/2, mem_size/2);
4763-
cb_mark_executable(ocb);
47644754

47654755
// Generate the interpreter exit code for leave
4766-
cb_mark_writeable(cb);
47674756
leave_exit_code = yjit_gen_leave_exit(cb);
47684757

47694758
// Generate full exit code for C func
47704759
gen_full_cfunc_return();
4771-
cb_mark_writeable(cb);
4760+
cb_mark_all_executable(cb);
47724761

47734762
// Map YARV opcodes to the corresponding codegen functions
47744763
yjit_reg_op(BIN(nop), gen_nop);

‎yjit_core.c

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -757,16 +757,15 @@ gen_entry_point(const rb_iseq_t *iseq, uint32_t insn_idx, rb_execution_context_t
757757
// The entry context makes no assumptions about types
758758
blockid_t blockid = { iseq, insn_idx };
759759

760-
rb_yjit_cb_mark_writeable(cb);
761-
rb_yjit_cb_mark_writeable(ocb);
760+
rb_vm_barrier();
762761
// Write the interpreter entry prologue
763762
uint8_t *code_ptr = yjit_entry_prologue(cb, iseq);
764763

765764
// Try to generate code for the entry block
766765
block_t *block = gen_block_version(blockid, &DEFAULT_CTX, ec);
767766

768-
cb_mark_executable(ocb);
769-
cb_mark_executable(cb);
767+
cb_mark_all_executable(ocb);
768+
cb_mark_all_executable(cb);
770769

771770
// If we couldn't generate any code
772771
if (block->end_idx == insn_idx)
@@ -801,8 +800,7 @@ branch_stub_hit(branch_t *branch, const uint32_t target_idx, rb_execution_contex
801800
}
802801
else
803802
{
804-
rb_yjit_cb_mark_writeable(cb);
805-
rb_yjit_cb_mark_writeable(ocb);
803+
rb_vm_barrier();
806804
//fprintf(stderr, "\nstub hit, branch: %p, target idx: %d\n", branch, target_idx);
807805
//fprintf(stderr, "blockid.iseq=%p, blockid.idx=%d\n", target.iseq, target.idx);
808806
//fprintf(stderr, "chain_depth=%d\n", target_ctx->chain_depth);
@@ -870,8 +868,8 @@ branch_stub_hit(branch_t *branch, const uint32_t target_idx, rb_execution_contex
870868
// Restore interprete 10000 r sp, since the code hitting the stub expects the original.
871869
ec->cfp->sp = original_interp_sp;
872870

873-
cb_mark_executable(ocb);
874-
cb_mark_executable(cb);
871+
cb_mark_all_executable(ocb);
872+
cb_mark_all_executable(cb);
875873
}
876874

877875
RB_VM_LOCK_LEAVE();
@@ -1111,8 +1109,6 @@ static void
11111109
invalidate_block_version(block_t *block)
11121110
{
11131111
ASSERT_vm_locking();
1114-
rb_yjit_cb_mark_writeable(cb);
1115-
rb_yjit_cb_mark_writeable(ocb);
11161112

11171113
// TODO: want to assert that all other ractors are stopped here. Can't patch
11181114
// machine code that some other thread is running.
@@ -1207,8 +1203,8 @@ invalidate_block_version(block_t *block)
12071203
yjit_runtime_counters.invalidation_count++;
12081204
#endif
12091205

1210-
cb_mark_executable(ocb);
1211-
cb_mark_executable(cb);
1206+
cb_mark_all_executable(ocb);
1207+
cb_mark_all_executable(cb);
12121208

12131209
// fprintf(stderr, "invalidation done\n");
12141210
}

‎yjit_iface.c

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -899,8 +899,8 @@ rb_yjit_iseq_update_references(const struct rb_iseq_constant_body *body)
899899
return;
900900
}
901901

902-
rb_yjit_cb_mark_writeable(cb);
903-
rb_yjit_cb_mark_writeable(ocb);
902+
rb_vm_barrier();
903+
904904
rb_darray_for(body->yjit_blocks, version_array_idx) {
905905
rb_yjit_block_array_t version_array = rb_darray_get(body->yjit_blocks, version_array_idx);
906906

@@ -942,8 +942,8 @@ rb_yjit_iseq_update_refe 1023B rences(const struct rb_iseq_constant_body *body)
942942
//block->code_page = rb_gc_location(block->code_page);
943943
}
944944
}
945-
cb_mark_executable(cb);
946-
cb_mark_executable(ocb);
945+
cb_mark_all_executable(cb);
946+
cb_mark_all_executable(ocb);
947947
}
948948

949949
// Free the yjit resources associated with an iseq
@@ -1128,17 +1128,6 @@ yjit_get_code_page(uint32_t cb_bytes_needed, uint32_t ocb_bytes_needed)
11281128
return yjit_cur_code_page;
11291129
}
11301130

1131-
// Mark a codeblock as writeable, but calls rb_vm_barrier to ensure that
1132-
// only one ractor is executing. Callers of this function need to call
1133-
// RB_VM_LOCK_ENTER()
1134-
static void
1135-
rb_yjit_cb_mark_writeable(codeblock_t *cb)
1136-
{
1137-
ASSERT_vm_locking();
1138-
rb_vm_barrier();
1139-
cb_mark_writeable(cb);
1140-
}
1141-
11421131
bool
11431132
rb_yjit_enabled_p(void)
11441133
{

‎yjit_iface.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,5 @@ static VALUE yjit_get_code_page(uint32_t cb_bytes_needed, uint32_t ocb_bytes_nee
3434
//code_page_t *rb_yjit_code_page_unwrap(VALUE cp_obj);
3535
//void rb_yjit_get_cb(codeblock_t* cb, uint8_t* code_ptr);
3636
//void rb_yjit_get_ocb(codeblock_t* cb, uint8_t* code_ptr);
37-
static void rb_yjit_cb_mark_writeable(codeblock_t *cb);
3837

3938
#endif // #ifndef YJIT_IFACE_H

0 commit comments

Comments
 (0)
0