diff --git a/ext/objspace/objspace_dump.c b/ext/objspace/objspace_dump.c index c29cecf7bd005c..ac8bafaea9c764 100644 --- a/ext/objspace/objspace_dump.c +++ b/ext/objspace/objspace_dump.c @@ -817,9 +817,6 @@ shape_id_i(shape_id_t shape_id, void *data) dump_append(dc, ",\"edge_name\":"); dump_append_id(dc, shape->edge_name); - break; - case SHAPE_T_OBJECT: - dump_append(dc, ", \"shape_type\":\"T_OBJECT\""); break; case SHAPE_OBJ_ID: dump_append(dc, ", \"shape_type\":\"OBJ_ID\""); diff --git a/gc.c b/gc.c index 9d314c74166a77..3e7d88209f364b 100644 --- a/gc.c +++ b/gc.c @@ -381,19 +381,9 @@ rb_gc_set_shape(VALUE obj, uint32_t shape_id) uint32_t rb_gc_rebuild_shape(VALUE obj, size_t heap_id) { - shape_id_t orig_shape_id = rb_obj_shape_id(obj); - if (rb_shape_too_complex_p(orig_shape_id)) { - return (uint32_t)orig_shape_id; - } - - shape_id_t initial_shape_id = rb_shape_root(heap_id); - shape_id_t new_shape_id = rb_shape_traverse_from_new_root(initial_shape_id, orig_shape_id); - - if (new_shape_id == INVALID_SHAPE_ID) { - return 0; - } + RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); - return (uint32_t)new_shape_id; + return (uint32_t)rb_shape_transition_heap(obj, heap_id); } void rb_vm_update_references(void *ptr); diff --git a/internal/variable.h b/internal/variable.h index fa27b1ef5c5187..a0608b22d17261 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -52,7 +52,7 @@ int rb_gen_fields_tbl_get(VALUE obj, ID id, struct gen_fields_tbl **fields_tbl); void rb_obj_copy_ivs_to_hash_table(VALUE obj, st_table *table); void rb_obj_init_too_complex(VALUE obj, st_table *table); void rb_evict_ivars_to_hash(VALUE obj); -void rb_evict_fields_to_hash(VALUE obj); +shape_id_t rb_evict_fields_to_hash(VALUE obj); VALUE rb_obj_field_get(VALUE obj, shape_id_t target_shape_id); void rb_ivar_set_internal(VALUE obj, ID id, VALUE val); void rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val); diff --git a/object.c b/object.c index cee423cc199e8e..fbd2f5d5570132 100644 --- a/object.c +++ b/object.c @@ -339,17 +339,15 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj) shape_id_t dest_shape_id = src_shape_id; shape_id_t initial_shape_id = RBASIC_SHAPE_ID(dest); - if (RSHAPE(initial_shape_id)->heap_index != RSHAPE(src_shape_id)->heap_index || !rb_shape_canonical_p(src_shape_id)) { - RUBY_ASSERT(RSHAPE(initial_shape_id)->type == SHAPE_T_OBJECT); + RUBY_ASSERT(RSHAPE(initial_shape_id)->type == SHAPE_ROOT); - dest_shape_id = rb_shape_rebuild(initial_shape_id, src_shape_id); - if (UNLIKELY(rb_shape_too_complex_p(dest_shape_id))) { - st_table *table = rb_st_init_numtable_with_size(src_num_ivs); - rb_obj_copy_ivs_to_hash_table(obj, table); - rb_obj_init_too_complex(dest, table); + dest_shape_id = rb_shape_rebuild(initial_shape_id, src_shape_id); + if (UNLIKELY(rb_shape_too_complex_p(dest_shape_id))) { + st_table *table = rb_st_init_numtable_with_size(src_num_ivs); + rb_obj_copy_ivs_to_hash_table(obj, table); + rb_obj_init_too_complex(dest, table); - return; - } + return; } VALUE *src_buf = ROBJECT_FIELDS(obj); diff --git a/shape.c b/shape.c index becd1aa8c5c1b7..668850cdd46dec 100644 --- a/shape.c +++ b/shape.c @@ -37,8 +37,6 @@ static ID id_frozen; static ID id_t_object; ID ruby_internal_object_id; // extern -static const attr_index_t *shape_capacities = NULL; - #define LEAF 0 #define BLACK 0x0 #define RED 0x1 @@ -306,14 +304,7 @@ shape_tree_mark(void *data) rb_shape_t *end = RSHAPE(GET_SHAPE_TREE()->next_shape_id - 1); while (cursor < end) { if (cursor->edges && !SINGLE_CHILD_P(cursor->edges)) { - // FIXME: GC compaction may call `rb_shape_traverse_from_new_root` - // to migrate objects from one object slot to another. - // Because of this if we don't pin `cursor->edges` it might be turned - // into a T_MOVED during GC. - // We'd need to eliminate `SHAPE_T_OBJECT` so that GC never need to lookup - // shapes this way. - // rb_gc_mark_movable(cursor->edges); - rb_gc_mark(cursor->edges); + rb_gc_mark_movable(cursor->edges); } cursor++; } @@ -357,18 +348,14 @@ static const rb_data_type_t shape_tree_type = { static inline shape_id_t raw_shape_id(rb_shape_t *shape) { - if (shape == NULL) { - return INVALID_SHAPE_ID; - } + RUBY_ASSERT(shape); return (shape_id_t)(shape - GET_SHAPE_TREE()->shape_list); } static inline shape_id_t shape_id(rb_shape_t *shape, shape_id_t previous_shape_id) { - if (shape == NULL) { - return INVALID_SHAPE_ID; - } + RUBY_ASSERT(shape); shape_id_t raw_id = (shape_id_t)(shape - GET_SHAPE_TREE()->shape_list); return raw_id | (previous_shape_id & SHAPE_ID_FLAGS_MASK); } @@ -457,7 +444,6 @@ rb_shape_alloc(ID edge_name, rb_shape_t *parent, enum shape_type type) { rb_shape_t *shape = rb_shape_alloc_with_parent_id(edge_name, raw_shape_id(parent)); shape->type = (uint8_t)type; - shape->heap_index = parent->heap_index; shape->capacity = parent->capacity; shape->edges = 0; return shape; @@ -501,7 +487,7 @@ redblack_cache_ancestors(rb_shape_t *shape) static attr_index_t shape_grow_capa(attr_index_t current_capa) { - const attr_index_t *capacities = shape_capacities; + const attr_index_t *capacities = GET_SHAPE_TREE()->capacities; // First try to use the next size that will be embeddable in a larger object slot. attr_index_t capa; @@ -536,7 +522,6 @@ rb_shape_alloc_new_child(ID id, rb_shape_t *shape, enum shape_type shape_type) } break; case SHAPE_ROOT: - case SHAPE_T_OBJECT: rb_bug("Unreachable"); break; } @@ -723,15 +708,70 @@ remove_shape_recursive(rb_shape_t *shape, ID id, rb_shape_t **removed_shape) } } +static inline shape_id_t transition_complex(shape_id_t shape_id); + +static shape_id_t +shape_transition_object_id(shape_id_t original_shape_id) +{ + RUBY_ASSERT(!rb_shape_has_object_id(original_shape_id)); + + bool dont_care; + rb_shape_t *shape = get_next_shape_internal(RSHAPE(original_shape_id), ruby_internal_object_id, SHAPE_OBJ_ID, &dont_care, true); + if (!shape) { + shape = RSHAPE(ROOT_SHAPE_WITH_OBJ_ID); + } + + RUBY_ASSERT(shape); + return shape_id(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID; +} + +shape_id_t +rb_shape_transition_object_id(VALUE obj) +{ + return shape_transition_object_id(RBASIC_SHAPE_ID(obj)); +} + +shape_id_t +rb_shape_object_id(shape_id_t original_shape_id) +{ + RUBY_ASSERT(rb_shape_has_object_id(original_shape_id)); + + rb_shape_t *shape = RSHAPE(original_shape_id); + while (shape->type != SHAPE_OBJ_ID) { + if (UNLIKELY(shape->parent_id == INVALID_SHAPE_ID)) { + rb_bug("Missing object_id in shape tree"); + } + shape = RSHAPE(shape->parent_id); + } + + return shape_id(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID; +} + static inline shape_id_t transition_complex(shape_id_t shape_id) { - if (rb_shape_has_object_id(shape_id)) { - return ROOT_TOO_COMPLEX_WITH_OBJ_ID | (shape_id & SHAPE_ID_FLAGS_MASK); + uint8_t heap_index = rb_shape_heap_index(shape_id); + shape_id_t next_shape_id; + + if (heap_index) { + next_shape_id = rb_shape_root(heap_index - 1) | SHAPE_ID_FL_TOO_COMPLEX; + if (rb_shape_has_object_id(shape_id)) { + next_shape_id = shape_transition_object_id(next_shape_id); + } + } + else { + if (rb_shape_has_object_id(shape_id)) { + next_shape_id = ROOT_TOO_COMPLEX_WITH_OBJ_ID | (shape_id & SHAPE_ID_FLAGS_MASK); + } + else { + next_shape_id = ROOT_TOO_COMPLEX_SHAPE_ID | (shape_id & SHAPE_ID_FLAGS_MASK); + } } - return ROOT_TOO_COMPLEX_SHAPE_ID | (shape_id & SHAPE_ID_FLAGS_MASK); -} + RUBY_ASSERT(rb_shape_has_object_id(shape_id) == rb_shape_has_object_id(next_shape_id)); + + return next_shape_id; +} shape_id_t rb_shape_transition_remove_ivar(VALUE obj, ID id, shape_id_t *removed_shape_id) @@ -754,7 +794,9 @@ rb_shape_transition_remove_ivar(VALUE obj, ID id, shape_id_t *removed_shape_id) else if (removed_shape) { // We found the shape to remove, but couldn't create a new variation. // We must transition to TOO_COMPLEX. - return transition_complex(original_shape_id); + shape_id_t next_shape_id = transition_complex(original_shape_id); + RUBY_ASSERT(rb_shape_has_object_id(next_shape_id) == rb_shape_has_object_id(original_shape_id)); + return next_shape_id; } return original_shape_id; } @@ -775,38 +817,9 @@ rb_shape_transition_complex(VALUE obj) } shape_id_t -rb_shape_transition_object_id(VALUE obj) +rb_shape_transition_heap(VALUE obj, size_t heap_index) { - shape_id_t original_shape_id = RBASIC_SHAPE_ID(obj); - - RUBY_ASSERT(!rb_shape_has_object_id(original_shape_id)); - - rb_shape_t *shape = NULL; - if (!rb_shape_too_complex_p(original_shape_id)) { - bool dont_care; - shape = get_next_shape_internal(RSHAPE(original_shape_id), ruby_internal_object_id, SHAPE_OBJ_ID, &dont_care, true); - } - - if (!shape) { - shape = RSHAPE(ROOT_TOO_COMPLEX_WITH_OBJ_ID); - } - return shape_id(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID; -} - -shape_id_t -rb_shape_object_id(shape_id_t original_shape_id) -{ - RUBY_ASSERT(rb_shape_has_object_id(original_shape_id)); - - rb_shape_t *shape = RSHAPE(original_shape_id); - while (shape->type != SHAPE_OBJ_ID) { - if (UNLIKELY(shape->parent_id == INVALID_SHAPE_ID)) { - rb_bug("Missing object_id in shape tree"); - } - shape = RSHAPE(shape->parent_id); - } - - return shape_id(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID; + return (RBASIC_SHAPE_ID(obj) & (~SHAPE_ID_HEAP_INDEX_MASK)) | rb_shape_root(heap_index); } /* @@ -843,7 +856,6 @@ shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) *value = shape->next_field_index - 1; return true; case SHAPE_ROOT: - case SHAPE_T_OBJECT: return false; case SHAPE_OBJ_ID: rb_bug("Ivar should not exist on transition"); @@ -918,7 +930,13 @@ rb_shape_transition_add_ivar(VALUE obj, ID id) shape_id_t original_shape_id = RBASIC_SHAPE_ID(obj); RUBY_ASSERT(!shape_frozen_p(original_shape_id)); - return shape_id(shape_get_next(RSHAPE(original_shape_id), obj, id, true), original_shape_id); + rb_shape_t *next_shape = shape_get_next(RSHAPE(original_shape_id), obj, id, true); + if (next_shape) { + return shape_id(next_shape, original_shape_id); + } + else { + return transition_complex(original_shape_id); + } } shape_id_t @@ -927,7 +945,13 @@ rb_shape_transition_add_ivar_no_warnings(VALUE obj, ID id) shape_id_t original_shape_id = RBASIC_SHAPE_ID(obj); RUBY_ASSERT(!shape_frozen_p(original_shape_id)); - return shape_id(shape_get_next(RSHAPE(original_shape_id), obj, id, false), original_shape_id); + rb_shape_t *next_shape = shape_get_next(RSHAPE(original_shape_id), obj, id, false); + if (next_shape) { + return shape_id(next_shape, original_shape_id); + } + else { + return transition_complex(original_shape_id); + } } // Same as rb_shape_get_iv_index, but uses a provided valid shape id and index @@ -1040,61 +1064,6 @@ rb_shape_id_offset(void) return sizeof(uintptr_t) - SHAPE_ID_NUM_BITS / sizeof(uintptr_t); } -static rb_shape_t * -shape_traverse_from_new_root(rb_shape_t *initial_shape, rb_shape_t *dest_shape) -{ - RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT); - rb_shape_t *next_shape = initial_shape; - - if (dest_shape->type != initial_shape->type) { - next_shape = shape_traverse_from_new_root(initial_shape, RSHAPE(dest_shape->parent_id)); - if (!next_shape) { - return NULL; - } - } - - switch ((enum shape_type)dest_shape->type) { - case SHAPE_IVAR: - case SHAPE_OBJ_ID: - if (!next_shape->edges) { - return NULL; - } - - VALUE lookup_result; - if (SINGLE_CHILD_P(next_shape->edges)) { - rb_shape_t *child = SINGLE_CHILD(next_shape->edges); - if (child->edge_name == dest_shape->edge_name) { - return child; - } - else { - return NULL; - } - } - else { - if (rb_managed_id_table_lookup(next_shape->edges, dest_shape->edge_name, &lookup_result)) { - next_shape = (rb_shape_t *)lookup_result; - } - else { - return NULL; - } - } - break; - case SHAPE_ROOT: - case SHAPE_T_OBJECT: - break; - } - - return next_shape; -} - -shape_id_t -rb_shape_traverse_from_new_root(shape_id_t initial_shape_id, shape_id_t dest_shape_id) -{ - rb_shape_t *initial_shape = RSHAPE(initial_shape_id); - rb_shape_t *dest_shape = RSHAPE(dest_shape_id); - return shape_id(shape_traverse_from_new_root(initial_shape, dest_shape), dest_shape_id); -} - // Rebuild a similar shape with the same ivars but starting from // a different SHAPE_T_OBJECT, and don't cary over non-canonical transitions // such as SHAPE_OBJ_ID. @@ -1103,7 +1072,7 @@ shape_rebuild(rb_shape_t *initial_shape, rb_shape_t *dest_shape) { rb_shape_t *midway_shape; - RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT || initial_shape->type == SHAPE_ROOT); + RUBY_ASSERT(initial_shape->type == SHAPE_ROOT); if (dest_shape->type != initial_shape->type) { midway_shape = shape_rebuild(initial_shape, RSHAPE(dest_shape->parent_id)); @@ -1121,7 +1090,6 @@ shape_rebuild(rb_shape_t *initial_shape, rb_shape_t *dest_shape) break; case SHAPE_OBJ_ID: case SHAPE_ROOT: - case SHAPE_T_OBJECT: break; } @@ -1136,7 +1104,13 @@ rb_shape_rebuild(shape_id_t initial_shape_id, shape_id_t dest_shape_id) RUBY_ASSERT(!rb_shape_too_complex_p(initial_shape_id)); RUBY_ASSERT(!rb_shape_too_complex_p(dest_shape_id)); - return raw_shape_id(shape_rebuild(RSHAPE(initial_shape_id), RSHAPE(dest_shape_id))); + rb_shape_t *next_shape = shape_rebuild(RSHAPE(initial_shape_id), RSHAPE(dest_shape_id)); + if (next_shape) { + return shape_id(next_shape, initial_shape_id); + } + else { + return transition_complex(initial_shape_id | (dest_shape_id & SHAPE_ID_FL_HAS_OBJECT_ID)); + } } void @@ -1214,6 +1188,10 @@ rb_shape_memsize(shape_id_t shape_id) bool rb_shape_verify_consistency(VALUE obj, shape_id_t shape_id) { + if (shape_id == INVALID_SHAPE_ID) { + rb_bug("Can't set INVALID_SHAPE_ID on an object"); + } + rb_shape_t *shape = RSHAPE(shape_id); bool has_object_id = false; @@ -1238,6 +1216,21 @@ rb_shape_verify_consistency(VALUE obj, shape_id_t shape_id) } } + uint8_t flags_heap_index = rb_shape_heap_index(shape_id); + if (RB_TYPE_P(obj, T_OBJECT)) { + size_t shape_id_slot_size = GET_SHAPE_TREE()->capacities[flags_heap_index - 1] * sizeof(VALUE) + sizeof(struct RBasic); + size_t actual_slot_size = rb_gc_obj_slot_size(obj); + + if (shape_id_slot_size != actual_slot_size) { + rb_bug("shape_id heap_index flags mismatch: shape_id_slot_size=%lu, gc_slot_size=%lu\n", shape_id_slot_size, actual_slot_size); + } + } + else { + if (flags_heap_index) { + rb_bug("shape_id indicate heap_index > 0 but objet is not T_OBJECT: %s", rb_obj_info(obj)); + } + } + return true; } #endif @@ -1288,12 +1281,13 @@ shape_id_t_to_rb_cShape(shape_id_t shape_id) VALUE obj = rb_struct_new(rb_cShape, INT2NUM(shape_id), + INT2NUM(shape_id & SHAPE_ID_OFFSET_MASK), INT2NUM(shape->parent_id), rb_shape_edge_name(shape), INT2NUM(shape->next_field_index), - INT2NUM(shape->heap_index), + INT2NUM(rb_shape_heap_index(shape_id)), INT2NUM(shape->type), - INT2NUM(shape->capacity)); + INT2NUM(RSHAPE_CAPACITY(shape_id))); rb_obj_freeze(obj); return obj; } @@ -1463,7 +1457,7 @@ Init_default_shapes(void) for (index = 0; index < heaps_count; index++) { capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); } - shape_capacities = capacities; + GET_SHAPE_TREE()->capacities = capacities; #ifdef HAVE_MMAP size_t shape_list_mmap_size = rb_size_mul_or_raise(SHAPE_BUFFER_SIZE, sizeof(rb_shape_t), rb_eRuntimeError); @@ -1512,7 +1506,6 @@ Init_default_shapes(void) rb_shape_t *root = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID); root->capacity = 0; root->type = SHAPE_ROOT; - root->heap_index = 0; GET_SHAPE_TREE()->root_shape = root; RUBY_ASSERT(raw_shape_id(GET_SHAPE_TREE()->root_shape) == ROOT_SHAPE_ID); @@ -1520,27 +1513,13 @@ Init_default_shapes(void) root_with_obj_id->type = SHAPE_OBJ_ID; root_with_obj_id->edge_name = ruby_internal_object_id; root_with_obj_id->next_field_index++; - root_with_obj_id->heap_index = 0; RUBY_ASSERT(raw_shape_id(root_with_obj_id) == ROOT_SHAPE_WITH_OBJ_ID); - - // Make shapes for T_OBJECT - size_t *sizes = rb_gc_heap_sizes(); - for (int i = 0; sizes[i] > 0; i++) { - rb_shape_t *t_object_shape = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID); - t_object_shape->type = SHAPE_T_OBJECT; - t_object_shape->heap_index = i; - t_object_shape->capacity = (uint32_t)((sizes[i] - offsetof(struct RObject, as.ary)) / sizeof(VALUE)); - t_object_shape->edges = rb_managed_id_table_new(256); - t_object_shape->ancestor_index = LEAF; - RUBY_ASSERT(t_object_shape == RSHAPE(rb_shape_root(i))); - } } void rb_shape_free_all(void) { - xfree((void *)shape_capacities); - shape_capacities = NULL; + xfree((void *)GET_SHAPE_TREE()->capacities); xfree(GET_SHAPE_TREE()); } @@ -1552,6 +1531,7 @@ Init_shape(void) * :nodoc: */ VALUE rb_cShape = rb_struct_define_under(rb_cRubyVM, "Shape", "id", + "raw_id", "parent_id", "edge_name", "next_field_index", @@ -1569,11 +1549,9 @@ Init_shape(void) rb_define_const(rb_cShape, "SHAPE_ROOT", INT2NUM(SHAPE_ROOT)); rb_define_const(rb_cShape, "SHAPE_IVAR", INT2NUM(SHAPE_IVAR)); - rb_define_const(rb_cShape, "SHAPE_T_OBJECT", INT2NUM(SHAPE_T_OBJECT)); rb_define_const(rb_cShape, "SHAPE_ID_NUM_BITS", INT2NUM(SHAPE_ID_NUM_BITS)); rb_define_const(rb_cShape, "SHAPE_FLAG_SHIFT", INT2NUM(SHAPE_FLAG_SHIFT)); rb_define_const(rb_cShape, "SPECIAL_CONST_SHAPE_ID", INT2NUM(SPECIAL_CONST_SHAPE_ID)); - rb_define_const(rb_cShape, "FIRST_T_OBJECT_SHAPE_ID", INT2NUM(FIRST_T_OBJECT_SHAPE_ID)); rb_define_const(rb_cShape, "SHAPE_MAX_VARIATIONS", INT2NUM(SHAPE_MAX_VARIATIONS)); rb_define_const(rb_cShape, "SIZEOF_RB_SHAPE_T", INT2NUM(sizeof(rb_shape_t))); rb_define_const(rb_cShape, "SIZEOF_REDBLACK_NODE_T", INT2NUM(sizeof(redblack_node_t))); diff --git a/shape.h b/shape.h index 194cd296a2e5aa..7fec93af9f7d2e 100644 --- a/shape.h +++ b/shape.h @@ -17,7 +17,16 @@ STATIC_ASSERT(shape_id_num_bits, SHAPE_ID_NUM_BITS == sizeof(shape_id_t) * CHAR_ #define SHAPE_ID_FL_HAS_OBJECT_ID (SHAPE_FL_HAS_OBJECT_ID << SHAPE_ID_OFFSET_NUM_BITS) #define SHAPE_ID_FL_TOO_COMPLEX (SHAPE_FL_TOO_COMPLEX << SHAPE_ID_OFFSET_NUM_BITS) #define SHAPE_ID_FL_NON_CANONICAL_MASK (SHAPE_FL_NON_CANONICAL_MASK << SHAPE_ID_OFFSET_NUM_BITS) -#define SHAPE_ID_READ_ONLY_MASK (~SHAPE_ID_FL_FROZEN) + +#define SHAPE_ID_HEAP_INDEX_BITS 3 +#define SHAPE_ID_HEAP_INDEX_OFFSET (SHAPE_ID_NUM_BITS - SHAPE_ID_HEAP_INDEX_BITS - 1) // FIXME: -1 to avoid crashing YJIT +#define SHAPE_ID_HEAP_INDEX_MAX ((1 << SHAPE_ID_HEAP_INDEX_BITS) - 1) +#define SHAPE_ID_HEAP_INDEX_MASK (SHAPE_ID_HEAP_INDEX_MAX << SHAPE_ID_HEAP_INDEX_OFFSET) + +// The interpreter doesn't care about frozen status or slot size when reading ivars. +// So we normalize shape_id by clearing these bits to improve cache hits. +// JITs however might care about it. +#define SHAPE_ID_READ_ONLY_MASK (~(SHAPE_ID_FL_FROZEN | SHAPE_ID_HEAP_INDEX_MASK)) typedef uint32_t redblack_id_t; @@ -35,7 +44,6 @@ typedef uint32_t redblack_id_t; #define ROOT_TOO_COMPLEX_SHAPE_ID (ROOT_SHAPE_ID | SHAPE_ID_FL_TOO_COMPLEX) #define ROOT_TOO_COMPLEX_WITH_OBJ_ID (ROOT_SHAPE_WITH_OBJ_ID | SHAPE_ID_FL_TOO_COMPLEX | SHAPE_ID_FL_HAS_OBJECT_ID) #define SPECIAL_CONST_SHAPE_ID (ROOT_SHAPE_ID | SHAPE_ID_FL_FROZEN) -#define FIRST_T_OBJECT_SHAPE_ID 0x2 extern ID ruby_internal_object_id; @@ -49,7 +57,6 @@ struct rb_shape { attr_index_t next_field_index; // Fields are either ivars or internal properties like `object_id` attr_index_t capacity; // Total capacity of the object with this shape uint8_t type; - uint8_t heap_index; }; typedef struct rb_shape rb_shape_t; @@ -65,7 +72,6 @@ enum shape_type { SHAPE_ROOT, SHAPE_IVAR, SHAPE_OBJ_ID, - SHAPE_T_OBJECT, }; enum shape_flags { @@ -80,6 +86,7 @@ typedef struct { /* object shapes */ rb_shape_t *shape_list; rb_shape_t *root_shape; + const attr_index_t *capacities; rb_atomic_t next_shape_id; redblack_node_t *shape_cache; @@ -157,6 +164,7 @@ shape_id_t rb_shape_transition_remove_ivar(VALUE obj, ID id, shape_id_t *removed shape_id_t rb_shape_transition_add_ivar(VALUE obj, ID id); shape_id_t rb_shape_transition_add_ivar_no_warnings(VALUE obj, ID id); shape_id_t rb_shape_transition_object_id(VALUE obj); +shape_id_t rb_shape_transition_heap(VALUE obj, size_t heap_index); shape_id_t rb_shape_object_id(shape_id_t original_shape_id); void rb_shape_free_all(void); @@ -189,10 +197,18 @@ rb_shape_canonical_p(shape_id_t shape_id) return !(shape_id & SHAPE_ID_FL_NON_CANONICAL_MASK); } +static inline uint8_t +rb_shape_heap_index(shape_id_t shape_id) +{ + return (uint8_t)((shape_id & SHAPE_ID_HEAP_INDEX_MASK) >> SHAPE_ID_HEAP_INDEX_OFFSET); +} + static inline shape_id_t rb_shape_root(size_t heap_id) { - return (shape_id_t)(heap_id + FIRST_T_OBJECT_SHAPE_ID); + shape_id_t heap_index = (shape_id_t)heap_id; + + return ROOT_SHAPE_ID | ((heap_index + 1) << SHAPE_ID_HEAP_INDEX_OFFSET); } static inline bool @@ -201,10 +217,27 @@ RSHAPE_TYPE_P(shape_id_t shape_id, enum shape_type type) return RSHAPE(shape_id)->type == type; } +static inline attr_index_t +RSHAPE_EMBEDDED_CAPACITY(shape_id_t shape_id) +{ + uint8_t heap_index = rb_shape_heap_index(shape_id); + if (heap_index) { + return GET_SHAPE_TREE()->capacities[heap_index - 1]; + } + return 0; +} + static inline attr_index_t RSHAPE_CAPACITY(shape_id_t shape_id) { - return RSHAPE(shape_id)->capacity; + attr_index_t embedded_capacity = RSHAPE_EMBEDDED_CAPACITY(shape_id); + + if (embedded_capacity > RSHAPE(shape_id)->capacity) { + return embedded_capacity; + } + else { + return RSHAPE(shape_id)->capacity; + } } static inline attr_index_t @@ -270,8 +303,6 @@ RBASIC_FIELDS_COUNT(VALUE obj) return RSHAPE(rb_obj_shape_id(obj))->next_field_index; } -shape_id_t rb_shape_traverse_from_new_root(shape_id_t initial_shape_id, shape_id_t orig_shape_id); - bool rb_obj_set_shape_id(VALUE obj, shape_id_t shape_id); static inline bool diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb index 7d9e28ba7a8eca..a4cf23c6d5ae89 100644 --- a/test/ruby/test_shapes.rb +++ b/test/ruby/test_shapes.rb @@ -905,13 +905,15 @@ def test_remove_instance_variable_when_out_of_shapes def test_remove_instance_variable_capacity_transition assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}") begin; - t_object_shape = RubyVM::Shape.find_by_id(RubyVM::Shape::FIRST_T_OBJECT_SHAPE_ID) - assert_equal(RubyVM::Shape::SHAPE_T_OBJECT, t_object_shape.type) - - initial_capacity = t_object_shape.capacity # a does not transition in capacity a = Class.new.new + root_shape = RubyVM::Shape.of(a) + + assert_equal(RubyVM::Shape::SHAPE_ROOT, root_shape.type) + initial_capacity = root_shape.capacity + refute_equal(0, initial_capacity) + initial_capacity.times do |i| a.instance_variable_set(:"@ivar#{i + 1}", i) end @@ -976,7 +978,7 @@ def test_iv_index example.add_foo # makes a transition add_foo_shape = RubyVM::Shape.of(example) assert_equal([:@foo], example.instance_variables) - assert_equal(initial_shape.id, add_foo_shape.parent.id) + assert_equal(initial_shape.raw_id, add_foo_shape.parent.raw_id) assert_equal(1, add_foo_shape.next_field_index) example.remove_foo # makes a transition @@ -987,7 +989,7 @@ def test_iv_index example.add_bar # makes a transition bar_shape = RubyVM::Shape.of(example) assert_equal([:@bar], example.instance_variables) - assert_equal(initial_shape.id, bar_shape.parent_id) + assert_equal(initial_shape.raw_id, bar_shape.parent_id) assert_equal(1, bar_shape.next_field_index) end @@ -1007,7 +1009,7 @@ class TestObject; end def test_new_obj_has_t_object_shape obj = TestObject.new shape = RubyVM::Shape.of(obj) - assert_equal RubyVM::Shape::SHAPE_T_OBJECT, shape.type + assert_equal RubyVM::Shape::SHAPE_ROOT, shape.type assert_nil shape.parent end @@ -1039,7 +1041,7 @@ def test_basic_shape_transition assert_equal RubyVM::Shape::SHAPE_IVAR, shape.type shape = shape.parent - assert_equal RubyVM::Shape::SHAPE_T_OBJECT, shape.type + assert_equal RubyVM::Shape::SHAPE_ROOT, shape.type assert_nil shape.parent assert_equal(1, obj.instance_variable_get(:@a)) diff --git a/variable.c b/variable.c index 0d01a349bc5e5d..6d0e9832e7b706 100644 --- a/variable.c +++ b/variable.c @@ -1611,7 +1611,7 @@ rb_attr_delete(VALUE obj, ID id) return rb_ivar_delete(obj, id, Qnil); } -static void +static shape_id_t obj_transition_too_complex(VALUE obj, st_table *table) { RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); @@ -1659,6 +1659,7 @@ obj_transition_too_complex(VALUE obj, st_table *table) } xfree(old_fields); + return shape_id; } void @@ -1673,7 +1674,7 @@ rb_obj_init_too_complex(VALUE obj, st_table *table) } // Copy all object fields, including ivars and internal object_id, etc -void +shape_id_t rb_evict_fields_to_hash(VALUE obj) { void rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table); @@ -1682,9 +1683,10 @@ rb_evict_fields_to_hash(VALUE obj) st_table *table = st_init_numtable_with_size(RSHAPE_LEN(RBASIC_SHAPE_ID(obj))); rb_obj_copy_fields_to_hash_table(obj, table); - obj_transition_too_complex(obj, table); + shape_id_t new_shape_id = obj_transition_too_complex(obj, table); RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); + return new_shape_id; } void @@ -1711,7 +1713,7 @@ general_ivar_set(VALUE obj, ID id, VALUE val, void *data, VALUE *(*shape_fields_func)(VALUE, void *), void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), void (*set_shape_id_func)(VALUE, shape_id_t, void *), - void (*transition_too_complex_func)(VALUE, void *), + shape_id_t (*transition_too_complex_func)(VALUE, void *), st_table *(*too_complex_table_func)(VALUE, void *)) { struct general_ivar_set_result result = { @@ -1736,7 +1738,7 @@ general_ivar_set(VALUE obj, ID id, VALUE val, void *data, shape_id_t next_shape_id = rb_shape_transition_add_ivar(obj, id); if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) { - transition_too_complex_func(obj, data); + current_shape_id = transition_too_complex_func(obj, data); goto too_complex; } else if (UNLIKELY(RSHAPE_CAPACITY(next_shape_id) != RSHAPE_CAPACITY(current_shape_id))) { @@ -1772,14 +1774,14 @@ general_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val, void *data, VALUE *(*shape_fields_func)(VALUE, void *), void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), void (*set_shape_id_func)(VALUE, shape_id_t, void *), - void (*transition_too_complex_func)(VALUE, void *), + shape_id_t (*transition_too_complex_func)(VALUE, void *), st_table *(*too_complex_table_func)(VALUE, void *)) { shape_id_t current_shape_id = RBASIC_SHAPE_ID(obj); if (UNLIKELY(rb_shape_too_complex_p(target_shape_id))) { if (UNLIKELY(!rb_shape_too_complex_p(current_shape_id))) { - transition_too_complex_func(obj, data); + current_shape_id = transition_too_complex_func(obj, data); } st_table *table = too_complex_table_func(obj, data); @@ -1788,6 +1790,7 @@ general_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val, void *data, RBASIC_SET_SHAPE_ID(obj, target_shape_id); } + RUBY_ASSERT(RSHAPE_EDGE_NAME(target_shape_id)); st_insert(table, (st_data_t)RSHAPE_EDGE_NAME(target_shape_id), (st_data_t)val); RB_OBJ_WRITTEN(obj, Qundef, val); } @@ -1877,11 +1880,12 @@ generic_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *data) fields_lookup->shape_id = shape_id; } -static void +static shape_id_t generic_ivar_set_transition_too_complex(VALUE obj, void *_data) { - rb_evict_fields_to_hash(obj); + shape_id_t new_shape_id = rb_evict_fields_to_hash(obj); FL_SET_RAW(obj, FL_EXIVAR); + return new_shape_id; } static st_table * @@ -1997,10 +2001,10 @@ obj_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *_data) rb_obj_set_shape_id(obj, shape_id); } -static void +static shape_id_t obj_ivar_set_transition_too_complex(VALUE obj, void *_data) { - rb_evict_fields_to_hash(obj); + return rb_evict_fields_to_hash(obj); } static st_table * @@ -2207,7 +2211,6 @@ iterate_over_shapes_with_callback(rb_shape_t *shape, rb_ivar_foreach_callback_fu { switch ((enum shape_type)shape->type) { case SHAPE_ROOT: - case SHAPE_T_OBJECT: return false; case SHAPE_OBJ_ID: if (itr_data->ivar_only) { @@ -4691,10 +4694,10 @@ class_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *_data) rb_obj_set_shape_id(obj, shape_id); } -static void +static shape_id_t class_ivar_set_transition_too_complex(VALUE obj, void *_data) { - rb_evict_fields_to_hash(obj); + return rb_evict_fields_to_hash(obj); } static st_table * diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index b2be5f37857afe..23682ac63ccdb1 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -698,7 +698,6 @@ pub struct rb_shape { pub next_field_index: attr_index_t, pub capacity: attr_index_t, pub type_: u8, - pub heap_index: u8, } pub type rb_shape_t = rb_shape; #[repr(C)] diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 5ab1355e124c71..0447f46fd0f26e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -406,7 +406,6 @@ pub struct rb_shape { pub next_field_index: attr_index_t, pub capacity: attr_index_t, pub type_: u8, - pub heap_index: u8, } pub type rb_shape_t = rb_shape; #[repr(C)]