remove_instance_variable: Handle running out of shapes

`remove_shape_recursive` wasn't considering that if we run out of shapes, it might have to transition to SHAPE_TOO_COMPLEX. When this happens, we now return with an error and the caller initiates the evacuation. 

1 files changed, 14 insertions, 1 deletions

diff --git a/shape.c b/shape.c
index 014d49b95f..e402344615 100644
--- a/shape.c
+++ b/shape.c
@@ -585,6 +585,10 @@ remove_shape_recursive(VALUE obj, ID id, rb_shape_t * shape, VALUE * removed)
 if (new_parent) {
 bool dont_care;
 rb_shape_t * new_child = get_next_shape_internal(new_parent, shape->edge_name, shape->type, &dont_care, true);
+ if (UNLIKELY(new_parent->type == SHAPE_OBJ_TOO_COMPLEX)) {
+ return new_parent;
+ }
+
 new_child->capacity = shape->capacity;
 if (new_child->type == SHAPE_IVAR) {
 move_iv(obj, id, shape->next_iv_index - 1, new_child->next_iv_index - 1);
@@ -601,13 +605,22 @@ remove_shape_recursive(VALUE obj, ID id, rb_shape_t * shape, VALUE * removed)
 }
 }

-void
+bool
 rb_shape_transition_shape_remove_ivar(VALUE obj, ID id, rb_shape_t *shape, VALUE * removed)
 {
+ if (UNLIKELY(shape->type == SHAPE_OBJ_TOO_COMPLEX)) {
+ return false;
+ }
+
 rb_shape_t * new_shape = remove_shape_recursive(obj, id, shape, removed);
 if (new_shape) {
+ if (UNLIKELY(new_shape->type == SHAPE_OBJ_TOO_COMPLEX)) {
+ return false;
+ }
+
 rb_shape_set_shape(obj, new_shape);
 }
+ return true;
 }

 rb_shape_t *