Use guard to check if array is packed or hash

Author: dstogov

ext/opcache/Optimizer/zend_inference.h

@@ -26,6 +26,7 @@
 /* Bitmask for type inference (zend_ssa_var_info.type) */
 #include "zend_type_info.h"
 
+#define MAY_BE_PACKED_GUARD         (1<<27) /* needs packed array guard */
 #define MAY_BE_CLASS_GUARD          (1<<27) /* needs class guard */
 #define MAY_BE_GUARD                (1<<28) /* needs type guard */
 #define MAY_BE_IN_REG               (1<<29) /* value allocated in CPU register */

ext/opcache/jit/zend_jit_internal.h

@@ -210,6 +210,7 @@ typedef enum _zend_jit_trace_stop {
 #define ZEND_JIT_EXIT_POLYMORPHISM  (1<<4) /* exit because of polymorphic call */
 #define ZEND_JIT_EXIT_FREE_OP1      (1<<5)
 #define ZEND_JIT_EXIT_FREE_OP2      (1<<6)
+#define ZEND_JIT_EXIT_PACKED_GUARD  (1<<7)
 
 typedef union _zend_op_trace_info {
 	zend_op dummy; /* the size of this structure must be the same as zend_op */

ext/opcache/jit/zend_jit_trace.c

@@ -1358,12 +1358,12 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
 	level = 0;
 	for (;;p++) {
 		if (p->op == ZEND_JIT_TRACE_VM) {
-			uint8_t op1_type, op2_type, op3_type;
+			uint8_t orig_op1_type, op1_type, op2_type, op3_type;
 
 			// TODO: range inference ???
 			opline = p->opline;
 
-			op1_type = p->op1_type;
+			op1_type = orig_op1_type = p->op1_type;
 			op2_type = p->op2_type;
 			op3_type = p->op3_type;
 			if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
@@ -1425,8 +1425,6 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
 				case ZEND_IS_IDENTICAL:
 				case ZEND_IS_NOT_IDENTICAL:
 				case ZEND_CASE_STRICT:
-				case ZEND_FETCH_DIM_R:
-				case ZEND_FETCH_DIM_IS:
 				case ZEND_BW_OR:
 				case ZEND_BW_AND:
 				case ZEND_BW_XOR:
@@ -1505,8 +1503,38 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
 						// TODO: support for empty() ???
 						break;
 					}
+					/* break missing intentionally */
+				case ZEND_FETCH_DIM_R:
+				case ZEND_FETCH_DIM_IS:
 					ADD_OP1_TRACE_GUARD();
 					ADD_OP2_TRACE_GUARD();
+
+					if (opline->op1_type != IS_CONST
+					 && op1_type == IS_ARRAY
+					 && ((opline->op2_type == IS_CONST
+					   && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
+					  || (opline->op2_type != IS_CONST
+					   && op2_type == IS_LONG))) {
+
+						zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
+
+						if ((info->type & MAY_BE_ARRAY_PACKED)
+						 && (info->type & MAY_BE_ARRAY_HASH)
+						 && orig_op1_type != IS_UNKNOWN
+						 && !(orig_op1_type & IS_TRACE_REFERENCE)) {
+							/* setup "packed" guards only for loop invariant or reused variables */
+							if ((trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
+							  && tssa->ops[idx].op1_use < trace_buffer->op_array->last_var)
+							 || tssa->ops[idx].op1_use_chain >= 0) {
+								info->type |= MAY_BE_PACKED_GUARD;
+								if (orig_op1_type & IS_TRACE_PACKED) {
+									info->type &= ~MAY_BE_ARRAY_HASH;
+								} else {
+									info->type &= ~MAY_BE_ARRAY_PACKED;
+								}
+							}
+						}
+					}
 					break;
 				case ZEND_SEND_VAL_EX:
 				case ZEND_SEND_VAR_EX:
@@ -3023,6 +3051,16 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 			} else {
 				SET_STACK_TYPE(stack, i, IS_UNKNOWN);
 			}
+
+			if ((info & MAY_BE_PACKED_GUARD) != 0
+			 && trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
+			 && ssa->vars[i].use_chain != -1) {
+				if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
+					goto jit_failure;
+				}
+				info &= ~MAY_BE_PACKED_GUARD;
+				ssa->var_info[i].type = info;
+			}
 		}
 
 		if (parent_trace) {
@@ -4023,6 +4061,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 						avoid_refcounting =
 							ssa_op->op1_use >= 0 &&
 							ssa->var_info[ssa_op->op1_use].avoid_refcounting;
+						if (op1_info & MAY_BE_PACKED_GUARD) {
+							ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
+						}
 						if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
 								op1_info, op1_addr, avoid_refcounting,
 								op2_info, res_info, RES_REG_ADDR(),
@@ -4090,6 +4131,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 						avoid_refcounting =
 							ssa_op->op1_use >= 0 &&
 							ssa->var_info[ssa_op->op1_use].avoid_refcounting;
+						if (op1_info & MAY_BE_PACKED_GUARD) {
+							ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
+						}
 						if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
 								op1_info, op1_addr, avoid_refcounting,
 								op2_info,

ext/opcache/jit/zend_jit_x86.dasc

@@ -3234,6 +3234,27 @@ static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t
 	return 1;
 }
 
+static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info)
+{
+	int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
+	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+	if (!exit_addr) {
+		return 0;
+	}
+
+	|	GET_ZVAL_LVAL ZREG_FCARG1a, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var)
+	if (op_info & MAY_BE_ARRAY_PACKED) {
+		|	test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+		|	jz &exit_addr
+	} else {
+		|	test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+		|	jnz &exit_addr
+	}
+
+	return 1;
+}
+
 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
 {
 	zend_jit_op_array_trace_extension *jit_extension =
@@ -4953,12 +4974,27 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
 			|	// if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
 			|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
 		}
+		if (op1_info & MAY_BE_PACKED_GUARD) {
+			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
+			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+			if (!exit_addr) {
+				return 0;
+			}
+			if (op1_info & MAY_BE_ARRAY_PACKED) {
+				|	test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+				|	jz &exit_addr
+			} else {
+				|	test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
+				|	jnz &exit_addr
+			}
+		}
 		if (type == BP_VAR_W) {
 			|	// hval = Z_LVAL_P(dim);
 			|	GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr
 			op2_loaded = 1;
 		}
-		if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_PACKED)) {
+		if (op1_info & MAY_BE_ARRAY_PACKED) {
 			if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
 				zend_long val = Z_LVAL_P(Z_ZV(op2_addr));
 				if (val >= 0 && val < HT_MAX_SIZE) {
@@ -5050,7 +5086,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
 		}
 		switch (type) {
 			case BP_JIT_IS:
-				if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_HASH)) {
+				if (op1_info & MAY_BE_ARRAY_HASH) {
 					|4:
 					if (!op2_loaded) {
 						|	// hval = Z_LVAL_P(dim);
@@ -5076,9 +5112,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
 			case BP_VAR_IS:
 			case BP_VAR_UNSET:
 				if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) ||
-						!(op1_info & MAY_BE_ARRAY_HASH) ||
-						Z_MODE(op2_addr) != IS_CONST_ZVAL ||
-						(Z_LVAL_P(Z_ZV(op2_addr)) >= 0 && Z_LVAL_P(Z_ZV(op2_addr)) < HT_MAX_SIZE)) {
+						((op1_info & MAY_BE_ARRAY_PACKED) &&
+							(Z_MODE(op2_addr) != IS_CONST_ZVAL ||
+							(Z_LVAL_P(Z_ZV(op2_addr)) >= 0 && Z_LVAL_P(Z_ZV(op2_addr)) < HT_MAX_SIZE)))) {
 					if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
 						|	jmp &exit_addr
 					} else if (type == BP_VAR_IS && not_found_exit_addr) {
@@ -5087,7 +5123,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
 						|	jmp >2 // NOT_FOUND
 					}
 				}
-				if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_HASH)) {
+				if (op1_info & MAY_BE_ARRAY_HASH) {
 					|4:
 					if (!op2_loaded) {
 						|	// hval = Z_LVAL_P(dim);
@@ -5140,20 +5176,27 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
 				|	jz >9
 				break;
 			case BP_VAR_W:
-				|2:
-				|	//retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval));
-				|.if X64
-					|	LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
-				|.else
-					|	sub r4, 12
-					|	PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
-				|.endif
-				|	EXT_CALL zend_hash_index_add_new, r0
-				|.if not(X64)
-				|	add r4, 12
-				|.endif
-				if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
-					|	jmp >8
+				if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) ||
+						((op1_info & MAY_BE_ARRAY_PACKED) &&
+							(Z_MODE(op2_addr) != IS_CONST_ZVAL ||
+							(Z_LVAL_P(Z_ZV(op2_addr)) >= 0 && Z_LVAL_P(Z_ZV(op2_addr)) < HT_MAX_SIZE)))) {
+					|2:
+					|	//retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval));
+					|.if X64
+						|	LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval
+					|.else
+						|	sub r4, 12
+						|	PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0
+					|.endif
+					|	EXT_CALL zend_hash_index_add_new, r0
+					|.if not(X64)
+					|	add r4, 12
+					|.endif
+					if (op1_info & MAY_BE_ARRAY_HASH) {
+						|	jmp >8
+					}
+				}
+				if (op1_info & MAY_BE_ARRAY_HASH) {
 					|4:
 					|	EXT_CALL zend_jit_hash_index_lookup_w, r0
 				}