diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 1fea866cc1d2955b362bc697d7952c904403bfd4..e755d4cee8819774899feeeb8e3d30c4d8c3d26d 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -505,7 +505,7 @@ class Address {
if (size == 0) // It's a byte
i->f(_ext.shift() >= 0, 12);
else {
- assert(_ext.shift() <= 0 || _ext.shift() == (int)size, "bad shift");
+ guarantee(_ext.shift() <= 0 || _ext.shift() == (int)size, "bad shift");
i->f(_ext.shift() > 0, 12);
}
i->f(0b10, 11, 10);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index a35b8b844817b59e9b5128fe3602a29a6000d789..7abb2205414fd21a19d54652b901161153b173de 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -1050,6 +1050,110 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
}
+// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
+// The target method is determined by <holder_klass, itable_index>.
+// The receiver klass is in recv_klass.
+// On success, the result will be in method_result, and execution falls through.
+// On failure, execution transfers to the given label.
+void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
+ Register holder_klass,
+ Register resolved_klass,
+ Register method_result,
+ Register temp_itbl_klass,
+ Register scan_temp,
+ int itable_index,
+ Label& L_no_such_interface) {
+ // 'method_result' is only used as output register at the very end of this method.
+ // Until then we can reuse it as 'holder_offset'.
+ Register holder_offset = method_result;
+ assert_different_registers(resolved_klass, recv_klass, holder_klass, temp_itbl_klass, scan_temp, holder_offset);
+
+ int vtable_start_offset = in_bytes(Klass::vtable_start_offset());
+ int itable_offset_entry_size = itableOffsetEntry::size() * wordSize;
+ int ioffset = itableOffsetEntry::interface_offset_in_bytes();
+ int ooffset = itableOffsetEntry::offset_offset_in_bytes();
+
+ Label L_loop_search_resolved_entry, L_resolved_found, L_holder_found;
+
+ ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
+ add(recv_klass, recv_klass, vtable_start_offset + ioffset);
+ // itableOffsetEntry[] itable = recv_klass + Klass::vtable_start_offset() + sizeof(vtableEntry) * recv_klass->_vtable_len;
+ // temp_itbl_klass = itable[0]._interface;
+ int vtblEntrySize = vtableEntry::size_in_bytes();
+ assert(vtblEntrySize == wordSize, "ldr lsl shift amount must be 3");
+ ldr(temp_itbl_klass, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize))));
+ mov(holder_offset, zr);
+ // scan_temp = &(itable[0]._interface)
+ lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize))));
+
+ // Initial checks:
+ // - if (holder_klass != resolved_klass), go to "scan for resolved"
+ // - if (itable[0] == holder_klass), shortcut to "holder found"
+ // - if (itable[0] == 0), no such interface
+ cmp(resolved_klass, holder_klass);
+ br(Assembler::NE, L_loop_search_resolved_entry);
+ cmp(holder_klass, temp_itbl_klass);
+ br(Assembler::EQ, L_holder_found);
+ cbz(temp_itbl_klass, L_no_such_interface);
+
+ // Loop: Look for holder_klass record in itable
+ // do {
+ // temp_itbl_klass = *(scan_temp += itable_offset_entry_size);
+ // if (temp_itbl_klass == holder_klass) {
+ // goto L_holder_found; // Found!
+ // }
+ // } while (temp_itbl_klass != 0);
+ // goto L_no_such_interface // Not found.
+ Label L_search_holder;
+ bind(L_search_holder);
+ ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size)));
+ cmp(holder_klass, temp_itbl_klass);
+ br(Assembler::EQ, L_holder_found);
+ cbnz(temp_itbl_klass, L_search_holder);
+
+ b(L_no_such_interface);
+
+ // Loop: Look for resolved_class record in itable
+ // while (true) {
+ // temp_itbl_klass = *(scan_temp += itable_offset_entry_size);
+ // if (temp_itbl_klass == 0) {
+ // goto L_no_such_interface;
+ // }
+ // if (temp_itbl_klass == resolved_klass) {
+ // goto L_resolved_found; // Found!
+ // }
+ // if (temp_itbl_klass == holder_klass) {
+ // holder_offset = scan_temp;
+ // }
+ // }
+ //
+ Label L_loop_search_resolved;
+ bind(L_loop_search_resolved);
+ ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size)));
+ bind(L_loop_search_resolved_entry);
+ cbz(temp_itbl_klass, L_no_such_interface);
+ cmp(resolved_klass, temp_itbl_klass);
+ br(Assembler::EQ, L_resolved_found);
+ cmp(holder_klass, temp_itbl_klass);
+ br(Assembler::NE, L_loop_search_resolved);
+ mov(holder_offset, scan_temp);
+ b(L_loop_search_resolved);
+
+ // See if we already have a holder klass. If not, go and scan for it.
+ bind(L_resolved_found);
+ cbz(holder_offset, L_search_holder);
+ mov(scan_temp, holder_offset);
+
+ // Finally, scan_temp contains holder_klass vtable offset
+ bind(L_holder_found);
+ ldrw(method_result, Address(scan_temp, ooffset - ioffset));
+ add(recv_klass, recv_klass, itable_index * wordSize + itableMethodEntry::method_offset_in_bytes()
+ - vtable_start_offset - ioffset); // substract offsets to restore the original value of recv_klass
+ ldr(method_result, Address(recv_klass, method_result, Address::uxtw(0)));
+}
+
// virtual method calling
void MacroAssembler::lookup_virtual_method(Register recv_klass,
RegisterOrConstant vtable_index,
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 3f6ebb41f2ac28a9d510a3cea4a812a7c2da918a..9b1c0a935394e88d0d5f0fc6ba2c720056a5b9ad 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -918,6 +918,15 @@ public:
Label& no_such_interface,
bool return_method = true);
+ void lookup_interface_method_stub(Register recv_klass,
+ Register holder_klass,
+ Register resolved_klass,
+ Register method_result,
+ Register temp_reg,
+ Register temp_reg2,
+ int itable_index,
+ Label& L_no_such_interface);
+
// virtual method calling
// n.b. x86 allows RegisterOrConstant for vtable_index
void lookup_virtual_method(Register recv_klass,
diff --git a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
index acef8d21abc8055629ae228544847a6eee0d0edf..94ac2a4ec9f4cfd6f061787d81f23ba7984ddb38 100644
--- a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
@@ -175,7 +175,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// so all registers except arguments are free at this point.
const Register recv_klass_reg = r10;
const Register holder_klass_reg = r16; // declaring interface klass (DECC)
- const Register resolved_klass_reg = rmethod; // resolved interface klass (REFC)
+ const Register resolved_klass_reg = r17; // resolved interface klass (REFC)
const Register temp_reg = r11;
const Register temp_reg2 = r15;
const Register icholder_reg = rscratch2;
@@ -192,28 +192,13 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass(recv_klass_reg, j_rarg0);
// Receiver subtype check against REFC.
- __ lookup_interface_method(// inputs: rec. class, interface
- recv_klass_reg, resolved_klass_reg, noreg,
- // outputs: scan temp. reg1, scan temp. reg2
- temp_reg2, temp_reg,
- L_no_such_interface,
- /*return_method=*/false);
-
- const ptrdiff_t typecheckSize = __ pc() - start_pc;
- start_pc = __ pc();
-
// Get selected method from declaring class and itable index
- __ lookup_interface_method(// inputs: rec. class, interface, itable index
- recv_klass_reg, holder_klass_reg, itable_index,
- // outputs: method, scan temp. reg
- rmethod, temp_reg,
- L_no_such_interface);
-
- const ptrdiff_t lookupSize = __ pc() - start_pc;
+ __ lookup_interface_method_stub(recv_klass_reg, holder_klass_reg, resolved_klass_reg, rmethod,
+ temp_reg, temp_reg2, itable_index, L_no_such_interface);
// Reduce "estimate" such that "padding" does not drop below 8.
const ptrdiff_t estimate = 124;
- const ptrdiff_t codesize = typecheckSize + lookupSize;
+ const ptrdiff_t codesize = __ pc() - start_pc;
slop_delta = (int)(estimate - codesize);
slop_bytes += slop_delta;
assert(slop_delta >= 0, "itable #%d: Code size estimate (%d) for lookup_interface_method too small, required: %d", itable_index, (int)estimate, (int)codesize);