1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
61 operand[1] = hi;
62 return operand;
63 }
64
65 // Buffer for 128-bits masks used by SSE instructions.
66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
67
68 // Static initialization during VM startup.
69 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
70 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
71 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
72 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
73
74
75 NEEDS_CLEANUP // remove this definitions ?
76 const Register SYNC_header = rax; // synchronization header
77 const Register SHIFT_count = rcx; // where count for shift operations must be
78
79 #define __ _masm->
80
81
82 static void select_different_registers(Register preserve,
83 Register extra,
84 Register &tmp1,
85 Register &tmp2) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, extra);
91 tmp2 = extra;
92 }
93 assert_different_registers(preserve, tmp1, tmp2);
94 }
95
96
97
98 static void select_different_registers(Register preserve,
99 Register extra,
100 Register &tmp1,
101 Register &tmp2,
102 Register &tmp3) {
103 if (tmp1 == preserve) {
104 assert_different_registers(tmp1, tmp2, tmp3, extra);
105 tmp1 = extra;
106 } else if (tmp2 == preserve) {
107 assert_different_registers(tmp1, tmp2, tmp3, extra);
108 tmp2 = extra;
109 } else if (tmp3 == preserve) {
110 assert_different_registers(tmp1, tmp2, tmp3, extra);
111 tmp3 = extra;
112 }
113 assert_different_registers(preserve, tmp1, tmp2, tmp3);
114 }
115
116
117
118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
119 if (opr->is_constant()) {
120 LIR_Const* constant = opr->as_constant_ptr();
121 switch (constant->type()) {
122 case T_INT: {
123 return true;
124 }
125
126 default:
127 return false;
128 }
129 }
130 return false;
131 }
132
133
134 LIR_Opr LIR_Assembler::receiverOpr() {
135 return FrameMap::receiver_opr;
136 }
137
138 LIR_Opr LIR_Assembler::osrBufferPointer() {
139 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
140 }
141
142 //--------------fpu register translations-----------------------
143
144
145 address LIR_Assembler::float_constant(float f) {
146 address const_addr = __ float_constant(f);
147 if (const_addr == nullptr) {
148 bailout("const section overflow");
149 return __ code()->consts()->start();
150 } else {
151 return const_addr;
152 }
153 }
154
155
156 address LIR_Assembler::double_constant(double d) {
157 address const_addr = __ double_constant(d);
158 if (const_addr == nullptr) {
159 bailout("const section overflow");
160 return __ code()->consts()->start();
161 } else {
162 return const_addr;
163 }
164 }
165
166 void LIR_Assembler::breakpoint() {
167 __ int3();
168 }
169
170 void LIR_Assembler::push(LIR_Opr opr) {
171 if (opr->is_single_cpu()) {
172 __ push_reg(opr->as_register());
173 } else if (opr->is_double_cpu()) {
174 NOT_LP64(__ push_reg(opr->as_register_hi()));
175 __ push_reg(opr->as_register_lo());
176 } else if (opr->is_stack()) {
177 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
178 } else if (opr->is_constant()) {
179 LIR_Const* const_opr = opr->as_constant_ptr();
180 if (const_opr->type() == T_OBJECT) {
181 __ push_oop(const_opr->as_jobject(), rscratch1);
182 } else if (const_opr->type() == T_INT) {
183 __ push_jint(const_opr->as_jint());
184 } else {
185 ShouldNotReachHere();
186 }
187
188 } else {
189 ShouldNotReachHere();
190 }
191 }
192
193 void LIR_Assembler::pop(LIR_Opr opr) {
194 if (opr->is_single_cpu()) {
195 __ pop_reg(opr->as_register());
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
202 return addr->base()->is_illegal() && addr->index()->is_illegal();
203 }
204
205 //-------------------------------------------
206
207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
208 return as_Address(addr, rscratch1);
209 }
210
211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
212 if (addr->base()->is_illegal()) {
213 assert(addr->index()->is_illegal(), "must be illegal too");
214 AddressLiteral laddr((address)addr->disp(), relocInfo::none);
215 if (! __ reachable(laddr)) {
216 __ movptr(tmp, laddr.addr());
217 Address res(tmp, 0);
218 return res;
219 } else {
220 return __ as_Address(laddr);
221 }
222 }
223
224 Register base = addr->base()->as_pointer_register();
225
226 if (addr->index()->is_illegal()) {
227 return Address( base, addr->disp());
228 } else if (addr->index()->is_cpu_register()) {
229 Register index = addr->index()->as_pointer_register();
230 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
231 } else if (addr->index()->is_constant()) {
232 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
233 assert(Assembler::is_simm32(addr_offset), "must be");
234
235 return Address(base, addr_offset);
236 } else {
237 Unimplemented();
238 return Address();
239 }
240 }
241
242
243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
244 Address base = as_Address(addr);
245 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
246 }
247
248
249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
250 return as_Address(addr);
251 }
252
253
254 void LIR_Assembler::osr_entry() {
255 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
256 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
257 ValueStack* entry_state = osr_entry->state();
258 int number_of_locks = entry_state->locks_size();
259
260 // we jump here if osr happens with the interpreter
261 // state set up to continue at the beginning of the
262 // loop that triggered osr - in particular, we have
263 // the following registers setup:
264 //
265 // rcx: osr buffer
266 //
267
268 // build frame
269 ciMethod* m = compilation()->method();
270 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
271
272 // OSR buffer is
273 //
274 // locals[nlocals-1..0]
275 // monitors[0..number_of_locks]
276 //
277 // locals is a direct copy of the interpreter frame so in the osr buffer
278 // so first slot in the local array is the last local from the interpreter
279 // and last slot is local[0] (receiver) from the interpreter
280 //
281 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
282 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
283 // in the interpreter frame (the method lock if a sync method)
284
285 // Initialize monitors in the compiled activation.
286 // rcx: pointer to osr buffer
287 //
288 // All other registers are dead at this point and the locals will be
289 // copied into place by code emitted in the IR.
290
291 Register OSR_buf = osrBufferPointer()->as_pointer_register();
292 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
293 int monitor_offset = BytesPerWord * method()->max_locals() +
294 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
295 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
296 // the OSR buffer using 2 word entries: first the lock and then
297 // the oop.
298 for (int i = 0; i < number_of_locks; i++) {
299 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
300 #ifdef ASSERT
301 // verify the interpreter's monitor has a non-null object
302 {
303 Label L;
304 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
305 __ jcc(Assembler::notZero, L);
306 __ stop("locked object is null");
307 __ bind(L);
308 }
309 #endif
310 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
311 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
312 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
313 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
314 }
315 }
316 }
317
318
319 // inline cache check; done before the frame is built.
320 int LIR_Assembler::check_icache() {
321 return __ ic_check(CodeEntryAlignment);
322 }
323
324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
325 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
326 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
327
328 Label L_skip_barrier;
329 Register klass = rscratch1;
330 Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );
331 assert(thread != noreg, "x86_32 not implemented");
332
333 __ mov_metadata(klass, method->holder()->constant_encoding());
334 __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);
335
336 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
337
338 __ bind(L_skip_barrier);
339 }
340
341 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
342 jobject o = nullptr;
343 PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
344 __ movoop(reg, o);
345 patching_epilog(patch, lir_patch_normal, reg, info);
346 }
347
348 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
349 Metadata* o = nullptr;
350 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
351 __ mov_metadata(reg, o);
352 patching_epilog(patch, lir_patch_normal, reg, info);
353 }
354
355 // This specifies the rsp decrement needed to build the frame
356 int LIR_Assembler::initial_frame_size_in_bytes() const {
357 // if rounding, must let FrameMap know!
358
359 // The frame_map records size in slots (32bit word)
360
361 // subtract two words to account for return address and link
362 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
363 }
364
365
366 int LIR_Assembler::emit_exception_handler() {
367 // generate code for exception handler
368 address handler_base = __ start_a_stub(exception_handler_size());
369 if (handler_base == nullptr) {
370 // not enough space left for the handler
371 bailout("exception handler overflow");
372 return -1;
373 }
374
375 int offset = code_offset();
376
377 // the exception oop and pc are in rax, and rdx
378 // no other registers need to be preserved, so invalidate them
379 __ invalidate_registers(false, true, true, false, true, true);
380
381 // check that there is really an exception
382 __ verify_not_null_oop(rax);
383
384 // search an exception handler (rax: exception oop, rdx: throwing pc)
385 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id)));
386 __ should_not_reach_here();
387 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
388 __ end_a_stub();
389
390 return offset;
391 }
392
393
394 // Emit the code to remove the frame from the stack in the exception
395 // unwind path.
396 int LIR_Assembler::emit_unwind_handler() {
397 #ifndef PRODUCT
398 if (CommentedAssembly) {
399 _masm->block_comment("Unwind handler");
400 }
401 #endif
402
403 int offset = code_offset();
404
405 // Fetch the exception from TLS and clear out exception related thread state
406 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
407 NOT_LP64(__ get_thread(thread));
408 __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
409 __ movptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
410 __ movptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
411
412 __ bind(_unwind_handler_entry);
413 __ verify_not_null_oop(rax);
414 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
415 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved)
416 }
417
418 // Perform needed unlocking
419 MonitorExitStub* stub = nullptr;
420 if (method()->is_synchronized()) {
421 monitor_address(0, FrameMap::rax_opr);
422 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
423 if (LockingMode == LM_MONITOR) {
424 __ jmp(*stub->entry());
425 } else {
426 __ unlock_object(rdi, rsi, rax, *stub->entry());
427 }
428 __ bind(*stub->continuation());
429 }
430
431 if (compilation()->env()->dtrace_method_probes()) {
432 #ifdef _LP64
433 __ mov(rdi, r15_thread);
434 __ mov_metadata(rsi, method()->constant_encoding());
435 #else
436 __ get_thread(rax);
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
439 #endif
440 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(rax, rbx); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
449 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != nullptr) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // generate code for exception handler
462 address handler_base = __ start_a_stub(deopt_handler_size());
463 if (handler_base == nullptr) {
464 // not enough space left for the handler
465 bailout("deopt handler overflow");
466 return -1;
467 }
468
469 int offset = code_offset();
470 InternalAddress here(__ pc());
471
472 __ pushptr(here.addr(), rscratch1);
473 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
474 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
475 __ end_a_stub();
476
477 return offset;
478 }
479
480 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
481 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
482 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
483 assert(result->fpu() == 0, "result must already be on TOS");
484 }
485 if (InlineTypeReturnedAsFields) {
486 #ifndef _LP64
487 Unimplemented();
488 #endif
489 // Check if we are returning an non-null inline type and load its fields into registers
490 ciType* return_type = compilation()->method()->return_type();
491 if (return_type->is_inlinetype()) {
492 ciInlineKlass* vk = return_type->as_inline_klass();
493 if (vk->can_be_returned_as_fields()) {
494 address unpack_handler = vk->unpack_handler();
495 assert(unpack_handler != nullptr, "must be");
496 __ call(RuntimeAddress(unpack_handler));
497 }
498 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
499 Label skip;
500 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
501
502 // Load fields from a buffered value with an inline class specific handler
503 __ load_klass(rdi, rax, rscratch1);
504 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
505 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
506 // Unpack handler can be null if inline type is not scalarizable in returns
507 __ testptr(rdi, rdi);
508 __ jcc(Assembler::zero, skip);
509 __ call(rdi);
510
511 __ bind(skip);
512 }
513 // At this point, rax points to the value object (for interpreter or C1 caller).
514 // The fields of the object are copied into registers (for C2 caller).
515 }
516
517 // Pop the stack before the safepoint code
518 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
519
520 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
521 __ reserved_stack_check();
522 }
523
524 // Note: we do not need to round double result; float result has the right precision
525 // the poll sets the condition code, but no data registers
526
527 #ifdef _LP64
528 const Register thread = r15_thread;
529 #else
530 const Register thread = rbx;
531 __ get_thread(thread);
532 #endif
533 code_stub->set_safepoint_offset(__ offset());
534 __ relocate(relocInfo::poll_return_type);
535 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
536 __ ret(0);
537 }
538
539
540 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
541 return (__ store_inline_type_fields_to_buf(vk, false));
542 }
543
544 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
545 guarantee(info != nullptr, "Shouldn't be null");
546 int offset = __ offset();
547 #ifdef _LP64
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 #else
551 assert(tmp->is_cpu_register(), "needed");
552 const Register poll_addr = tmp->as_register();
553 __ get_thread(poll_addr);
554 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
555 #endif
556 add_debug_info_for_branch(info);
557 __ relocate(relocInfo::poll_type);
558 address pre_pc = __ pc();
559 __ testl(rax, Address(poll_addr, 0));
560 address post_pc = __ pc();
561 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
562 return offset;
563 }
564
565
566 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
567 if (from_reg != to_reg) __ mov(to_reg, from_reg);
568 }
569
570 void LIR_Assembler::swap_reg(Register a, Register b) {
571 __ xchgptr(a, b);
572 }
573
574
575 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
576 assert(src->is_constant(), "should not call otherwise");
577 assert(dest->is_register(), "should not call otherwise");
578 LIR_Const* c = src->as_constant_ptr();
579
580 switch (c->type()) {
581 case T_INT: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movl(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_ADDRESS: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ movptr(dest->as_register(), c->as_jint());
590 break;
591 }
592
593 case T_LONG: {
594 assert(patch_code == lir_patch_none, "no patching handled here");
595 #ifdef _LP64
596 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
597 #else
598 __ movptr(dest->as_register_lo(), c->as_jint_lo());
599 __ movptr(dest->as_register_hi(), c->as_jint_hi());
600 #endif // _LP64
601 break;
602 }
603
604 case T_OBJECT: {
605 if (patch_code != lir_patch_none) {
606 jobject2reg_with_patching(dest->as_register(), info);
607 } else {
608 __ movoop(dest->as_register(), c->as_jobject());
609 }
610 break;
611 }
612
613 case T_METADATA: {
614 if (patch_code != lir_patch_none) {
615 klass2reg_with_patching(dest->as_register(), info);
616 } else {
617 __ mov_metadata(dest->as_register(), c->as_metadata());
618 }
619 break;
620 }
621
622 case T_FLOAT: {
623 if (dest->is_single_xmm()) {
624 if (UseAVX <= 2 && c->is_zero_float()) {
625 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
626 } else {
627 __ movflt(dest->as_xmm_float_reg(),
628 InternalAddress(float_constant(c->as_jfloat())));
629 }
630 } else {
631 ShouldNotReachHere();
632 }
633 break;
634 }
635
636 case T_DOUBLE: {
637 if (dest->is_double_xmm()) {
638 if (UseAVX <= 2 && c->is_zero_double()) {
639 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
640 } else {
641 __ movdbl(dest->as_xmm_double_reg(),
642 InternalAddress(double_constant(c->as_jdouble())));
643 }
644 } else {
645 ShouldNotReachHere();
646 }
647 break;
648 }
649
650 default:
651 ShouldNotReachHere();
652 }
653 }
654
655 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
656 assert(src->is_constant(), "should not call otherwise");
657 assert(dest->is_stack(), "should not call otherwise");
658 LIR_Const* c = src->as_constant_ptr();
659
660 switch (c->type()) {
661 case T_INT: // fall through
662 case T_FLOAT:
663 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
664 break;
665
666 case T_ADDRESS:
667 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
668 break;
669
670 case T_OBJECT:
671 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
672 break;
673
674 case T_LONG: // fall through
675 case T_DOUBLE:
676 #ifdef _LP64
677 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
678 lo_word_offset_in_bytes),
679 (intptr_t)c->as_jlong_bits(),
680 rscratch1);
681 #else
682 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
683 lo_word_offset_in_bytes), c->as_jint_lo_bits());
684 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
685 hi_word_offset_in_bytes), c->as_jint_hi_bits());
686 #endif // _LP64
687 break;
688
689 default:
690 ShouldNotReachHere();
691 }
692 }
693
694 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
695 assert(src->is_constant(), "should not call otherwise");
696 assert(dest->is_address(), "should not call otherwise");
697 LIR_Const* c = src->as_constant_ptr();
698 LIR_Address* addr = dest->as_address_ptr();
699
700 int null_check_here = code_offset();
701 switch (type) {
702 case T_INT: // fall through
703 case T_FLOAT:
704 __ movl(as_Address(addr), c->as_jint_bits());
705 break;
706
707 case T_ADDRESS:
708 __ movptr(as_Address(addr), c->as_jint_bits());
709 break;
710
711 case T_OBJECT: // fall through
712 case T_ARRAY:
713 if (c->as_jobject() == nullptr) {
714 if (UseCompressedOops && !wide) {
715 __ movl(as_Address(addr), NULL_WORD);
716 } else {
717 #ifdef _LP64
718 __ xorptr(rscratch1, rscratch1);
719 null_check_here = code_offset();
720 __ movptr(as_Address(addr), rscratch1);
721 #else
722 __ movptr(as_Address(addr), NULL_WORD);
723 #endif
724 }
725 } else {
726 if (is_literal_address(addr)) {
727 ShouldNotReachHere();
728 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
729 } else {
730 #ifdef _LP64
731 __ movoop(rscratch1, c->as_jobject());
732 if (UseCompressedOops && !wide) {
733 __ encode_heap_oop(rscratch1);
734 null_check_here = code_offset();
735 __ movl(as_Address_lo(addr), rscratch1);
736 } else {
737 null_check_here = code_offset();
738 __ movptr(as_Address_lo(addr), rscratch1);
739 }
740 #else
741 __ movoop(as_Address(addr), c->as_jobject(), noreg);
742 #endif
743 }
744 }
745 break;
746
747 case T_LONG: // fall through
748 case T_DOUBLE:
749 #ifdef _LP64
750 if (is_literal_address(addr)) {
751 ShouldNotReachHere();
752 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
753 } else {
754 __ movptr(r10, (intptr_t)c->as_jlong_bits());
755 null_check_here = code_offset();
756 __ movptr(as_Address_lo(addr), r10);
757 }
758 #else
759 // Always reachable in 32bit so this doesn't produce useless move literal
760 __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
761 __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
762 #endif // _LP64
763 break;
764
765 case T_BOOLEAN: // fall through
766 case T_BYTE:
767 __ movb(as_Address(addr), c->as_jint() & 0xFF);
768 break;
769
770 case T_CHAR: // fall through
771 case T_SHORT:
772 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
773 break;
774
775 default:
776 ShouldNotReachHere();
777 };
778
779 if (info != nullptr) {
780 add_debug_info_for_null_check(null_check_here, info);
781 }
782 }
783
784
785 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
786 assert(src->is_register(), "should not call otherwise");
787 assert(dest->is_register(), "should not call otherwise");
788
789 // move between cpu-registers
790 if (dest->is_single_cpu()) {
791 #ifdef _LP64
792 if (src->type() == T_LONG) {
793 // Can do LONG -> OBJECT
794 move_regs(src->as_register_lo(), dest->as_register());
795 return;
796 }
797 #endif
798 assert(src->is_single_cpu(), "must match");
799 if (src->type() == T_OBJECT) {
800 __ verify_oop(src->as_register());
801 }
802 move_regs(src->as_register(), dest->as_register());
803
804 } else if (dest->is_double_cpu()) {
805 #ifdef _LP64
806 if (is_reference_type(src->type())) {
807 // Surprising to me but we can see move of a long to t_object
808 __ verify_oop(src->as_register());
809 move_regs(src->as_register(), dest->as_register_lo());
810 return;
811 }
812 #endif
813 assert(src->is_double_cpu(), "must match");
814 Register f_lo = src->as_register_lo();
815 Register f_hi = src->as_register_hi();
816 Register t_lo = dest->as_register_lo();
817 Register t_hi = dest->as_register_hi();
818 #ifdef _LP64
819 assert(f_hi == f_lo, "must be same");
820 assert(t_hi == t_lo, "must be same");
821 move_regs(f_lo, t_lo);
822 #else
823 assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
824
825
826 if (f_lo == t_hi && f_hi == t_lo) {
827 swap_reg(f_lo, f_hi);
828 } else if (f_hi == t_lo) {
829 assert(f_lo != t_hi, "overwriting register");
830 move_regs(f_hi, t_hi);
831 move_regs(f_lo, t_lo);
832 } else {
833 assert(f_hi != t_lo, "overwriting register");
834 move_regs(f_lo, t_lo);
835 move_regs(f_hi, t_hi);
836 }
837 #endif // LP64
838
839 // move between xmm-registers
840 } else if (dest->is_single_xmm()) {
841 assert(src->is_single_xmm(), "must match");
842 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
843 } else if (dest->is_double_xmm()) {
844 assert(src->is_double_xmm(), "must match");
845 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
846
847 } else {
848 ShouldNotReachHere();
849 }
850 }
851
852 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
853 assert(src->is_register(), "should not call otherwise");
854 assert(dest->is_stack(), "should not call otherwise");
855
856 if (src->is_single_cpu()) {
857 Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
858 if (is_reference_type(type)) {
859 __ verify_oop(src->as_register());
860 __ movptr (dst, src->as_register());
861 } else if (type == T_METADATA || type == T_ADDRESS) {
862 __ movptr (dst, src->as_register());
863 } else {
864 __ movl (dst, src->as_register());
865 }
866
867 } else if (src->is_double_cpu()) {
868 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
869 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
870 __ movptr (dstLO, src->as_register_lo());
871 NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
872
873 } else if (src->is_single_xmm()) {
874 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
875 __ movflt(dst_addr, src->as_xmm_float_reg());
876
877 } else if (src->is_double_xmm()) {
878 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
879 __ movdbl(dst_addr, src->as_xmm_double_reg());
880
881 } else {
882 ShouldNotReachHere();
883 }
884 }
885
886
887 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
888 LIR_Address* to_addr = dest->as_address_ptr();
889 PatchingStub* patch = nullptr;
890 Register compressed_src = rscratch1;
891
892 if (is_reference_type(type)) {
893 __ verify_oop(src->as_register());
894 #ifdef _LP64
895 if (UseCompressedOops && !wide) {
896 __ movptr(compressed_src, src->as_register());
897 __ encode_heap_oop(compressed_src);
898 if (patch_code != lir_patch_none) {
899 info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
900 }
901 }
902 #endif
903 }
904
905 if (patch_code != lir_patch_none) {
906 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
907 Address toa = as_Address(to_addr);
908 assert(toa.disp() != 0, "must have");
909 }
910
911 int null_check_here = code_offset();
912 switch (type) {
913 case T_FLOAT: {
914 assert(src->is_single_xmm(), "not a float");
915 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
916 break;
917 }
918
919 case T_DOUBLE: {
920 assert(src->is_double_xmm(), "not a double");
921 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
922 break;
923 }
924
925 case T_ARRAY: // fall through
926 case T_OBJECT: // fall through
927 if (UseCompressedOops && !wide) {
928 __ movl(as_Address(to_addr), compressed_src);
929 } else {
930 __ movptr(as_Address(to_addr), src->as_register());
931 }
932 break;
933 case T_METADATA:
934 // We get here to store a method pointer to the stack to pass to
935 // a dtrace runtime call. This can't work on 64 bit with
936 // compressed klass ptrs: T_METADATA can be a compressed klass
937 // ptr or a 64 bit method pointer.
938 LP64_ONLY(ShouldNotReachHere());
939 __ movptr(as_Address(to_addr), src->as_register());
940 break;
941 case T_ADDRESS:
942 __ movptr(as_Address(to_addr), src->as_register());
943 break;
944 case T_INT:
945 __ movl(as_Address(to_addr), src->as_register());
946 break;
947
948 case T_LONG: {
949 Register from_lo = src->as_register_lo();
950 Register from_hi = src->as_register_hi();
951 #ifdef _LP64
952 __ movptr(as_Address_lo(to_addr), from_lo);
953 #else
954 Register base = to_addr->base()->as_register();
955 Register index = noreg;
956 if (to_addr->index()->is_register()) {
957 index = to_addr->index()->as_register();
958 }
959 if (base == from_lo || index == from_lo) {
960 assert(base != from_hi, "can't be");
961 assert(index == noreg || (index != base && index != from_hi), "can't handle this");
962 __ movl(as_Address_hi(to_addr), from_hi);
963 if (patch != nullptr) {
964 patching_epilog(patch, lir_patch_high, base, info);
965 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
966 patch_code = lir_patch_low;
967 }
968 __ movl(as_Address_lo(to_addr), from_lo);
969 } else {
970 assert(index == noreg || (index != base && index != from_lo), "can't handle this");
971 __ movl(as_Address_lo(to_addr), from_lo);
972 if (patch != nullptr) {
973 patching_epilog(patch, lir_patch_low, base, info);
974 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
975 patch_code = lir_patch_high;
976 }
977 __ movl(as_Address_hi(to_addr), from_hi);
978 }
979 #endif // _LP64
980 break;
981 }
982
983 case T_BYTE: // fall through
984 case T_BOOLEAN: {
985 Register src_reg = src->as_register();
986 Address dst_addr = as_Address(to_addr);
987 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
988 __ movb(dst_addr, src_reg);
989 break;
990 }
991
992 case T_CHAR: // fall through
993 case T_SHORT:
994 __ movw(as_Address(to_addr), src->as_register());
995 break;
996
997 default:
998 ShouldNotReachHere();
999 }
1000 if (info != nullptr) {
1001 add_debug_info_for_null_check(null_check_here, info);
1002 }
1003
1004 if (patch_code != lir_patch_none) {
1005 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1006 }
1007 }
1008
1009
1010 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1011 assert(src->is_stack(), "should not call otherwise");
1012 assert(dest->is_register(), "should not call otherwise");
1013
1014 if (dest->is_single_cpu()) {
1015 if (is_reference_type(type)) {
1016 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1017 __ verify_oop(dest->as_register());
1018 } else if (type == T_METADATA || type == T_ADDRESS) {
1019 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1020 } else {
1021 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1022 }
1023
1024 } else if (dest->is_double_cpu()) {
1025 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1026 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1027 __ movptr(dest->as_register_lo(), src_addr_LO);
1028 NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1029
1030 } else if (dest->is_single_xmm()) {
1031 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1032 __ movflt(dest->as_xmm_float_reg(), src_addr);
1033
1034 } else if (dest->is_double_xmm()) {
1035 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1036 __ movdbl(dest->as_xmm_double_reg(), src_addr);
1037
1038 } else {
1039 ShouldNotReachHere();
1040 }
1041 }
1042
1043
1044 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1045 if (src->is_single_stack()) {
1046 if (is_reference_type(type)) {
1047 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1048 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1049 } else {
1050 #ifndef _LP64
1051 __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1052 __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1053 #else
1054 //no pushl on 64bits
1055 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1056 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1057 #endif
1058 }
1059
1060 } else if (src->is_double_stack()) {
1061 #ifdef _LP64
1062 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1063 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1064 #else
1065 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1066 // push and pop the part at src + wordSize, adding wordSize for the previous push
1067 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1068 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1069 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1070 #endif // _LP64
1071
1072 } else {
1073 ShouldNotReachHere();
1074 }
1075 }
1076
1077
1078 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
1079 assert(src->is_address(), "should not call otherwise");
1080 assert(dest->is_register(), "should not call otherwise");
1081
1082 LIR_Address* addr = src->as_address_ptr();
1083 Address from_addr = as_Address(addr);
1084
1085 if (addr->base()->type() == T_OBJECT) {
1086 __ verify_oop(addr->base()->as_pointer_register());
1087 }
1088
1089 switch (type) {
1090 case T_BOOLEAN: // fall through
1091 case T_BYTE: // fall through
1092 case T_CHAR: // fall through
1093 case T_SHORT:
1094 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1095 // on pre P6 processors we may get partial register stalls
1096 // so blow away the value of to_rinfo before loading a
1097 // partial word into it. Do it here so that it precedes
1098 // the potential patch point below.
1099 __ xorptr(dest->as_register(), dest->as_register());
1100 }
1101 break;
1102 default:
1103 break;
1104 }
1105
1106 PatchingStub* patch = nullptr;
1107 if (patch_code != lir_patch_none) {
1108 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1109 assert(from_addr.disp() != 0, "must have");
1110 }
1111 if (info != nullptr) {
1112 add_debug_info_for_null_check_here(info);
1113 }
1114
1115 switch (type) {
1116 case T_FLOAT: {
1117 if (dest->is_single_xmm()) {
1118 __ movflt(dest->as_xmm_float_reg(), from_addr);
1119 } else {
1120 ShouldNotReachHere();
1121 }
1122 break;
1123 }
1124
1125 case T_DOUBLE: {
1126 if (dest->is_double_xmm()) {
1127 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1128 } else {
1129 ShouldNotReachHere();
1130 }
1131 break;
1132 }
1133
1134 case T_OBJECT: // fall through
1135 case T_ARRAY: // fall through
1136 if (UseCompressedOops && !wide) {
1137 __ movl(dest->as_register(), from_addr);
1138 } else {
1139 __ movptr(dest->as_register(), from_addr);
1140 }
1141 break;
1142
1143 case T_ADDRESS:
1144 __ movptr(dest->as_register(), from_addr);
1145 break;
1146 case T_INT:
1147 __ movl(dest->as_register(), from_addr);
1148 break;
1149
1150 case T_LONG: {
1151 Register to_lo = dest->as_register_lo();
1152 Register to_hi = dest->as_register_hi();
1153 #ifdef _LP64
1154 __ movptr(to_lo, as_Address_lo(addr));
1155 #else
1156 Register base = addr->base()->as_register();
1157 Register index = noreg;
1158 if (addr->index()->is_register()) {
1159 index = addr->index()->as_register();
1160 }
1161 if ((base == to_lo && index == to_hi) ||
1162 (base == to_hi && index == to_lo)) {
1163 // addresses with 2 registers are only formed as a result of
1164 // array access so this code will never have to deal with
1165 // patches or null checks.
1166 assert(info == nullptr && patch == nullptr, "must be");
1167 __ lea(to_hi, as_Address(addr));
1168 __ movl(to_lo, Address(to_hi, 0));
1169 __ movl(to_hi, Address(to_hi, BytesPerWord));
1170 } else if (base == to_lo || index == to_lo) {
1171 assert(base != to_hi, "can't be");
1172 assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1173 __ movl(to_hi, as_Address_hi(addr));
1174 if (patch != nullptr) {
1175 patching_epilog(patch, lir_patch_high, base, info);
1176 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1177 patch_code = lir_patch_low;
1178 }
1179 __ movl(to_lo, as_Address_lo(addr));
1180 } else {
1181 assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1182 __ movl(to_lo, as_Address_lo(addr));
1183 if (patch != nullptr) {
1184 patching_epilog(patch, lir_patch_low, base, info);
1185 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1186 patch_code = lir_patch_high;
1187 }
1188 __ movl(to_hi, as_Address_hi(addr));
1189 }
1190 #endif // _LP64
1191 break;
1192 }
1193
1194 case T_BOOLEAN: // fall through
1195 case T_BYTE: {
1196 Register dest_reg = dest->as_register();
1197 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1198 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1199 __ movsbl(dest_reg, from_addr);
1200 } else {
1201 __ movb(dest_reg, from_addr);
1202 __ shll(dest_reg, 24);
1203 __ sarl(dest_reg, 24);
1204 }
1205 break;
1206 }
1207
1208 case T_CHAR: {
1209 Register dest_reg = dest->as_register();
1210 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1211 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1212 __ movzwl(dest_reg, from_addr);
1213 } else {
1214 __ movw(dest_reg, from_addr);
1215 }
1216 break;
1217 }
1218
1219 case T_SHORT: {
1220 Register dest_reg = dest->as_register();
1221 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1222 __ movswl(dest_reg, from_addr);
1223 } else {
1224 __ movw(dest_reg, from_addr);
1225 __ shll(dest_reg, 16);
1226 __ sarl(dest_reg, 16);
1227 }
1228 break;
1229 }
1230
1231 default:
1232 ShouldNotReachHere();
1233 }
1234
1235 if (patch != nullptr) {
1236 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1237 }
1238
1239 if (is_reference_type(type)) {
1240 #ifdef _LP64
1241 if (UseCompressedOops && !wide) {
1242 __ decode_heap_oop(dest->as_register());
1243 }
1244 #endif
1245
1246 __ verify_oop(dest->as_register());
1247 }
1248 }
1249
1250
1251 NEEDS_CLEANUP; // This could be static?
1252 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1253 int elem_size = type2aelembytes(type);
1254 switch (elem_size) {
1255 case 1: return Address::times_1;
1256 case 2: return Address::times_2;
1257 case 4: return Address::times_4;
1258 case 8: return Address::times_8;
1259 }
1260 ShouldNotReachHere();
1261 return Address::no_scale;
1262 }
1263
1264
1265 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1266 switch (op->code()) {
1267 case lir_idiv:
1268 case lir_irem:
1269 arithmetic_idiv(op->code(),
1270 op->in_opr1(),
1271 op->in_opr2(),
1272 op->in_opr3(),
1273 op->result_opr(),
1274 op->info());
1275 break;
1276 case lir_fmad:
1277 __ fmad(op->result_opr()->as_xmm_double_reg(),
1278 op->in_opr1()->as_xmm_double_reg(),
1279 op->in_opr2()->as_xmm_double_reg(),
1280 op->in_opr3()->as_xmm_double_reg());
1281 break;
1282 case lir_fmaf:
1283 __ fmaf(op->result_opr()->as_xmm_float_reg(),
1284 op->in_opr1()->as_xmm_float_reg(),
1285 op->in_opr2()->as_xmm_float_reg(),
1286 op->in_opr3()->as_xmm_float_reg());
1287 break;
1288 default: ShouldNotReachHere(); break;
1289 }
1290 }
1291
1292 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1293 #ifdef ASSERT
1294 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1295 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1296 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1297 #endif
1298
1299 if (op->cond() == lir_cond_always) {
1300 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1301 __ jmp (*(op->label()));
1302 } else {
1303 Assembler::Condition acond = Assembler::zero;
1304 if (op->code() == lir_cond_float_branch) {
1305 assert(op->ublock() != nullptr, "must have unordered successor");
1306 __ jcc(Assembler::parity, *(op->ublock()->label()));
1307 switch(op->cond()) {
1308 case lir_cond_equal: acond = Assembler::equal; break;
1309 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1310 case lir_cond_less: acond = Assembler::below; break;
1311 case lir_cond_lessEqual: acond = Assembler::belowEqual; break;
1312 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1313 case lir_cond_greater: acond = Assembler::above; break;
1314 default: ShouldNotReachHere();
1315 }
1316 } else {
1317 switch (op->cond()) {
1318 case lir_cond_equal: acond = Assembler::equal; break;
1319 case lir_cond_notEqual: acond = Assembler::notEqual; break;
1320 case lir_cond_less: acond = Assembler::less; break;
1321 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
1322 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1323 case lir_cond_greater: acond = Assembler::greater; break;
1324 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
1325 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
1326 default: ShouldNotReachHere();
1327 }
1328 }
1329 __ jcc(acond,*(op->label()));
1330 }
1331 }
1332
1333 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1334 LIR_Opr src = op->in_opr();
1335 LIR_Opr dest = op->result_opr();
1336
1337 switch (op->bytecode()) {
1338 case Bytecodes::_i2l:
1339 #ifdef _LP64
1340 __ movl2ptr(dest->as_register_lo(), src->as_register());
1341 #else
1342 move_regs(src->as_register(), dest->as_register_lo());
1343 move_regs(src->as_register(), dest->as_register_hi());
1344 __ sarl(dest->as_register_hi(), 31);
1345 #endif // LP64
1346 break;
1347
1348 case Bytecodes::_l2i:
1349 #ifdef _LP64
1350 __ movl(dest->as_register(), src->as_register_lo());
1351 #else
1352 move_regs(src->as_register_lo(), dest->as_register());
1353 #endif
1354 break;
1355
1356 case Bytecodes::_i2b:
1357 move_regs(src->as_register(), dest->as_register());
1358 __ sign_extend_byte(dest->as_register());
1359 break;
1360
1361 case Bytecodes::_i2c:
1362 move_regs(src->as_register(), dest->as_register());
1363 __ andl(dest->as_register(), 0xFFFF);
1364 break;
1365
1366 case Bytecodes::_i2s:
1367 move_regs(src->as_register(), dest->as_register());
1368 __ sign_extend_short(dest->as_register());
1369 break;
1370
1371 case Bytecodes::_f2d:
1372 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1373 break;
1374
1375 case Bytecodes::_d2f:
1376 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1377 break;
1378
1379 case Bytecodes::_i2f:
1380 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1381 break;
1382
1383 case Bytecodes::_i2d:
1384 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1385 break;
1386
1387 case Bytecodes::_l2f:
1388 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1389 break;
1390
1391 case Bytecodes::_l2d:
1392 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1393 break;
1394
1395 case Bytecodes::_f2i:
1396 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1397 break;
1398
1399 case Bytecodes::_d2i:
1400 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1401 break;
1402
1403 case Bytecodes::_f2l:
1404 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1405 break;
1406
1407 case Bytecodes::_d2l:
1408 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1409 break;
1410
1411 default: ShouldNotReachHere();
1412 }
1413 }
1414
1415 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1416 if (op->init_check()) {
1417 add_debug_info_for_null_check_here(op->stub()->info());
1418 // init_state needs acquire, but x86 is TSO, and so we are already good.
1419 __ cmpb(Address(op->klass()->as_register(),
1420 InstanceKlass::init_state_offset()),
1421 InstanceKlass::fully_initialized);
1422 __ jcc(Assembler::notEqual, *op->stub()->entry());
1423 }
1424 __ allocate_object(op->obj()->as_register(),
1425 op->tmp1()->as_register(),
1426 op->tmp2()->as_register(),
1427 op->header_size(),
1428 op->object_size(),
1429 op->klass()->as_register(),
1430 *op->stub()->entry());
1431 __ bind(*op->stub()->continuation());
1432 }
1433
1434 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1435 Register len = op->len()->as_register();
1436 LP64_ONLY( __ movslq(len, len); )
1437
1438 if (UseSlowPath || op->is_null_free() ||
1439 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1440 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1441 __ jmp(*op->stub()->entry());
1442 } else {
1443 Register tmp1 = op->tmp1()->as_register();
1444 Register tmp2 = op->tmp2()->as_register();
1445 Register tmp3 = op->tmp3()->as_register();
1446 if (len == tmp1) {
1447 tmp1 = tmp3;
1448 } else if (len == tmp2) {
1449 tmp2 = tmp3;
1450 } else if (len == tmp3) {
1451 // everything is ok
1452 } else {
1453 __ mov(tmp3, len);
1454 }
1455 __ allocate_array(op->obj()->as_register(),
1456 len,
1457 tmp1,
1458 tmp2,
1459 arrayOopDesc::base_offset_in_bytes(op->type()),
1460 array_element_size(op->type()),
1461 op->klass()->as_register(),
1462 *op->stub()->entry(),
1463 op->zero_array());
1464 }
1465 __ bind(*op->stub()->continuation());
1466 }
1467
1468 void LIR_Assembler::type_profile_helper(Register mdo,
1469 ciMethodData *md, ciProfileData *data,
1470 Register recv, Label* update_done) {
1471 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1472 Label next_test;
1473 // See if the receiver is receiver[n].
1474 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1475 __ jccb(Assembler::notEqual, next_test);
1476 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1477 __ addptr(data_addr, DataLayout::counter_increment);
1478 __ jmp(*update_done);
1479 __ bind(next_test);
1480 }
1481
1482 // Didn't find receiver; find next empty slot and fill it in
1483 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1484 Label next_test;
1485 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1486 __ cmpptr(recv_addr, NULL_WORD);
1487 __ jccb(Assembler::notEqual, next_test);
1488 __ movptr(recv_addr, recv);
1489 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1490 __ jmp(*update_done);
1491 __ bind(next_test);
1492 }
1493 }
1494
1495 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1496 // we always need a stub for the failure case.
1497 CodeStub* stub = op->stub();
1498 Register obj = op->object()->as_register();
1499 Register k_RInfo = op->tmp1()->as_register();
1500 Register klass_RInfo = op->tmp2()->as_register();
1501 Register dst = op->result_opr()->as_register();
1502 ciKlass* k = op->klass();
1503 Register Rtmp1 = noreg;
1504 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1505
1506 // check if it needs to be profiled
1507 ciMethodData* md = nullptr;
1508 ciProfileData* data = nullptr;
1509
1510 if (op->should_profile()) {
1511 ciMethod* method = op->profiled_method();
1512 assert(method != nullptr, "Should have method");
1513 int bci = op->profiled_bci();
1514 md = method->method_data_or_null();
1515 assert(md != nullptr, "Sanity");
1516 data = md->bci_to_data(bci);
1517 assert(data != nullptr, "need data for type check");
1518 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1519 }
1520 Label* success_target = success;
1521 Label* failure_target = failure;
1522
1523 if (obj == k_RInfo) {
1524 k_RInfo = dst;
1525 } else if (obj == klass_RInfo) {
1526 klass_RInfo = dst;
1527 }
1528 if (k->is_loaded() && !UseCompressedClassPointers) {
1529 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1530 } else {
1531 Rtmp1 = op->tmp3()->as_register();
1532 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1533 }
1534
1535 assert_different_registers(obj, k_RInfo, klass_RInfo);
1536
1537 if (op->need_null_check()) {
1538 __ testptr(obj, obj);
1539 if (op->should_profile()) {
1540 Label not_null;
1541 Register mdo = klass_RInfo;
1542 __ mov_metadata(mdo, md->constant_encoding());
1543 __ jccb(Assembler::notEqual, not_null);
1544 // Object is null; update MDO and exit
1545 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1546 int header_bits = BitData::null_seen_byte_constant();
1547 __ orb(data_addr, header_bits);
1548 __ jmp(*obj_is_null);
1549 __ bind(not_null);
1550
1551 Label update_done;
1552 Register recv = k_RInfo;
1553 __ load_klass(recv, obj, tmp_load_klass);
1554 type_profile_helper(mdo, md, data, recv, &update_done);
1555
1556 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1557 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1558
1559 __ bind(update_done);
1560 } else {
1561 __ jcc(Assembler::equal, *obj_is_null);
1562 }
1563 }
1564
1565 if (!k->is_loaded()) {
1566 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1567 } else {
1568 #ifdef _LP64
1569 __ mov_metadata(k_RInfo, k->constant_encoding());
1570 #endif // _LP64
1571 }
1572 __ verify_oop(obj);
1573
1574 if (op->fast_check()) {
1575 // get object class
1576 // not a safepoint as obj null check happens earlier
1577 #ifdef _LP64
1578 if (UseCompressedClassPointers) {
1579 __ load_klass(Rtmp1, obj, tmp_load_klass);
1580 __ cmpptr(k_RInfo, Rtmp1);
1581 } else {
1582 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1583 }
1584 #else
1585 if (k->is_loaded()) {
1586 __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1587 } else {
1588 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1589 }
1590 #endif
1591 __ jcc(Assembler::notEqual, *failure_target);
1592 // successful cast, fall through to profile or jump
1593 } else {
1594 // get object class
1595 // not a safepoint as obj null check happens earlier
1596 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1597 if (k->is_loaded()) {
1598 // See if we get an immediate positive hit
1599 #ifdef _LP64
1600 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1601 #else
1602 __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1603 #endif // _LP64
1604 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1605 __ jcc(Assembler::notEqual, *failure_target);
1606 // successful cast, fall through to profile or jump
1607 } else {
1608 // See if we get an immediate positive hit
1609 __ jcc(Assembler::equal, *success_target);
1610 // check for self
1611 #ifdef _LP64
1612 __ cmpptr(klass_RInfo, k_RInfo);
1613 #else
1614 __ cmpklass(klass_RInfo, k->constant_encoding());
1615 #endif // _LP64
1616 __ jcc(Assembler::equal, *success_target);
1617
1618 __ push(klass_RInfo);
1619 #ifdef _LP64
1620 __ push(k_RInfo);
1621 #else
1622 __ pushklass(k->constant_encoding(), noreg);
1623 #endif // _LP64
1624 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1625 __ pop(klass_RInfo);
1626 __ pop(klass_RInfo);
1627 // result is a boolean
1628 __ testl(klass_RInfo, klass_RInfo);
1629 __ jcc(Assembler::equal, *failure_target);
1630 // successful cast, fall through to profile or jump
1631 }
1632 } else {
1633 // perform the fast part of the checking logic
1634 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1635 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1636 __ push(klass_RInfo);
1637 __ push(k_RInfo);
1638 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1639 __ pop(klass_RInfo);
1640 __ pop(k_RInfo);
1641 // result is a boolean
1642 __ testl(k_RInfo, k_RInfo);
1643 __ jcc(Assembler::equal, *failure_target);
1644 // successful cast, fall through to profile or jump
1645 }
1646 }
1647 __ jmp(*success);
1648 }
1649
1650
1651 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1652 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1653 LIR_Code code = op->code();
1654 if (code == lir_store_check) {
1655 Register value = op->object()->as_register();
1656 Register array = op->array()->as_register();
1657 Register k_RInfo = op->tmp1()->as_register();
1658 Register klass_RInfo = op->tmp2()->as_register();
1659 Register Rtmp1 = op->tmp3()->as_register();
1660
1661 CodeStub* stub = op->stub();
1662
1663 // check if it needs to be profiled
1664 ciMethodData* md = nullptr;
1665 ciProfileData* data = nullptr;
1666
1667 if (op->should_profile()) {
1668 ciMethod* method = op->profiled_method();
1669 assert(method != nullptr, "Should have method");
1670 int bci = op->profiled_bci();
1671 md = method->method_data_or_null();
1672 assert(md != nullptr, "Sanity");
1673 data = md->bci_to_data(bci);
1674 assert(data != nullptr, "need data for type check");
1675 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1676 }
1677 Label done;
1678 Label* success_target = &done;
1679 Label* failure_target = stub->entry();
1680
1681 __ testptr(value, value);
1682 if (op->should_profile()) {
1683 Label not_null;
1684 Register mdo = klass_RInfo;
1685 __ mov_metadata(mdo, md->constant_encoding());
1686 __ jccb(Assembler::notEqual, not_null);
1687 // Object is null; update MDO and exit
1688 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1689 int header_bits = BitData::null_seen_byte_constant();
1690 __ orb(data_addr, header_bits);
1691 __ jmp(done);
1692 __ bind(not_null);
1693
1694 Label update_done;
1695 Register recv = k_RInfo;
1696 __ load_klass(recv, value, tmp_load_klass);
1697 type_profile_helper(mdo, md, data, recv, &update_done);
1698
1699 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1700 __ addptr(counter_addr, DataLayout::counter_increment);
1701 __ bind(update_done);
1702 } else {
1703 __ jcc(Assembler::equal, done);
1704 }
1705
1706 add_debug_info_for_null_check_here(op->info_for_exception());
1707 __ load_klass(k_RInfo, array, tmp_load_klass);
1708 __ load_klass(klass_RInfo, value, tmp_load_klass);
1709
1710 // get instance klass (it's already uncompressed)
1711 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1712 // perform the fast part of the checking logic
1713 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1714 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1715 __ push(klass_RInfo);
1716 __ push(k_RInfo);
1717 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1718 __ pop(klass_RInfo);
1719 __ pop(k_RInfo);
1720 // result is a boolean
1721 __ testl(k_RInfo, k_RInfo);
1722 __ jcc(Assembler::equal, *failure_target);
1723 // fall through to the success case
1724
1725 __ bind(done);
1726 } else
1727 if (code == lir_checkcast) {
1728 Register obj = op->object()->as_register();
1729 Register dst = op->result_opr()->as_register();
1730 Label success;
1731 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1732 __ bind(success);
1733 if (dst != obj) {
1734 __ mov(dst, obj);
1735 }
1736 } else
1737 if (code == lir_instanceof) {
1738 Register obj = op->object()->as_register();
1739 Register dst = op->result_opr()->as_register();
1740 Label success, failure, done;
1741 emit_typecheck_helper(op, &success, &failure, &failure);
1742 __ bind(failure);
1743 __ xorptr(dst, dst);
1744 __ jmpb(done);
1745 __ bind(success);
1746 __ movptr(dst, 1);
1747 __ bind(done);
1748 } else {
1749 ShouldNotReachHere();
1750 }
1751
1752 }
1753
1754 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1755 // We are loading/storing from/to an array that *may* be a flat array (the
1756 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1757 // If this array is a flat array, take the slow path.
1758 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1759 if (!op->value()->is_illegal()) {
1760 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1761 // The array is not a flat array, but it might be null-free. If we are storing
1762 // a null into a null-free array, take the slow path (which will throw NPE).
1763 Label skip;
1764 __ cmpptr(op->value()->as_register(), NULL_WORD);
1765 __ jcc(Assembler::notEqual, skip);
1766 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1767 __ bind(skip);
1768 }
1769 }
1770
1771 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1772 // We are storing into an array that *may* be null-free (the declared type is
1773 // Object[], abstract[], interface[] or VT.ref[]).
1774 Label test_mark_word;
1775 Register tmp = op->tmp()->as_register();
1776 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1777 __ testl(tmp, markWord::unlocked_value);
1778 __ jccb(Assembler::notZero, test_mark_word);
1779 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1780 __ bind(test_mark_word);
1781 __ testl(tmp, markWord::null_free_array_bit_in_place);
1782 }
1783
1784 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1785 Label L_oops_equal;
1786 Label L_oops_not_equal;
1787 Label L_end;
1788
1789 Register left = op->left()->as_register();
1790 Register right = op->right()->as_register();
1791
1792 __ cmpptr(left, right);
1793 __ jcc(Assembler::equal, L_oops_equal);
1794
1795 // (1) Null check -- if one of the operands is null, the other must not be null (because
1796 // the two references are not equal), so they are not substitutable,
1797 // FIXME: do null check only if the operand is nullable
1798 __ testptr(left, right);
1799 __ jcc(Assembler::zero, L_oops_not_equal);
1800
1801 ciKlass* left_klass = op->left_klass();
1802 ciKlass* right_klass = op->right_klass();
1803
1804 // (2) Inline type check -- if either of the operands is not a inline type,
1805 // they are not substitutable. We do this only if we are not sure that the
1806 // operands are inline type
1807 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1808 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1809 Register tmp1 = op->tmp1()->as_register();
1810 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1811 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1812 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1813 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1814 __ jcc(Assembler::notEqual, L_oops_not_equal);
1815 }
1816
1817 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1818 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1819 // No need to load klass -- the operands are statically known to be the same inline klass.
1820 __ jmp(*op->stub()->entry());
1821 } else {
1822 Register left_klass_op = op->left_klass_op()->as_register();
1823 Register right_klass_op = op->right_klass_op()->as_register();
1824
1825 if (UseCompressedClassPointers) {
1826 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1827 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1828 __ cmpl(left_klass_op, right_klass_op);
1829 } else {
1830 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1831 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1832 __ cmpptr(left_klass_op, right_klass_op);
1833 }
1834
1835 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1836 // fall through to L_oops_not_equal
1837 }
1838
1839 __ bind(L_oops_not_equal);
1840 move(op->not_equal_result(), op->result_opr());
1841 __ jmp(L_end);
1842
1843 __ bind(L_oops_equal);
1844 move(op->equal_result(), op->result_opr());
1845 __ jmp(L_end);
1846
1847 // We've returned from the stub. RAX contains 0x0 IFF the two
1848 // operands are not substitutable. (Don't compare against 0x1 in case the
1849 // C compiler is naughty)
1850 __ bind(*op->stub()->continuation());
1851 __ cmpl(rax, 0);
1852 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1853 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1854 // fall-through
1855 __ bind(L_end);
1856 }
1857
1858 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1859 if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1860 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1861 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1862 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1863 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1864 Register addr = op->addr()->as_register();
1865 __ lock();
1866 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1867
1868 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1869 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1870 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1871 Register newval = op->new_value()->as_register();
1872 Register cmpval = op->cmp_value()->as_register();
1873 assert(cmpval == rax, "wrong register");
1874 assert(newval != noreg, "new val must be register");
1875 assert(cmpval != newval, "cmp and new values must be in different registers");
1876 assert(cmpval != addr, "cmp and addr must be in different registers");
1877 assert(newval != addr, "new value and addr must be in different registers");
1878
1879 if ( op->code() == lir_cas_obj) {
1880 #ifdef _LP64
1881 if (UseCompressedOops) {
1882 __ encode_heap_oop(cmpval);
1883 __ mov(rscratch1, newval);
1884 __ encode_heap_oop(rscratch1);
1885 __ lock();
1886 // cmpval (rax) is implicitly used by this instruction
1887 __ cmpxchgl(rscratch1, Address(addr, 0));
1888 } else
1889 #endif
1890 {
1891 __ lock();
1892 __ cmpxchgptr(newval, Address(addr, 0));
1893 }
1894 } else {
1895 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1896 __ lock();
1897 __ cmpxchgl(newval, Address(addr, 0));
1898 }
1899 #ifdef _LP64
1900 } else if (op->code() == lir_cas_long) {
1901 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1902 Register newval = op->new_value()->as_register_lo();
1903 Register cmpval = op->cmp_value()->as_register_lo();
1904 assert(cmpval == rax, "wrong register");
1905 assert(newval != noreg, "new val must be register");
1906 assert(cmpval != newval, "cmp and new values must be in different registers");
1907 assert(cmpval != addr, "cmp and addr must be in different registers");
1908 assert(newval != addr, "new value and addr must be in different registers");
1909 __ lock();
1910 __ cmpxchgq(newval, Address(addr, 0));
1911 #endif // _LP64
1912 } else {
1913 Unimplemented();
1914 }
1915 }
1916
1917 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1918 assert(dst->is_cpu_register(), "must be");
1919 assert(dst->type() == src->type(), "must be");
1920
1921 if (src->is_cpu_register()) {
1922 reg2reg(src, dst);
1923 } else if (src->is_stack()) {
1924 stack2reg(src, dst, dst->type());
1925 } else if (src->is_constant()) {
1926 const2reg(src, dst, lir_patch_none, nullptr);
1927 } else {
1928 ShouldNotReachHere();
1929 }
1930 }
1931
1932 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1933 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1934 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1935
1936 Assembler::Condition acond, ncond;
1937 switch (condition) {
1938 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1939 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1940 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1941 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1942 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1943 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1944 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1945 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1946 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1947 ShouldNotReachHere();
1948 }
1949
1950 if (opr1->is_cpu_register()) {
1951 reg2reg(opr1, result);
1952 } else if (opr1->is_stack()) {
1953 stack2reg(opr1, result, result->type());
1954 } else if (opr1->is_constant()) {
1955 const2reg(opr1, result, lir_patch_none, nullptr);
1956 } else {
1957 ShouldNotReachHere();
1958 }
1959
1960 if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1961 // optimized version that does not require a branch
1962 if (opr2->is_single_cpu()) {
1963 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1964 __ cmov(ncond, result->as_register(), opr2->as_register());
1965 } else if (opr2->is_double_cpu()) {
1966 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1967 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1968 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1969 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
1970 } else if (opr2->is_single_stack()) {
1971 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1972 } else if (opr2->is_double_stack()) {
1973 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1974 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
1975 } else {
1976 ShouldNotReachHere();
1977 }
1978
1979 } else {
1980 Label skip;
1981 __ jccb(acond, skip);
1982 if (opr2->is_cpu_register()) {
1983 reg2reg(opr2, result);
1984 } else if (opr2->is_stack()) {
1985 stack2reg(opr2, result, result->type());
1986 } else if (opr2->is_constant()) {
1987 const2reg(opr2, result, lir_patch_none, nullptr);
1988 } else {
1989 ShouldNotReachHere();
1990 }
1991 __ bind(skip);
1992 }
1993 }
1994
1995
1996 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1997 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1998
1999 if (left->is_single_cpu()) {
2000 assert(left == dest, "left and dest must be equal");
2001 Register lreg = left->as_register();
2002
2003 if (right->is_single_cpu()) {
2004 // cpu register - cpu register
2005 Register rreg = right->as_register();
2006 switch (code) {
2007 case lir_add: __ addl (lreg, rreg); break;
2008 case lir_sub: __ subl (lreg, rreg); break;
2009 case lir_mul: __ imull(lreg, rreg); break;
2010 default: ShouldNotReachHere();
2011 }
2012
2013 } else if (right->is_stack()) {
2014 // cpu register - stack
2015 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2016 switch (code) {
2017 case lir_add: __ addl(lreg, raddr); break;
2018 case lir_sub: __ subl(lreg, raddr); break;
2019 default: ShouldNotReachHere();
2020 }
2021
2022 } else if (right->is_constant()) {
2023 // cpu register - constant
2024 jint c = right->as_constant_ptr()->as_jint();
2025 switch (code) {
2026 case lir_add: {
2027 __ incrementl(lreg, c);
2028 break;
2029 }
2030 case lir_sub: {
2031 __ decrementl(lreg, c);
2032 break;
2033 }
2034 default: ShouldNotReachHere();
2035 }
2036
2037 } else {
2038 ShouldNotReachHere();
2039 }
2040
2041 } else if (left->is_double_cpu()) {
2042 assert(left == dest, "left and dest must be equal");
2043 Register lreg_lo = left->as_register_lo();
2044 Register lreg_hi = left->as_register_hi();
2045
2046 if (right->is_double_cpu()) {
2047 // cpu register - cpu register
2048 Register rreg_lo = right->as_register_lo();
2049 Register rreg_hi = right->as_register_hi();
2050 NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2051 LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2052 switch (code) {
2053 case lir_add:
2054 __ addptr(lreg_lo, rreg_lo);
2055 NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2056 break;
2057 case lir_sub:
2058 __ subptr(lreg_lo, rreg_lo);
2059 NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2060 break;
2061 case lir_mul:
2062 #ifdef _LP64
2063 __ imulq(lreg_lo, rreg_lo);
2064 #else
2065 assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2066 __ imull(lreg_hi, rreg_lo);
2067 __ imull(rreg_hi, lreg_lo);
2068 __ addl (rreg_hi, lreg_hi);
2069 __ mull (rreg_lo);
2070 __ addl (lreg_hi, rreg_hi);
2071 #endif // _LP64
2072 break;
2073 default:
2074 ShouldNotReachHere();
2075 }
2076
2077 } else if (right->is_constant()) {
2078 // cpu register - constant
2079 #ifdef _LP64
2080 jlong c = right->as_constant_ptr()->as_jlong_bits();
2081 __ movptr(r10, (intptr_t) c);
2082 switch (code) {
2083 case lir_add:
2084 __ addptr(lreg_lo, r10);
2085 break;
2086 case lir_sub:
2087 __ subptr(lreg_lo, r10);
2088 break;
2089 default:
2090 ShouldNotReachHere();
2091 }
2092 #else
2093 jint c_lo = right->as_constant_ptr()->as_jint_lo();
2094 jint c_hi = right->as_constant_ptr()->as_jint_hi();
2095 switch (code) {
2096 case lir_add:
2097 __ addptr(lreg_lo, c_lo);
2098 __ adcl(lreg_hi, c_hi);
2099 break;
2100 case lir_sub:
2101 __ subptr(lreg_lo, c_lo);
2102 __ sbbl(lreg_hi, c_hi);
2103 break;
2104 default:
2105 ShouldNotReachHere();
2106 }
2107 #endif // _LP64
2108
2109 } else {
2110 ShouldNotReachHere();
2111 }
2112
2113 } else if (left->is_single_xmm()) {
2114 assert(left == dest, "left and dest must be equal");
2115 XMMRegister lreg = left->as_xmm_float_reg();
2116
2117 if (right->is_single_xmm()) {
2118 XMMRegister rreg = right->as_xmm_float_reg();
2119 switch (code) {
2120 case lir_add: __ addss(lreg, rreg); break;
2121 case lir_sub: __ subss(lreg, rreg); break;
2122 case lir_mul: __ mulss(lreg, rreg); break;
2123 case lir_div: __ divss(lreg, rreg); break;
2124 default: ShouldNotReachHere();
2125 }
2126 } else {
2127 Address raddr;
2128 if (right->is_single_stack()) {
2129 raddr = frame_map()->address_for_slot(right->single_stack_ix());
2130 } else if (right->is_constant()) {
2131 // hack for now
2132 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2133 } else {
2134 ShouldNotReachHere();
2135 }
2136 switch (code) {
2137 case lir_add: __ addss(lreg, raddr); break;
2138 case lir_sub: __ subss(lreg, raddr); break;
2139 case lir_mul: __ mulss(lreg, raddr); break;
2140 case lir_div: __ divss(lreg, raddr); break;
2141 default: ShouldNotReachHere();
2142 }
2143 }
2144
2145 } else if (left->is_double_xmm()) {
2146 assert(left == dest, "left and dest must be equal");
2147
2148 XMMRegister lreg = left->as_xmm_double_reg();
2149 if (right->is_double_xmm()) {
2150 XMMRegister rreg = right->as_xmm_double_reg();
2151 switch (code) {
2152 case lir_add: __ addsd(lreg, rreg); break;
2153 case lir_sub: __ subsd(lreg, rreg); break;
2154 case lir_mul: __ mulsd(lreg, rreg); break;
2155 case lir_div: __ divsd(lreg, rreg); break;
2156 default: ShouldNotReachHere();
2157 }
2158 } else {
2159 Address raddr;
2160 if (right->is_double_stack()) {
2161 raddr = frame_map()->address_for_slot(right->double_stack_ix());
2162 } else if (right->is_constant()) {
2163 // hack for now
2164 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2165 } else {
2166 ShouldNotReachHere();
2167 }
2168 switch (code) {
2169 case lir_add: __ addsd(lreg, raddr); break;
2170 case lir_sub: __ subsd(lreg, raddr); break;
2171 case lir_mul: __ mulsd(lreg, raddr); break;
2172 case lir_div: __ divsd(lreg, raddr); break;
2173 default: ShouldNotReachHere();
2174 }
2175 }
2176
2177 } else if (left->is_single_stack() || left->is_address()) {
2178 assert(left == dest, "left and dest must be equal");
2179
2180 Address laddr;
2181 if (left->is_single_stack()) {
2182 laddr = frame_map()->address_for_slot(left->single_stack_ix());
2183 } else if (left->is_address()) {
2184 laddr = as_Address(left->as_address_ptr());
2185 } else {
2186 ShouldNotReachHere();
2187 }
2188
2189 if (right->is_single_cpu()) {
2190 Register rreg = right->as_register();
2191 switch (code) {
2192 case lir_add: __ addl(laddr, rreg); break;
2193 case lir_sub: __ subl(laddr, rreg); break;
2194 default: ShouldNotReachHere();
2195 }
2196 } else if (right->is_constant()) {
2197 jint c = right->as_constant_ptr()->as_jint();
2198 switch (code) {
2199 case lir_add: {
2200 __ incrementl(laddr, c);
2201 break;
2202 }
2203 case lir_sub: {
2204 __ decrementl(laddr, c);
2205 break;
2206 }
2207 default: ShouldNotReachHere();
2208 }
2209 } else {
2210 ShouldNotReachHere();
2211 }
2212
2213 } else {
2214 ShouldNotReachHere();
2215 }
2216 }
2217
2218
2219 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2220 if (value->is_double_xmm()) {
2221 switch(code) {
2222 case lir_abs :
2223 {
2224 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2225 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2226 }
2227 assert(!tmp->is_valid(), "do not need temporary");
2228 __ andpd(dest->as_xmm_double_reg(),
2229 ExternalAddress((address)double_signmask_pool),
2230 rscratch1);
2231 }
2232 break;
2233
2234 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2235 // all other intrinsics are not available in the SSE instruction set, so FPU is used
2236 default : ShouldNotReachHere();
2237 }
2238
2239 } else if (code == lir_f2hf) {
2240 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2241 } else if (code == lir_hf2f) {
2242 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2243 } else {
2244 Unimplemented();
2245 }
2246 }
2247
2248 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2249 // assert(left->destroys_register(), "check");
2250 if (left->is_single_cpu()) {
2251 Register reg = left->as_register();
2252 if (right->is_constant()) {
2253 int val = right->as_constant_ptr()->as_jint();
2254 switch (code) {
2255 case lir_logic_and: __ andl (reg, val); break;
2256 case lir_logic_or: __ orl (reg, val); break;
2257 case lir_logic_xor: __ xorl (reg, val); break;
2258 default: ShouldNotReachHere();
2259 }
2260 } else if (right->is_stack()) {
2261 // added support for stack operands
2262 Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2263 switch (code) {
2264 case lir_logic_and: __ andl (reg, raddr); break;
2265 case lir_logic_or: __ orl (reg, raddr); break;
2266 case lir_logic_xor: __ xorl (reg, raddr); break;
2267 default: ShouldNotReachHere();
2268 }
2269 } else {
2270 Register rright = right->as_register();
2271 switch (code) {
2272 case lir_logic_and: __ andptr (reg, rright); break;
2273 case lir_logic_or : __ orptr (reg, rright); break;
2274 case lir_logic_xor: __ xorptr (reg, rright); break;
2275 default: ShouldNotReachHere();
2276 }
2277 }
2278 move_regs(reg, dst->as_register());
2279 } else {
2280 Register l_lo = left->as_register_lo();
2281 Register l_hi = left->as_register_hi();
2282 if (right->is_constant()) {
2283 #ifdef _LP64
2284 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2285 switch (code) {
2286 case lir_logic_and:
2287 __ andq(l_lo, rscratch1);
2288 break;
2289 case lir_logic_or:
2290 __ orq(l_lo, rscratch1);
2291 break;
2292 case lir_logic_xor:
2293 __ xorq(l_lo, rscratch1);
2294 break;
2295 default: ShouldNotReachHere();
2296 }
2297 #else
2298 int r_lo = right->as_constant_ptr()->as_jint_lo();
2299 int r_hi = right->as_constant_ptr()->as_jint_hi();
2300 switch (code) {
2301 case lir_logic_and:
2302 __ andl(l_lo, r_lo);
2303 __ andl(l_hi, r_hi);
2304 break;
2305 case lir_logic_or:
2306 __ orl(l_lo, r_lo);
2307 __ orl(l_hi, r_hi);
2308 break;
2309 case lir_logic_xor:
2310 __ xorl(l_lo, r_lo);
2311 __ xorl(l_hi, r_hi);
2312 break;
2313 default: ShouldNotReachHere();
2314 }
2315 #endif // _LP64
2316 } else {
2317 #ifdef _LP64
2318 Register r_lo;
2319 if (is_reference_type(right->type())) {
2320 r_lo = right->as_register();
2321 } else {
2322 r_lo = right->as_register_lo();
2323 }
2324 #else
2325 Register r_lo = right->as_register_lo();
2326 Register r_hi = right->as_register_hi();
2327 assert(l_lo != r_hi, "overwriting registers");
2328 #endif
2329 switch (code) {
2330 case lir_logic_and:
2331 __ andptr(l_lo, r_lo);
2332 NOT_LP64(__ andptr(l_hi, r_hi);)
2333 break;
2334 case lir_logic_or:
2335 __ orptr(l_lo, r_lo);
2336 NOT_LP64(__ orptr(l_hi, r_hi);)
2337 break;
2338 case lir_logic_xor:
2339 __ xorptr(l_lo, r_lo);
2340 NOT_LP64(__ xorptr(l_hi, r_hi);)
2341 break;
2342 default: ShouldNotReachHere();
2343 }
2344 }
2345
2346 Register dst_lo = dst->as_register_lo();
2347 Register dst_hi = dst->as_register_hi();
2348
2349 #ifdef _LP64
2350 move_regs(l_lo, dst_lo);
2351 #else
2352 if (dst_lo == l_hi) {
2353 assert(dst_hi != l_lo, "overwriting registers");
2354 move_regs(l_hi, dst_hi);
2355 move_regs(l_lo, dst_lo);
2356 } else {
2357 assert(dst_lo != l_hi, "overwriting registers");
2358 move_regs(l_lo, dst_lo);
2359 move_regs(l_hi, dst_hi);
2360 }
2361 #endif // _LP64
2362 }
2363 }
2364
2365
2366 // we assume that rax, and rdx can be overwritten
2367 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2368
2369 assert(left->is_single_cpu(), "left must be register");
2370 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2371 assert(result->is_single_cpu(), "result must be register");
2372
2373 // assert(left->destroys_register(), "check");
2374 // assert(right->destroys_register(), "check");
2375
2376 Register lreg = left->as_register();
2377 Register dreg = result->as_register();
2378
2379 if (right->is_constant()) {
2380 jint divisor = right->as_constant_ptr()->as_jint();
2381 assert(divisor > 0 && is_power_of_2(divisor), "must be");
2382 if (code == lir_idiv) {
2383 assert(lreg == rax, "must be rax,");
2384 assert(temp->as_register() == rdx, "tmp register must be rdx");
2385 __ cdql(); // sign extend into rdx:rax
2386 if (divisor == 2) {
2387 __ subl(lreg, rdx);
2388 } else {
2389 __ andl(rdx, divisor - 1);
2390 __ addl(lreg, rdx);
2391 }
2392 __ sarl(lreg, log2i_exact(divisor));
2393 move_regs(lreg, dreg);
2394 } else if (code == lir_irem) {
2395 Label done;
2396 __ mov(dreg, lreg);
2397 __ andl(dreg, 0x80000000 | (divisor - 1));
2398 __ jcc(Assembler::positive, done);
2399 __ decrement(dreg);
2400 __ orl(dreg, ~(divisor - 1));
2401 __ increment(dreg);
2402 __ bind(done);
2403 } else {
2404 ShouldNotReachHere();
2405 }
2406 } else {
2407 Register rreg = right->as_register();
2408 assert(lreg == rax, "left register must be rax,");
2409 assert(rreg != rdx, "right register must not be rdx");
2410 assert(temp->as_register() == rdx, "tmp register must be rdx");
2411
2412 move_regs(lreg, rax);
2413
2414 int idivl_offset = __ corrected_idivl(rreg);
2415 if (ImplicitDiv0Checks) {
2416 add_debug_info_for_div0(idivl_offset, info);
2417 }
2418 if (code == lir_irem) {
2419 move_regs(rdx, dreg); // result is in rdx
2420 } else {
2421 move_regs(rax, dreg);
2422 }
2423 }
2424 }
2425
2426
2427 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2428 if (opr1->is_single_cpu()) {
2429 Register reg1 = opr1->as_register();
2430 if (opr2->is_single_cpu()) {
2431 // cpu register - cpu register
2432 if (is_reference_type(opr1->type())) {
2433 __ cmpoop(reg1, opr2->as_register());
2434 } else {
2435 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2436 __ cmpl(reg1, opr2->as_register());
2437 }
2438 } else if (opr2->is_stack()) {
2439 // cpu register - stack
2440 if (is_reference_type(opr1->type())) {
2441 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2442 } else {
2443 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2444 }
2445 } else if (opr2->is_constant()) {
2446 // cpu register - constant
2447 LIR_Const* c = opr2->as_constant_ptr();
2448 if (c->type() == T_INT) {
2449 jint i = c->as_jint();
2450 if (i == 0) {
2451 __ testl(reg1, reg1);
2452 } else {
2453 __ cmpl(reg1, i);
2454 }
2455 } else if (c->type() == T_METADATA) {
2456 // All we need for now is a comparison with null for equality.
2457 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2458 Metadata* m = c->as_metadata();
2459 if (m == nullptr) {
2460 __ testptr(reg1, reg1);
2461 } else {
2462 ShouldNotReachHere();
2463 }
2464 } else if (is_reference_type(c->type())) {
2465 // In 64bit oops are single register
2466 jobject o = c->as_jobject();
2467 if (o == nullptr) {
2468 __ testptr(reg1, reg1);
2469 } else {
2470 __ cmpoop(reg1, o, rscratch1);
2471 }
2472 } else {
2473 fatal("unexpected type: %s", basictype_to_str(c->type()));
2474 }
2475 // cpu register - address
2476 } else if (opr2->is_address()) {
2477 if (op->info() != nullptr) {
2478 add_debug_info_for_null_check_here(op->info());
2479 }
2480 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2481 } else {
2482 ShouldNotReachHere();
2483 }
2484
2485 } else if(opr1->is_double_cpu()) {
2486 Register xlo = opr1->as_register_lo();
2487 Register xhi = opr1->as_register_hi();
2488 if (opr2->is_double_cpu()) {
2489 #ifdef _LP64
2490 __ cmpptr(xlo, opr2->as_register_lo());
2491 #else
2492 // cpu register - cpu register
2493 Register ylo = opr2->as_register_lo();
2494 Register yhi = opr2->as_register_hi();
2495 __ subl(xlo, ylo);
2496 __ sbbl(xhi, yhi);
2497 if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2498 __ orl(xhi, xlo);
2499 }
2500 #endif // _LP64
2501 } else if (opr2->is_constant()) {
2502 // cpu register - constant 0
2503 assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2504 #ifdef _LP64
2505 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2506 #else
2507 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2508 __ orl(xhi, xlo);
2509 #endif // _LP64
2510 } else {
2511 ShouldNotReachHere();
2512 }
2513
2514 } else if (opr1->is_single_xmm()) {
2515 XMMRegister reg1 = opr1->as_xmm_float_reg();
2516 if (opr2->is_single_xmm()) {
2517 // xmm register - xmm register
2518 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2519 } else if (opr2->is_stack()) {
2520 // xmm register - stack
2521 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2522 } else if (opr2->is_constant()) {
2523 // xmm register - constant
2524 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2525 } else if (opr2->is_address()) {
2526 // xmm register - address
2527 if (op->info() != nullptr) {
2528 add_debug_info_for_null_check_here(op->info());
2529 }
2530 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2531 } else {
2532 ShouldNotReachHere();
2533 }
2534
2535 } else if (opr1->is_double_xmm()) {
2536 XMMRegister reg1 = opr1->as_xmm_double_reg();
2537 if (opr2->is_double_xmm()) {
2538 // xmm register - xmm register
2539 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2540 } else if (opr2->is_stack()) {
2541 // xmm register - stack
2542 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2543 } else if (opr2->is_constant()) {
2544 // xmm register - constant
2545 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2546 } else if (opr2->is_address()) {
2547 // xmm register - address
2548 if (op->info() != nullptr) {
2549 add_debug_info_for_null_check_here(op->info());
2550 }
2551 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2552 } else {
2553 ShouldNotReachHere();
2554 }
2555
2556 } else if (opr1->is_address() && opr2->is_constant()) {
2557 LIR_Const* c = opr2->as_constant_ptr();
2558 #ifdef _LP64
2559 if (is_reference_type(c->type())) {
2560 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2561 __ movoop(rscratch1, c->as_jobject());
2562 }
2563 #endif // LP64
2564 if (op->info() != nullptr) {
2565 add_debug_info_for_null_check_here(op->info());
2566 }
2567 // special case: address - constant
2568 LIR_Address* addr = opr1->as_address_ptr();
2569 if (c->type() == T_INT) {
2570 __ cmpl(as_Address(addr), c->as_jint());
2571 } else if (is_reference_type(c->type())) {
2572 #ifdef _LP64
2573 // %%% Make this explode if addr isn't reachable until we figure out a
2574 // better strategy by giving noreg as the temp for as_Address
2575 __ cmpoop(rscratch1, as_Address(addr, noreg));
2576 #else
2577 __ cmpoop(as_Address(addr), c->as_jobject());
2578 #endif // _LP64
2579 } else {
2580 ShouldNotReachHere();
2581 }
2582
2583 } else {
2584 ShouldNotReachHere();
2585 }
2586 }
2587
2588 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2589 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2590 if (left->is_single_xmm()) {
2591 assert(right->is_single_xmm(), "must match");
2592 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2593 } else if (left->is_double_xmm()) {
2594 assert(right->is_double_xmm(), "must match");
2595 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2596
2597 } else {
2598 ShouldNotReachHere();
2599 }
2600 } else {
2601 assert(code == lir_cmp_l2i, "check");
2602 #ifdef _LP64
2603 Label done;
2604 Register dest = dst->as_register();
2605 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2606 __ movl(dest, -1);
2607 __ jccb(Assembler::less, done);
2608 __ setb(Assembler::notZero, dest);
2609 __ movzbl(dest, dest);
2610 __ bind(done);
2611 #else
2612 __ lcmp2int(left->as_register_hi(),
2613 left->as_register_lo(),
2614 right->as_register_hi(),
2615 right->as_register_lo());
2616 move_regs(left->as_register_hi(), dst->as_register());
2617 #endif // _LP64
2618 }
2619 }
2620
2621
2622 void LIR_Assembler::align_call(LIR_Code code) {
2623 // make sure that the displacement word of the call ends up word aligned
2624 int offset = __ offset();
2625 switch (code) {
2626 case lir_static_call:
2627 case lir_optvirtual_call:
2628 case lir_dynamic_call:
2629 offset += NativeCall::displacement_offset;
2630 break;
2631 case lir_icvirtual_call:
2632 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2633 break;
2634 default: ShouldNotReachHere();
2635 }
2636 __ align(BytesPerWord, offset);
2637 }
2638
2639
2640 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2641 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2642 "must be aligned");
2643 __ call(AddressLiteral(op->addr(), rtype));
2644 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2645 __ post_call_nop();
2646 }
2647
2648
2649 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2650 __ ic_call(op->addr());
2651 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2652 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2653 "must be aligned");
2654 __ post_call_nop();
2655 }
2656
2657
2658 void LIR_Assembler::emit_static_call_stub() {
2659 address call_pc = __ pc();
2660 address stub = __ start_a_stub(call_stub_size());
2661 if (stub == nullptr) {
2662 bailout("static call stub overflow");
2663 return;
2664 }
2665
2666 int start = __ offset();
2667
2668 // make sure that the displacement word of the call ends up word aligned
2669 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2670 __ relocate(static_stub_Relocation::spec(call_pc));
2671 __ mov_metadata(rbx, (Metadata*)nullptr);
2672 // must be set to -1 at code generation time
2673 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2674 // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2675 __ jump(RuntimeAddress(__ pc()));
2676
2677 assert(__ offset() - start <= call_stub_size(), "stub too big");
2678 __ end_a_stub();
2679 }
2680
2681
2682 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2683 assert(exceptionOop->as_register() == rax, "must match");
2684 assert(exceptionPC->as_register() == rdx, "must match");
2685
2686 // exception object is not added to oop map by LinearScan
2687 // (LinearScan assumes that no oops are in fixed registers)
2688 info->add_register_oop(exceptionOop);
2689 C1StubId unwind_id;
2690
2691 // get current pc information
2692 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2693 int pc_for_athrow_offset = __ offset();
2694 InternalAddress pc_for_athrow(__ pc());
2695 __ lea(exceptionPC->as_register(), pc_for_athrow);
2696 add_call_info(pc_for_athrow_offset, info); // for exception handler
2697
2698 __ verify_not_null_oop(rax);
2699 // search an exception handler (rax: exception oop, rdx: throwing pc)
2700 if (compilation()->has_fpu_code()) {
2701 unwind_id = C1StubId::handle_exception_id;
2702 } else {
2703 unwind_id = C1StubId::handle_exception_nofpu_id;
2704 }
2705 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2706
2707 // enough room for two byte trap
2708 __ nop();
2709 }
2710
2711
2712 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2713 assert(exceptionOop->as_register() == rax, "must match");
2714
2715 __ jmp(_unwind_handler_entry);
2716 }
2717
2718
2719 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2720
2721 // optimized version for linear scan:
2722 // * count must be already in ECX (guaranteed by LinearScan)
2723 // * left and dest must be equal
2724 // * tmp must be unused
2725 assert(count->as_register() == SHIFT_count, "count must be in ECX");
2726 assert(left == dest, "left and dest must be equal");
2727 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2728
2729 if (left->is_single_cpu()) {
2730 Register value = left->as_register();
2731 assert(value != SHIFT_count, "left cannot be ECX");
2732
2733 switch (code) {
2734 case lir_shl: __ shll(value); break;
2735 case lir_shr: __ sarl(value); break;
2736 case lir_ushr: __ shrl(value); break;
2737 default: ShouldNotReachHere();
2738 }
2739 } else if (left->is_double_cpu()) {
2740 Register lo = left->as_register_lo();
2741 Register hi = left->as_register_hi();
2742 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2743 #ifdef _LP64
2744 switch (code) {
2745 case lir_shl: __ shlptr(lo); break;
2746 case lir_shr: __ sarptr(lo); break;
2747 case lir_ushr: __ shrptr(lo); break;
2748 default: ShouldNotReachHere();
2749 }
2750 #else
2751
2752 switch (code) {
2753 case lir_shl: __ lshl(hi, lo); break;
2754 case lir_shr: __ lshr(hi, lo, true); break;
2755 case lir_ushr: __ lshr(hi, lo, false); break;
2756 default: ShouldNotReachHere();
2757 }
2758 #endif // LP64
2759 } else {
2760 ShouldNotReachHere();
2761 }
2762 }
2763
2764
2765 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2766 if (dest->is_single_cpu()) {
2767 // first move left into dest so that left is not destroyed by the shift
2768 Register value = dest->as_register();
2769 count = count & 0x1F; // Java spec
2770
2771 move_regs(left->as_register(), value);
2772 switch (code) {
2773 case lir_shl: __ shll(value, count); break;
2774 case lir_shr: __ sarl(value, count); break;
2775 case lir_ushr: __ shrl(value, count); break;
2776 default: ShouldNotReachHere();
2777 }
2778 } else if (dest->is_double_cpu()) {
2779 #ifndef _LP64
2780 Unimplemented();
2781 #else
2782 // first move left into dest so that left is not destroyed by the shift
2783 Register value = dest->as_register_lo();
2784 count = count & 0x1F; // Java spec
2785
2786 move_regs(left->as_register_lo(), value);
2787 switch (code) {
2788 case lir_shl: __ shlptr(value, count); break;
2789 case lir_shr: __ sarptr(value, count); break;
2790 case lir_ushr: __ shrptr(value, count); break;
2791 default: ShouldNotReachHere();
2792 }
2793 #endif // _LP64
2794 } else {
2795 ShouldNotReachHere();
2796 }
2797 }
2798
2799
2800 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2801 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2802 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2803 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2804 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2805 }
2806
2807
2808 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2809 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2810 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2811 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2812 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2813 }
2814
2815
2816 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2817 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2818 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2819 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2820 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2821 }
2822
2823
2824 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2825 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2826 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2827 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2828 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2829 }
2830
2831
2832 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2833 if (null_check) {
2834 __ testptr(obj, obj);
2835 __ jcc(Assembler::zero, *slow_path->entry());
2836 }
2837 if (is_dest) {
2838 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2839 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2840 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2841 } else {
2842 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2843 }
2844 }
2845
2846
2847 // This code replaces a call to arraycopy; no exception may
2848 // be thrown in this code, they must be thrown in the System.arraycopy
2849 // activation frame; we could save some checks if this would not be the case
2850 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2851 ciArrayKlass* default_type = op->expected_type();
2852 Register src = op->src()->as_register();
2853 Register dst = op->dst()->as_register();
2854 Register src_pos = op->src_pos()->as_register();
2855 Register dst_pos = op->dst_pos()->as_register();
2856 Register length = op->length()->as_register();
2857 Register tmp = op->tmp()->as_register();
2858 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2859 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2860
2861 CodeStub* stub = op->stub();
2862 int flags = op->flags();
2863 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2864 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2865
2866 if (flags & LIR_OpArrayCopy::always_slow_path) {
2867 __ jmp(*stub->entry());
2868 __ bind(*stub->continuation());
2869 return;
2870 }
2871
2872 // if we don't know anything, just go through the generic arraycopy
2873 if (default_type == nullptr) {
2874 // save outgoing arguments on stack in case call to System.arraycopy is needed
2875 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2876 // for interpreter calling conventions. Now we have to do it in new style conventions.
2877 // For the moment until C1 gets the new register allocator I just force all the
2878 // args to the right place (except the register args) and then on the back side
2879 // reload the register args properly if we go slow path. Yuck
2880
2881 // These are proper for the calling convention
2882 store_parameter(length, 2);
2883 store_parameter(dst_pos, 1);
2884 store_parameter(dst, 0);
2885
2886 // these are just temporary placements until we need to reload
2887 store_parameter(src_pos, 3);
2888 store_parameter(src, 4);
2889 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2890
2891 address copyfunc_addr = StubRoutines::generic_arraycopy();
2892 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2893
2894 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2895 #ifdef _LP64
2896 // The arguments are in java calling convention so we can trivially shift them to C
2897 // convention
2898 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2899 __ mov(c_rarg0, j_rarg0);
2900 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2901 __ mov(c_rarg1, j_rarg1);
2902 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2903 __ mov(c_rarg2, j_rarg2);
2904 assert_different_registers(c_rarg3, j_rarg4);
2905 __ mov(c_rarg3, j_rarg3);
2906 #ifdef _WIN64
2907 // Allocate abi space for args but be sure to keep stack aligned
2908 __ subptr(rsp, 6*wordSize);
2909 store_parameter(j_rarg4, 4);
2910 #ifndef PRODUCT
2911 if (PrintC1Statistics) {
2912 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2913 }
2914 #endif
2915 __ call(RuntimeAddress(copyfunc_addr));
2916 __ addptr(rsp, 6*wordSize);
2917 #else
2918 __ mov(c_rarg4, j_rarg4);
2919 #ifndef PRODUCT
2920 if (PrintC1Statistics) {
2921 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2922 }
2923 #endif
2924 __ call(RuntimeAddress(copyfunc_addr));
2925 #endif // _WIN64
2926 #else
2927 __ push(length);
2928 __ push(dst_pos);
2929 __ push(dst);
2930 __ push(src_pos);
2931 __ push(src);
2932
2933 #ifndef PRODUCT
2934 if (PrintC1Statistics) {
2935 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2936 }
2937 #endif
2938 __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
2939
2940 #endif // _LP64
2941
2942 __ testl(rax, rax);
2943 __ jcc(Assembler::equal, *stub->continuation());
2944
2945 __ mov(tmp, rax);
2946 __ xorl(tmp, -1);
2947
2948 // Reload values from the stack so they are where the stub
2949 // expects them.
2950 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2951 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2952 __ movptr (length, Address(rsp, 2*BytesPerWord));
2953 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2954 __ movptr (src, Address(rsp, 4*BytesPerWord));
2955
2956 __ subl(length, tmp);
2957 __ addl(src_pos, tmp);
2958 __ addl(dst_pos, tmp);
2959 __ jmp(*stub->entry());
2960
2961 __ bind(*stub->continuation());
2962 return;
2963 }
2964
2965 // Handle inline type arrays
2966 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2967 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2968 }
2969 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2970 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2971 }
2972
2973 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2974
2975 int elem_size = type2aelembytes(basic_type);
2976 Address::ScaleFactor scale;
2977
2978 switch (elem_size) {
2979 case 1 :
2980 scale = Address::times_1;
2981 break;
2982 case 2 :
2983 scale = Address::times_2;
2984 break;
2985 case 4 :
2986 scale = Address::times_4;
2987 break;
2988 case 8 :
2989 scale = Address::times_8;
2990 break;
2991 default:
2992 scale = Address::no_scale;
2993 ShouldNotReachHere();
2994 }
2995
2996 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2997 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2998
2999 // length and pos's are all sign extended at this point on 64bit
3000
3001 // test for null
3002 if (flags & LIR_OpArrayCopy::src_null_check) {
3003 __ testptr(src, src);
3004 __ jcc(Assembler::zero, *stub->entry());
3005 }
3006 if (flags & LIR_OpArrayCopy::dst_null_check) {
3007 __ testptr(dst, dst);
3008 __ jcc(Assembler::zero, *stub->entry());
3009 }
3010
3011 // If the compiler was not able to prove that exact type of the source or the destination
3012 // of the arraycopy is an array type, check at runtime if the source or the destination is
3013 // an instance type.
3014 if (flags & LIR_OpArrayCopy::type_check) {
3015 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3016 __ load_klass(tmp, dst, tmp_load_klass);
3017 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3018 __ jcc(Assembler::greaterEqual, *stub->entry());
3019 }
3020
3021 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3022 __ load_klass(tmp, src, tmp_load_klass);
3023 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3024 __ jcc(Assembler::greaterEqual, *stub->entry());
3025 }
3026 }
3027
3028 // check if negative
3029 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3030 __ testl(src_pos, src_pos);
3031 __ jcc(Assembler::less, *stub->entry());
3032 }
3033 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3034 __ testl(dst_pos, dst_pos);
3035 __ jcc(Assembler::less, *stub->entry());
3036 }
3037
3038 if (flags & LIR_OpArrayCopy::src_range_check) {
3039 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3040 __ cmpl(tmp, src_length_addr);
3041 __ jcc(Assembler::above, *stub->entry());
3042 }
3043 if (flags & LIR_OpArrayCopy::dst_range_check) {
3044 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3045 __ cmpl(tmp, dst_length_addr);
3046 __ jcc(Assembler::above, *stub->entry());
3047 }
3048
3049 if (flags & LIR_OpArrayCopy::length_positive_check) {
3050 __ testl(length, length);
3051 __ jcc(Assembler::less, *stub->entry());
3052 }
3053
3054 #ifdef _LP64
3055 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3056 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3057 #endif
3058
3059 if (flags & LIR_OpArrayCopy::type_check) {
3060 // We don't know the array types are compatible
3061 if (basic_type != T_OBJECT) {
3062 // Simple test for basic type arrays
3063 __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
3064 __ jcc(Assembler::notEqual, *stub->entry());
3065 } else {
3066 // For object arrays, if src is a sub class of dst then we can
3067 // safely do the copy.
3068 Label cont, slow;
3069
3070 __ push(src);
3071 __ push(dst);
3072
3073 __ load_klass(src, src, tmp_load_klass);
3074 __ load_klass(dst, dst, tmp_load_klass);
3075
3076 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
3077
3078 __ push(src);
3079 __ push(dst);
3080 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
3081 __ pop(dst);
3082 __ pop(src);
3083
3084 __ testl(src, src);
3085 __ jcc(Assembler::notEqual, cont);
3086
3087 __ bind(slow);
3088 __ pop(dst);
3089 __ pop(src);
3090
3091 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3092 if (copyfunc_addr != nullptr) { // use stub if available
3093 // src is not a sub class of dst so we have to do a
3094 // per-element check.
3095
3096 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3097 if ((flags & mask) != mask) {
3098 // Check that at least both of them object arrays.
3099 assert(flags & mask, "one of the two should be known to be an object array");
3100
3101 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3102 __ load_klass(tmp, src, tmp_load_klass);
3103 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3104 __ load_klass(tmp, dst, tmp_load_klass);
3105 }
3106 int lh_offset = in_bytes(Klass::layout_helper_offset());
3107 Address klass_lh_addr(tmp, lh_offset);
3108 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3109 __ cmpl(klass_lh_addr, objArray_lh);
3110 __ jcc(Assembler::notEqual, *stub->entry());
3111 }
3112
3113 // Spill because stubs can use any register they like and it's
3114 // easier to restore just those that we care about.
3115 store_parameter(dst, 0);
3116 store_parameter(dst_pos, 1);
3117 store_parameter(length, 2);
3118 store_parameter(src_pos, 3);
3119 store_parameter(src, 4);
3120
3121 #ifndef _LP64
3122 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3123 __ movptr(tmp, dst_klass_addr);
3124 __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3125 __ push(tmp);
3126 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3127 __ push(tmp);
3128 __ push(length);
3129 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3130 __ push(tmp);
3131 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3132 __ push(tmp);
3133
3134 __ call_VM_leaf(copyfunc_addr, 5);
3135 #else
3136 __ movl2ptr(length, length); //higher 32bits must be null
3137
3138 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3139 assert_different_registers(c_rarg0, dst, dst_pos, length);
3140 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3141 assert_different_registers(c_rarg1, dst, length);
3142
3143 __ mov(c_rarg2, length);
3144 assert_different_registers(c_rarg2, dst);
3145
3146 #ifdef _WIN64
3147 // Allocate abi space for args but be sure to keep stack aligned
3148 __ subptr(rsp, 6*wordSize);
3149 __ load_klass(c_rarg3, dst, tmp_load_klass);
3150 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3151 store_parameter(c_rarg3, 4);
3152 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3153 __ call(RuntimeAddress(copyfunc_addr));
3154 __ addptr(rsp, 6*wordSize);
3155 #else
3156 __ load_klass(c_rarg4, dst, tmp_load_klass);
3157 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3158 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3159 __ call(RuntimeAddress(copyfunc_addr));
3160 #endif
3161
3162 #endif
3163
3164 #ifndef PRODUCT
3165 if (PrintC1Statistics) {
3166 Label failed;
3167 __ testl(rax, rax);
3168 __ jcc(Assembler::notZero, failed);
3169 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
3170 __ bind(failed);
3171 }
3172 #endif
3173
3174 __ testl(rax, rax);
3175 __ jcc(Assembler::zero, *stub->continuation());
3176
3177 #ifndef PRODUCT
3178 if (PrintC1Statistics) {
3179 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
3180 }
3181 #endif
3182
3183 __ mov(tmp, rax);
3184
3185 __ xorl(tmp, -1);
3186
3187 // Restore previously spilled arguments
3188 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3189 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3190 __ movptr (length, Address(rsp, 2*BytesPerWord));
3191 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3192 __ movptr (src, Address(rsp, 4*BytesPerWord));
3193
3194
3195 __ subl(length, tmp);
3196 __ addl(src_pos, tmp);
3197 __ addl(dst_pos, tmp);
3198 }
3199
3200 __ jmp(*stub->entry());
3201
3202 __ bind(cont);
3203 __ pop(dst);
3204 __ pop(src);
3205 }
3206 }
3207
3208 #ifdef ASSERT
3209 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3210 // Sanity check the known type with the incoming class. For the
3211 // primitive case the types must match exactly with src.klass and
3212 // dst.klass each exactly matching the default type. For the
3213 // object array case, if no type check is needed then either the
3214 // dst type is exactly the expected type and the src type is a
3215 // subtype which we can't check or src is the same array as dst
3216 // but not necessarily exactly of type default_type.
3217 Label known_ok, halt;
3218 __ mov_metadata(tmp, default_type->constant_encoding());
3219 #ifdef _LP64
3220 if (UseCompressedClassPointers) {
3221 __ encode_klass_not_null(tmp, rscratch1);
3222 }
3223 #endif
3224
3225 if (basic_type != T_OBJECT) {
3226 __ cmp_klass(tmp, dst, tmp2);
3227 __ jcc(Assembler::notEqual, halt);
3228 __ cmp_klass(tmp, src, tmp2);
3229 __ jcc(Assembler::equal, known_ok);
3230 } else {
3231 __ cmp_klass(tmp, dst, tmp2);
3232 __ jcc(Assembler::equal, known_ok);
3233 __ cmpptr(src, dst);
3234 __ jcc(Assembler::equal, known_ok);
3235 }
3236 __ bind(halt);
3237 __ stop("incorrect type information in arraycopy");
3238 __ bind(known_ok);
3239 }
3240 #endif
3241
3242 #ifndef PRODUCT
3243 if (PrintC1Statistics) {
3244 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
3245 }
3246 #endif
3247
3248 #ifdef _LP64
3249 assert_different_registers(c_rarg0, dst, dst_pos, length);
3250 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3251 assert_different_registers(c_rarg1, length);
3252 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3253 __ mov(c_rarg2, length);
3254
3255 #else
3256 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3257 store_parameter(tmp, 0);
3258 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3259 store_parameter(tmp, 1);
3260 store_parameter(length, 2);
3261 #endif // _LP64
3262
3263 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3264 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3265 const char *name;
3266 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3267 __ call_VM_leaf(entry, 0);
3268
3269 if (stub != nullptr) {
3270 __ bind(*stub->continuation());
3271 }
3272 }
3273
3274 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3275 assert(op->crc()->is_single_cpu(), "crc must be register");
3276 assert(op->val()->is_single_cpu(), "byte value must be register");
3277 assert(op->result_opr()->is_single_cpu(), "result must be register");
3278 Register crc = op->crc()->as_register();
3279 Register val = op->val()->as_register();
3280 Register res = op->result_opr()->as_register();
3281
3282 assert_different_registers(val, crc, res);
3283
3284 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3285 __ notl(crc); // ~crc
3286 __ update_byte_crc32(crc, val, res);
3287 __ notl(crc); // ~crc
3288 __ mov(res, crc);
3289 }
3290
3291 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3292 Register obj = op->obj_opr()->as_register(); // may not be an oop
3293 Register hdr = op->hdr_opr()->as_register();
3294 Register lock = op->lock_opr()->as_register();
3295 if (LockingMode == LM_MONITOR) {
3296 if (op->info() != nullptr) {
3297 add_debug_info_for_null_check_here(op->info());
3298 __ null_check(obj);
3299 }
3300 __ jmp(*op->stub()->entry());
3301 } else if (op->code() == lir_lock) {
3302 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3303 Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
3304 // add debug info for NullPointerException only if one is possible
3305 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3306 if (op->info() != nullptr) {
3307 add_debug_info_for_null_check(null_check_offset, op->info());
3308 }
3309 // done
3310 } else if (op->code() == lir_unlock) {
3311 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3312 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3313 } else {
3314 Unimplemented();
3315 }
3316 __ bind(*op->stub()->continuation());
3317 }
3318
3319 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
3320 Register obj = op->obj()->as_pointer_register();
3321 Register result = op->result_opr()->as_pointer_register();
3322
3323 CodeEmitInfo* info = op->info();
3324 if (info != nullptr) {
3325 add_debug_info_for_null_check_here(info);
3326 }
3327
3328 __ load_klass(result, obj, rscratch1);
3329 }
3330
3331 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3332 ciMethod* method = op->profiled_method();
3333 int bci = op->profiled_bci();
3334 ciMethod* callee = op->profiled_callee();
3335 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3336
3337 // Update counter for all call types
3338 ciMethodData* md = method->method_data_or_null();
3339 assert(md != nullptr, "Sanity");
3340 ciProfileData* data = md->bci_to_data(bci);
3341 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
3342 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
3343 Register mdo = op->mdo()->as_register();
3344 __ mov_metadata(mdo, md->constant_encoding());
3345 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3346 // Perform additional virtual call profiling for invokevirtual and
3347 // invokeinterface bytecodes
3348 if (op->should_profile_receiver_type()) {
3349 assert(op->recv()->is_single_cpu(), "recv must be allocated");
3350 Register recv = op->recv()->as_register();
3351 assert_different_registers(mdo, recv);
3352 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3353 ciKlass* known_klass = op->known_holder();
3354 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
3355 // We know the type that will be seen at this call site; we can
3356 // statically update the MethodData* rather than needing to do
3357 // dynamic tests on the receiver type
3358
3359 // NOTE: we should probably put a lock around this search to
3360 // avoid collisions by concurrent compilations
3361 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3362 uint i;
3363 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3364 ciKlass* receiver = vc_data->receiver(i);
3365 if (known_klass->equals(receiver)) {
3366 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3367 __ addptr(data_addr, DataLayout::counter_increment);
3368 return;
3369 }
3370 }
3371
3372 // Receiver type not found in profile data; select an empty slot
3373
3374 // Note that this is less efficient than it should be because it
3375 // always does a write to the receiver part of the
3376 // VirtualCallData rather than just the first time
3377 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3378 ciKlass* receiver = vc_data->receiver(i);
3379 if (receiver == nullptr) {
3380 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3381 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3382 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3383 __ addptr(data_addr, DataLayout::counter_increment);
3384 return;
3385 }
3386 }
3387 } else {
3388 __ load_klass(recv, recv, tmp_load_klass);
3389 Label update_done;
3390 type_profile_helper(mdo, md, data, recv, &update_done);
3391 // Receiver did not match any saved receiver and there is no empty row for it.
3392 // Increment total counter to indicate polymorphic case.
3393 __ addptr(counter_addr, DataLayout::counter_increment);
3394
3395 __ bind(update_done);
3396 }
3397 } else {
3398 // Static call
3399 __ addptr(counter_addr, DataLayout::counter_increment);
3400 }
3401 }
3402
3403 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3404 Register obj = op->obj()->as_register();
3405 Register tmp = op->tmp()->as_pointer_register();
3406 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3407 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3408 ciKlass* exact_klass = op->exact_klass();
3409 intptr_t current_klass = op->current_klass();
3410 bool not_null = op->not_null();
3411 bool no_conflict = op->no_conflict();
3412
3413 Label update, next, none;
3414
3415 bool do_null = !not_null;
3416 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3417 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3418
3419 assert(do_null || do_update, "why are we here?");
3420 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3421
3422 __ verify_oop(obj);
3423
3424 #ifdef ASSERT
3425 if (obj == tmp) {
3426 #ifdef _LP64
3427 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3428 #else
3429 assert_different_registers(obj, mdo_addr.base(), mdo_addr.index());
3430 #endif
3431 } else {
3432 #ifdef _LP64
3433 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3434 #else
3435 assert_different_registers(obj, tmp, mdo_addr.base(), mdo_addr.index());
3436 #endif
3437 }
3438 #endif
3439 if (do_null) {
3440 __ testptr(obj, obj);
3441 __ jccb(Assembler::notZero, update);
3442 if (!TypeEntries::was_null_seen(current_klass)) {
3443 __ testptr(mdo_addr, TypeEntries::null_seen);
3444 #ifndef ASSERT
3445 __ jccb(Assembler::notZero, next); // already set
3446 #else
3447 __ jcc(Assembler::notZero, next); // already set
3448 #endif
3449 // atomic update to prevent overwriting Klass* with 0
3450 __ lock();
3451 __ orptr(mdo_addr, TypeEntries::null_seen);
3452 }
3453 if (do_update) {
3454 #ifndef ASSERT
3455 __ jmpb(next);
3456 }
3457 #else
3458 __ jmp(next);
3459 }
3460 } else {
3461 __ testptr(obj, obj);
3462 __ jcc(Assembler::notZero, update);
3463 __ stop("unexpected null obj");
3464 #endif
3465 }
3466
3467 __ bind(update);
3468
3469 if (do_update) {
3470 #ifdef ASSERT
3471 if (exact_klass != nullptr) {
3472 Label ok;
3473 __ load_klass(tmp, obj, tmp_load_klass);
3474 __ push(tmp);
3475 __ mov_metadata(tmp, exact_klass->constant_encoding());
3476 __ cmpptr(tmp, Address(rsp, 0));
3477 __ jcc(Assembler::equal, ok);
3478 __ stop("exact klass and actual klass differ");
3479 __ bind(ok);
3480 __ pop(tmp);
3481 }
3482 #endif
3483 if (!no_conflict) {
3484 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3485 if (exact_klass != nullptr) {
3486 __ mov_metadata(tmp, exact_klass->constant_encoding());
3487 } else {
3488 __ load_klass(tmp, obj, tmp_load_klass);
3489 }
3490 #ifdef _LP64
3491 __ mov(rscratch1, tmp); // save original value before XOR
3492 #endif
3493 __ xorptr(tmp, mdo_addr);
3494 __ testptr(tmp, TypeEntries::type_klass_mask);
3495 // klass seen before, nothing to do. The unknown bit may have been
3496 // set already but no need to check.
3497 __ jccb(Assembler::zero, next);
3498
3499 __ testptr(tmp, TypeEntries::type_unknown);
3500 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3501
3502 if (TypeEntries::is_type_none(current_klass)) {
3503 __ testptr(mdo_addr, TypeEntries::type_mask);
3504 __ jccb(Assembler::zero, none);
3505 #ifdef _LP64
3506 // There is a chance that the checks above (re-reading profiling
3507 // data from memory) fail if another thread has just set the
3508 // profiling to this obj's klass
3509 __ mov(tmp, rscratch1); // get back original value before XOR
3510 __ xorptr(tmp, mdo_addr);
3511 __ testptr(tmp, TypeEntries::type_klass_mask);
3512 __ jccb(Assembler::zero, next);
3513 #endif
3514 }
3515 } else {
3516 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3517 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3518
3519 __ testptr(mdo_addr, TypeEntries::type_unknown);
3520 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3521 }
3522
3523 // different than before. Cannot keep accurate profile.
3524 __ orptr(mdo_addr, TypeEntries::type_unknown);
3525
3526 if (TypeEntries::is_type_none(current_klass)) {
3527 __ jmpb(next);
3528
3529 __ bind(none);
3530 // first time here. Set profile type.
3531 __ movptr(mdo_addr, tmp);
3532 #ifdef ASSERT
3533 __ andptr(tmp, TypeEntries::type_klass_mask);
3534 __ verify_klass_ptr(tmp);
3535 #endif
3536 }
3537 } else {
3538 // There's a single possible klass at this profile point
3539 assert(exact_klass != nullptr, "should be");
3540 if (TypeEntries::is_type_none(current_klass)) {
3541 __ mov_metadata(tmp, exact_klass->constant_encoding());
3542 __ xorptr(tmp, mdo_addr);
3543 __ testptr(tmp, TypeEntries::type_klass_mask);
3544 #ifdef ASSERT
3545 __ jcc(Assembler::zero, next);
3546
3547 {
3548 Label ok;
3549 __ push(tmp);
3550 __ testptr(mdo_addr, TypeEntries::type_mask);
3551 __ jcc(Assembler::zero, ok);
3552 // may have been set by another thread
3553 __ mov_metadata(tmp, exact_klass->constant_encoding());
3554 __ xorptr(tmp, mdo_addr);
3555 __ testptr(tmp, TypeEntries::type_mask);
3556 __ jcc(Assembler::zero, ok);
3557
3558 __ stop("unexpected profiling mismatch");
3559 __ bind(ok);
3560 __ pop(tmp);
3561 }
3562 #else
3563 __ jccb(Assembler::zero, next);
3564 #endif
3565 // first time here. Set profile type.
3566 __ movptr(mdo_addr, tmp);
3567 #ifdef ASSERT
3568 __ andptr(tmp, TypeEntries::type_klass_mask);
3569 __ verify_klass_ptr(tmp);
3570 #endif
3571 } else {
3572 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3573 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3574
3575 __ testptr(mdo_addr, TypeEntries::type_unknown);
3576 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3577
3578 __ orptr(mdo_addr, TypeEntries::type_unknown);
3579 }
3580 }
3581 }
3582 __ bind(next);
3583 }
3584
3585 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3586 Register obj = op->obj()->as_register();
3587 Register tmp = op->tmp()->as_pointer_register();
3588 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3589 bool not_null = op->not_null();
3590 int flag = op->flag();
3591
3592 Label not_inline_type;
3593 if (!not_null) {
3594 __ testptr(obj, obj);
3595 __ jccb(Assembler::zero, not_inline_type);
3596 }
3597
3598 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3599
3600 __ orb(mdo_addr, flag);
3601
3602 __ bind(not_inline_type);
3603 }
3604
3605 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3606 Unimplemented();
3607 }
3608
3609
3610 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3611 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3612 }
3613
3614
3615 void LIR_Assembler::align_backward_branch_target() {
3616 __ align(BytesPerWord);
3617 }
3618
3619
3620 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3621 if (left->is_single_cpu()) {
3622 __ negl(left->as_register());
3623 move_regs(left->as_register(), dest->as_register());
3624
3625 } else if (left->is_double_cpu()) {
3626 Register lo = left->as_register_lo();
3627 #ifdef _LP64
3628 Register dst = dest->as_register_lo();
3629 __ movptr(dst, lo);
3630 __ negptr(dst);
3631 #else
3632 Register hi = left->as_register_hi();
3633 __ lneg(hi, lo);
3634 if (dest->as_register_lo() == hi) {
3635 assert(dest->as_register_hi() != lo, "destroying register");
3636 move_regs(hi, dest->as_register_hi());
3637 move_regs(lo, dest->as_register_lo());
3638 } else {
3639 move_regs(lo, dest->as_register_lo());
3640 move_regs(hi, dest->as_register_hi());
3641 }
3642 #endif // _LP64
3643
3644 } else if (dest->is_single_xmm()) {
3645 assert(!tmp->is_valid(), "do not need temporary");
3646 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3647 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3648 }
3649 __ xorps(dest->as_xmm_float_reg(),
3650 ExternalAddress((address)float_signflip_pool),
3651 rscratch1);
3652 } else if (dest->is_double_xmm()) {
3653 assert(!tmp->is_valid(), "do not need temporary");
3654 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3655 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3656 }
3657 __ xorpd(dest->as_xmm_double_reg(),
3658 ExternalAddress((address)double_signflip_pool),
3659 rscratch1);
3660 } else {
3661 ShouldNotReachHere();
3662 }
3663 }
3664
3665
3666 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3667 assert(src->is_address(), "must be an address");
3668 assert(dest->is_register(), "must be a register");
3669
3670 PatchingStub* patch = nullptr;
3671 if (patch_code != lir_patch_none) {
3672 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3673 }
3674
3675 Register reg = dest->as_pointer_register();
3676 LIR_Address* addr = src->as_address_ptr();
3677 __ lea(reg, as_Address(addr));
3678
3679 if (patch != nullptr) {
3680 patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3681 }
3682 }
3683
3684
3685
3686 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3687 assert(!tmp->is_valid(), "don't need temporary");
3688 __ call(RuntimeAddress(dest));
3689 if (info != nullptr) {
3690 add_call_info_here(info);
3691 }
3692 __ post_call_nop();
3693 }
3694
3695
3696 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3697 assert(type == T_LONG, "only for volatile long fields");
3698
3699 if (info != nullptr) {
3700 add_debug_info_for_null_check_here(info);
3701 }
3702
3703 if (src->is_double_xmm()) {
3704 if (dest->is_double_cpu()) {
3705 #ifdef _LP64
3706 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3707 #else
3708 __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3709 __ psrlq(src->as_xmm_double_reg(), 32);
3710 __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3711 #endif // _LP64
3712 } else if (dest->is_double_stack()) {
3713 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3714 } else if (dest->is_address()) {
3715 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3716 } else {
3717 ShouldNotReachHere();
3718 }
3719
3720 } else if (dest->is_double_xmm()) {
3721 if (src->is_double_stack()) {
3722 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3723 } else if (src->is_address()) {
3724 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3725 } else {
3726 ShouldNotReachHere();
3727 }
3728 } else {
3729 ShouldNotReachHere();
3730 }
3731 }
3732
3733 #ifdef ASSERT
3734 // emit run-time assertion
3735 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3736 assert(op->code() == lir_assert, "must be");
3737
3738 if (op->in_opr1()->is_valid()) {
3739 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3740 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3741 } else {
3742 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3743 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3744 }
3745
3746 Label ok;
3747 if (op->condition() != lir_cond_always) {
3748 Assembler::Condition acond = Assembler::zero;
3749 switch (op->condition()) {
3750 case lir_cond_equal: acond = Assembler::equal; break;
3751 case lir_cond_notEqual: acond = Assembler::notEqual; break;
3752 case lir_cond_less: acond = Assembler::less; break;
3753 case lir_cond_lessEqual: acond = Assembler::lessEqual; break;
3754 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3755 case lir_cond_greater: acond = Assembler::greater; break;
3756 case lir_cond_belowEqual: acond = Assembler::belowEqual; break;
3757 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break;
3758 default: ShouldNotReachHere();
3759 }
3760 __ jcc(acond, ok);
3761 }
3762 if (op->halt()) {
3763 const char* str = __ code_string(op->msg());
3764 __ stop(str);
3765 } else {
3766 breakpoint();
3767 }
3768 __ bind(ok);
3769 }
3770 #endif
3771
3772 void LIR_Assembler::membar() {
3773 // QQQ sparc TSO uses this,
3774 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3775 }
3776
3777 void LIR_Assembler::membar_acquire() {
3778 // No x86 machines currently require load fences
3779 }
3780
3781 void LIR_Assembler::membar_release() {
3782 // No x86 machines currently require store fences
3783 }
3784
3785 void LIR_Assembler::membar_loadload() {
3786 // no-op
3787 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3788 }
3789
3790 void LIR_Assembler::membar_storestore() {
3791 // no-op
3792 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3793 }
3794
3795 void LIR_Assembler::membar_loadstore() {
3796 // no-op
3797 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3798 }
3799
3800 void LIR_Assembler::membar_storeload() {
3801 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3802 }
3803
3804 void LIR_Assembler::on_spin_wait() {
3805 __ pause ();
3806 }
3807
3808 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3809 assert(result_reg->is_register(), "check");
3810 #ifdef _LP64
3811 // __ get_thread(result_reg->as_register_lo());
3812 __ mov(result_reg->as_register(), r15_thread);
3813 #else
3814 __ get_thread(result_reg->as_register());
3815 #endif // _LP64
3816 }
3817
3818 void LIR_Assembler::check_orig_pc() {
3819 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3820 }
3821
3822 void LIR_Assembler::peephole(LIR_List*) {
3823 // do nothing for now
3824 }
3825
3826 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3827 assert(data == dest, "xchg/xadd uses only 2 operands");
3828
3829 if (data->type() == T_INT) {
3830 if (code == lir_xadd) {
3831 __ lock();
3832 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3833 } else {
3834 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3835 }
3836 } else if (data->is_oop()) {
3837 assert (code == lir_xchg, "xadd for oops");
3838 Register obj = data->as_register();
3839 #ifdef _LP64
3840 if (UseCompressedOops) {
3841 __ encode_heap_oop(obj);
3842 __ xchgl(obj, as_Address(src->as_address_ptr()));
3843 __ decode_heap_oop(obj);
3844 } else {
3845 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3846 }
3847 #else
3848 __ xchgl(obj, as_Address(src->as_address_ptr()));
3849 #endif
3850 } else if (data->type() == T_LONG) {
3851 #ifdef _LP64
3852 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3853 if (code == lir_xadd) {
3854 __ lock();
3855 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3856 } else {
3857 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3858 }
3859 #else
3860 ShouldNotReachHere();
3861 #endif
3862 } else {
3863 ShouldNotReachHere();
3864 }
3865 }
3866
3867 #undef __