1 /*
2 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/aotCodeCache.hpp"
27 #include "gc/g1/g1BarrierSet.hpp"
28 #include "gc/g1/g1BarrierSetAssembler.hpp"
29 #include "gc/g1/g1BarrierSetRuntime.hpp"
30 #include "gc/g1/g1CardTable.hpp"
31 #include "gc/g1/g1HeapRegion.hpp"
32 #include "gc/g1/g1ThreadLocalData.hpp"
33 #include "interpreter/interp_masm.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "utilities/debug.hpp"
36 #include "utilities/macros.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_LIRAssembler.hpp"
39 #include "c1/c1_MacroAssembler.hpp"
40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
41 #endif // COMPILER1
42 #ifdef COMPILER2
43 #include "gc/g1/c2/g1BarrierSetC2.hpp"
44 #endif // COMPILER2
45
46 #define __ masm->
47
48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
49 Register addr, Register count) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51
52 if (!dest_uninitialized) {
53 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
54 #ifndef _LP64
55 __ push(thread);
56 __ get_thread(thread);
57 #endif
58
59 Label filtered;
60 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
61 // Is marking active?
62 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
63 __ cmpl(in_progress, 0);
64 } else {
65 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
66 __ cmpb(in_progress, 0);
67 }
68
69 NOT_LP64(__ pop(thread);)
70
71 __ jcc(Assembler::equal, filtered);
72
73 __ push_call_clobbered_registers(false /* save_fpu */);
74 #ifdef _LP64
75 if (count == c_rarg0) {
76 if (addr == c_rarg1) {
77 // exactly backwards!!
78 __ xchgptr(c_rarg1, c_rarg0);
79 } else {
80 __ movptr(c_rarg1, count);
81 __ movptr(c_rarg0, addr);
82 }
83 } else {
84 __ movptr(c_rarg0, addr);
85 __ movptr(c_rarg1, count);
86 }
87 if (UseCompressedOops) {
88 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
89 } else {
90 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
91 }
92 #else
93 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
94 addr, count);
95 #endif
96 __ pop_call_clobbered_registers(false /* save_fpu */);
97
98 __ bind(filtered);
99 }
100 }
101
102 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
103 Register addr, Register count, Register tmp) {
104 __ push_call_clobbered_registers(false /* save_fpu */);
105 #ifdef _LP64
106 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
107 assert_different_registers(c_rarg1, addr);
108 __ mov(c_rarg1, count);
109 __ mov(c_rarg0, addr);
110 } else {
111 assert_different_registers(c_rarg0, count);
112 __ mov(c_rarg0, addr);
113 __ mov(c_rarg1, count);
114 }
115 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
116 #else
117 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
118 addr, count);
119 #endif
120 __ pop_call_clobbered_registers(false /* save_fpu */);
121 }
122
123 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
124 Register dst, Address src, Register tmp1, Register tmp_thread) {
125 bool on_oop = is_reference_type(type);
126 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
127 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
128 bool on_reference = on_weak || on_phantom;
129 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
130 if (on_oop && on_reference) {
131 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
132
133 #ifndef _LP64
134 // Work around the x86_32 bug that only manifests with Loom for some reason.
135 // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg.
136 if (thread == noreg) {
137 if (dst != rcx && tmp1 != rcx) {
138 thread = rcx;
139 } else if (dst != rdx && tmp1 != rdx) {
140 thread = rdx;
141 } else if (dst != rdi && tmp1 != rdi) {
142 thread = rdi;
143 }
144 }
145 assert_different_registers(dst, tmp1, thread);
146 __ push(thread);
147 __ get_thread(thread);
148 #endif
149
150 // Generate the G1 pre-barrier code to log the value of
151 // the referent field in an SATB buffer.
152 g1_write_barrier_pre(masm /* masm */,
153 noreg /* obj */,
154 dst /* pre_val */,
155 thread /* thread */,
156 tmp1 /* tmp */,
157 true /* tosca_live */,
158 true /* expand_call */);
159
160 #ifndef _LP64
161 __ pop(thread);
162 #endif
163 }
164 }
165
166 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
167 const Register thread, const Register value, const Register temp) {
168 // This code assumes that buffer index is pointer sized.
169 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
170 // Can we store a value in the given thread's buffer?
171 // (The index field is typed as size_t.)
172 __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address)
173 __ testptr(temp, temp); // index == 0?
174 __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer)
175 // The buffer is not full, store value into it.
176 __ subptr(temp, wordSize); // temp := next index
177 __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index
178 __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index
179 __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value
180 }
181
182 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
183 const Register thread) {
184 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
185 // Is marking active?
186 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
187 __ cmpl(in_progress, 0);
188 } else {
189 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
190 __ cmpb(in_progress, 0);
191 }
192 }
193
194 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
195 const Register obj,
196 const Register pre_val,
197 const Register thread,
198 const Register tmp,
199 Label& done,
200 Label& runtime) {
201 // Do we need to load the previous value?
202 if (obj != noreg) {
203 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
204 }
205 // Is the previous value null?
206 __ cmpptr(pre_val, NULL_WORD);
207 __ jcc(Assembler::equal, done);
208 generate_queue_insertion(masm,
209 G1ThreadLocalData::satb_mark_queue_index_offset(),
210 G1ThreadLocalData::satb_mark_queue_buffer_offset(),
211 runtime,
212 thread, pre_val, tmp);
213 __ jmp(done);
214 }
215
216 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
217 Register obj,
218 Register pre_val,
219 Register thread,
220 Register tmp,
221 bool tosca_live,
222 bool expand_call) {
223 // If expand_call is true then we expand the call_VM_leaf macro
224 // directly to skip generating the check by
225 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
226
227 #ifdef _LP64
228 assert(thread == r15_thread, "must be");
229 #endif // _LP64
230
231 Label done;
232 Label runtime;
233
234 assert(pre_val != noreg, "check this code");
235
236 if (obj != noreg) {
237 assert_different_registers(obj, pre_val, tmp);
238 assert(pre_val != rax, "check this code");
239 }
240
241 generate_pre_barrier_fast_path(masm, thread);
242 // If marking is not active (*(mark queue active address) == 0), jump to done
243 __ jcc(Assembler::equal, done);
244 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
245
246 __ bind(runtime);
247
248 // Determine and save the live input values
249 __ push_call_clobbered_registers();
250
251 // Calling the runtime using the regular call_VM_leaf mechanism generates
252 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
253 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
254 //
255 // If we care generating the pre-barrier without a frame (e.g. in the
256 // intrinsified Reference.get() routine) then ebp might be pointing to
257 // the caller frame and so this check will most likely fail at runtime.
258 //
259 // Expanding the call directly bypasses the generation of the check.
260 // So when we do not have have a full interpreter frame on the stack
261 // expand_call should be passed true.
262
263 if (expand_call) {
264 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
265 #ifdef _LP64
266 if (c_rarg1 != thread) {
267 __ mov(c_rarg1, thread);
268 }
269 if (c_rarg0 != pre_val) {
270 __ mov(c_rarg0, pre_val);
271 }
272 #else
273 __ push(thread);
274 __ push(pre_val);
275 #endif
276 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
277 } else {
278 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
279 }
280
281 __ pop_call_clobbered_registers();
282
283 __ bind(done);
284 }
285
286 // return a register that differs from reg1, reg2, reg3 and is not rcx
287
288 static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
289 RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
290 RegSet::of(reg1, reg2, reg3, reg4));
291 return *(available.begin());
292 }
293
294 static void generate_post_barrier_fast_path(MacroAssembler* masm,
295 const Register store_addr,
296 const Register new_val,
297 const Register tmp,
298 const Register tmp2,
299 Label& done,
300 bool new_val_may_be_null) {
301 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
302 // Does store cross heap regions?
303 #if INCLUDE_CDS
304 // AOT code needs to load the barrier grain shift from the aot
305 // runtime constants area in the code cache otherwise we can compile
306 // it as an immediate operand
307
308 if (AOTCodeCache::is_on_for_write()) {
309 address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
310 Register save = pick_different_reg(rcx, tmp, new_val, store_addr);
311 __ push(save);
312 __ movptr(save, store_addr);
313 __ xorptr(save, new_val);
314 __ push(rcx);
315 __ lea(rcx, ExternalAddress(grain_shift_addr));
316 __ movptr(rcx, Address(rcx, 0));
317 __ shrptr(save);
318 __ pop(rcx);
319 __ mov(tmp, save);
320 __ pop(save);
321 __ jcc(Assembler::equal, done);
322 } else
323 #endif // INCLUDE_CDS
324 {
325 __ movptr(tmp, store_addr); // tmp := store address
326 __ xorptr(tmp, new_val); // tmp := store address ^ new value
327 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
328 __ jcc(Assembler::equal, done);
329 }
330
331 // Crosses regions, storing null?
332 if (new_val_may_be_null) {
333 __ cmpptr(new_val, NULL_WORD); // new value == null?
334 __ jcc(Assembler::equal, done);
335 }
336 // Storing region crossing non-null, is card young?
337 __ movptr(tmp, store_addr); // tmp := store address
338 #if INCLUDE_CDS
339 // AOT code needs to load the barrier card shift from the aot
340 // runtime constants area in the code cache otherwise we can compile
341 // it as an immediate operand
342 if (AOTCodeCache::is_on_for_write()) {
343 address card_shift_addr = AOTRuntimeConstants::card_shift_address();
344 Register save = pick_different_reg(rcx, tmp);
345 __ push(save);
346 __ mov(save, tmp);
347 __ push(rcx);
348 __ lea(rcx, ExternalAddress(card_shift_addr));
349 __ movptr(rcx, Address(rcx, 0));
350 __ shrptr(save);
351 __ pop(rcx);
352 __ mov(tmp, save);
353 __ pop(save);
354 } else
355 #endif // INCLUDE_CDS
356 {
357 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
358 }
359 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
360 // a valid address and therefore is not properly handled by the relocation code.
361 if (AOTCodeCache::is_on_for_write()) {
362 // AOT code needs relocation info for this address
363 __ lea(tmp2, ExternalAddress((address)ct->card_table()->byte_map_base())); // tmp2 := card table base address
364 } else {
365 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
366 }
367 __ addptr(tmp, tmp2); // tmp := card address
368 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
369 }
370
371 static void generate_post_barrier_slow_path(MacroAssembler* masm,
372 const Register thread,
373 const Register tmp,
374 const Register tmp2,
375 Label& done,
376 Label& runtime) {
377 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
378 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
379 __ jcc(Assembler::equal, done);
380 // Storing a region crossing, non-null oop, card is clean.
381 // Dirty card and log.
382 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
383 generate_queue_insertion(masm,
384 G1ThreadLocalData::dirty_card_queue_index_offset(),
385 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
386 runtime,
387 thread, tmp, tmp2);
388 __ jmp(done);
389 }
390
391 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
392 Register store_addr,
393 Register new_val,
394 Register thread,
395 Register tmp,
396 Register tmp2) {
397 #ifdef _LP64
398 assert(thread == r15_thread, "must be");
399 #endif // _LP64
400
401 Label done;
402 Label runtime;
403
404 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
405 // If card is young, jump to done
406 __ jcc(Assembler::equal, done);
407 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
408
409 __ bind(runtime);
410 // save the live input values
411 RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
412 __ push_set(saved);
413 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
414 __ pop_set(saved);
415
416 __ bind(done);
417 }
418
419 #if defined(COMPILER2)
420
421 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
422 #ifdef _LP64
423 SaveLiveRegisters save_registers(masm, stub);
424 if (c_rarg0 != arg) {
425 __ mov(c_rarg0, arg);
426 }
427 __ mov(c_rarg1, r15_thread);
428 // rax is a caller-saved, non-argument-passing register, so it does not
429 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
430 // entering this stub, it is saved at this point, and restored after the
431 // call. If it did not contain any live value, it is free to be used. In
432 // either case, it is safe to use it here as a call scratch register.
433 __ call(RuntimeAddress(runtime_path), rax);
434 #else
435 Unimplemented();
436 #endif // _LP64
437 }
438
439 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
440 Register obj,
441 Register pre_val,
442 Register thread,
443 Register tmp,
444 G1PreBarrierStubC2* stub) {
445 #ifdef _LP64
446 assert(thread == r15_thread, "must be");
447 #endif // _LP64
448 assert(pre_val != noreg, "check this code");
449 if (obj != noreg) {
450 assert_different_registers(obj, pre_val, tmp);
451 }
452
453 stub->initialize_registers(obj, pre_val, thread, tmp);
454
455 generate_pre_barrier_fast_path(masm, thread);
456 // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
457 __ jcc(Assembler::notEqual, *stub->entry());
458
459 __ bind(*stub->continuation());
460 }
461
462 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
463 G1PreBarrierStubC2* stub) const {
464 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
465 Label runtime;
466 Register obj = stub->obj();
467 Register pre_val = stub->pre_val();
468 Register thread = stub->thread();
469 Register tmp = stub->tmp1();
470 assert(stub->tmp2() == noreg, "not needed in this platform");
471
472 __ bind(*stub->entry());
473 generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
474
475 __ bind(runtime);
476 generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
477 __ jmp(*stub->continuation());
478 }
479
480 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
481 Register store_addr,
482 Register new_val,
483 Register thread,
484 Register tmp,
485 Register tmp2,
486 G1PostBarrierStubC2* stub) {
487 #ifdef _LP64
488 assert(thread == r15_thread, "must be");
489 #endif // _LP64
490
491 stub->initialize_registers(thread, tmp, tmp2);
492
493 bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
494 generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
495 // If card is not young, jump to stub (slow path)
496 __ jcc(Assembler::notEqual, *stub->entry());
497
498 __ bind(*stub->continuation());
499 }
500
501 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
502 G1PostBarrierStubC2* stub) const {
503 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
504 Label runtime;
505 Register thread = stub->thread();
506 Register tmp = stub->tmp1(); // tmp holds the card address.
507 Register tmp2 = stub->tmp2();
508 assert(stub->tmp3() == noreg, "not needed in this platform");
509
510 __ bind(*stub->entry());
511 generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
512
513 __ bind(runtime);
514 generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
515 __ jmp(*stub->continuation());
516 }
517
518 #endif // COMPILER2
519
520 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
521 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
522 bool in_heap = (decorators & IN_HEAP) != 0;
523 bool as_normal = (decorators & AS_NORMAL) != 0;
524
525 bool needs_pre_barrier = as_normal;
526 bool needs_post_barrier = val != noreg && in_heap;
527
528 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
529 // flatten object address if needed
530 // We do it regardless of precise because we need the registers
531 if (dst.index() == noreg && dst.disp() == 0) {
532 if (dst.base() != tmp1) {
533 __ movptr(tmp1, dst.base());
534 }
535 } else {
536 __ lea(tmp1, dst);
537 }
538
539 #ifndef _LP64
540 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
541 #endif
542
543 NOT_LP64(__ get_thread(rcx));
544 NOT_LP64(imasm->save_bcp());
545
546 if (needs_pre_barrier) {
547 g1_write_barrier_pre(masm /*masm*/,
548 tmp1 /* obj */,
549 tmp2 /* pre_val */,
550 rthread /* thread */,
551 tmp3 /* tmp */,
552 val != noreg /* tosca_live */,
553 false /* expand_call */);
554 }
555 if (val == noreg) {
556 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
557 } else {
558 Register new_val = val;
559 if (needs_post_barrier) {
560 // G1 barrier needs uncompressed oop for region cross check.
561 if (UseCompressedOops) {
562 new_val = tmp2;
563 __ movptr(new_val, val);
564 }
565 }
566 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
567 if (needs_post_barrier) {
568 g1_write_barrier_post(masm /*masm*/,
569 tmp1 /* store_adr */,
570 new_val /* new_val */,
571 rthread /* thread */,
572 tmp3 /* tmp */,
573 tmp2 /* tmp2 */);
574 }
575 }
576 NOT_LP64(imasm->restore_bcp());
577 }
578
579 #ifdef COMPILER1
580
581 #undef __
582 #define __ ce->masm()->
583
584 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
585 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
586 // At this point we know that marking is in progress.
587 // If do_load() is true then we have to emit the
588 // load of the previous value; otherwise it has already
589 // been loaded into _pre_val.
590
591 __ bind(*stub->entry());
592 assert(stub->pre_val()->is_register(), "Precondition.");
593
594 Register pre_val_reg = stub->pre_val()->as_register();
595
596 if (stub->do_load()) {
597 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
598 }
599
600 __ cmpptr(pre_val_reg, NULL_WORD);
601 __ jcc(Assembler::equal, *stub->continuation());
602 ce->store_parameter(stub->pre_val()->as_register(), 0);
603 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
604 __ jmp(*stub->continuation());
605
606 }
607
608 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
609 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
610 __ bind(*stub->entry());
611 assert(stub->addr()->is_register(), "Precondition.");
612 assert(stub->new_val()->is_register(), "Precondition.");
613 Register new_val_reg = stub->new_val()->as_register();
614 __ cmpptr(new_val_reg, NULL_WORD);
615 __ jcc(Assembler::equal, *stub->continuation());
616 ce->store_parameter(stub->addr()->as_pointer_register(), 0);
617 __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
618 __ jmp(*stub->continuation());
619 }
620
621 #undef __
622
623 #define __ sasm->
624
625 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
626 // Generated code assumes that buffer index is pointer sized.
627 STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
628
629 __ prologue("g1_pre_barrier", false);
630 // arg0 : previous value of memory
631
632 __ push(rax);
633 __ push(rdx);
634
635 const Register pre_val = rax;
636 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
637 const Register tmp = rdx;
638
639 NOT_LP64(__ get_thread(thread);)
640
641 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
642 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
643 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
644
645 Label done;
646 Label runtime;
647
648 // Is marking still active?
649 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
650 __ cmpl(queue_active, 0);
651 } else {
652 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
653 __ cmpb(queue_active, 0);
654 }
655 __ jcc(Assembler::equal, done);
656
657 // Can we store original value in the thread's buffer?
658
659 __ movptr(tmp, queue_index);
660 __ testptr(tmp, tmp);
661 __ jcc(Assembler::zero, runtime);
662 __ subptr(tmp, wordSize);
663 __ movptr(queue_index, tmp);
664 __ addptr(tmp, buffer);
665
666 // prev_val (rax)
667 __ load_parameter(0, pre_val);
668 __ movptr(Address(tmp, 0), pre_val);
669 __ jmp(done);
670
671 __ bind(runtime);
672
673 __ push_call_clobbered_registers();
674
675 // load the pre-value
676 __ load_parameter(0, rcx);
677 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
678
679 __ pop_call_clobbered_registers();
680
681 __ bind(done);
682
683 __ pop(rdx);
684 __ pop(rax);
685
686 __ epilogue();
687 }
688
689 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
690 __ prologue("g1_post_barrier", false);
691
692 CardTableBarrierSet* ct =
693 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
694
695 Label done;
696 Label enqueued;
697 Label runtime;
698
699 // At this point we know new_value is non-null and the new_value crosses regions.
700 // Must check to see if card is already dirty
701
702 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
703
704 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
705 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
706
707 __ push(rax);
708 __ push(rcx);
709
710 const Register cardtable = rax;
711 const Register card_addr = rcx;
712
713 __ load_parameter(0, card_addr);
714 __ shrptr(card_addr, CardTable::card_shift());
715 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
716 // a valid address and therefore is not properly handled by the relocation code.
717 if (AOTCodeCache::is_on()) {
718 // AOT code needs relocation info for this address
719 __ lea(cardtable, ExternalAddress((address)ct->card_table()->byte_map_base()));
720 } else {
721 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
722 }
723 __ addptr(card_addr, cardtable);
724
725 NOT_LP64(__ get_thread(thread);)
726
727 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
728 __ jcc(Assembler::equal, done);
729
730 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
731 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
732 __ jcc(Assembler::equal, done);
733
734 // storing region crossing non-null, card is clean.
735 // dirty card and log.
736
737 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
738
739 const Register tmp = rdx;
740 __ push(rdx);
741
742 __ movptr(tmp, queue_index);
743 __ testptr(tmp, tmp);
744 __ jcc(Assembler::zero, runtime);
745 __ subptr(tmp, wordSize);
746 __ movptr(queue_index, tmp);
747 __ addptr(tmp, buffer);
748 __ movptr(Address(tmp, 0), card_addr);
749 __ jmp(enqueued);
750
751 __ bind(runtime);
752 __ push_call_clobbered_registers();
753
754 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
755
756 __ pop_call_clobbered_registers();
757
758 __ bind(enqueued);
759 __ pop(rdx);
760
761 __ bind(done);
762 __ pop(rcx);
763 __ pop(rax);
764
765 __ epilogue();
766 }
767
768 #undef __
769
770 #endif // COMPILER1