1 //
2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp1,
41 Register tmp2,
42 RegSet preserve = RegSet(),
43 RegSet no_preserve = RegSet()) {
44 if (!G1PreBarrierStubC2::needs_barrier(node)) {
45 return;
46 }
47 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
48 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
49 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
50 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
51 stub->preserve(*reg);
52 }
53 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
54 stub->dont_preserve(*reg);
55 }
56 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
57 }
58
59 static void write_barrier_post(MacroAssembler* masm,
60 const MachNode* node,
61 Register store_addr,
62 Register new_val,
63 Register tmp1,
64 Register tmp2,
65 RegSet preserve = RegSet()) {
66 if (!G1PostBarrierStubC2::needs_barrier(node)) {
67 return;
68 }
69 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
70 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
71 G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
72 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
73 stub->preserve(*reg);
74 }
75 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
76 }
77
78 %}
79
80 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
81 // - Can we use an unbound register for src?
82 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
83 // - Is the zero-extend really required in all the places?
84 // - Move this into the .m4?
85 instruct g1StoreLSpecialOneOop(indirect mem, iRegL_R11 src, immI off, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
86 %{
87 predicate(UseG1GC);
88 match(Set mem (StoreLSpecial mem (Binary src off)));
89 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
90 ins_cost(INSN_COST);
91 format %{ "str $src, $mem\t# g1StoreLSpecialOneOop" %}
92 ins_encode %{
93 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
94
95 // Adjust address to point to narrow oop
96 __ add($tmp4$$Register, $mem$$Register, $off$$constant);
97 write_barrier_pre(masm, this,
98 $tmp4$$Register /* obj */,
99 $tmp1$$Register /* pre_val */,
100 $tmp2$$Register /* tmp1 */,
101 $tmp3$$Register /* tmp2 */,
102 RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
103
104 __ str($src$$Register, $mem$$Register);
105
106 // Shift long value to extract the narrow oop field value and zero-extend it
107 __ lsr($src$$Register, $src$$Register, $off$$constant << LogBitsPerByte);
108 __ ubfm($src$$Register, $src$$Register, 0, 31);
109
110 write_barrier_post(masm, this,
111 $tmp4$$Register /* store_addr */,
112 $src$$Register /* new_val */,
113 $tmp2$$Register /* tmp1 */,
114 $tmp3$$Register /* tmp2 */);
115 %}
116 ins_pipe(istore_reg_mem);
117 %}
118
119 instruct g1StoreLSpecialTwoOops(indirect mem, iRegL_R11 src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
120 %{
121 predicate(UseG1GC);
122 match(Set mem (StoreLSpecial mem src));
123 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
124 ins_cost(INSN_COST);
125 format %{ "str $src, $mem\t# g1StoreLSpecialTwoOops" %}
126 ins_encode %{
127 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
128
129 write_barrier_pre(masm, this,
130 $mem$$Register /* obj */,
131 $tmp1$$Register /* pre_val */,
132 $tmp2$$Register /* tmp1 */,
133 $tmp3$$Register /* tmp2 */,
134 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
135 // Adjust address to point to the second narrow oop in the long value
136 __ add($tmp4$$Register, $mem$$Register, 4);
137 write_barrier_pre(masm, this,
138 $tmp4$$Register /* obj */,
139 $tmp1$$Register /* pre_val */,
140 $tmp2$$Register /* tmp1 */,
141 $tmp3$$Register /* tmp2 */,
142 RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
143
144 __ str($src$$Register, $mem$$Register);
145
146 // Zero-extend first narrow oop to long
147 __ ubfm($tmp1$$Register, $src$$Register, 0, 31);
148
149 // Shift long value to extract the second narrow oop field value
150 __ lsr($src$$Register, $src$$Register, 32);
151 write_barrier_post(masm, this,
152 $mem$$Register /* store_addr */,
153 $tmp1$$Register /* new_val */,
154 $tmp2$$Register /* tmp1 */,
155 $tmp3$$Register /* tmp2 */,
156 RegSet::of($src$$Register, $tmp4$$Register) /* preserve */);
157 write_barrier_post(masm, this,
158 $tmp4$$Register /* store_addr */,
159 $src$$Register /* new_val */,
160 $tmp2$$Register /* tmp1 */,
161 $tmp3$$Register /* tmp2 */);
162 %}
163 ins_pipe(istore_reg_mem);
164 %}
165
166
167 // BEGIN This section of the file is automatically generated. Do not edit --------------
168
169 // This section is generated from g1_aarch64.m4
170
171
172 // This pattern is generated automatically from g1_aarch64.m4.
173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
174 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
175 %{
176 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
177 match(Set mem (StoreP mem src));
178 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
179 ins_cost(INSN_COST);
180 format %{ "str $src, $mem\t# ptr" %}
181 ins_encode %{
182 write_barrier_pre(masm, this,
183 $mem$$Register /* obj */,
184 $tmp1$$Register /* pre_val */,
185 $tmp2$$Register /* tmp1 */,
186 $tmp3$$Register /* tmp2 */,
187 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
188 __ str($src$$Register, $mem$$Register);
189 write_barrier_post(masm, this,
190 $mem$$Register /* store_addr */,
191 $src$$Register /* new_val */,
192 $tmp2$$Register /* tmp1 */,
193 $tmp3$$Register /* tmp2 */);
194 %}
195 ins_pipe(istore_reg_mem);
196 %}
197
198 // This pattern is generated automatically from g1_aarch64.m4.
199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
200 instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
201 %{
202 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
203 match(Set mem (StoreP mem src));
204 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
205 ins_cost(VOLATILE_REF_COST);
206 format %{ "stlr $src, $mem\t# ptr" %}
207 ins_encode %{
208 write_barrier_pre(masm, this,
209 $mem$$Register /* obj */,
210 $tmp1$$Register /* pre_val */,
211 $tmp2$$Register /* tmp1 */,
212 $tmp3$$Register /* tmp2 */,
213 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
214 __ stlr($src$$Register, $mem$$Register);
215 write_barrier_post(masm, this,
216 $mem$$Register /* store_addr */,
217 $src$$Register /* new_val */,
218 $tmp2$$Register /* tmp1 */,
219 $tmp3$$Register /* tmp2 */);
220 %}
221 ins_pipe(pipe_class_memory);
222 %}
223
224 // This pattern is generated automatically from g1_aarch64.m4.
225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
226 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
227 %{
228 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
229 match(Set mem (StoreN mem src));
230 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
231 ins_cost(INSN_COST);
232 format %{ "strw $src, $mem\t# compressed ptr" %}
233 ins_encode %{
234 write_barrier_pre(masm, this,
235 $mem$$Register /* obj */,
236 $tmp1$$Register /* pre_val */,
237 $tmp2$$Register /* tmp1 */,
238 $tmp3$$Register /* tmp2 */,
239 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
240 __ strw($src$$Register, $mem$$Register);
241 if ((barrier_data() & G1C2BarrierPost) != 0) {
242 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
243 __ decode_heap_oop($tmp1$$Register, $src$$Register);
244 } else {
245 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
246 }
247 }
248 write_barrier_post(masm, this,
249 $mem$$Register /* store_addr */,
250 $tmp1$$Register /* new_val */,
251 $tmp2$$Register /* tmp1 */,
252 $tmp3$$Register /* tmp2 */);
253 %}
254 ins_pipe(istore_reg_mem);
255 %}
256
257 // This pattern is generated automatically from g1_aarch64.m4.
258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
259 instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
260 %{
261 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
262 match(Set mem (StoreN mem src));
263 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
264 ins_cost(VOLATILE_REF_COST);
265 format %{ "stlrw $src, $mem\t# compressed ptr" %}
266 ins_encode %{
267 write_barrier_pre(masm, this,
268 $mem$$Register /* obj */,
269 $tmp1$$Register /* pre_val */,
270 $tmp2$$Register /* tmp1 */,
271 $tmp3$$Register /* tmp2 */,
272 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
273 __ stlrw($src$$Register, $mem$$Register);
274 if ((barrier_data() & G1C2BarrierPost) != 0) {
275 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
276 __ decode_heap_oop($tmp1$$Register, $src$$Register);
277 } else {
278 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
279 }
280 }
281 write_barrier_post(masm, this,
282 $mem$$Register /* store_addr */,
283 $tmp1$$Register /* new_val */,
284 $tmp2$$Register /* tmp1 */,
285 $tmp3$$Register /* tmp2 */);
286 %}
287 ins_pipe(pipe_class_memory);
288 %}
289
290 // This pattern is generated automatically from g1_aarch64.m4.
291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
292 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
293 %{
294 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
295 match(Set mem (StoreN mem (EncodeP src)));
296 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
297 ins_cost(INSN_COST);
298 format %{ "encode_heap_oop $tmp1, $src\n\t"
299 "strw $tmp1, $mem\t# compressed ptr" %}
300 ins_encode %{
301 write_barrier_pre(masm, this,
302 $mem$$Register /* obj */,
303 $tmp1$$Register /* pre_val */,
304 $tmp2$$Register /* tmp1 */,
305 $tmp3$$Register /* tmp2 */,
306 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
307 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
308 __ encode_heap_oop($tmp1$$Register, $src$$Register);
309 } else {
310 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
311 }
312 __ strw($tmp1$$Register, $mem$$Register);
313 write_barrier_post(masm, this,
314 $mem$$Register /* store_addr */,
315 $src$$Register /* new_val */,
316 $tmp2$$Register /* tmp1 */,
317 $tmp3$$Register /* tmp2 */);
318 %}
319 ins_pipe(istore_reg_mem);
320 %}
321
322 // This pattern is generated automatically from g1_aarch64.m4.
323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
324 instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
325 %{
326 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
327 match(Set mem (StoreN mem (EncodeP src)));
328 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
329 ins_cost(VOLATILE_REF_COST);
330 format %{ "encode_heap_oop $tmp1, $src\n\t"
331 "stlrw $tmp1, $mem\t# compressed ptr" %}
332 ins_encode %{
333 write_barrier_pre(masm, this,
334 $mem$$Register /* obj */,
335 $tmp1$$Register /* pre_val */,
336 $tmp2$$Register /* tmp1 */,
337 $tmp3$$Register /* tmp2 */,
338 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
339 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
340 __ encode_heap_oop($tmp1$$Register, $src$$Register);
341 } else {
342 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
343 }
344 __ stlrw($tmp1$$Register, $mem$$Register);
345 write_barrier_post(masm, this,
346 $mem$$Register /* store_addr */,
347 $src$$Register /* new_val */,
348 $tmp2$$Register /* tmp1 */,
349 $tmp3$$Register /* tmp2 */);
350 %}
351 ins_pipe(pipe_class_memory);
352 %}
353
354 // This pattern is generated automatically from g1_aarch64.m4.
355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
356 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
357 %{
358 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
359 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
360 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
361 ins_cost(2 * VOLATILE_REF_COST);
362 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
363 ins_encode %{
364 assert_different_registers($oldval$$Register, $mem$$Register);
365 assert_different_registers($newval$$Register, $mem$$Register);
366 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
367 // $oldval is the only value that can be overwritten.
368 // The same holds for g1CompareAndSwapP and its Acq variant.
369 write_barrier_pre(masm, this,
370 noreg /* obj */,
371 $oldval$$Register /* pre_val */,
372 $tmp1$$Register /* tmp1 */,
373 $tmp2$$Register /* tmp2 */,
374 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
375 RegSet::of($res$$Register) /* no_preserve */);
376 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
377 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
378 write_barrier_post(masm, this,
379 $mem$$Register /* store_addr */,
380 $newval$$Register /* new_val */,
381 $tmp1$$Register /* tmp1 */,
382 $tmp2$$Register /* tmp2 */);
383 %}
384 ins_pipe(pipe_slow);
385 %}
386
387 // This pattern is generated automatically from g1_aarch64.m4.
388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
389 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
390 %{
391 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
392 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
393 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
394 ins_cost(VOLATILE_REF_COST);
395 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
396 ins_encode %{
397 assert_different_registers($oldval$$Register, $mem$$Register);
398 assert_different_registers($newval$$Register, $mem$$Register);
399 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
400 // $oldval is the only value that can be overwritten.
401 // The same holds for g1CompareAndSwapP and its Acq variant.
402 write_barrier_pre(masm, this,
403 noreg /* obj */,
404 $oldval$$Register /* pre_val */,
405 $tmp1$$Register /* tmp1 */,
406 $tmp2$$Register /* tmp2 */,
407 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
408 RegSet::of($res$$Register) /* no_preserve */);
409 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
410 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
411 write_barrier_post(masm, this,
412 $mem$$Register /* store_addr */,
413 $newval$$Register /* new_val */,
414 $tmp1$$Register /* tmp1 */,
415 $tmp2$$Register /* tmp2 */);
416 %}
417 ins_pipe(pipe_slow);
418 %}
419
420 // This pattern is generated automatically from g1_aarch64.m4.
421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
422 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
423 %{
424 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
425 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
426 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
427 ins_cost(2 * VOLATILE_REF_COST);
428 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
429 ins_encode %{
430 assert_different_registers($oldval$$Register, $mem$$Register);
431 assert_different_registers($newval$$Register, $mem$$Register);
432 write_barrier_pre(masm, this,
433 $mem$$Register /* obj */,
434 $tmp1$$Register /* pre_val */,
435 $tmp2$$Register /* tmp1 */,
436 $tmp3$$Register /* tmp2 */,
437 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
438 RegSet::of($res$$Register) /* no_preserve */);
439 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
440 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
441 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
442 write_barrier_post(masm, this,
443 $mem$$Register /* store_addr */,
444 $tmp1$$Register /* new_val */,
445 $tmp2$$Register /* tmp1 */,
446 $tmp3$$Register /* tmp2 */);
447 %}
448 ins_pipe(pipe_slow);
449 %}
450
451 // This pattern is generated automatically from g1_aarch64.m4.
452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
453 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
454 %{
455 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
456 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
457 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
458 ins_cost(VOLATILE_REF_COST);
459 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
460 ins_encode %{
461 assert_different_registers($oldval$$Register, $mem$$Register);
462 assert_different_registers($newval$$Register, $mem$$Register);
463 write_barrier_pre(masm, this,
464 $mem$$Register /* obj */,
465 $tmp1$$Register /* pre_val */,
466 $tmp2$$Register /* tmp1 */,
467 $tmp3$$Register /* tmp2 */,
468 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
469 RegSet::of($res$$Register) /* no_preserve */);
470 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
471 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
472 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
473 write_barrier_post(masm, this,
474 $mem$$Register /* store_addr */,
475 $tmp1$$Register /* new_val */,
476 $tmp2$$Register /* tmp1 */,
477 $tmp3$$Register /* tmp2 */);
478 %}
479 ins_pipe(pipe_slow);
480 %}
481
482 // This pattern is generated automatically from g1_aarch64.m4.
483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
484 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
485 %{
486 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
487 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
488 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
489 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
490 ins_cost(2 * VOLATILE_REF_COST);
491 format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
492 "cset $res, EQ" %}
493 ins_encode %{
494 assert_different_registers($oldval$$Register, $mem$$Register);
495 assert_different_registers($newval$$Register, $mem$$Register);
496 write_barrier_pre(masm, this,
497 noreg /* obj */,
498 $oldval$$Register /* pre_val */,
499 $tmp1$$Register /* tmp1 */,
500 $tmp2$$Register /* tmp2 */,
501 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
502 RegSet::of($res$$Register) /* no_preserve */);
503 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
504 false /* acquire */, true /* release */, false /* weak */, noreg);
505 __ cset($res$$Register, Assembler::EQ);
506 write_barrier_post(masm, this,
507 $mem$$Register /* store_addr */,
508 $newval$$Register /* new_val */,
509 $tmp1$$Register /* tmp1 */,
510 $tmp2$$Register /* tmp2 */);
511 %}
512 ins_pipe(pipe_slow);
513 %}
514
515 // This pattern is generated automatically from g1_aarch64.m4.
516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
517 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
518 %{
519 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
520 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
521 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
522 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
523 ins_cost(VOLATILE_REF_COST);
524 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
525 "cset $res, EQ" %}
526 ins_encode %{
527 assert_different_registers($oldval$$Register, $mem$$Register);
528 assert_different_registers($newval$$Register, $mem$$Register);
529 write_barrier_pre(masm, this,
530 noreg /* obj */,
531 $oldval$$Register /* pre_val */,
532 $tmp1$$Register /* tmp1 */,
533 $tmp2$$Register /* tmp2 */,
534 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
535 RegSet::of($res$$Register) /* no_preserve */);
536 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
537 true /* acquire */, true /* release */, false /* weak */, noreg);
538 __ cset($res$$Register, Assembler::EQ);
539 write_barrier_post(masm, this,
540 $mem$$Register /* store_addr */,
541 $newval$$Register /* new_val */,
542 $tmp1$$Register /* tmp1 */,
543 $tmp2$$Register /* tmp2 */);
544 %}
545 ins_pipe(pipe_slow);
546 %}
547
548 // This pattern is generated automatically from g1_aarch64.m4.
549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
550 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
551 %{
552 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
553 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
554 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
555 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
556 ins_cost(2 * VOLATILE_REF_COST);
557 format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
558 "cset $res, EQ" %}
559 ins_encode %{
560 assert_different_registers($oldval$$Register, $mem$$Register);
561 assert_different_registers($newval$$Register, $mem$$Register);
562 write_barrier_pre(masm, this,
563 $mem$$Register /* obj */,
564 $tmp1$$Register /* pre_val */,
565 $tmp2$$Register /* tmp1 */,
566 $tmp3$$Register /* tmp2 */,
567 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
568 RegSet::of($res$$Register) /* no_preserve */);
569 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
570 false /* acquire */, true /* release */, false /* weak */, noreg);
571 __ cset($res$$Register, Assembler::EQ);
572 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
573 write_barrier_post(masm, this,
574 $mem$$Register /* store_addr */,
575 $tmp1$$Register /* new_val */,
576 $tmp2$$Register /* tmp1 */,
577 $tmp3$$Register /* tmp2 */);
578 %}
579 ins_pipe(pipe_slow);
580 %}
581
582 // This pattern is generated automatically from g1_aarch64.m4.
583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
584 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
585 %{
586 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
587 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
588 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
589 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
590 ins_cost(VOLATILE_REF_COST);
591 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
592 "cset $res, EQ" %}
593 ins_encode %{
594 assert_different_registers($oldval$$Register, $mem$$Register);
595 assert_different_registers($newval$$Register, $mem$$Register);
596 write_barrier_pre(masm, this,
597 $mem$$Register /* obj */,
598 $tmp1$$Register /* pre_val */,
599 $tmp2$$Register /* tmp1 */,
600 $tmp3$$Register /* tmp2 */,
601 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
602 RegSet::of($res$$Register) /* no_preserve */);
603 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
604 true /* acquire */, true /* release */, false /* weak */, noreg);
605 __ cset($res$$Register, Assembler::EQ);
606 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
607 write_barrier_post(masm, this,
608 $mem$$Register /* store_addr */,
609 $tmp1$$Register /* new_val */,
610 $tmp2$$Register /* tmp1 */,
611 $tmp3$$Register /* tmp2 */);
612 %}
613 ins_pipe(pipe_slow);
614 %}
615
616 // This pattern is generated automatically from g1_aarch64.m4.
617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
618 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
619 %{
620 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
621 match(Set preval (GetAndSetP mem newval));
622 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
623 ins_cost(2 * VOLATILE_REF_COST);
624 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
625 ins_encode %{
626 assert_different_registers($mem$$Register, $newval$$Register);
627 write_barrier_pre(masm, this,
628 $mem$$Register /* obj */,
629 $preval$$Register /* pre_val (as a temporary register) */,
630 $tmp1$$Register /* tmp1 */,
631 $tmp2$$Register /* tmp2 */,
632 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
633 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
634 write_barrier_post(masm, this,
635 $mem$$Register /* store_addr */,
636 $newval$$Register /* new_val */,
637 $tmp1$$Register /* tmp1 */,
638 $tmp2$$Register /* tmp2 */);
639 %}
640 ins_pipe(pipe_serial);
641 %}
642
643 // This pattern is generated automatically from g1_aarch64.m4.
644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
645 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
646 %{
647 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
648 match(Set preval (GetAndSetP mem newval));
649 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
650 ins_cost(VOLATILE_REF_COST);
651 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
652 ins_encode %{
653 assert_different_registers($mem$$Register, $newval$$Register);
654 write_barrier_pre(masm, this,
655 $mem$$Register /* obj */,
656 $preval$$Register /* pre_val (as a temporary register) */,
657 $tmp1$$Register /* tmp1 */,
658 $tmp2$$Register /* tmp2 */,
659 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
660 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
661 write_barrier_post(masm, this,
662 $mem$$Register /* store_addr */,
663 $newval$$Register /* new_val */,
664 $tmp1$$Register /* tmp1 */,
665 $tmp2$$Register /* tmp2 */);
666 %}
667 ins_pipe(pipe_serial);
668 %}
669
670 // This pattern is generated automatically from g1_aarch64.m4.
671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
672 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
673 %{
674 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
675 match(Set preval (GetAndSetN mem newval));
676 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
677 ins_cost(2 * VOLATILE_REF_COST);
678 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
679 ins_encode %{
680 assert_different_registers($mem$$Register, $newval$$Register);
681 write_barrier_pre(masm, this,
682 $mem$$Register /* obj */,
683 $tmp1$$Register /* pre_val */,
684 $tmp2$$Register /* tmp1 */,
685 $tmp3$$Register /* tmp2 */,
686 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
687 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
688 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
689 write_barrier_post(masm, this,
690 $mem$$Register /* store_addr */,
691 $tmp1$$Register /* new_val */,
692 $tmp2$$Register /* tmp1 */,
693 $tmp3$$Register /* tmp2 */);
694 %}
695 ins_pipe(pipe_serial);
696 %}
697
698 // This pattern is generated automatically from g1_aarch64.m4.
699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
700 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
701 %{
702 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
703 match(Set preval (GetAndSetN mem newval));
704 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
705 ins_cost(VOLATILE_REF_COST);
706 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
707 ins_encode %{
708 assert_different_registers($mem$$Register, $newval$$Register);
709 write_barrier_pre(masm, this,
710 $mem$$Register /* obj */,
711 $tmp1$$Register /* pre_val */,
712 $tmp2$$Register /* tmp1 */,
713 $tmp3$$Register /* tmp2 */,
714 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
715 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
716 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
717 write_barrier_post(masm, this,
718 $mem$$Register /* store_addr */,
719 $tmp1$$Register /* new_val */,
720 $tmp2$$Register /* tmp1 */,
721 $tmp3$$Register /* tmp2 */);
722 %}
723 ins_pipe(pipe_serial);
724 %}
725
726 // This pattern is generated automatically from g1_aarch64.m4.
727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
728 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
729 %{
730 // This instruction does not need an acquiring counterpart because it is only
731 // used for reference loading (Reference::get()). The same holds for g1LoadN.
732 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
733 match(Set dst (LoadP mem));
734 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
735 ins_cost(4 * INSN_COST);
736 format %{ "ldr $dst, $mem\t# ptr" %}
737 ins_encode %{
738 __ ldr($dst$$Register, $mem$$Register);
739 write_barrier_pre(masm, this,
740 noreg /* obj */,
741 $dst$$Register /* pre_val */,
742 $tmp1$$Register /* tmp1 */,
743 $tmp2$$Register /* tmp2 */);
744 %}
745 ins_pipe(iload_reg_mem);
746 %}
747
748 // This pattern is generated automatically from g1_aarch64.m4.
749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
750 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
751 %{
752 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
753 match(Set dst (LoadN mem));
754 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
755 ins_cost(4 * INSN_COST);
756 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
757 ins_encode %{
758 __ ldrw($dst$$Register, $mem$$Register);
759 if ((barrier_data() & G1C2BarrierPre) != 0) {
760 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
761 write_barrier_pre(masm, this,
762 noreg /* obj */,
763 $tmp1$$Register /* pre_val */,
764 $tmp2$$Register /* tmp1 */,
765 $tmp3$$Register /* tmp2 */);
766 }
767 %}
768 ins_pipe(iload_reg_mem);
769 %}
770
771 // END This section of the file is automatically generated. Do not edit --------------