1 /*
2 * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zAddress.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29
30 #include "gc/shared/accessBarrierSupport.inline.hpp"
31 #include "gc/z/zAddress.inline.hpp"
32 #include "gc/z/zBarrier.inline.hpp"
33 #include "gc/z/zIterator.inline.hpp"
34 #include "gc/z/zNMethod.hpp"
35 #include "memory/iterator.inline.hpp"
36 #include "oops/inlineKlass.inline.hpp"
37 #include "utilities/debug.hpp"
38
39 template <DecoratorSet decorators, typename BarrierSetT>
40 template <DecoratorSet expected>
41 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
42 if ((decorators & expected) == 0) {
43 fatal("Using unsupported access decorators");
44 }
45 }
46
47 template <DecoratorSet decorators, typename BarrierSetT>
48 template <DecoratorSet expected>
49 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
50 if ((decorators & expected) != 0) {
51 fatal("Using unsupported access decorators");
52 }
53 }
54
55 template <DecoratorSet decorators, typename BarrierSetT>
56 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
57 ShouldNotReachHere();
58 }
59
60 template <DecoratorSet decorators, typename BarrierSetT>
61 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
62 assert(base != nullptr, "Invalid base");
63 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
64 }
65
66 template <DecoratorSet decorators, typename BarrierSetT>
67 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
68 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
69
70 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
71 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
72 // Load barriers on strong oop refs don't keep objects alive
73 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
74 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
75 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
76 } else {
77 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
78 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
79 }
80 } else {
81 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
82 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
83 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
84 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
85 } else {
86 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
87 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
88 }
89 }
90 }
91
92 template <DecoratorSet decorators, typename BarrierSetT>
93 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
94 verify_decorators_present<ON_UNKNOWN_OOP_REF>();
95
96 const DecoratorSet decorators_known_strength =
97 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
98
99 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
100 if (decorators_known_strength & ON_STRONG_OOP_REF) {
101 // Load barriers on strong oop refs don't keep objects alive
102 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
103 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
104 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
105 } else {
106 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
107 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
108 }
109 } else {
110 if (decorators_known_strength & ON_STRONG_OOP_REF) {
111 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
112 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
113 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
114 } else {
115 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
116 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
117 }
118 }
119 }
120
121 inline zpointer ZBarrierSet::store_good(oop obj) {
122 assert(ZPointerStoreGoodMask != 0, "sanity");
123
124 const zaddress addr = to_zaddress(obj);
125 return ZAddress::store_good(addr);
126 }
127
128 template <DecoratorSet decorators, typename BarrierSetT>
129 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
130 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
131 ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
132 } else {
133 assert(false, "Should not be used on uninitialized memory");
134 }
135 }
136
137 template <DecoratorSet decorators, typename BarrierSetT>
138 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
139 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
140 ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
141 }
142 }
143
144 template <DecoratorSet decorators, typename BarrierSetT>
145 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
146 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
147 ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
148 }
149 }
150
151 template <DecoratorSet decorators, typename BarrierSetT>
152 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
153 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
154 ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
155 } else {
156 assert(false, "Should not be used on uninitialized memory");
157 }
158 }
159
160 template <DecoratorSet decorators, typename BarrierSetT>
161 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
162 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
163 ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
164 }
165 }
166
167 //
168 // In heap
169 //
170 template <DecoratorSet decorators, typename BarrierSetT>
171 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
172 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
173
174 const zpointer o = Raw::load_in_heap(p);
175 assert_is_valid(o);
176
177 return to_oop(load_barrier(p, o));
178 }
179
180 template <DecoratorSet decorators, typename BarrierSetT>
181 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
182 zpointer* const p = field_addr(base, offset);
183
184 const zpointer o = Raw::load_in_heap(p);
185 assert_is_valid(o);
186
187 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
188 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
189 }
190
191 return to_oop(load_barrier(p, o));
192 }
193
194 template <DecoratorSet decorators>
195 bool is_store_barrier_no_keep_alive() {
196 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
197 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
198 }
199
200 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
201 return true;
202 }
203
204 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
205 return true;
206 }
207
208 template <DecoratorSet decorators>
209 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
210 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
211 return is_store_barrier_no_keep_alive<decorators>();
212 }
213
214 const DecoratorSet decorators_known_strength =
215 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
216
217 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
218 return (decorators & AS_NO_KEEPALIVE) != 0;
219 }
220
221 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
222 return true;
223 }
224
225 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
226 return true;
227 }
228
229 template <DecoratorSet decorators, typename BarrierSetT>
230 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
231 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
232
233 if (is_store_barrier_no_keep_alive<decorators>()) {
234 no_keep_alive_store_barrier_heap(p);
235 } else {
236 store_barrier_heap_without_healing(p);
237 }
238
239 Raw::store_in_heap(p, store_good(value));
240 }
241
242 template <DecoratorSet decorators, typename BarrierSetT>
243 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
244 zpointer* const p = field_addr(base, offset);
245
246 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
247 no_keep_alive_store_barrier_heap(p);
248 } else {
249 store_barrier_heap_without_healing(p);
250 }
251
252 Raw::store_in_heap(p, store_good(value));
253 }
254
255 template <DecoratorSet decorators, typename BarrierSetT>
256 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
257 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
258
259 if (!is_store_barrier_no_keep_alive<decorators>()) {
260 store_barrier_native_without_healing(p);
261 }
262
263 Raw::store(p, store_good(value));
264 }
265
266 template <DecoratorSet decorators, typename BarrierSetT>
267 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
268 verify_decorators_present<ON_STRONG_OOP_REF>();
269 verify_decorators_absent<AS_NO_KEEPALIVE>();
270
271 store_barrier_heap_with_healing(p);
272
273 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
274 assert_is_valid(o);
275
276 return to_oop(ZPointer::uncolor_store_good(o));
277 }
278
279 template <DecoratorSet decorators, typename BarrierSetT>
280 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
281 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
282 verify_decorators_absent<AS_NO_KEEPALIVE>();
283
284 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
285 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
286 // with the motivation that if you're doing Unsafe operations on a Reference.referent
287 // field, then you're on your own anyway.
288 zpointer* const p = field_addr(base, offset);
289
290 store_barrier_heap_with_healing(p);
291
292 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
293 assert_is_valid(o);
294
295 return to_oop(ZPointer::uncolor_store_good(o));
296 }
297
298 template <DecoratorSet decorators, typename BarrierSetT>
299 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
300 verify_decorators_present<ON_STRONG_OOP_REF>();
301 verify_decorators_absent<AS_NO_KEEPALIVE>();
302
303 store_barrier_heap_with_healing(p);
304
305 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
306 assert_is_valid(o);
307
308 return to_oop(ZPointer::uncolor_store_good(o));
309 }
310
311 template <DecoratorSet decorators, typename BarrierSetT>
312 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
313 verify_decorators_present<ON_STRONG_OOP_REF>();
314 verify_decorators_absent<AS_NO_KEEPALIVE>();
315
316 zpointer* const p = field_addr(base, offset);
317
318 store_barrier_heap_with_healing(p);
319
320 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
321 assert_is_valid(o);
322
323 return to_oop(ZPointer::uncolor_store_good(o));
324 }
325
326 template <DecoratorSet decorators, typename BarrierSetT>
327 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
328 store_barrier_heap_without_healing(dst);
329
330 return ZBarrier::load_barrier_on_oop_field(src);
331 }
332
333 template <DecoratorSet decorators, typename BarrierSetT>
334 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
335 const zaddress obj = oop_copy_one_barriers(dst, src);
336
337 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
338 return oop_copy_check_null;
339 }
340
341 Atomic::store(dst, ZAddress::store_good(obj));
342 return oop_copy_check_ok;
343 }
344
345 template <DecoratorSet decorators, typename BarrierSetT>
346 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
347 const zaddress obj = oop_copy_one_barriers(dst, src);
348 const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
349
350 if (null_check && is_null(obj)) {
351 return oop_copy_check_null;
352 }
353 else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
354 // Check cast failed
355 return oop_copy_check_class_cast;
356 }
357
358 Atomic::store(dst, ZAddress::store_good(obj));
359
360 return oop_copy_check_ok;
361 }
362
363
364 template <DecoratorSet decorators, typename BarrierSetT>
365 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
366 // Check cast and copy each elements
367 OopCopyCheckStatus check_status = oop_copy_check_ok;
368 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
369 check_status = oop_copy_one_check_cast(dst, src, dst_klass);
370 }
371 return check_status;
372 }
373
374 template <DecoratorSet decorators, typename BarrierSetT>
375 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
376 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
377 OopCopyCheckStatus check_status = oop_copy_check_ok;
378 if (is_disjoint || src > dst) {
379 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
380 check_status = oop_copy_one(dst, src);
381 }
382 return check_status;
383 }
384
385 if (src < dst) {
386 const zpointer* const end = src;
387 src += length - 1;
388 dst += length - 1;
389 for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
390 check_status = oop_copy_one(dst, src);
391 }
392 return check_status;
393 }
394
395 // src and dst are the same; nothing to do
396 return check_status;
397 }
398
399 template <DecoratorSet decorators, typename BarrierSetT>
400 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
401 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
402 size_t length) {
403 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
404 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
405 OopCopyCheckStatus check_status;
406
407 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
408 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
409 check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
410 } else {
411 check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
412 }
413
414 switch (check_status) {
415 case oop_copy_check_ok:
416 return;
417 case oop_copy_check_class_cast:
418 throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
419 break;
420 case oop_copy_check_null:
421 throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
422 break;
423 default:
424 ShouldNotReachHere();
425 return;
426 }
427 }
428
429 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
430 public:
431 virtual void do_oop(oop* p_) {
432 volatile zpointer* const p = (volatile zpointer*)p_;
433 const zpointer ptr = ZBarrier::load_atomic(p);
434 const zaddress addr = ZPointer::uncolor(ptr);
435 Atomic::store(p, ZAddress::store_good(addr));
436 }
437
438 virtual void do_oop(narrowOop* p) {
439 ShouldNotReachHere();
440 }
441 };
442
443 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
444 public:
445 virtual void do_oop(oop* p) {
446 ZBarrier::load_barrier_on_oop_field((zpointer*)p);
447 }
448
449 virtual void do_oop(narrowOop* p) {
450 ShouldNotReachHere();
451 }
452 };
453
454 template <DecoratorSet decorators, typename BarrierSetT>
455 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
456 check_is_valid_zaddress(src);
457
458 if (dst->is_objArray()) {
459 // Cloning an object array is similar to performing array copy.
460 // If an array is large enough to have its allocation segmented,
461 // this operation might require GC barriers. However, the intrinsics
462 // for cloning arrays transform the clone to an optimized allocation
463 // and arraycopy sequence, so the performance of this runtime call
464 // does not matter for object arrays.
465 clone_obj_array(objArrayOop(src), objArrayOop(dst));
466 return;
467 }
468
469 // Fix the oops
470 ZLoadBarrierOopClosure cl;
471 ZIterator::oop_iterate(src, &cl);
472
473 // Clone the object
474 Raw::clone_in_heap(src, dst, size);
475
476 assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
477
478 // Color store good before handing out
479 ZColorStoreGoodOopClosure cl_sg;
480 ZIterator::oop_iterate(dst, &cl_sg);
481 }
482
483 static inline void copy_primitive_payload(const void* src, const void* dst, const size_t payload_size_bytes, size_t& copied_bytes) {
484 if (payload_size_bytes == 0) {
485 return;
486 }
487 void* src_payload = (void*)(address(src) + copied_bytes);
488 void* dst_payload = (void*)(address(dst) + copied_bytes);
489 Copy::copy_value_content(src_payload, dst_payload, payload_size_bytes);
490 copied_bytes += payload_size_bytes;
491 }
492
493 template <DecoratorSet decorators, typename BarrierSetT>
494 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
495 if (md->contains_oops()) {
496 // Iterate over each oop map, performing:
497 // 1) possibly raw copy for any primitive payload before each map
498 // 2) load and store barrier for each oop
499 // 3) possibly raw copy for any primitive payload trailer
500
501 // src/dst may not be oops, need offset to adjust oop map offset
502 const address src_oop_addr_offset = ((address) src) - md->first_field_offset();
503 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
504 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
505 size_t size_in_bytes = md->layout_size_in_bytes(lk);
506 size_t copied_bytes = 0;
507 while (map != end) {
508 zpointer *src_p = (zpointer*)(src_oop_addr_offset + map->offset());
509 const uintptr_t oop_offset = uintptr_t(src_p) - uintptr_t(src);
510 zpointer *dst_p = (zpointer*)(uintptr_t(dst) + oop_offset);
511
512 // Copy any leading primitive payload before every cluster of oops
513 assert(copied_bytes < oop_offset || copied_bytes == oop_offset, "Negative sized leading payload segment");
514 copy_primitive_payload(src, dst, oop_offset - copied_bytes, copied_bytes);
515
516 // Copy a cluster of oops
517 for (const zpointer* const src_end = src_p + map->count(); src_p < src_end; src_p++, dst_p++) {
518 oop_copy_one(dst_p, src_p);
519 copied_bytes += sizeof(zpointer);
520 }
521 map++;
522 }
523
524 // Copy trailing primitive payload after potential oops
525 assert(copied_bytes < size_in_bytes || copied_bytes == size_in_bytes, "Negative sized trailing payload segment");
526 copy_primitive_payload(src, dst, size_in_bytes - copied_bytes, copied_bytes);
527 } else {
528 Raw::value_copy_in_heap(src, dst, md, lk);
529 }
530 }
531
532 //
533 // Not in heap
534 //
535 template <DecoratorSet decorators, typename BarrierSetT>
536 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
537 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
538
539 const zpointer o = Raw::template load<zpointer>(p);
540 assert_is_valid(o);
541 return to_oop(load_barrier(p, o));
542 }
543
544 template <DecoratorSet decorators, typename BarrierSetT>
545 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
546 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
547
548 if (HasDecorator<decorators, IN_NMETHOD>::value) {
549 return ZNMethod::load_oop(p, decorators);
550 } else {
551 return oop_load_not_in_heap((zpointer*)p);
552 }
553 }
554
555 template <DecoratorSet decorators, typename BarrierSetT>
556 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
557 verify_decorators_present<ON_STRONG_OOP_REF>();
558 verify_decorators_absent<AS_NO_KEEPALIVE>();
559
560 store_barrier_native_with_healing(p);
561
562 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
563 assert_is_valid(o);
564
565 return to_oop(ZPointer::uncolor_store_good(o));
566 }
567
568 template <DecoratorSet decorators, typename BarrierSetT>
569 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
570 verify_decorators_present<ON_STRONG_OOP_REF>();
571 verify_decorators_absent<AS_NO_KEEPALIVE>();
572
573 store_barrier_native_with_healing(p);
574
575 const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
576 assert_is_valid(o);
577
578 return to_oop(ZPointer::uncolor_store_good(o));
579 }
580
581 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP