1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
78 VectorMaskUseType mask_use_type, bool has_scalar_args) {
79 bool is_supported = true;
80
81 // has_scalar_args flag is true only for non-constant scalar shift count,
82 // since in this case shift needs to be broadcasted.
83 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
84 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
85 is_supported = false;
86 }
87
88 if (is_supported) {
89 // Check if mask unboxing is supported, this is a two step process which first loads the contents
90 // of boolean array into vector followed by either lane expansion to match the lane size of masked
91 // vector operation or populate the predicate register.
92 if ((mask_use_type & VecMaskUseLoad) != 0) {
93 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
94 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
95 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
96 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
97 return false;
98 }
99 }
100
101 if ((mask_use_type & VecMaskUsePred) != 0) {
102 if (!Matcher::has_predicated_vectors() ||
103 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
104 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
105 NodeClassNames[opc], type2name(elem_bt), num_elem);
106 return false;
107 }
108 }
109 }
110
111 int lshiftopc, rshiftopc;
112 switch(elem_bt) {
113 case T_BYTE:
114 lshiftopc = Op_LShiftI;
115 rshiftopc = Op_URShiftB;
116 break;
117 case T_SHORT:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftS;
120 break;
121 case T_INT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftI;
124 break;
125 case T_LONG:
126 lshiftopc = Op_LShiftL;
127 rshiftopc = Op_URShiftL;
128 break;
129 default: fatal("Unexpected type: %s", type2name(elem_bt));
130 }
131 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
132 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
133 if (!is_supported &&
134 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
136 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
137 is_supported = true;
138 }
139 return is_supported;
140 }
141
142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
143 assert(EnableVectorSupport, "");
144
145 PreserveReexecuteState preexecs(this);
146 jvms()->set_should_reexecute(true);
147
148 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
149 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
150 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
151 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
152 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
153 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
154
155 assert(check_vbox(vbox_type), "");
156 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
157 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
158 return gvn().transform(vbox);
159 }
160
161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
162 assert(EnableVectorSupport, "");
163 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
164 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
165 return nullptr; // arguments don't agree on vector shapes
166 }
167 if (vbox_type_v->maybe_null()) {
168 return nullptr; // no nulls are allowed
169 }
170 assert(check_vbox(vbox_type), "");
171 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
172 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
173 return unbox;
174 }
175
176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
177 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
178 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
179 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
180 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
181 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
182 }
183
184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
185 // Check that the operation is valid.
186 if (sopc <= 0) {
187 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
188 return false;
189 }
190
191 if (VectorNode::is_vector_rotate(sopc)) {
192 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
193 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
194 NodeClassNames[sopc], type2name(type), num_elem);
195 return false;
196 }
197 } else if (VectorNode::is_vector_integral_negate(sopc)) {
198 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
199 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
200 NodeClassNames[sopc], type2name(type), num_elem);
201 return false;
202 }
203 } else {
204 // Check that architecture supports this op-size-type combination.
205 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
206 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
207 NodeClassNames[sopc], type2name(type), num_elem);
208 return false;
209 } else {
210 assert(Matcher::match_rule_supported(sopc), "must be supported");
211 }
212 }
213
214 if (num_elem == 1) {
215 if (mask_use_type != VecMaskNotUsed) {
216 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
217 NodeClassNames[sopc], type2name(type), num_elem);
218 return false;
219 }
220
221 if (sopc != 0) {
222 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
223 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
224 NodeClassNames[sopc], type2name(type), num_elem);
225 return false;
226 }
227 }
228 }
229
230 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
231 Matcher::supports_vector_variable_shifts() == false) {
232 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
233 NodeClassNames[sopc], type2name(type), num_elem);
234 return false;
235 }
236
237 // Check if mask unboxing is supported, this is a two step process which first loads the contents
238 // of boolean array into vector followed by either lane expansion to match the lane size of masked
239 // vector operation or populate the predicate register.
240 if ((mask_use_type & VecMaskUseLoad) != 0) {
241 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
242 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
243 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
244 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
245 return false;
246 }
247 }
248
249 // Check if mask boxing is supported, this is a two step process which first stores the contents
250 // of mask vector / predicate register into a boolean vector followed by vector store operation to
251 // transfer the contents to underlined storage of mask boxes which is a boolean array.
252 if ((mask_use_type & VecMaskUseStore) != 0) {
253 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
254 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
255 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
256 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
257 return false;
258 }
259 }
260
261 if ((mask_use_type & VecMaskUsePred) != 0) {
262 bool is_supported = false;
263 if (Matcher::has_predicated_vectors()) {
264 if (VectorNode::is_vector_integral_negate(sopc)) {
265 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
266 } else {
267 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
268 }
269 }
270 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
271
272 if (!is_supported) {
273 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
274 NodeClassNames[sopc], type2name(type), num_elem);
275 return false;
276 }
277 }
278
279 return true;
280 }
281
282 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
283 if (vec_klass->const_oop() == nullptr) {
284 return false; // uninitialized or some kind of unsafe access
285 }
286 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
287 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
288 return klass->is_initialized();
289 }
290
291 // public static
292 // <V extends Vector<E>,
293 // M extends VectorMask<E>,
294 // E>
295 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
296 // int length, V v, M m,
297 // UnaryOperation<V, M> defaultImpl)
298 //
299 // public static
300 // <V,
301 // M extends VectorMask<E>,
302 // E>
303 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
304 // int length, V v1, V v2, M m,
305 // BinaryOperation<V, M> defaultImpl)
306 //
307 // public static
308 // <V extends Vector<E>,
309 // M extends VectorMask<E>,
310 // E>
311 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
312 // int length, V v1, V v2, V v3, M m,
313 // TernaryOperation<V, M> defaultImpl)
314 //
315 bool LibraryCallKit::inline_vector_nary_operation(int n) {
316 const TypeInt* opr = gvn().type(argument(0))->isa_int();
317 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
318 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
319 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
320 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
321
322 if (opr == nullptr || !opr->is_con() ||
323 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
324 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
325 vlen == nullptr || !vlen->is_con()) {
326 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
327 NodeClassNames[argument(0)->Opcode()],
328 NodeClassNames[argument(1)->Opcode()],
329 NodeClassNames[argument(3)->Opcode()],
330 NodeClassNames[argument(4)->Opcode()]);
331 return false; // not enough info for intrinsification
332 }
333
334 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
335 if (!elem_type->is_primitive_type()) {
336 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
337 return false; // should be primitive type
338 }
339 if (!is_klass_initialized(vector_klass)) {
340 log_if_needed(" ** klass argument not initialized");
341 return false;
342 }
343
344 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
345 // is used to control this operation.
346 const Type* vmask_type = gvn().type(argument(n + 5));
347 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
348 if (is_masked_op) {
349 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
350 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
351 return false; // not enough info for intrinsification
352 }
353
354 if (!is_klass_initialized(mask_klass)) {
355 log_if_needed(" ** mask klass argument not initialized");
356 return false;
357 }
358
359 if (vmask_type->maybe_null()) {
360 log_if_needed(" ** null mask values are not allowed for masked op");
361 return false;
362 }
363 }
364
365 BasicType elem_bt = elem_type->basic_type();
366 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
367 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
368
369 int num_elem = vlen->get_con();
370 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
371 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
372 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
373 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
374 return false; // operation not supported
375 }
376 if (num_elem == 1) {
377 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
378 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
379 n, opc, num_elem, type2name(elem_bt));
380 return false;
381 }
382 }
383 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
384 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
385
386 if (is_vector_mask(vbox_klass)) {
387 assert(!is_masked_op, "mask operations do not need mask to control");
388 }
389
390 if (opc == Op_CallLeafVector) {
391 if (!UseVectorStubs) {
392 log_if_needed(" ** vector stubs support is disabled");
393 return false;
394 }
395 if (!Matcher::supports_vector_calling_convention()) {
396 log_if_needed(" ** no vector calling conventions supported");
397 return false;
398 }
399 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
400 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
401 num_elem, type2name(elem_bt));
402 return false;
403 }
404 }
405
406 // When using mask, mask use type needs to be VecMaskUseLoad.
407 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
408 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
409 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
410 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
411 n, sopc, num_elem, type2name(elem_bt),
412 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
413 return false; // not supported
414 }
415
416 // Return true if current platform has implemented the masked operation with predicate feature.
417 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
418 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
419 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
420 n, sopc, num_elem, type2name(elem_bt));
421 return false;
422 }
423
424 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
425 switch (n) {
426 case 3: {
427 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
428 if (opd3 == nullptr) {
429 log_if_needed(" ** unbox failed v3=%s",
430 NodeClassNames[argument(7)->Opcode()]);
431 return false;
432 }
433 // fall-through
434 }
435 case 2: {
436 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
437 if (opd2 == nullptr) {
438 log_if_needed(" ** unbox failed v2=%s",
439 NodeClassNames[argument(6)->Opcode()]);
440 return false;
441 }
442 // fall-through
443 }
444 case 1: {
445 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
446 if (opd1 == nullptr) {
447 log_if_needed(" ** unbox failed v1=%s",
448 NodeClassNames[argument(5)->Opcode()]);
449 return false;
450 }
451 break;
452 }
453 default: fatal("unsupported arity: %d", n);
454 }
455
456 Node* mask = nullptr;
457 if (is_masked_op) {
458 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
459 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
460 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
461 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
462 if (mask == nullptr) {
463 log_if_needed(" ** unbox failed mask=%s",
464 NodeClassNames[argument(n + 5)->Opcode()]);
465 return false;
466 }
467 }
468
469 Node* operation = nullptr;
470 if (opc == Op_CallLeafVector) {
471 assert(UseVectorStubs, "sanity");
472 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
473 if (operation == nullptr) {
474 log_if_needed(" ** Vector math call failed for %s_%s_%d",
475 (elem_bt == T_FLOAT) ? "float" : "double",
476 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
477 num_elem * type2aelembytes(elem_bt));
478 return false;
479 }
480 } else {
481 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
482 switch (n) {
483 case 1:
484 case 2: {
485 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
486 break;
487 }
488 case 3: {
489 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
490 break;
491 }
492 default: fatal("unsupported arity: %d", n);
493 }
494 }
495
496 if (is_masked_op && mask != nullptr) {
497 if (use_predicate) {
498 operation->add_req(mask);
499 operation->add_flag(Node::Flag_is_predicated_vector);
500 } else {
501 operation->add_flag(Node::Flag_is_predicated_using_blend);
502 operation = gvn().transform(operation);
503 operation = new VectorBlendNode(opd1, operation, mask);
504 }
505 }
506 operation = gvn().transform(operation);
507
508 // Wrap it up in VectorBox to keep object type information.
509 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
510 set_result(vbox);
511 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
512 return true;
513 }
514
515 // public static
516 // <E, M>
517 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
518 // int length, M m, VectorMaskOp<M> defaultImpl)
519 bool LibraryCallKit::inline_vector_mask_operation() {
520 const TypeInt* oper = gvn().type(argument(0))->isa_int();
521 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
522 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
523 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
524 Node* mask = argument(4);
525
526 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
527 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
528 vlen == nullptr || !vlen->is_con() ||
529 oper == nullptr || !oper->is_con() ||
530 mask->is_top()) {
531 return false; // dead code
532 }
533
534 if (!is_klass_initialized(mask_klass)) {
535 log_if_needed(" ** klass argument not initialized");
536 return false;
537 }
538
539 int num_elem = vlen->get_con();
540 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
541 BasicType elem_bt = elem_type->basic_type();
542
543 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
544 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
545 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
546 mopc, num_elem, type2name(elem_bt));
547 return false; // not supported
548 }
549
550 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
551 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
552 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
553 if (mask_vec == nullptr) {
554 log_if_needed(" ** unbox failed mask=%s",
555 NodeClassNames[argument(4)->Opcode()]);
556 return false;
557 }
558
559 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
560 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
561 }
562 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
563 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
564 if (mopc != Op_VectorMaskToLong) {
565 maskoper = ConvI2L(maskoper);
566 }
567 set_result(maskoper);
568
569 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
570 return true;
571 }
572
573 // public static
574 // <M,
575 // S extends VectorSpecies<E>,
576 // E>
577 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
578 // long bits, int mode, S s,
579 // BroadcastOperation<M, E, S> defaultImpl)
580 bool LibraryCallKit::inline_vector_frombits_coerced() {
581 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
582 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
583 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
584 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
585 // Mode argument determines the mode of operation it can take following values:-
586 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
587 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
588 const TypeInt* mode = gvn().type(argument(5))->isa_int();
589
590 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
591 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
592 vlen == nullptr || !vlen->is_con() ||
593 bits_type == nullptr ||
594 mode == nullptr || !mode->is_con()) {
595 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
596 NodeClassNames[argument(0)->Opcode()],
597 NodeClassNames[argument(1)->Opcode()],
598 NodeClassNames[argument(2)->Opcode()],
599 NodeClassNames[argument(5)->Opcode()]);
600 return false; // not enough info for intrinsification
601 }
602
603 if (!is_klass_initialized(vector_klass)) {
604 log_if_needed(" ** klass argument not initialized");
605 return false;
606 }
607 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
608 if (!elem_type->is_primitive_type()) {
609 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
610 return false; // should be primitive type
611 }
612 BasicType elem_bt = elem_type->basic_type();
613 int num_elem = vlen->get_con();
614 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
615 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
616
617 bool is_mask = is_vector_mask(vbox_klass);
618 int bcast_mode = mode->get_con();
619 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
620 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
621
622 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
623 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
624 num_elem, type2name(elem_bt),
625 is_mask ? 1 : 0,
626 bcast_mode);
627 return false; // not supported
628 }
629
630 Node* broadcast = nullptr;
631 Node* bits = argument(3);
632 Node* elem = bits;
633
634 if (opc == Op_VectorLongToMask) {
635 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
636 if (vt->isa_vectmask()) {
637 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
638 } else {
639 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
640 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
641 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
642 }
643 } else {
644 switch (elem_bt) {
645 case T_BOOLEAN: // fall-through
646 case T_BYTE: // fall-through
647 case T_SHORT: // fall-through
648 case T_CHAR: // fall-through
649 case T_INT: {
650 elem = gvn().transform(new ConvL2INode(bits));
651 break;
652 }
653 case T_DOUBLE: {
654 elem = gvn().transform(new MoveL2DNode(bits));
655 break;
656 }
657 case T_FLOAT: {
658 bits = gvn().transform(new ConvL2INode(bits));
659 elem = gvn().transform(new MoveI2FNode(bits));
660 break;
661 }
662 case T_LONG: {
663 // no conversion needed
664 break;
665 }
666 default: fatal("%s", type2name(elem_bt));
667 }
668 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
669 broadcast = gvn().transform(broadcast);
670 }
671
672 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
673 set_result(box);
674 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
675 return true;
676 }
677
678 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
679 assert(arr_type != nullptr, "unexpected");
680 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
681 if (elem_bt == arr_elem_bt) {
682 return true;
683 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
684 // Load/store of short vector from/to char[] is supported
685 return true;
686 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
687 // Load/store of byte vector from/to boolean[] is supported
688 return true;
689 } else {
690 return mismatched_ms;
691 }
692 }
693
694 // public static
695 // <C,
696 // VM extends VectorPayload,
697 // E,
698 // S extends VectorSpecies<E>>
699 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
700 // int length,
701 // Object base, long offset, // Unsafe addressing
702 // boolean fromSegment,
703 // C container, long index, S s, // Arguments for default implementation
704 // LoadOperation<C, VM, S> defaultImpl) {
705 // public static
706 // <C,
707 // V extends VectorPayload>
708 // void store(Class<?> vClass, Class<?> eClass,
709 // int length,
710 // Object base, long offset, // Unsafe addressing
711 // boolean fromSegment,
712 // V v, C container, long index, // Arguments for default implementation
713 // StoreVectorOperation<C, V> defaultImpl) {
714 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
715 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
716 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
717 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
718 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
719
720 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
721 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
722 vlen == nullptr || !vlen->is_con() ||
723 from_ms == nullptr || !from_ms->is_con()) {
724 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
725 NodeClassNames[argument(0)->Opcode()],
726 NodeClassNames[argument(1)->Opcode()],
727 NodeClassNames[argument(2)->Opcode()],
728 NodeClassNames[argument(6)->Opcode()]);
729 return false; // not enough info for intrinsification
730 }
731 if (!is_klass_initialized(vector_klass)) {
732 log_if_needed(" ** klass argument not initialized");
733 return false;
734 }
735
736 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
737 if (!elem_type->is_primitive_type()) {
738 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
739 return false; // should be primitive type
740 }
741 BasicType elem_bt = elem_type->basic_type();
742 int num_elem = vlen->get_con();
743
744 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
745 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
746 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
747 is_store, is_store ? "store" : "load",
748 num_elem, type2name(elem_bt));
749 return false; // not supported
750 }
751
752 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
753 bool is_mask = is_vector_mask(vbox_klass);
754
755 Node* base = argument(3);
756 Node* offset = ConvL2X(argument(4));
757
758 // Save state and restore on bailout
759 uint old_sp = sp();
760 SafePointNode* old_map = clone_map();
761
762 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
763
764 // The memory barrier checks are based on ones for unsafe access.
765 // This is not 1-1 implementation.
766 const Type *const base_type = gvn().type(base);
767
768 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
769 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
770
771 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
772 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
773
774 const bool is_mixed_access = !in_heap && !in_native;
775
776 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
777
778 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
779
780 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
781 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
782 // loading from or before storing to backing storage which is mandatory for semantic correctness of
783 // big-endian memory layout.
784 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
785 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
786 arr_type->elem()->array_element_basic_type() != elem_bt);
787 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
788 if (!is_java_primitive(mem_elem_bt)) {
789 log_if_needed(" ** non-primitive array element type");
790 return false;
791 }
792 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
793 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
794 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
795 is_store, is_store ? "store" : "load",
796 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
797 set_map(old_map);
798 set_sp(old_sp);
799 return false;
800 }
801
802 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
803 if (mismatched_ms) {
804 if (is_store) {
805 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
806 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
807 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
808 is_store, "store",
809 num_elem, type2name(elem_bt));
810 set_map(old_map);
811 set_sp(old_sp);
812 return false; // not supported
813 }
814 } else {
815 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
816 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
817 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
818 is_store, "load",
819 mem_num_elem, type2name(mem_elem_bt));
820 set_map(old_map);
821 set_sp(old_sp);
822 return false; // not supported
823 }
824 }
825 }
826 if (is_mask) {
827 if (!is_store) {
828 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
829 set_map(old_map);
830 set_sp(old_sp);
831 return false; // not supported
832 }
833 } else {
834 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
835 set_map(old_map);
836 set_sp(old_sp);
837 return false; // not supported
838 }
839 }
840 }
841
842 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
843
844 if (needs_cpu_membar) {
845 insert_mem_bar(Op_MemBarCPUOrder);
846 }
847
848 if (is_store) {
849 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
850 if (val == nullptr) {
851 set_map(old_map);
852 set_sp(old_sp);
853 return false; // operand unboxing failed
854 }
855 set_all_memory(reset_memory());
856
857 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
858 int store_num_elem = num_elem;
859 if (mismatched_ms) {
860 store_num_elem = mem_num_elem;
861 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
862 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
863 }
864 if (is_mask) {
865 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
866 }
867 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
868 set_memory(vstore, addr_type);
869 } else {
870 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
871 Node* vload = nullptr;
872 if (mismatched_ms) {
873 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
874 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
875 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
876 } else {
877 // Special handle for masks
878 if (is_mask) {
879 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
880 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
881 } else {
882 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
883 }
884 }
885 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
886 set_result(box);
887 }
888
889 destruct_map_clone(old_map);
890
891 if (needs_cpu_membar) {
892 insert_mem_bar(Op_MemBarCPUOrder);
893 }
894
895 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
896 return true;
897 }
898
899 // public static
900 // <C,
901 // V extends Vector<?>,
902 // E,
903 // S extends VectorSpecies<E>,
904 // M extends VectorMask<E>>
905 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
906 // int length, Object base, long offset, // Unsafe addressing
907 // boolean fromSegment,
908 // M m, int offsetInRange,
909 // C container, long index, S s, // Arguments for default implementation
910 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
911 // public static
912 // <C,
913 // V extends Vector<E>,
914 // M extends VectorMask<E>,
915 // E>
916 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
917 // int length,
918 // Object base, long offset, // Unsafe addressing
919 // boolean fromSegment,
920 // V v, M m, C container, long index, // Arguments for default implementation
921 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
922
923 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
924 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
925 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
926 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
927 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
928 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
929
930 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
931 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
932 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
933 vlen == nullptr || !vlen->is_con() ||
934 from_ms == nullptr || !from_ms->is_con()) {
935 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
936 NodeClassNames[argument(0)->Opcode()],
937 NodeClassNames[argument(1)->Opcode()],
938 NodeClassNames[argument(2)->Opcode()],
939 NodeClassNames[argument(3)->Opcode()],
940 NodeClassNames[argument(7)->Opcode()]);
941 return false; // not enough info for intrinsification
942 }
943 if (!is_klass_initialized(vector_klass)) {
944 log_if_needed(" ** klass argument not initialized");
945 return false;
946 }
947
948 if (!is_klass_initialized(mask_klass)) {
949 log_if_needed(" ** mask klass argument not initialized");
950 return false;
951 }
952
953 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
954 if (!elem_type->is_primitive_type()) {
955 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
956 return false; // should be primitive type
957 }
958
959 BasicType elem_bt = elem_type->basic_type();
960 int num_elem = vlen->get_con();
961
962 Node* base = argument(4);
963 Node* offset = ConvL2X(argument(5));
964
965 // Save state and restore on bailout
966 uint old_sp = sp();
967 SafePointNode* old_map = clone_map();
968
969 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
970 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
971 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
972
973 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
974 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
975 // If there is no consistency between array and vector element types, it must be special byte array case
976 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
977 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
978 is_store, is_store ? "storeMasked" : "loadMasked",
979 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
980 set_map(old_map);
981 set_sp(old_sp);
982 return false;
983 }
984
985 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
986 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
987 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
988 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
989
990 // If current arch does not support the predicated operations, we have to bail
991 // out when current case uses the predicate feature.
992 if (!supports_predicate) {
993 bool needs_predicate = false;
994 if (is_store) {
995 // Masked vector store always uses the predicated store.
996 needs_predicate = true;
997 } else {
998 // Masked vector load with IOOBE always uses the predicated load.
999 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1000 if (!offset_in_range->is_con()) {
1001 log_if_needed(" ** missing constant: offsetInRange=%s",
1002 NodeClassNames[argument(8)->Opcode()]);
1003 set_map(old_map);
1004 set_sp(old_sp);
1005 return false;
1006 }
1007 needs_predicate = (offset_in_range->get_con() == 0);
1008 }
1009
1010 if (needs_predicate) {
1011 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1012 is_store ? "storeMasked" : "loadMasked",
1013 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1014 set_map(old_map);
1015 set_sp(old_sp);
1016 return false;
1017 }
1018 }
1019
1020 // This only happens for masked vector load. If predicate is not supported, then check whether
1021 // the normal vector load and blend operations are supported by backend.
1022 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1023 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1024 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1025 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1026 set_map(old_map);
1027 set_sp(old_sp);
1028 return false;
1029 }
1030
1031 // Since we are using byte array, we need to double check that the vector reinterpret operation
1032 // with byte type is supported by backend.
1033 if (mismatched_ms) {
1034 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1035 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1036 is_store, is_store ? "storeMasked" : "loadMasked",
1037 num_elem, type2name(elem_bt));
1038 set_map(old_map);
1039 set_sp(old_sp);
1040 return false;
1041 }
1042 }
1043
1044 // Since it needs to unbox the mask, we need to double check that the related load operations
1045 // for mask are supported by backend.
1046 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1047 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1048 is_store, is_store ? "storeMasked" : "loadMasked",
1049 num_elem, type2name(elem_bt));
1050 set_map(old_map);
1051 set_sp(old_sp);
1052 return false;
1053 }
1054
1055 // Can base be null? Otherwise, always on-heap access.
1056 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1057 if (can_access_non_heap) {
1058 insert_mem_bar(Op_MemBarCPUOrder);
1059 }
1060
1061 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1062 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1063 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1064 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1065 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1066
1067 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1068 if (mask == nullptr) {
1069 log_if_needed(" ** unbox failed mask=%s",
1070 is_store ? NodeClassNames[argument(9)->Opcode()]
1071 : NodeClassNames[argument(8)->Opcode()]);
1072 set_map(old_map);
1073 set_sp(old_sp);
1074 return false;
1075 }
1076
1077 if (is_store) {
1078 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1079 if (val == nullptr) {
1080 log_if_needed(" ** unbox failed vector=%s",
1081 NodeClassNames[argument(8)->Opcode()]);
1082 set_map(old_map);
1083 set_sp(old_sp);
1084 return false; // operand unboxing failed
1085 }
1086 set_all_memory(reset_memory());
1087
1088 if (mismatched_ms) {
1089 // Reinterpret the incoming vector to byte vector.
1090 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1091 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1092 // Reinterpret the vector mask to byte type.
1093 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1094 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1095 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1096 }
1097 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1098 set_memory(vstore, addr_type);
1099 } else {
1100 Node* vload = nullptr;
1101
1102 if (mismatched_ms) {
1103 // Reinterpret the vector mask to byte type.
1104 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1105 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1106 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1107 }
1108
1109 if (supports_predicate) {
1110 // Generate masked load vector node if predicate feature is supported.
1111 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1112 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1113 } else {
1114 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1115 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1116 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1117 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1118 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1119 }
1120
1121 if (mismatched_ms) {
1122 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1123 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1124 }
1125
1126 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1127 set_result(box);
1128 }
1129
1130 destruct_map_clone(old_map);
1131
1132 if (can_access_non_heap) {
1133 insert_mem_bar(Op_MemBarCPUOrder);
1134 }
1135
1136 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1137 return true;
1138 }
1139
1140 // <C,
1141 // V extends Vector<?>,
1142 // W extends Vector<Integer>,
1143 // S extends VectorSpecies<E>,
1144 // M extends VectorMask<E>,
1145 // E>
1146 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1147 // Class<? extends Vector<Integer>> vectorIndexClass,
1148 // Object base, long offset, // Unsafe addressing
1149 // W index_vector, M m,
1150 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1151 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1152 //
1153 // <C,
1154 // V extends Vector<E>,
1155 // W extends Vector<Integer>,
1156 // M extends VectorMask<E>,
1157 // E>
1158 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1159 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1160 // W index_vector, V v, M m,
1161 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1162 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1163 //
1164 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1165 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1166 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1167 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1168 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1169 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1170
1171 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1172 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1173 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1174 vlen == nullptr || !vlen->is_con() ||
1175 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1176 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1177 NodeClassNames[argument(0)->Opcode()],
1178 NodeClassNames[argument(2)->Opcode()],
1179 NodeClassNames[argument(3)->Opcode()],
1180 NodeClassNames[argument(4)->Opcode()]);
1181 return false; // not enough info for intrinsification
1182 }
1183
1184 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1185 log_if_needed(" ** klass argument not initialized");
1186 return false;
1187 }
1188
1189 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1190 if (!elem_type->is_primitive_type()) {
1191 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1192 return false; // should be primitive type
1193 }
1194
1195 BasicType elem_bt = elem_type->basic_type();
1196 int num_elem = vlen->get_con();
1197
1198 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1199 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1200 if (is_masked_op) {
1201 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1202 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1203 return false; // not enough info for intrinsification
1204 }
1205
1206 if (!is_klass_initialized(mask_klass)) {
1207 log_if_needed(" ** mask klass argument not initialized");
1208 return false;
1209 }
1210
1211 if (vmask_type->maybe_null()) {
1212 log_if_needed(" ** null mask values are not allowed for masked op");
1213 return false;
1214 }
1215
1216 // Check whether the predicated gather/scatter node is supported by architecture.
1217 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1218 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1219 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1220 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1221 num_elem, type2name(elem_bt));
1222 return false; // not supported
1223 }
1224 } else {
1225 // Check whether the normal gather/scatter node is supported for non-masked operation.
1226 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1227 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1228 is_scatter, is_scatter ? "scatter" : "gather",
1229 num_elem, type2name(elem_bt));
1230 return false; // not supported
1231 }
1232 }
1233
1234 // Check that the vector holding indices is supported by architecture
1235 // For sub-word gathers expander receive index array.
1236 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1237 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1238 is_scatter, is_scatter ? "scatter" : "gather",
1239 num_elem, is_masked_op ? 1 : 0);
1240 return false; // not supported
1241 }
1242
1243 Node* base = argument(5);
1244 Node* offset = ConvL2X(argument(6));
1245
1246 // Save state and restore on bailout
1247 uint old_sp = sp();
1248 SafePointNode* old_map = clone_map();
1249
1250 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1251
1252 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1253 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1254
1255 // The array must be consistent with vector type
1256 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1257 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1258 is_scatter, is_scatter ? "scatter" : "gather",
1259 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1260 set_map(old_map);
1261 set_sp(old_sp);
1262 return false;
1263 }
1264
1265 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1266 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1267 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1268 if (vbox_idx_klass == nullptr) {
1269 set_map(old_map);
1270 set_sp(old_sp);
1271 return false;
1272 }
1273
1274 Node* index_vect = nullptr;
1275 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1276 if (!is_subword_type(elem_bt)) {
1277 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1278 if (index_vect == nullptr) {
1279 set_map(old_map);
1280 set_sp(old_sp);
1281 return false;
1282 }
1283 }
1284
1285 Node* mask = nullptr;
1286 if (is_masked_op) {
1287 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1288 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1289 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1290 if (mask == nullptr) {
1291 log_if_needed(" ** unbox failed mask=%s",
1292 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1293 : NodeClassNames[argument(9)->Opcode()]);
1294 set_map(old_map);
1295 set_sp(old_sp);
1296 return false;
1297 }
1298 }
1299
1300 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1301 if (is_scatter) {
1302 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1303 if (val == nullptr) {
1304 set_map(old_map);
1305 set_sp(old_sp);
1306 return false; // operand unboxing failed
1307 }
1308 set_all_memory(reset_memory());
1309
1310 Node* vstore = nullptr;
1311 if (mask != nullptr) {
1312 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1313 } else {
1314 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1315 }
1316 set_memory(vstore, addr_type);
1317 } else {
1318 Node* vload = nullptr;
1319 Node* index = argument(11);
1320 Node* indexMap = argument(12);
1321 Node* indexM = argument(13);
1322 if (mask != nullptr) {
1323 if (is_subword_type(elem_bt)) {
1324 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1325 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1326 } else {
1327 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1328 }
1329 } else {
1330 if (is_subword_type(elem_bt)) {
1331 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1332 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1333 } else {
1334 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1335 }
1336 }
1337 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1338 set_result(box);
1339 }
1340
1341 destruct_map_clone(old_map);
1342
1343 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1344 return true;
1345 }
1346
1347 // public static
1348 // <V extends Vector<E>,
1349 // M extends VectorMask<E>,
1350 // E>
1351 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1352 // Class<E> elementType, int length, V v, M m,
1353 // ReductionOperation<V, M> defaultImpl)
1354 bool LibraryCallKit::inline_vector_reduction() {
1355 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1356 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1357 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1358 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1359 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1360
1361 if (opr == nullptr || !opr->is_con() ||
1362 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1363 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1364 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1365 vlen == nullptr || !vlen->is_con()) {
1366 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1367 NodeClassNames[argument(0)->Opcode()],
1368 NodeClassNames[argument(1)->Opcode()],
1369 NodeClassNames[argument(3)->Opcode()],
1370 NodeClassNames[argument(4)->Opcode()]);
1371 return false; // not enough info for intrinsification
1372 }
1373 if (!is_klass_initialized(vector_klass)) {
1374 log_if_needed(" ** klass argument not initialized");
1375 return false;
1376 }
1377 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1378 if (!elem_type->is_primitive_type()) {
1379 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1380 return false; // should be primitive type
1381 }
1382
1383 const Type* vmask_type = gvn().type(argument(6));
1384 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1385 if (is_masked_op) {
1386 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1387 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1388 return false; // not enough info for intrinsification
1389 }
1390
1391 if (!is_klass_initialized(mask_klass)) {
1392 log_if_needed(" ** mask klass argument not initialized");
1393 return false;
1394 }
1395
1396 if (vmask_type->maybe_null()) {
1397 log_if_needed(" ** null mask values are not allowed for masked op");
1398 return false;
1399 }
1400 }
1401
1402 BasicType elem_bt = elem_type->basic_type();
1403 int num_elem = vlen->get_con();
1404 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1405 int sopc = ReductionNode::opcode(opc, elem_bt);
1406
1407 // Ensure reduction operation for lanewise operation
1408 // When using mask, mask use type needs to be VecMaskUseLoad.
1409 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1410 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1411 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1412 return false;
1413 }
1414
1415 // Return true if current platform has implemented the masked operation with predicate feature.
1416 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1417 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1418 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1419 sopc, num_elem, type2name(elem_bt));
1420 return false;
1421 }
1422
1423 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1424 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1425
1426 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1427 if (opd == nullptr) {
1428 return false; // operand unboxing failed
1429 }
1430
1431 Node* mask = nullptr;
1432 if (is_masked_op) {
1433 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1434 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1435 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1436 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1437 if (mask == nullptr) {
1438 log_if_needed(" ** unbox failed mask=%s",
1439 NodeClassNames[argument(6)->Opcode()]);
1440 return false;
1441 }
1442 }
1443
1444 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1445 Node* value = opd;
1446
1447 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1448 if (mask != nullptr && !use_predicate) {
1449 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1450 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1451 }
1452
1453 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1454 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1455 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1456
1457 if (mask != nullptr && use_predicate) {
1458 value->add_req(mask);
1459 value->add_flag(Node::Flag_is_predicated_vector);
1460 }
1461
1462 value = gvn().transform(value);
1463
1464 Node* bits = nullptr;
1465 switch (elem_bt) {
1466 case T_BYTE:
1467 case T_SHORT:
1468 case T_INT: {
1469 bits = gvn().transform(new ConvI2LNode(value));
1470 break;
1471 }
1472 case T_FLOAT: {
1473 value = gvn().transform(new MoveF2INode(value));
1474 bits = gvn().transform(new ConvI2LNode(value));
1475 break;
1476 }
1477 case T_DOUBLE: {
1478 bits = gvn().transform(new MoveD2LNode(value));
1479 break;
1480 }
1481 case T_LONG: {
1482 bits = value; // no conversion needed
1483 break;
1484 }
1485 default: fatal("%s", type2name(elem_bt));
1486 }
1487 set_result(bits);
1488 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1489 return true;
1490 }
1491
1492 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1493 // V v1, V v2,
1494 // BiFunction<V, V, Boolean> defaultImpl)
1495 //
1496 bool LibraryCallKit::inline_vector_test() {
1497 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1498 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1499 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1500 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1501
1502 if (cond == nullptr || !cond->is_con() ||
1503 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1504 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1505 vlen == nullptr || !vlen->is_con()) {
1506 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1507 NodeClassNames[argument(0)->Opcode()],
1508 NodeClassNames[argument(1)->Opcode()],
1509 NodeClassNames[argument(2)->Opcode()],
1510 NodeClassNames[argument(3)->Opcode()]);
1511 return false; // not enough info for intrinsification
1512 }
1513 if (!is_klass_initialized(vector_klass)) {
1514 log_if_needed(" ** klass argument not initialized");
1515 return false;
1516 }
1517 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1518 if (!elem_type->is_primitive_type()) {
1519 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1520 return false; // should be primitive type
1521 }
1522 BasicType elem_bt = elem_type->basic_type();
1523 int num_elem = vlen->get_con();
1524 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1525 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1526 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1527
1528 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1529 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1530 cond->get_con(), num_elem, type2name(elem_bt),
1531 is_vector_mask(vbox_klass));
1532 return false;
1533 }
1534
1535 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1536 Node* opd2;
1537 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1538 opd1->bottom_type()->isa_vectmask())) {
1539 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1540 } else {
1541 opd2 = opd1;
1542 }
1543 if (opd1 == nullptr || opd2 == nullptr) {
1544 return false; // operand unboxing failed
1545 }
1546
1547 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1548 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1549 opd1->bottom_type()->isa_vectmask(), num_elem);
1550 Node* bol = gvn().transform(new BoolNode(cmp, test));
1551 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1552
1553 set_result(res);
1554 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1555 return true;
1556 }
1557
1558 // public static
1559 // <V extends Vector<E>,
1560 // M extends VectorMask<E>,
1561 // E>
1562 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1563 // V v1, V v2, M m,
1564 // VectorBlendOp<V, M, E> defaultImpl)
1565 bool LibraryCallKit::inline_vector_blend() {
1566 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1567 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1568 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1569 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1570
1571 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1572 return false; // dead code
1573 }
1574 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1575 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1576 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1577 NodeClassNames[argument(0)->Opcode()],
1578 NodeClassNames[argument(1)->Opcode()],
1579 NodeClassNames[argument(2)->Opcode()],
1580 NodeClassNames[argument(3)->Opcode()]);
1581 return false; // not enough info for intrinsification
1582 }
1583 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1584 log_if_needed(" ** klass argument not initialized");
1585 return false;
1586 }
1587 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1588 if (!elem_type->is_primitive_type()) {
1589 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1590 return false; // should be primitive type
1591 }
1592 BasicType elem_bt = elem_type->basic_type();
1593 BasicType mask_bt = elem_bt;
1594 int num_elem = vlen->get_con();
1595
1596 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1597 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1598 num_elem, type2name(elem_bt));
1599 return false; // not supported
1600 }
1601 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1602 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1603
1604 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1605 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1606
1607 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1608 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1609 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1610
1611 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1612 return false; // operand unboxing failed
1613 }
1614
1615 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1616
1617 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1618 set_result(box);
1619 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1620 return true;
1621 }
1622
1623 // public static
1624 // <V extends Vector<E>,
1625 // M extends VectorMask<E>,
1626 // E>
1627 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1628 // V v1, V v2, M m,
1629 // VectorCompareOp<V,M> defaultImpl)
1630 bool LibraryCallKit::inline_vector_compare() {
1631 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1632 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1633 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1634 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1635 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1636
1637 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1638 return false; // dead code
1639 }
1640 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1641 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1642 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1643 NodeClassNames[argument(0)->Opcode()],
1644 NodeClassNames[argument(1)->Opcode()],
1645 NodeClassNames[argument(2)->Opcode()],
1646 NodeClassNames[argument(3)->Opcode()],
1647 NodeClassNames[argument(4)->Opcode()]);
1648 return false; // not enough info for intrinsification
1649 }
1650 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1651 log_if_needed(" ** klass argument not initialized");
1652 return false;
1653 }
1654 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1655 if (!elem_type->is_primitive_type()) {
1656 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1657 return false; // should be primitive type
1658 }
1659
1660 int num_elem = vlen->get_con();
1661 BasicType elem_bt = elem_type->basic_type();
1662 BasicType mask_bt = elem_bt;
1663
1664 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1665 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1666 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1667 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1668 return false;
1669 }
1670 }
1671
1672 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1673 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1674 cond->get_con(), num_elem, type2name(elem_bt));
1675 return false;
1676 }
1677
1678 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1679 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1680
1681 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1682 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1683
1684 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1685 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1686
1687 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1688 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1689 if (is_masked_op && mask == nullptr) {
1690 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1691 cond->get_con(), num_elem, type2name(elem_bt));
1692 return false;
1693 }
1694
1695 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1696 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1697 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1698 cond->get_con(), num_elem, type2name(elem_bt));
1699 return false;
1700 }
1701
1702 if (v1 == nullptr || v2 == nullptr) {
1703 return false; // operand unboxing failed
1704 }
1705 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1706 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1707
1708 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1709 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1710
1711 if (is_masked_op) {
1712 if (use_predicate) {
1713 operation->add_req(mask);
1714 operation->add_flag(Node::Flag_is_predicated_vector);
1715 } else {
1716 operation = gvn().transform(operation);
1717 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1718 }
1719 }
1720
1721 operation = gvn().transform(operation);
1722
1723 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1724 set_result(box);
1725 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1726 return true;
1727 }
1728
1729 // public static
1730 // <V extends Vector<E>,
1731 // Sh extends VectorShuffle<E>,
1732 // M extends VectorMask<E>,
1733 // E>
1734 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1735 // V v1, Sh sh, M m,
1736 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1737 bool LibraryCallKit::inline_vector_rearrange() {
1738 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1739 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1740 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1741 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1742 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1743
1744 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1745 return false; // dead code
1746 }
1747 if (shuffle_klass->const_oop() == nullptr ||
1748 vector_klass->const_oop() == nullptr ||
1749 elem_klass->const_oop() == nullptr ||
1750 !vlen->is_con()) {
1751 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1752 NodeClassNames[argument(0)->Opcode()],
1753 NodeClassNames[argument(1)->Opcode()],
1754 NodeClassNames[argument(3)->Opcode()],
1755 NodeClassNames[argument(4)->Opcode()]);
1756 return false; // not enough info for intrinsification
1757 }
1758 if (!is_klass_initialized(vector_klass) ||
1759 !is_klass_initialized(shuffle_klass)) {
1760 log_if_needed(" ** klass argument not initialized");
1761 return false;
1762 }
1763 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1764 if (!elem_type->is_primitive_type()) {
1765 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1766 return false; // should be primitive type
1767 }
1768
1769 BasicType elem_bt = elem_type->basic_type();
1770 BasicType shuffle_bt = elem_bt;
1771 if (shuffle_bt == T_FLOAT) {
1772 shuffle_bt = T_INT;
1773 } else if (shuffle_bt == T_DOUBLE) {
1774 shuffle_bt = T_LONG;
1775 }
1776
1777 int num_elem = vlen->get_con();
1778 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1779
1780 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1781 if (C->print_intrinsics()) {
1782 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1783 num_elem, type2name(shuffle_bt));
1784 }
1785 return false; // not supported
1786 }
1787
1788 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1789 bool use_predicate = is_masked_op;
1790 if (is_masked_op &&
1791 (mask_klass == nullptr ||
1792 mask_klass->const_oop() == nullptr ||
1793 !is_klass_initialized(mask_klass))) {
1794 log_if_needed(" ** mask_klass argument not initialized");
1795 }
1796 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1797 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1798 num_elem, type2name(elem_bt));
1799 return false;
1800 }
1801 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1802 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1803 use_predicate = false;
1804 if(!is_masked_op ||
1805 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1806 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1807 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1808 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1809 num_elem, type2name(elem_bt));
1810 return false; // not supported
1811 }
1812 }
1813 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1814 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1815
1816 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1817 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1818
1819 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1820 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1821 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1822
1823 if (v1 == nullptr || shuffle == nullptr) {
1824 return false; // operand unboxing failed
1825 }
1826
1827 assert(is_power_of_2(num_elem), "wrapping invalid");
1828 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1829 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1830 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1831
1832 Node* mask = nullptr;
1833 if (is_masked_op) {
1834 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1835 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1836 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1837 if (mask == nullptr) {
1838 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1839 num_elem, type2name(elem_bt));
1840 return false;
1841 }
1842 }
1843
1844 if (need_load_shuffle) {
1845 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1846 }
1847
1848 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1849 if (is_masked_op) {
1850 if (use_predicate) {
1851 rearrange->add_req(mask);
1852 rearrange->add_flag(Node::Flag_is_predicated_vector);
1853 } else {
1854 rearrange = gvn().transform(rearrange);
1855 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1856 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1857 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1858 }
1859 }
1860 rearrange = gvn().transform(rearrange);
1861
1862 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1863 set_result(box);
1864 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1865 return true;
1866 }
1867
1868 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1869 address addr = nullptr;
1870 assert(UseVectorStubs, "sanity");
1871 assert(name_ptr != nullptr, "unexpected");
1872 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
1873 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
1874
1875 switch(bits) {
1876 case 64: //fallthough
1877 case 128: //fallthough
1878 case 256: //fallthough
1879 case 512:
1880 if (bt == T_FLOAT) {
1881 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
1882 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
1883 } else {
1884 assert(bt == T_DOUBLE, "must be FP type only");
1885 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
1886 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
1887 }
1888 break;
1889 default:
1890 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
1891 snprintf(name_ptr, name_len, "invalid");
1892 addr = nullptr;
1893 Unimplemented();
1894 }
1895 break;
1896 }
1897
1898 if (addr == nullptr && Matcher::supports_scalable_vector()) {
1899 if (bt == T_FLOAT) {
1900 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
1901 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1902 } else {
1903 assert(bt == T_DOUBLE, "must be FP type only");
1904 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
1905 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1906 }
1907 }
1908
1909 return addr;
1910 }
1911
1912 // public static
1913 // <V extends Vector<E>,
1914 // M extends VectorMask<E>,
1915 // E>
1916 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1917 // int length, V v1, V v2, M m,
1918 // VectorSelectFromOp<V, M> defaultImpl)
1919 bool LibraryCallKit::inline_vector_select_from() {
1920 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1921 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1922 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1923 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1924
1925 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1926 vector_klass->const_oop() == nullptr ||
1927 elem_klass->const_oop() == nullptr ||
1928 !vlen->is_con()) {
1929 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1930 NodeClassNames[argument(0)->Opcode()],
1931 NodeClassNames[argument(2)->Opcode()],
1932 NodeClassNames[argument(3)->Opcode()]);
1933 return false; // not enough info for intrinsification
1934 }
1935 if (!is_klass_initialized(vector_klass)) {
1936 log_if_needed(" ** klass argument not initialized");
1937 return false;
1938 }
1939 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1940 if (!elem_type->is_primitive_type()) {
1941 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1942 return false; // should be primitive type
1943 }
1944 BasicType elem_bt = elem_type->basic_type();
1945 int num_elem = vlen->get_con();
1946 if (!is_power_of_2(num_elem)) {
1947 log_if_needed(" ** vlen not power of two=%d", num_elem);
1948 return false;
1949 }
1950
1951 BasicType shuffle_bt = elem_bt;
1952 if (shuffle_bt == T_FLOAT) {
1953 shuffle_bt = T_INT;
1954 } else if (shuffle_bt == T_DOUBLE) {
1955 shuffle_bt = T_LONG;
1956 }
1957 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1958
1959 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1960 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1961 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1962 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1963 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1964 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1965 num_elem, type2name(elem_bt));
1966 return false; // not supported
1967 }
1968
1969 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1970 bool use_predicate = is_masked_op;
1971 if (is_masked_op &&
1972 (mask_klass == nullptr ||
1973 mask_klass->const_oop() == nullptr ||
1974 !is_klass_initialized(mask_klass))) {
1975 log_if_needed(" ** mask_klass argument not initialized");
1976 return false; // not supported
1977 }
1978 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1979 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1980 use_predicate = false;
1981 if(!is_masked_op ||
1982 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1983 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1984 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1985 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1986 num_elem, type2name(elem_bt), is_masked_op);
1987 return false; // not supported
1988 }
1989 }
1990 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1991 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1992
1993 // v1 is the index vector
1994 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1995 // v2 is the vector being rearranged
1996 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1997
1998 if (v1 == nullptr) {
1999 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
2000 return false; // operand unboxing failed
2001 }
2002
2003 if (v2 == nullptr) {
2004 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2005 return false; // operand unboxing failed
2006 }
2007
2008 Node* mask = nullptr;
2009 if (is_masked_op) {
2010 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2011 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2012 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2013 if (mask == nullptr) {
2014 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2015 return false;
2016 }
2017 }
2018
2019 // cast index vector from elem_bt vector to byte vector
2020 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2021 Node* shuffle = v1;
2022
2023 if (shuffle_bt != elem_bt) {
2024 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2025 }
2026
2027 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2028 // this is a simple AND operation as we come here only for power of two vector length
2029 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2030 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2031 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2032
2033 // load the shuffle to use in rearrange
2034 if (need_load_shuffle) {
2035 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2036 }
2037
2038 // and finally rearrange
2039 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2040 if (is_masked_op) {
2041 if (use_predicate) {
2042 // masked rearrange is supported so use that directly
2043 rearrange->add_req(mask);
2044 rearrange->add_flag(Node::Flag_is_predicated_vector);
2045 } else {
2046 // masked rearrange is not supported so emulate usig blend
2047 const TypeVect* vt = v1->bottom_type()->is_vect();
2048 rearrange = gvn().transform(rearrange);
2049
2050 // create a zero vector with each lane element set as zero
2051 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2052 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2053
2054 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2055 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2056 }
2057 }
2058 rearrange = gvn().transform(rearrange);
2059
2060 // box the result
2061 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2062 set_result(box);
2063
2064 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2065 return true;
2066 }
2067
2068 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2069 assert(UseVectorStubs, "sanity");
2070 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2071 assert(opd1 != nullptr, "must not be null");
2072 const TypeVect* vt = TypeVect::make(bt, num_elem);
2073 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2074 char name[100] = "";
2075
2076 // Get address for vector math method.
2077 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2078
2079 if (addr == nullptr) {
2080 return nullptr;
2081 }
2082
2083 assert(name[0] != '\0', "name must not be null");
2084 Node* operation = make_runtime_call(RC_VECTOR,
2085 call_type,
2086 addr,
2087 name,
2088 TypePtr::BOTTOM,
2089 opd1,
2090 opd2);
2091 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2092 }
2093
2094 // public static
2095 // <V extends Vector<E>,
2096 // M extends VectorMask<E>,
2097 // E>
2098 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2099 // Class<E> elementType, int length,
2100 // V v, int n, M m,
2101 // VectorBroadcastIntOp<V, M> defaultImpl)
2102 bool LibraryCallKit::inline_vector_broadcast_int() {
2103 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2104 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2105 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2106 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2107 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2108
2109 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2110 return false; // dead code
2111 }
2112 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2113 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2114 NodeClassNames[argument(0)->Opcode()],
2115 NodeClassNames[argument(1)->Opcode()],
2116 NodeClassNames[argument(3)->Opcode()],
2117 NodeClassNames[argument(4)->Opcode()]);
2118 return false; // not enough info for intrinsification
2119 }
2120 if (!is_klass_initialized(vector_klass)) {
2121 log_if_needed(" ** klass argument not initialized");
2122 return false;
2123 }
2124
2125 const Type* vmask_type = gvn().type(argument(7));
2126 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2127 if (is_masked_op) {
2128 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2129 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2130 return false; // not enough info for intrinsification
2131 }
2132
2133 if (!is_klass_initialized(mask_klass)) {
2134 log_if_needed(" ** mask klass argument not initialized");
2135 return false;
2136 }
2137
2138 if (vmask_type->maybe_null()) {
2139 log_if_needed(" ** null mask values are not allowed for masked op");
2140 return false;
2141 }
2142 }
2143
2144 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2145 if (!elem_type->is_primitive_type()) {
2146 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2147 return false; // should be primitive type
2148 }
2149
2150 int num_elem = vlen->get_con();
2151 BasicType elem_bt = elem_type->basic_type();
2152 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2153
2154 bool is_shift = VectorNode::is_shift_opcode(opc);
2155 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2156
2157 if (opc == 0 || (!is_shift && !is_rotate)) {
2158 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2159 return false; // operation not supported
2160 }
2161
2162 int sopc = VectorNode::opcode(opc, elem_bt);
2163 if (sopc == 0) {
2164 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2165 return false; // operation not supported
2166 }
2167
2168 Node* cnt = argument(6);
2169 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2170
2171 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2172 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2173
2174 // If CPU supports vector constant rotate instructions pass it directly
2175 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2176 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2177 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2178
2179 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2180 bool use_predicate = is_masked_op;
2181
2182 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2183 use_predicate = false;
2184 if (!is_masked_op ||
2185 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2186 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2187
2188 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2189 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2190 return false; // not supported
2191 }
2192 }
2193
2194 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2195 Node* opd2 = nullptr;
2196 if (is_shift) {
2197 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2198 } else {
2199 assert(is_rotate, "unexpected operation");
2200 if (!is_const_rotate) {
2201 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2202 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2203 } else {
2204 // Constant shift value.
2205 opd2 = cnt;
2206 }
2207 }
2208
2209 if (opd1 == nullptr || opd2 == nullptr) {
2210 return false;
2211 }
2212
2213 Node* mask = nullptr;
2214 if (is_masked_op) {
2215 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2216 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2217 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2218 if (mask == nullptr) {
2219 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2220 return false;
2221 }
2222 }
2223
2224 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2225 if (is_masked_op && mask != nullptr) {
2226 if (use_predicate) {
2227 operation->add_req(mask);
2228 operation->add_flag(Node::Flag_is_predicated_vector);
2229 } else {
2230 operation = gvn().transform(operation);
2231 operation = new VectorBlendNode(opd1, operation, mask);
2232 }
2233 }
2234 operation = gvn().transform(operation);
2235 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2236 set_result(vbox);
2237 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2238 return true;
2239 }
2240
2241 // public static <VOUT extends VectorPayload,
2242 // VIN extends VectorPayload,
2243 // S extends VectorSpecies>
2244 // VOUT convert(int oprId,
2245 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2246 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2247 // VIN v, S s,
2248 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2249 //
2250 bool LibraryCallKit::inline_vector_convert() {
2251 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2252
2253 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2254 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2255 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2256
2257 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2258 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2259 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2260
2261 if (opr == nullptr ||
2262 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2263 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2264 return false; // dead code
2265 }
2266 if (!opr->is_con() ||
2267 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2268 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2269 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2270 NodeClassNames[argument(0)->Opcode()],
2271 NodeClassNames[argument(1)->Opcode()],
2272 NodeClassNames[argument(2)->Opcode()],
2273 NodeClassNames[argument(3)->Opcode()],
2274 NodeClassNames[argument(4)->Opcode()],
2275 NodeClassNames[argument(5)->Opcode()],
2276 NodeClassNames[argument(6)->Opcode()]);
2277 return false; // not enough info for intrinsification
2278 }
2279 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2280 log_if_needed(" ** klass argument not initialized");
2281 return false;
2282 }
2283
2284 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2285 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2286 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2287 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2288 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2289
2290 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2291 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2292
2293 bool is_mask = is_vector_mask(vbox_klass_from);
2294
2295 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2296 if (!elem_type_from->is_primitive_type()) {
2297 return false; // should be primitive type
2298 }
2299 BasicType elem_bt_from = elem_type_from->basic_type();
2300 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2301 if (!elem_type_to->is_primitive_type()) {
2302 return false; // should be primitive type
2303 }
2304 BasicType elem_bt_to = elem_type_to->basic_type();
2305
2306 int num_elem_from = vlen_from->get_con();
2307 int num_elem_to = vlen_to->get_con();
2308
2309 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2310 // since we may need to change size.
2311 if (!arch_supports_vector(Op_VectorReinterpret,
2312 num_elem_from,
2313 elem_bt_from,
2314 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2315 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2316 is_cast ? "cast" : "reinterpret",
2317 num_elem_from, type2name(elem_bt_from), is_mask);
2318 return false;
2319 }
2320
2321 // Check whether we can support resizing/reinterpreting to the new size.
2322 if (!arch_supports_vector(Op_VectorReinterpret,
2323 num_elem_to,
2324 elem_bt_to,
2325 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2326 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2327 is_cast ? "cast" : "reinterpret",
2328 num_elem_to, type2name(elem_bt_to), is_mask);
2329 return false;
2330 }
2331
2332 // At this point, we know that both input and output vector registers are supported
2333 // by the architecture. Next check if the casted type is simply to same type - which means
2334 // that it is actually a resize and not a cast.
2335 if (is_cast && elem_bt_from == elem_bt_to) {
2336 is_cast = false;
2337 }
2338
2339 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2340
2341 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2342 if (opd1 == nullptr) {
2343 return false;
2344 }
2345
2346 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2347 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2348
2349 // Safety check to prevent casting if source mask is of type vector
2350 // and destination mask of type predicate vector and vice-versa.
2351 // From X86 standpoint, this case will only arise over KNL target,
2352 // where certain masks (depending on the species) are either propagated
2353 // through a vector or predicate register.
2354 if (is_mask &&
2355 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2356 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2357 return false;
2358 }
2359
2360 Node* op = opd1;
2361 if (is_cast) {
2362 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2363 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2364
2365 // Make sure that vector cast is implemented to particular type/size combination if it is
2366 // not a mask casting.
2367 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2368 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2369 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2370 return false;
2371 }
2372
2373 if (num_elem_from < num_elem_to) {
2374 // Since input and output number of elements are not consistent, we need to make sure we
2375 // properly size. Thus, first make a cast that retains the number of elements from source.
2376 int num_elem_for_cast = num_elem_from;
2377
2378 // It is possible that arch does not support this intermediate vector size
2379 // TODO More complex logic required here to handle this corner case for the sizes.
2380 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2381 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2382 cast_vopc,
2383 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2384 return false;
2385 }
2386
2387 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2388 // Now ensure that the destination gets properly resized to needed size.
2389 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2390 } else if (num_elem_from > num_elem_to) {
2391 // Since number of elements from input is larger than output, simply reduce size of input
2392 // (we are supposed to drop top elements anyway).
2393 int num_elem_for_resize = num_elem_to;
2394
2395 // It is possible that arch does not support this intermediate vector size
2396 // TODO More complex logic required here to handle this corner case for the sizes.
2397 if (!arch_supports_vector(Op_VectorReinterpret,
2398 num_elem_for_resize,
2399 elem_bt_from,
2400 VecMaskNotUsed)) {
2401 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2402 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2403 return false;
2404 }
2405
2406 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2407 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2408 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2409 } else { // num_elem_from == num_elem_to
2410 if (is_mask) {
2411 // Make sure that cast for vector mask is implemented to particular type/size combination.
2412 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2413 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2414 num_elem_to, type2name(elem_bt_to), is_mask);
2415 return false;
2416 }
2417 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2418 } else {
2419 // Since input and output number of elements match, and since we know this vector size is
2420 // supported, simply do a cast with no resize needed.
2421 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2422 }
2423 }
2424 } else if (!Type::equals(src_type, dst_type)) {
2425 assert(!is_cast, "must be reinterpret");
2426 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2427 }
2428
2429 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2430 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2431 set_result(vbox);
2432 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2433 return true;
2434 }
2435
2436 // public static
2437 // <V extends Vector<E>,
2438 // E>
2439 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2440 // V vec, int ix, long val,
2441 // VecInsertOp<V> defaultImpl)
2442 bool LibraryCallKit::inline_vector_insert() {
2443 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2444 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2445 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2446 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2447
2448 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2449 return false; // dead code
2450 }
2451 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2452 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2453 NodeClassNames[argument(0)->Opcode()],
2454 NodeClassNames[argument(1)->Opcode()],
2455 NodeClassNames[argument(2)->Opcode()],
2456 NodeClassNames[argument(4)->Opcode()]);
2457 return false; // not enough info for intrinsification
2458 }
2459 if (!is_klass_initialized(vector_klass)) {
2460 log_if_needed(" ** klass argument not initialized");
2461 return false;
2462 }
2463 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2464 if (!elem_type->is_primitive_type()) {
2465 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2466 return false; // should be primitive type
2467 }
2468 BasicType elem_bt = elem_type->basic_type();
2469 int num_elem = vlen->get_con();
2470 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2471 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2472 num_elem, type2name(elem_bt));
2473 return false; // not supported
2474 }
2475
2476 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2477 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2478
2479 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2480 if (opd == nullptr) {
2481 return false;
2482 }
2483
2484 Node* insert_val = argument(5);
2485 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2486
2487 // Convert insert value back to its appropriate type.
2488 switch (elem_bt) {
2489 case T_BYTE:
2490 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2491 break;
2492 case T_SHORT:
2493 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2494 break;
2495 case T_INT:
2496 insert_val = gvn().transform(new ConvL2INode(insert_val));
2497 break;
2498 case T_FLOAT:
2499 insert_val = gvn().transform(new ConvL2INode(insert_val));
2500 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2501 break;
2502 case T_DOUBLE:
2503 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2504 break;
2505 case T_LONG:
2506 // no conversion needed
2507 break;
2508 default: fatal("%s", type2name(elem_bt)); break;
2509 }
2510
2511 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2512
2513 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2514 set_result(vbox);
2515 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2516 return true;
2517 }
2518
2519 // public static
2520 // <VM extends VectorPayload,
2521 // E>
2522 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2523 // int length,
2524 // VM vm, int i,
2525 // VecExtractOp<VM> defaultImpl)
2526 bool LibraryCallKit::inline_vector_extract() {
2527 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2528 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2529 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2530 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2531
2532 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2533 return false; // dead code
2534 }
2535 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2536 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2537 NodeClassNames[argument(0)->Opcode()],
2538 NodeClassNames[argument(1)->Opcode()],
2539 NodeClassNames[argument(2)->Opcode()]);
2540 return false; // not enough info for intrinsification
2541 }
2542 if (!is_klass_initialized(vector_klass)) {
2543 log_if_needed(" ** klass argument not initialized");
2544 return false;
2545 }
2546 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2547 if (!elem_type->is_primitive_type()) {
2548 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2549 return false; // should be primitive type
2550 }
2551 BasicType elem_bt = elem_type->basic_type();
2552 int num_elem = vlen->get_con();
2553
2554 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2555 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2556
2557 Node* opd = nullptr;
2558
2559 if (is_vector_mask(vbox_klass)) {
2560 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2561
2562 Node* pos = argument(4); // can be variable
2563 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2564 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2565 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2566 if (opd == nullptr) {
2567 return false;
2568 }
2569 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2570 opd = gvn().transform(new ExtractUBNode(opd, pos));
2571 opd = gvn().transform(new ConvI2LNode(opd));
2572 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2573 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2574 if (opd == nullptr) {
2575 return false;
2576 }
2577 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2578 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2579 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2580 }
2581 // ((toLong() >>> pos) & 1L
2582 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2583 opd = gvn().transform(new URShiftLNode(opd, pos));
2584 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2585 } else {
2586 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2587 return false; // not supported
2588 }
2589 } else {
2590 // vbox_klass is vector. This is used for Vector.lane(int).
2591 if (!idx->is_con()) {
2592 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2593 return false; // not enough info for intrinsification
2594 }
2595
2596 int vopc = ExtractNode::opcode(elem_bt);
2597 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2598 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2599 num_elem, type2name(elem_bt));
2600 return false; // not supported
2601 }
2602
2603 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2604 if (opd == nullptr) {
2605 return false;
2606 }
2607 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2608
2609 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2610 switch (elem_bt) {
2611 case T_BYTE:
2612 case T_SHORT:
2613 case T_INT: {
2614 opd = gvn().transform(new ConvI2LNode(opd));
2615 break;
2616 }
2617 case T_FLOAT: {
2618 opd = gvn().transform(new MoveF2INode(opd));
2619 opd = gvn().transform(new ConvI2LNode(opd));
2620 break;
2621 }
2622 case T_DOUBLE: {
2623 opd = gvn().transform(new MoveD2LNode(opd));
2624 break;
2625 }
2626 case T_LONG: {
2627 // no conversion needed
2628 break;
2629 }
2630 default: fatal("%s", type2name(elem_bt));
2631 }
2632 }
2633 set_result(opd);
2634 return true;
2635 }
2636
2637 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2638 int num_elem = vt->length();
2639 BasicType elem_bt = vt->element_basic_type();
2640
2641 // Lower selectFrom operation into its constituent operations.
2642 // SelectFromTwoVectorNode =
2643 // (VectorBlend
2644 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2645 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2646 // MASK)
2647 // Where
2648 // WRAPED_INDEX are computed by wrapping incoming indexes
2649 // to two vector index range [0, VLEN*2) and
2650 // MASK = WRAPED_INDEX < VLEN
2651 //
2652 // IR lowering prevents intrinsification failure and associated argument
2653 // boxing penalties.
2654 //
2655
2656 BasicType shuffle_bt = elem_bt;
2657 if (shuffle_bt == T_FLOAT) {
2658 shuffle_bt = T_INT;
2659 } else if (shuffle_bt == T_DOUBLE) {
2660 shuffle_bt = T_LONG;
2661 }
2662 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2663
2664 // Cast index vector to the corresponding bit type
2665 if (elem_bt != shuffle_bt) {
2666 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2667 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2668 }
2669
2670 // Wrap indexes into two vector index range [0, VLEN * 2)
2671 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2672 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2673 shuffle_bt, false));
2674 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2675
2676 // Compute the blend mask for merging two independently permitted vectors
2677 // using shuffle index in two vector index range [0, VLEN * 2).
2678 BoolTest::mask pred = BoolTest::le;
2679 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2680 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2681 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2682 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2683 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2684
2685 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2686 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2687
2688 // Load indexes from byte vector and appropriately transform them to target
2689 // specific permutation index format.
2690 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2691 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2692 }
2693
2694 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2695 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2696
2697 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2698 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2699
2700 return new VectorBlendNode(p2, p1, mask);
2701 }
2702
2703 // public static
2704 // <V extends Vector<E>,
2705 // E>
2706 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2707 // V v1, V v2, V v3,
2708 // SelectFromTwoVector<V> defaultImpl)
2709 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2710 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2711 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2712 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2713
2714 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2715 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2716 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2717 NodeClassNames[argument(0)->Opcode()],
2718 NodeClassNames[argument(1)->Opcode()],
2719 NodeClassNames[argument(2)->Opcode()]);
2720 return false; // not enough info for intrinsification
2721 }
2722
2723 if (!is_klass_initialized(vector_klass)) {
2724 log_if_needed(" ** klass argument not initialized");
2725 return false;
2726 }
2727
2728 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2729 if (!elem_type->is_primitive_type()) {
2730 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2731 return false; // should be primitive type
2732 }
2733
2734 int num_elem = vlen->get_con();
2735 if (!is_power_of_2(num_elem)) {
2736 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2737 return false;
2738 }
2739
2740 BasicType elem_bt = elem_type->basic_type();
2741 BasicType index_elem_bt = elem_bt;
2742 if (elem_bt == T_FLOAT) {
2743 index_elem_bt = T_INT;
2744 } else if (elem_bt == T_DOUBLE) {
2745 index_elem_bt = T_LONG;
2746 }
2747
2748 bool lowerSelectFromOp = false;
2749 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2750 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2751 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2752 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2753 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2754 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2755 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2756 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2757 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2758 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2759 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2760 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2761 return false; // not supported
2762 }
2763 lowerSelectFromOp = true;
2764 }
2765
2766 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2767 if (!lowerSelectFromOp) {
2768 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2769 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2770 (is_floating_point_type(elem_bt) &&
2771 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2772 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2773 num_elem, type2name(elem_bt));
2774 return false; // not supported
2775 }
2776 }
2777
2778 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2779 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2780
2781 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2782 if (opd1 == nullptr) {
2783 log_if_needed(" ** unbox failed v1=%s",
2784 NodeClassNames[argument(3)->Opcode()]);
2785 return false;
2786 }
2787 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2788 if (opd2 == nullptr) {
2789 log_if_needed(" ** unbox failed v2=%s",
2790 NodeClassNames[argument(4)->Opcode()]);
2791 return false;
2792 }
2793 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2794 if (opd3 == nullptr) {
2795 log_if_needed(" ** unbox failed v3=%s",
2796 NodeClassNames[argument(5)->Opcode()]);
2797 return false;
2798 }
2799
2800 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2801
2802 Node* operation = nullptr;
2803 if (lowerSelectFromOp) {
2804 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2805 } else {
2806 if (index_elem_bt != elem_bt) {
2807 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2808 }
2809 int indexRangeMask = 2 * num_elem - 1;
2810 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2811 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2812 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2813 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2814 }
2815
2816 // Wrap it up in VectorBox to keep object type information.
2817 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2818 set_result(vbox);
2819 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2820 return true;
2821 }
2822
2823 // public static
2824 // <V extends Vector<E>,
2825 // M extends VectorMask<E>,
2826 // E>
2827 // V compressExpandOp(int opr,
2828 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2829 // int length, V v, M m,
2830 // CompressExpandOperation<V, M> defaultImpl)
2831 bool LibraryCallKit::inline_vector_compress_expand() {
2832 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2833 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2834 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2835 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2836 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2837
2838 if (opr == nullptr || !opr->is_con() ||
2839 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2840 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2841 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2842 vlen == nullptr || !vlen->is_con()) {
2843 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2844 NodeClassNames[argument(0)->Opcode()],
2845 NodeClassNames[argument(1)->Opcode()],
2846 NodeClassNames[argument(2)->Opcode()],
2847 NodeClassNames[argument(3)->Opcode()],
2848 NodeClassNames[argument(4)->Opcode()]);
2849 return false; // not enough info for intrinsification
2850 }
2851
2852 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2853 log_if_needed(" ** klass argument not initialized");
2854 return false;
2855 }
2856
2857 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2858 if (!elem_type->is_primitive_type()) {
2859 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2860 return false; // should be primitive type
2861 }
2862
2863 int num_elem = vlen->get_con();
2864 BasicType elem_bt = elem_type->basic_type();
2865 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2866
2867 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2868 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2869 opc, num_elem, type2name(elem_bt));
2870 return false; // not supported
2871 }
2872
2873 Node* opd1 = nullptr;
2874 const TypeInstPtr* vbox_type = nullptr;
2875 if (opc != Op_CompressM) {
2876 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2877 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2878 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2879 if (opd1 == nullptr) {
2880 log_if_needed(" ** unbox failed vector=%s",
2881 NodeClassNames[argument(5)->Opcode()]);
2882 return false;
2883 }
2884 }
2885
2886 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2887 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2888 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2889
2890 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2891 if (mask == nullptr) {
2892 log_if_needed(" ** unbox failed mask=%s",
2893 NodeClassNames[argument(6)->Opcode()]);
2894 return false;
2895 }
2896
2897 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2898 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2899
2900 // Wrap it up in VectorBox to keep object type information.
2901 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2902 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2903 set_result(vbox);
2904 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2905 return true;
2906 }
2907
2908 // public static
2909 // <V extends Vector<E>,
2910 // E,
2911 // S extends VectorSpecies<E>>
2912 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2913 // int length,
2914 // V v, int step, S s,
2915 // IndexOperation<V, S> defaultImpl)
2916 bool LibraryCallKit::inline_index_vector() {
2917 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2918 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2919 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2920
2921 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2922 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2923 vlen == nullptr || !vlen->is_con() ) {
2924 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2925 NodeClassNames[argument(0)->Opcode()],
2926 NodeClassNames[argument(1)->Opcode()],
2927 NodeClassNames[argument(2)->Opcode()]);
2928 return false; // not enough info for intrinsification
2929 }
2930
2931 if (!is_klass_initialized(vector_klass)) {
2932 log_if_needed(" ** klass argument not initialized");
2933 return false;
2934 }
2935
2936 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2937 if (!elem_type->is_primitive_type()) {
2938 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2939 return false; // should be primitive type
2940 }
2941
2942 int num_elem = vlen->get_con();
2943 BasicType elem_bt = elem_type->basic_type();
2944
2945 // Check whether the iota index generation op is supported by the current hardware
2946 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2947 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2948 return false; // not supported
2949 }
2950
2951 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2952 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2953 bool needs_mul = true;
2954 Node* scale = argument(4);
2955 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2956 // Multiply is not needed if the scale is a constant "1".
2957 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2958 needs_mul = false;
2959 } else {
2960 // Check whether the vector multiply op is supported by the current hardware
2961 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2962 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2963 return false; // not supported
2964 }
2965
2966 // Check whether the scalar cast op is supported by the current hardware
2967 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2968 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2969 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2970 if (!Matcher::match_rule_supported(cast_op)) {
2971 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2972 NodeClassNames[cast_op]);
2973 return false; // not supported
2974 }
2975 }
2976 }
2977
2978 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2979 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2980 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2981 if (opd == nullptr) {
2982 log_if_needed(" ** unbox failed vector=%s",
2983 NodeClassNames[argument(3)->Opcode()]);
2984 return false;
2985 }
2986
2987 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2988 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2989 bool needs_add = true;
2990 // The addition is not needed if all the element values of "opd" are zero
2991 if (VectorNode::is_all_zeros_vector(opd)) {
2992 needs_add = false;
2993 } else {
2994 // Check whether the vector addition op is supported by the current hardware
2995 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2996 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2997 return false; // not supported
2998 }
2999 }
3000
3001 // Compute the iota indice vector
3002 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3003 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3004
3005 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3006 if (needs_mul) {
3007 switch (elem_bt) {
3008 case T_BOOLEAN: // fall-through
3009 case T_BYTE: // fall-through
3010 case T_SHORT: // fall-through
3011 case T_CHAR: // fall-through
3012 case T_INT: {
3013 // no conversion needed
3014 break;
3015 }
3016 case T_LONG: {
3017 scale = gvn().transform(new ConvI2LNode(scale));
3018 break;
3019 }
3020 case T_FLOAT: {
3021 scale = gvn().transform(new ConvI2FNode(scale));
3022 break;
3023 }
3024 case T_DOUBLE: {
3025 scale = gvn().transform(new ConvI2DNode(scale));
3026 break;
3027 }
3028 default: fatal("%s", type2name(elem_bt));
3029 }
3030 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3031 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3032 }
3033
3034 // Add "opd" if addition is needed.
3035 if (needs_add) {
3036 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3037 }
3038 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3039 set_result(vbox);
3040 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3041 return true;
3042 }
3043
3044 // public static
3045 // <E,
3046 // M extends VectorMask<E>>
3047 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3048 // long offset, long limit,
3049 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3050 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3051 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3052 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3053 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3054
3055 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3056 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3057 vlen == nullptr || !vlen->is_con()) {
3058 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3059 NodeClassNames[argument(0)->Opcode()],
3060 NodeClassNames[argument(1)->Opcode()],
3061 NodeClassNames[argument(2)->Opcode()]);
3062 return false; // not enough info for intrinsification
3063 }
3064
3065 if (!is_klass_initialized(mask_klass)) {
3066 log_if_needed(" ** klass argument not initialized");
3067 return false;
3068 }
3069
3070 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3071 if (!elem_type->is_primitive_type()) {
3072 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3073 return false; // should be primitive type
3074 }
3075
3076 int num_elem = vlen->get_con();
3077 BasicType elem_bt = elem_type->basic_type();
3078
3079 // Check whether the necessary ops are supported by current hardware.
3080 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3081 if (!supports_mask_gen) {
3082 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3083 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3084 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3085 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3086 return false; // not supported
3087 }
3088
3089 // Check whether the scalar cast operation is supported by current hardware.
3090 if (elem_bt != T_LONG) {
3091 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3092 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3093 if (!Matcher::match_rule_supported(cast_op)) {
3094 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3095 NodeClassNames[cast_op]);
3096 return false; // not supported
3097 }
3098 }
3099 }
3100
3101 Node* offset = argument(3);
3102 Node* limit = argument(5);
3103 if (offset == nullptr || limit == nullptr) {
3104 log_if_needed(" ** offset or limit argument is null");
3105 return false; // not supported
3106 }
3107
3108 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3109 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3110 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3111
3112 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3113 // So directly get indexLimit with "indexLimit = limit - offset".
3114 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3115 Node* mask = nullptr;
3116 if (supports_mask_gen) {
3117 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3118 } else {
3119 // Generate the vector mask based on "mask = iota < indexLimit".
3120 // Broadcast "indexLimit" to a vector.
3121 switch (elem_bt) {
3122 case T_BOOLEAN: // fall-through
3123 case T_BYTE: // fall-through
3124 case T_SHORT: // fall-through
3125 case T_CHAR: // fall-through
3126 case T_INT: {
3127 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3128 break;
3129 }
3130 case T_DOUBLE: {
3131 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3132 break;
3133 }
3134 case T_FLOAT: {
3135 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3136 break;
3137 }
3138 case T_LONG: {
3139 // no conversion needed
3140 break;
3141 }
3142 default: fatal("%s", type2name(elem_bt));
3143 }
3144 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3145
3146 // Load the "iota" vector.
3147 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3148 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3149
3150 // Compute the vector mask with "mask = iota < indexLimit".
3151 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3152 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3153 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3154 }
3155 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3156 set_result(vbox);
3157 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3158 return true;
3159 }
3160
3161 #undef non_product_log_if_needed
3162 #undef log_if_needed