1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2636 return true;
2637 }
2638
2639 Node *off = m->in(AddPNode::Offset);
2640 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2641 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2642 // Are there other uses besides address expressions?
2643 !is_visited(off)) {
2644 address_visited.set(off->_idx); // Flag as address_visited
2645 mstack.push(off->in(2), Visit);
2646 Node *conv = off->in(1);
2647 if (conv->Opcode() == Op_ConvI2L &&
2648 // Are there other uses besides address expressions?
2649 !is_visited(conv)) {
2650 address_visited.set(conv->_idx); // Flag as address_visited
2651 mstack.push(conv->in(1), Pre_Visit);
2652 } else {
2653 mstack.push(conv, Pre_Visit);
2654 }
2655 address_visited.test_set(m->_idx); // Flag as address_visited
2656 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2657 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2658 return true;
2659 } else if (off->Opcode() == Op_ConvI2L &&
2660 // Are there other uses besides address expressions?
2661 !is_visited(off)) {
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 address_visited.set(off->_idx); // Flag as address_visited
2664 mstack.push(off->in(1), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 }
2669 return false;
2670 }
2671
2672 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2673 { \
2674 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2675 guarantee(DISP == 0, "mode not permitted for volatile"); \
2676 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2677 __ INSN(REG, as_Register(BASE)); \
2678 }
2679
2680
2681 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2682 {
2683 Address::extend scale;
2684
2685 // Hooboy, this is fugly. We need a way to communicate to the
2686 // encoder that the index needs to be sign extended, so we have to
2687 // enumerate all the cases.
2688 switch (opcode) {
2689 case INDINDEXSCALEDI2L:
2690 case INDINDEXSCALEDI2LN:
2691 case INDINDEXI2L:
2692 case INDINDEXI2LN:
2693 scale = Address::sxtw(size);
2694 break;
2695 default:
2696 scale = Address::lsl(size);
2697 }
2698
2699 if (index == -1) {
2700 return Address(base, disp);
2701 } else {
2702 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2703 return Address(base, as_Register(index), scale);
2704 }
2705 }
2706
2707
2708 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2709 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2710 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2711 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2712 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2713
2714 // Used for all non-volatile memory accesses. The use of
2715 // $mem->opcode() to discover whether this pattern uses sign-extended
2716 // offsets is something of a kludge.
2717 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2718 Register reg, int opcode,
2719 Register base, int index, int scale, int disp,
2720 int size_in_memory)
2721 {
2722 Address addr = mem2address(opcode, base, index, scale, disp);
2723 if (addr.getMode() == Address::base_plus_offset) {
2724 /* Fix up any out-of-range offsets. */
2725 assert_different_registers(rscratch1, base);
2726 assert_different_registers(rscratch1, reg);
2727 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2728 }
2729 (masm->*insn)(reg, addr);
2730 }
2731
2732 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2733 FloatRegister reg, int opcode,
2734 Register base, int index, int size, int disp,
2735 int size_in_memory)
2736 {
2737 Address::extend scale;
2738
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 scale = Address::sxtw(size);
2743 break;
2744 default:
2745 scale = Address::lsl(size);
2746 }
2747
2748 if (index == -1) {
2749 // Fix up any out-of-range offsets.
2750 assert_different_registers(rscratch1, base);
2751 Address addr = Address(base, disp);
2752 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2753 (masm->*insn)(reg, addr);
2754 } else {
2755 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2756 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2757 }
2758 }
2759
2760 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2761 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2762 int opcode, Register base, int index, int size, int disp)
2763 {
2764 if (index == -1) {
2765 (masm->*insn)(reg, T, Address(base, disp));
2766 } else {
2767 assert(disp == 0, "unsupported address mode");
2768 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2769 }
2770 }
2771
2772 %}
2773
2774
2775
2776 //----------ENCODING BLOCK-----------------------------------------------------
2777 // This block specifies the encoding classes used by the compiler to
2778 // output byte streams. Encoding classes are parameterized macros
2779 // used by Machine Instruction Nodes in order to generate the bit
2780 // encoding of the instruction. Operands specify their base encoding
2781 // interface with the interface keyword. There are currently
2782 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2783 // COND_INTER. REG_INTER causes an operand to generate a function
2784 // which returns its register number when queried. CONST_INTER causes
2785 // an operand to generate a function which returns the value of the
2786 // constant when queried. MEMORY_INTER causes an operand to generate
2787 // four functions which return the Base Register, the Index Register,
2788 // the Scale Value, and the Offset Value of the operand when queried.
2789 // COND_INTER causes an operand to generate six functions which return
2790 // the encoding code (ie - encoding bits for the instruction)
2791 // associated with each basic boolean condition for a conditional
2792 // instruction.
2793 //
2794 // Instructions specify two basic values for encoding. Again, a
2795 // function is available to check if the constant displacement is an
2796 // oop. They use the ins_encode keyword to specify their encoding
2797 // classes (which must be a sequence of enc_class names, and their
2798 // parameters, specified in the encoding block), and they use the
2799 // opcode keyword to specify, in order, their primary, secondary, and
2800 // tertiary opcode. Only the opcode sections which a particular
2801 // instruction needs for encoding need to be specified.
2802 encode %{
2803 // Build emit functions for each basic byte or larger field in the
2804 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2805 // from C++ code in the enc_class source block. Emit functions will
2806 // live in the main source block for now. In future, we can
2807 // generalize this by adding a syntax that specifies the sizes of
2808 // fields in an order, so that the adlc can build the emit functions
2809 // automagically
2810
2811 // catch all for unimplemented encodings
2812 enc_class enc_unimplemented %{
2813 __ unimplemented("C2 catch all");
2814 %}
2815
2816 // BEGIN Non-volatile memory access
2817
2818 // This encoding class is generated automatically from ad_encode.m4.
2819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2820 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2821 Register dst_reg = as_Register($dst$$reg);
2822 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2823 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2824 %}
2825
2826 // This encoding class is generated automatically from ad_encode.m4.
2827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2828 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2829 Register dst_reg = as_Register($dst$$reg);
2830 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2831 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2832 %}
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2917 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2925 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2933 Register src_reg = as_Register($src$$reg);
2934 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_strb0(memory1 mem) %{
2941 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2948 Register src_reg = as_Register($src$$reg);
2949 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strh0(memory2 mem) %{
2956 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2963 Register src_reg = as_Register($src$$reg);
2964 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strw0(memory4 mem) %{
2971 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2978 Register src_reg = as_Register($src$$reg);
2979 // we sometimes get asked to store the stack pointer into the
2980 // current thread -- we cannot do that directly on AArch64
2981 if (src_reg == r31_sp) {
2982 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2983 __ mov(rscratch2, sp);
2984 src_reg = rscratch2;
2985 }
2986 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_str0(memory8 mem) %{
2993 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3000 FloatRegister src_reg = as_FloatRegister($src$$reg);
3001 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3008 FloatRegister src_reg = as_FloatRegister($src$$reg);
3009 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3016 __ membar(Assembler::StoreStore);
3017 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3019 %}
3020
3021 // END Non-volatile memory access
3022
3023 // Vector loads and stores
3024 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3025 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3026 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3031 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3032 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3037 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3038 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3043 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3044 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3045 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3046 %}
3047
3048 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3051 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3052 %}
3053
3054 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3055 FloatRegister src_reg = as_FloatRegister($src$$reg);
3056 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3057 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3058 %}
3059
3060 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3061 FloatRegister src_reg = as_FloatRegister($src$$reg);
3062 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3063 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3064 %}
3065
3066 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3069 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3070 %}
3071
3072 // volatile loads and stores
3073
3074 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3075 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, stlrb);
3077 %}
3078
3079 enc_class aarch64_enc_stlrb0(memory mem) %{
3080 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3081 rscratch1, stlrb);
3082 %}
3083
3084 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3085 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3086 rscratch1, stlrh);
3087 %}
3088
3089 enc_class aarch64_enc_stlrh0(memory mem) %{
3090 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3091 rscratch1, stlrh);
3092 %}
3093
3094 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3095 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3096 rscratch1, stlrw);
3097 %}
3098
3099 enc_class aarch64_enc_stlrw0(memory mem) %{
3100 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, stlrw);
3102 %}
3103
3104 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3105 Register dst_reg = as_Register($dst$$reg);
3106 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, ldarb);
3108 __ sxtbw(dst_reg, dst_reg);
3109 %}
3110
3111 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3112 Register dst_reg = as_Register($dst$$reg);
3113 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, ldarb);
3115 __ sxtb(dst_reg, dst_reg);
3116 %}
3117
3118 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3119 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, ldarb);
3121 %}
3122
3123 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3124 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, ldarb);
3126 %}
3127
3128 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarh);
3132 __ sxthw(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3136 Register dst_reg = as_Register($dst$$reg);
3137 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarh);
3139 __ sxth(dst_reg, dst_reg);
3140 %}
3141
3142 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3143 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, ldarh);
3145 %}
3146
3147 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarh);
3150 %}
3151
3152 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarw);
3155 %}
3156
3157 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3158 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, ldarw);
3160 %}
3161
3162 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3163 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldar);
3165 %}
3166
3167 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3168 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarw);
3170 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3171 %}
3172
3173 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3174 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldar);
3176 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3177 %}
3178
3179 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3180 Register src_reg = as_Register($src$$reg);
3181 // we sometimes get asked to store the stack pointer into the
3182 // current thread -- we cannot do that directly on AArch64
3183 if (src_reg == r31_sp) {
3184 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3185 __ mov(rscratch2, sp);
3186 src_reg = rscratch2;
3187 }
3188 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlr);
3190 %}
3191
3192 enc_class aarch64_enc_stlr0(memory mem) %{
3193 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, stlr);
3195 %}
3196
3197 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3198 {
3199 FloatRegister src_reg = as_FloatRegister($src$$reg);
3200 __ fmovs(rscratch2, src_reg);
3201 }
3202 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlrw);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovd(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 // synchronized read/update encodings
3216
3217 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3218 Register dst_reg = as_Register($dst$$reg);
3219 Register base = as_Register($mem$$base);
3220 int index = $mem$$index;
3221 int scale = $mem$$scale;
3222 int disp = $mem$$disp;
3223 if (index == -1) {
3224 if (disp != 0) {
3225 __ lea(rscratch1, Address(base, disp));
3226 __ ldaxr(dst_reg, rscratch1);
3227 } else {
3228 // TODO
3229 // should we ever get anything other than this case?
3230 __ ldaxr(dst_reg, base);
3231 }
3232 } else {
3233 Register index_reg = as_Register(index);
3234 if (disp == 0) {
3235 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3236 __ ldaxr(dst_reg, rscratch1);
3237 } else {
3238 __ lea(rscratch1, Address(base, disp));
3239 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3240 __ ldaxr(dst_reg, rscratch1);
3241 }
3242 }
3243 %}
3244
3245 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3246 Register src_reg = as_Register($src$$reg);
3247 Register base = as_Register($mem$$base);
3248 int index = $mem$$index;
3249 int scale = $mem$$scale;
3250 int disp = $mem$$disp;
3251 if (index == -1) {
3252 if (disp != 0) {
3253 __ lea(rscratch2, Address(base, disp));
3254 __ stlxr(rscratch1, src_reg, rscratch2);
3255 } else {
3256 // TODO
3257 // should we ever get anything other than this case?
3258 __ stlxr(rscratch1, src_reg, base);
3259 }
3260 } else {
3261 Register index_reg = as_Register(index);
3262 if (disp == 0) {
3263 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3264 __ stlxr(rscratch1, src_reg, rscratch2);
3265 } else {
3266 __ lea(rscratch2, Address(base, disp));
3267 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3268 __ stlxr(rscratch1, src_reg, rscratch2);
3269 }
3270 }
3271 __ cmpw(rscratch1, zr);
3272 %}
3273
3274 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3275 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3276 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3277 Assembler::xword, /*acquire*/ false, /*release*/ true,
3278 /*weak*/ false, noreg);
3279 %}
3280
3281 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3282 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3283 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3284 Assembler::word, /*acquire*/ false, /*release*/ true,
3285 /*weak*/ false, noreg);
3286 %}
3287
3288 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3289 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3290 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3291 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3292 /*weak*/ false, noreg);
3293 %}
3294
3295 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::byte, /*acquire*/ false, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302
3303 // The only difference between aarch64_enc_cmpxchg and
3304 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3305 // CompareAndSwap sequence to serve as a barrier on acquiring a
3306 // lock.
3307 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3308 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3309 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3310 Assembler::xword, /*acquire*/ true, /*release*/ true,
3311 /*weak*/ false, noreg);
3312 %}
3313
3314 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3315 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3316 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3317 Assembler::word, /*acquire*/ true, /*release*/ true,
3318 /*weak*/ false, noreg);
3319 %}
3320
3321 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3322 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3323 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3324 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3325 /*weak*/ false, noreg);
3326 %}
3327
3328 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3329 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3330 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3331 Assembler::byte, /*acquire*/ true, /*release*/ true,
3332 /*weak*/ false, noreg);
3333 %}
3334
3335 // auxiliary used for CompareAndSwapX to set result register
3336 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3337 Register res_reg = as_Register($res$$reg);
3338 __ cset(res_reg, Assembler::EQ);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none, "unexpected reloc type");
3400 if (! __ is_valid_AArch64_address(con) ||
3401 con < (address)(uintptr_t)os::vm_page_size()) {
3402 __ mov(dst_reg, con);
3403 } else {
3404 uint64_t offset;
3405 __ adrp(dst_reg, con, offset);
3406 __ add(dst_reg, dst_reg, offset);
3407 }
3408 }
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 __ mov(dst_reg, zr);
3415 %}
3416
3417 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, (uint64_t)1);
3420 %}
3421
3422 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3423 __ load_byte_map_base($dst$$Register);
3424 %}
3425
3426 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 address con = (address)$src$$constant;
3429 if (con == nullptr) {
3430 ShouldNotReachHere();
3431 } else {
3432 relocInfo::relocType rtype = $src->constant_reloc();
3433 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3434 __ set_narrow_oop(dst_reg, (jobject)con);
3435 }
3436 %}
3437
3438 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3439 Register dst_reg = as_Register($dst$$reg);
3440 __ mov(dst_reg, zr);
3441 %}
3442
3443 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 address con = (address)$src$$constant;
3446 if (con == nullptr) {
3447 ShouldNotReachHere();
3448 } else {
3449 relocInfo::relocType rtype = $src->constant_reloc();
3450 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3451 __ set_narrow_klass(dst_reg, (Klass *)con);
3452 }
3453 %}
3454
3455 // arithmetic encodings
3456
3457 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3458 Register dst_reg = as_Register($dst$$reg);
3459 Register src_reg = as_Register($src1$$reg);
3460 int32_t con = (int32_t)$src2$$constant;
3461 // add has primary == 0, subtract has primary == 1
3462 if ($primary) { con = -con; }
3463 if (con < 0) {
3464 __ subw(dst_reg, src_reg, -con);
3465 } else {
3466 __ addw(dst_reg, src_reg, con);
3467 }
3468 %}
3469
3470 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3471 Register dst_reg = as_Register($dst$$reg);
3472 Register src_reg = as_Register($src1$$reg);
3473 int32_t con = (int32_t)$src2$$constant;
3474 // add has primary == 0, subtract has primary == 1
3475 if ($primary) { con = -con; }
3476 if (con < 0) {
3477 __ sub(dst_reg, src_reg, -con);
3478 } else {
3479 __ add(dst_reg, src_reg, con);
3480 }
3481 %}
3482
3483 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src1_reg = as_Register($src1$$reg);
3486 Register src2_reg = as_Register($src2$$reg);
3487 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3488 %}
3489
3490 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3491 Register dst_reg = as_Register($dst$$reg);
3492 Register src1_reg = as_Register($src1$$reg);
3493 Register src2_reg = as_Register($src2$$reg);
3494 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3495 %}
3496
3497 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3498 Register dst_reg = as_Register($dst$$reg);
3499 Register src1_reg = as_Register($src1$$reg);
3500 Register src2_reg = as_Register($src2$$reg);
3501 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3502 %}
3503
3504 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3505 Register dst_reg = as_Register($dst$$reg);
3506 Register src1_reg = as_Register($src1$$reg);
3507 Register src2_reg = as_Register($src2$$reg);
3508 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3509 %}
3510
3511 // compare instruction encodings
3512
3513 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3514 Register reg1 = as_Register($src1$$reg);
3515 Register reg2 = as_Register($src2$$reg);
3516 __ cmpw(reg1, reg2);
3517 %}
3518
3519 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3520 Register reg = as_Register($src1$$reg);
3521 int32_t val = $src2$$constant;
3522 if (val >= 0) {
3523 __ subsw(zr, reg, val);
3524 } else {
3525 __ addsw(zr, reg, -val);
3526 }
3527 %}
3528
3529 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3530 Register reg1 = as_Register($src1$$reg);
3531 uint32_t val = (uint32_t)$src2$$constant;
3532 __ movw(rscratch1, val);
3533 __ cmpw(reg1, rscratch1);
3534 %}
3535
3536 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3537 Register reg1 = as_Register($src1$$reg);
3538 Register reg2 = as_Register($src2$$reg);
3539 __ cmp(reg1, reg2);
3540 %}
3541
3542 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3543 Register reg = as_Register($src1$$reg);
3544 int64_t val = $src2$$constant;
3545 if (val >= 0) {
3546 __ subs(zr, reg, val);
3547 } else if (val != -val) {
3548 __ adds(zr, reg, -val);
3549 } else {
3550 // aargh, Long.MIN_VALUE is a special case
3551 __ orr(rscratch1, zr, (uint64_t)val);
3552 __ subs(zr, reg, rscratch1);
3553 }
3554 %}
3555
3556 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3557 Register reg1 = as_Register($src1$$reg);
3558 uint64_t val = (uint64_t)$src2$$constant;
3559 __ mov(rscratch1, val);
3560 __ cmp(reg1, rscratch1);
3561 %}
3562
3563 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmp(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3570 Register reg1 = as_Register($src1$$reg);
3571 Register reg2 = as_Register($src2$$reg);
3572 __ cmpw(reg1, reg2);
3573 %}
3574
3575 enc_class aarch64_enc_testp(iRegP src) %{
3576 Register reg = as_Register($src$$reg);
3577 __ cmp(reg, zr);
3578 %}
3579
3580 enc_class aarch64_enc_testn(iRegN src) %{
3581 Register reg = as_Register($src$$reg);
3582 __ cmpw(reg, zr);
3583 %}
3584
3585 enc_class aarch64_enc_b(label lbl) %{
3586 Label *L = $lbl$$label;
3587 __ b(*L);
3588 %}
3589
3590 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3591 Label *L = $lbl$$label;
3592 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3593 %}
3594
3595 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3596 Label *L = $lbl$$label;
3597 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3598 %}
3599
3600 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3601 %{
3602 Register sub_reg = as_Register($sub$$reg);
3603 Register super_reg = as_Register($super$$reg);
3604 Register temp_reg = as_Register($temp$$reg);
3605 Register result_reg = as_Register($result$$reg);
3606
3607 Label miss;
3608 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3609 nullptr, &miss,
3610 /*set_cond_codes:*/ true);
3611 if ($primary) {
3612 __ mov(result_reg, zr);
3613 }
3614 __ bind(miss);
3615 %}
3616
3617 enc_class aarch64_enc_java_static_call(method meth) %{
3618 address addr = (address)$meth$$method;
3619 address call;
3620 if (!_method) {
3621 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3622 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3623 if (call == nullptr) {
3624 ciEnv::current()->record_failure("CodeCache is full");
3625 return;
3626 }
3627 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3628 // The NOP here is purely to ensure that eliding a call to
3629 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3630 __ nop();
3631 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3632 } else {
3633 int method_index = resolved_method_index(masm);
3634 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3635 : static_call_Relocation::spec(method_index);
3636 call = __ trampoline_call(Address(addr, rspec));
3637 if (call == nullptr) {
3638 ciEnv::current()->record_failure("CodeCache is full");
3639 return;
3640 }
3641 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3642 // Calls of the same statically bound method can share
3643 // a stub to the interpreter.
3644 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3645 } else {
3646 // Emit stub for static call
3647 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3648 if (stub == nullptr) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 }
3653 }
3654
3655 __ post_call_nop();
3656
3657 // Only non uncommon_trap calls need to reinitialize ptrue.
3658 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3659 __ reinitialize_ptrue();
3660 }
3661 %}
3662
3663 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3664 int method_index = resolved_method_index(masm);
3665 address call = __ ic_call((address)$meth$$method, method_index);
3666 if (call == nullptr) {
3667 ciEnv::current()->record_failure("CodeCache is full");
3668 return;
3669 }
3670 __ post_call_nop();
3671 if (Compile::current()->max_vector_size() > 0) {
3672 __ reinitialize_ptrue();
3673 }
3674 %}
3675
3676 enc_class aarch64_enc_call_epilog() %{
3677 if (VerifyStackAtCalls) {
3678 // Check that stack depth is unchanged: find majik cookie on stack
3679 __ call_Unimplemented();
3680 }
3681 %}
3682
3683 enc_class aarch64_enc_java_to_runtime(method meth) %{
3684 // some calls to generated routines (arraycopy code) are scheduled
3685 // by C2 as runtime calls. if so we can call them using a br (they
3686 // will be in a reachable segment) otherwise we have to use a blr
3687 // which loads the absolute address into a register.
3688 address entry = (address)$meth$$method;
3689 CodeBlob *cb = CodeCache::find_blob(entry);
3690 if (cb) {
3691 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3692 if (call == nullptr) {
3693 ciEnv::current()->record_failure("CodeCache is full");
3694 return;
3695 }
3696 __ post_call_nop();
3697 } else {
3698 Label retaddr;
3699 __ adr(rscratch2, retaddr);
3700 __ lea(rscratch1, RuntimeAddress(entry));
3701 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3702 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3703 __ blr(rscratch1);
3704 __ bind(retaddr);
3705 __ post_call_nop();
3706 __ add(sp, sp, 2 * wordSize);
3707 }
3708 if (Compile::current()->max_vector_size() > 0) {
3709 __ reinitialize_ptrue();
3710 }
3711 %}
3712
3713 enc_class aarch64_enc_rethrow() %{
3714 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3715 %}
3716
3717 enc_class aarch64_enc_ret() %{
3718 #ifdef ASSERT
3719 if (Compile::current()->max_vector_size() > 0) {
3720 __ verify_ptrue();
3721 }
3722 #endif
3723 __ ret(lr);
3724 %}
3725
3726 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3727 Register target_reg = as_Register($jump_target$$reg);
3728 __ br(target_reg);
3729 %}
3730
3731 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3732 Register target_reg = as_Register($jump_target$$reg);
3733 // exception oop should be in r0
3734 // ret addr has been popped into lr
3735 // callee expects it in r3
3736 __ mov(r3, lr);
3737 __ br(target_reg);
3738 %}
3739
3740 %}
3741
3742 //----------FRAME--------------------------------------------------------------
3743 // Definition of frame structure and management information.
3744 //
3745 // S T A C K L A Y O U T Allocators stack-slot number
3746 // | (to get allocators register number
3747 // G Owned by | | v add OptoReg::stack0())
3748 // r CALLER | |
3749 // o | +--------+ pad to even-align allocators stack-slot
3750 // w V | pad0 | numbers; owned by CALLER
3751 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3752 // h ^ | in | 5
3753 // | | args | 4 Holes in incoming args owned by SELF
3754 // | | | | 3
3755 // | | +--------+
3756 // V | | old out| Empty on Intel, window on Sparc
3757 // | old |preserve| Must be even aligned.
3758 // | SP-+--------+----> Matcher::_old_SP, even aligned
3759 // | | in | 3 area for Intel ret address
3760 // Owned by |preserve| Empty on Sparc.
3761 // SELF +--------+
3762 // | | pad2 | 2 pad to align old SP
3763 // | +--------+ 1
3764 // | | locks | 0
3765 // | +--------+----> OptoReg::stack0(), even aligned
3766 // | | pad1 | 11 pad to align new SP
3767 // | +--------+
3768 // | | | 10
3769 // | | spills | 9 spills
3770 // V | | 8 (pad0 slot for callee)
3771 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3772 // ^ | out | 7
3773 // | | args | 6 Holes in outgoing args owned by CALLEE
3774 // Owned by +--------+
3775 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3776 // | new |preserve| Must be even-aligned.
3777 // | SP-+--------+----> Matcher::_new_SP, even aligned
3778 // | | |
3779 //
3780 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3781 // known from SELF's arguments and the Java calling convention.
3782 // Region 6-7 is determined per call site.
3783 // Note 2: If the calling convention leaves holes in the incoming argument
3784 // area, those holes are owned by SELF. Holes in the outgoing area
3785 // are owned by the CALLEE. Holes should not be necessary in the
3786 // incoming area, as the Java calling convention is completely under
3787 // the control of the AD file. Doubles can be sorted and packed to
3788 // avoid holes. Holes in the outgoing arguments may be necessary for
3789 // varargs C calling conventions.
3790 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3791 // even aligned with pad0 as needed.
3792 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3793 // (the latter is true on Intel but is it false on AArch64?)
3794 // region 6-11 is even aligned; it may be padded out more so that
3795 // the region from SP to FP meets the minimum stack alignment.
3796 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3797 // alignment. Region 11, pad1, may be dynamically extended so that
3798 // SP meets the minimum alignment.
3799
3800 frame %{
3801 // These three registers define part of the calling convention
3802 // between compiled code and the interpreter.
3803
3804 // Inline Cache Register or Method for I2C.
3805 inline_cache_reg(R12);
3806
3807 // Number of stack slots consumed by locking an object
3808 sync_stack_slots(2);
3809
3810 // Compiled code's Frame Pointer
3811 frame_pointer(R31);
3812
3813 // Interpreter stores its frame pointer in a register which is
3814 // stored to the stack by I2CAdaptors.
3815 // I2CAdaptors convert from interpreted java to compiled java.
3816 interpreter_frame_pointer(R29);
3817
3818 // Stack alignment requirement
3819 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3820
3821 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3822 // for calls to C. Supports the var-args backing area for register parms.
3823 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3824
3825 // The after-PROLOG location of the return address. Location of
3826 // return address specifies a type (REG or STACK) and a number
3827 // representing the register number (i.e. - use a register name) or
3828 // stack slot.
3829 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3830 // Otherwise, it is above the locks and verification slot and alignment word
3831 // TODO this may well be correct but need to check why that - 2 is there
3832 // ppc port uses 0 but we definitely need to allow for fixed_slots
3833 // which folds in the space used for monitors
3834 return_addr(STACK - 2 +
3835 align_up((Compile::current()->in_preserve_stack_slots() +
3836 Compile::current()->fixed_slots()),
3837 stack_alignment_in_slots()));
3838
3839 // Location of compiled Java return values. Same as C for now.
3840 return_value
3841 %{
3842 // TODO do we allow ideal_reg == Op_RegN???
3843 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3844 "only return normal values");
3845
3846 static const int lo[Op_RegL + 1] = { // enum name
3847 0, // Op_Node
3848 0, // Op_Set
3849 R0_num, // Op_RegN
3850 R0_num, // Op_RegI
3851 R0_num, // Op_RegP
3852 V0_num, // Op_RegF
3853 V0_num, // Op_RegD
3854 R0_num // Op_RegL
3855 };
3856
3857 static const int hi[Op_RegL + 1] = { // enum name
3858 0, // Op_Node
3859 0, // Op_Set
3860 OptoReg::Bad, // Op_RegN
3861 OptoReg::Bad, // Op_RegI
3862 R0_H_num, // Op_RegP
3863 OptoReg::Bad, // Op_RegF
3864 V0_H_num, // Op_RegD
3865 R0_H_num // Op_RegL
3866 };
3867
3868 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3869 %}
3870 %}
3871
3872 //----------ATTRIBUTES---------------------------------------------------------
3873 //----------Operand Attributes-------------------------------------------------
3874 op_attrib op_cost(1); // Required cost attribute
3875
3876 //----------Instruction Attributes---------------------------------------------
3877 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3878 ins_attrib ins_size(32); // Required size attribute (in bits)
3879 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3880 // a non-matching short branch variant
3881 // of some long branch?
3882 ins_attrib ins_alignment(4); // Required alignment attribute (must
3883 // be a power of 2) specifies the
3884 // alignment that some part of the
3885 // instruction (not necessarily the
3886 // start) requires. If > 1, a
3887 // compute_padding() function must be
3888 // provided for the instruction
3889
3890 //----------OPERANDS-----------------------------------------------------------
3891 // Operand definitions must precede instruction definitions for correct parsing
3892 // in the ADLC because operands constitute user defined types which are used in
3893 // instruction definitions.
3894
3895 //----------Simple Operands----------------------------------------------------
3896
3897 // Integer operands 32 bit
3898 // 32 bit immediate
3899 operand immI()
3900 %{
3901 match(ConI);
3902
3903 op_cost(0);
3904 format %{ %}
3905 interface(CONST_INTER);
3906 %}
3907
3908 // 32 bit zero
3909 operand immI0()
3910 %{
3911 predicate(n->get_int() == 0);
3912 match(ConI);
3913
3914 op_cost(0);
3915 format %{ %}
3916 interface(CONST_INTER);
3917 %}
3918
3919 // 32 bit unit increment
3920 operand immI_1()
3921 %{
3922 predicate(n->get_int() == 1);
3923 match(ConI);
3924
3925 op_cost(0);
3926 format %{ %}
3927 interface(CONST_INTER);
3928 %}
3929
3930 // 32 bit unit decrement
3931 operand immI_M1()
3932 %{
3933 predicate(n->get_int() == -1);
3934 match(ConI);
3935
3936 op_cost(0);
3937 format %{ %}
3938 interface(CONST_INTER);
3939 %}
3940
3941 // Shift values for add/sub extension shift
3942 operand immIExt()
3943 %{
3944 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3945 match(ConI);
3946
3947 op_cost(0);
3948 format %{ %}
3949 interface(CONST_INTER);
3950 %}
3951
3952 operand immI_gt_1()
3953 %{
3954 predicate(n->get_int() > 1);
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_le_4()
3963 %{
3964 predicate(n->get_int() <= 4);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_16()
3973 %{
3974 predicate(n->get_int() == 16);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_24()
3983 %{
3984 predicate(n->get_int() == 24);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_32()
3993 %{
3994 predicate(n->get_int() == 32);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_48()
4003 %{
4004 predicate(n->get_int() == 48);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_56()
4013 %{
4014 predicate(n->get_int() == 56);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_255()
4023 %{
4024 predicate(n->get_int() == 255);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_65535()
4033 %{
4034 predicate(n->get_int() == 65535);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_positive()
4043 %{
4044 predicate(n->get_int() > 0);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 // BoolTest condition for signed compare
4053 operand immI_cmp_cond()
4054 %{
4055 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4056 match(ConI);
4057
4058 op_cost(0);
4059 format %{ %}
4060 interface(CONST_INTER);
4061 %}
4062
4063 // BoolTest condition for unsigned compare
4064 operand immI_cmpU_cond()
4065 %{
4066 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4067 match(ConI);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immL_255()
4075 %{
4076 predicate(n->get_long() == 255L);
4077 match(ConL);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_65535()
4085 %{
4086 predicate(n->get_long() == 65535L);
4087 match(ConL);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 operand immL_4294967295()
4095 %{
4096 predicate(n->get_long() == 4294967295L);
4097 match(ConL);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immL_bitmask()
4105 %{
4106 predicate((n->get_long() != 0)
4107 && ((n->get_long() & 0xc000000000000000l) == 0)
4108 && is_power_of_2(n->get_long() + 1));
4109 match(ConL);
4110
4111 op_cost(0);
4112 format %{ %}
4113 interface(CONST_INTER);
4114 %}
4115
4116 operand immI_bitmask()
4117 %{
4118 predicate((n->get_int() != 0)
4119 && ((n->get_int() & 0xc0000000) == 0)
4120 && is_power_of_2(n->get_int() + 1));
4121 match(ConI);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immL_positive_bitmaskI()
4129 %{
4130 predicate((n->get_long() != 0)
4131 && ((julong)n->get_long() < 0x80000000ULL)
4132 && is_power_of_2(n->get_long() + 1));
4133 match(ConL);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 // Scale values for scaled offset addressing modes (up to long but not quad)
4141 operand immIScale()
4142 %{
4143 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4144 match(ConI);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 // 5 bit signed integer
4152 operand immI5()
4153 %{
4154 predicate(Assembler::is_simm(n->get_int(), 5));
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 // 7 bit unsigned integer
4163 operand immIU7()
4164 %{
4165 predicate(Assembler::is_uimm(n->get_int(), 7));
4166 match(ConI);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 // Offset for scaled or unscaled immediate loads and stores
4174 operand immIOffset()
4175 %{
4176 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immIOffset1()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset2()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset4()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset8()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset16()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immLOffset()
4235 %{
4236 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4237 match(ConL);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLoffset1()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset2()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset4()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset8()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset16()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 // 5 bit signed long integer
4295 operand immL5()
4296 %{
4297 predicate(Assembler::is_simm(n->get_long(), 5));
4298 match(ConL);
4299
4300 op_cost(0);
4301 format %{ %}
4302 interface(CONST_INTER);
4303 %}
4304
4305 // 7 bit unsigned long integer
4306 operand immLU7()
4307 %{
4308 predicate(Assembler::is_uimm(n->get_long(), 7));
4309 match(ConL);
4310
4311 op_cost(0);
4312 format %{ %}
4313 interface(CONST_INTER);
4314 %}
4315
4316 // 8 bit signed value.
4317 operand immI8()
4318 %{
4319 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4320 match(ConI);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 // 8 bit signed value (simm8), or #simm8 LSL 8.
4328 operand immI8_shift8()
4329 %{
4330 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4331 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4332 match(ConI);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 8 bit signed value (simm8), or #simm8 LSL 8.
4340 operand immL8_shift8()
4341 %{
4342 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4343 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4344 match(ConL);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 8 bit integer valid for vector add sub immediate
4352 operand immBAddSubV()
4353 %{
4354 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4355 match(ConI);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // 32 bit integer valid for add sub immediate
4363 operand immIAddSub()
4364 %{
4365 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4366 match(ConI);
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 // 32 bit integer valid for vector add sub immediate
4373 operand immIAddSubV()
4374 %{
4375 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4376 match(ConI);
4377
4378 op_cost(0);
4379 format %{ %}
4380 interface(CONST_INTER);
4381 %}
4382
4383 // 32 bit unsigned integer valid for logical immediate
4384
4385 operand immBLog()
4386 %{
4387 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4388 match(ConI);
4389
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 operand immSLog()
4396 %{
4397 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4398 match(ConI);
4399
4400 op_cost(0);
4401 format %{ %}
4402 interface(CONST_INTER);
4403 %}
4404
4405 operand immILog()
4406 %{
4407 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4408 match(ConI);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // Integer operands 64 bit
4416 // 64 bit immediate
4417 operand immL()
4418 %{
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 64 bit zero
4427 operand immL0()
4428 %{
4429 predicate(n->get_long() == 0);
4430 match(ConL);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 64 bit unit decrement
4438 operand immL_M1()
4439 %{
4440 predicate(n->get_long() == -1);
4441 match(ConL);
4442
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 // 64 bit integer valid for add sub immediate
4449 operand immLAddSub()
4450 %{
4451 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4452 match(ConL);
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 64 bit integer valid for addv subv immediate
4459 operand immLAddSubV()
4460 %{
4461 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4462 match(ConL);
4463
4464 op_cost(0);
4465 format %{ %}
4466 interface(CONST_INTER);
4467 %}
4468
4469 // 64 bit integer valid for logical immediate
4470 operand immLLog()
4471 %{
4472 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4473 match(ConL);
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 // Long Immediate: low 32-bit mask
4480 operand immL_32bits()
4481 %{
4482 predicate(n->get_long() == 0xFFFFFFFFL);
4483 match(ConL);
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // Pointer operands
4490 // Pointer Immediate
4491 operand immP()
4492 %{
4493 match(ConP);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 // nullptr Pointer Immediate
4501 operand immP0()
4502 %{
4503 predicate(n->get_ptr() == 0);
4504 match(ConP);
4505
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 // Pointer Immediate One
4512 // this is used in object initialization (initial object header)
4513 operand immP_1()
4514 %{
4515 predicate(n->get_ptr() == 1);
4516 match(ConP);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // Card Table Byte Map Base
4524 operand immByteMapBase()
4525 %{
4526 // Get base of card map
4527 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4528 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4529 match(ConP);
4530
4531 op_cost(0);
4532 format %{ %}
4533 interface(CONST_INTER);
4534 %}
4535
4536 // Float and Double operands
4537 // Double Immediate
4538 operand immD()
4539 %{
4540 match(ConD);
4541 op_cost(0);
4542 format %{ %}
4543 interface(CONST_INTER);
4544 %}
4545
4546 // Double Immediate: +0.0d
4547 operand immD0()
4548 %{
4549 predicate(jlong_cast(n->getd()) == 0);
4550 match(ConD);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // constant 'double +0.0'.
4558 operand immDPacked()
4559 %{
4560 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4561 match(ConD);
4562 op_cost(0);
4563 format %{ %}
4564 interface(CONST_INTER);
4565 %}
4566
4567 // Float Immediate
4568 operand immF()
4569 %{
4570 match(ConF);
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Float Immediate: +0.0f.
4577 operand immF0()
4578 %{
4579 predicate(jint_cast(n->getf()) == 0);
4580 match(ConF);
4581
4582 op_cost(0);
4583 format %{ %}
4584 interface(CONST_INTER);
4585 %}
4586
4587 //
4588 operand immFPacked()
4589 %{
4590 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4591 match(ConF);
4592 op_cost(0);
4593 format %{ %}
4594 interface(CONST_INTER);
4595 %}
4596
4597 // Narrow pointer operands
4598 // Narrow Pointer Immediate
4599 operand immN()
4600 %{
4601 match(ConN);
4602
4603 op_cost(0);
4604 format %{ %}
4605 interface(CONST_INTER);
4606 %}
4607
4608 // Narrow nullptr Pointer Immediate
4609 operand immN0()
4610 %{
4611 predicate(n->get_narrowcon() == 0);
4612 match(ConN);
4613
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 operand immNKlass()
4620 %{
4621 match(ConNKlass);
4622
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // Integer 32 bit Register Operands
4629 // Integer 32 bitRegister (excludes SP)
4630 operand iRegI()
4631 %{
4632 constraint(ALLOC_IN_RC(any_reg32));
4633 match(RegI);
4634 match(iRegINoSp);
4635 op_cost(0);
4636 format %{ %}
4637 interface(REG_INTER);
4638 %}
4639
4640 // Integer 32 bit Register not Special
4641 operand iRegINoSp()
4642 %{
4643 constraint(ALLOC_IN_RC(no_special_reg32));
4644 match(RegI);
4645 op_cost(0);
4646 format %{ %}
4647 interface(REG_INTER);
4648 %}
4649
4650 // Integer 64 bit Register Operands
4651 // Integer 64 bit Register (includes SP)
4652 operand iRegL()
4653 %{
4654 constraint(ALLOC_IN_RC(any_reg));
4655 match(RegL);
4656 match(iRegLNoSp);
4657 op_cost(0);
4658 format %{ %}
4659 interface(REG_INTER);
4660 %}
4661
4662 // Integer 64 bit Register not Special
4663 operand iRegLNoSp()
4664 %{
4665 constraint(ALLOC_IN_RC(no_special_reg));
4666 match(RegL);
4667 match(iRegL_R0);
4668 format %{ %}
4669 interface(REG_INTER);
4670 %}
4671
4672 // Pointer Register Operands
4673 // Pointer Register
4674 operand iRegP()
4675 %{
4676 constraint(ALLOC_IN_RC(ptr_reg));
4677 match(RegP);
4678 match(iRegPNoSp);
4679 match(iRegP_R0);
4680 //match(iRegP_R2);
4681 //match(iRegP_R4);
4682 match(iRegP_R5);
4683 match(thread_RegP);
4684 op_cost(0);
4685 format %{ %}
4686 interface(REG_INTER);
4687 %}
4688
4689 // Pointer 64 bit Register not Special
4690 operand iRegPNoSp()
4691 %{
4692 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4693 match(RegP);
4694 // match(iRegP);
4695 // match(iRegP_R0);
4696 // match(iRegP_R2);
4697 // match(iRegP_R4);
4698 // match(iRegP_R5);
4699 // match(thread_RegP);
4700 op_cost(0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // This operand is not allowed to use rfp even if
4706 // rfp is not used to hold the frame pointer.
4707 operand iRegPNoSpNoRfp()
4708 %{
4709 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4710 match(RegP);
4711 match(iRegPNoSp);
4712 op_cost(0);
4713 format %{ %}
4714 interface(REG_INTER);
4715 %}
4716
4717 // Pointer 64 bit Register R0 only
4718 operand iRegP_R0()
4719 %{
4720 constraint(ALLOC_IN_RC(r0_reg));
4721 match(RegP);
4722 // match(iRegP);
4723 match(iRegPNoSp);
4724 op_cost(0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // Pointer 64 bit Register R1 only
4730 operand iRegP_R1()
4731 %{
4732 constraint(ALLOC_IN_RC(r1_reg));
4733 match(RegP);
4734 // match(iRegP);
4735 match(iRegPNoSp);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer 64 bit Register R2 only
4742 operand iRegP_R2()
4743 %{
4744 constraint(ALLOC_IN_RC(r2_reg));
4745 match(RegP);
4746 // match(iRegP);
4747 match(iRegPNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Pointer 64 bit Register R3 only
4754 operand iRegP_R3()
4755 %{
4756 constraint(ALLOC_IN_RC(r3_reg));
4757 match(RegP);
4758 // match(iRegP);
4759 match(iRegPNoSp);
4760 op_cost(0);
4761 format %{ %}
4762 interface(REG_INTER);
4763 %}
4764
4765 // Pointer 64 bit Register R4 only
4766 operand iRegP_R4()
4767 %{
4768 constraint(ALLOC_IN_RC(r4_reg));
4769 match(RegP);
4770 // match(iRegP);
4771 match(iRegPNoSp);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 // Pointer 64 bit Register R5 only
4778 operand iRegP_R5()
4779 %{
4780 constraint(ALLOC_IN_RC(r5_reg));
4781 match(RegP);
4782 // match(iRegP);
4783 match(iRegPNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register R10 only
4790 operand iRegP_R10()
4791 %{
4792 constraint(ALLOC_IN_RC(r10_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 match(iRegPNoSp);
4796 op_cost(0);
4797 format %{ %}
4798 interface(REG_INTER);
4799 %}
4800
4801 // Long 64 bit Register R0 only
4802 operand iRegL_R0()
4803 %{
4804 constraint(ALLOC_IN_RC(r0_reg));
4805 match(RegL);
4806 match(iRegLNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Long 64 bit Register R11 only
4813 operand iRegL_R11()
4814 %{
4815 constraint(ALLOC_IN_RC(r11_reg));
4816 match(RegL);
4817 match(iRegLNoSp);
4818 op_cost(0);
4819 format %{ %}
4820 interface(REG_INTER);
4821 %}
4822
4823 // Register R0 only
4824 operand iRegI_R0()
4825 %{
4826 constraint(ALLOC_IN_RC(int_r0_reg));
4827 match(RegI);
4828 match(iRegINoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Register R2 only
4835 operand iRegI_R2()
4836 %{
4837 constraint(ALLOC_IN_RC(int_r2_reg));
4838 match(RegI);
4839 match(iRegINoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Register R3 only
4846 operand iRegI_R3()
4847 %{
4848 constraint(ALLOC_IN_RC(int_r3_reg));
4849 match(RegI);
4850 match(iRegINoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856
4857 // Register R4 only
4858 operand iRegI_R4()
4859 %{
4860 constraint(ALLOC_IN_RC(int_r4_reg));
4861 match(RegI);
4862 match(iRegINoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868
4869 // Pointer Register Operands
4870 // Narrow Pointer Register
4871 operand iRegN()
4872 %{
4873 constraint(ALLOC_IN_RC(any_reg32));
4874 match(RegN);
4875 match(iRegNNoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 // Integer 64 bit Register not Special
4882 operand iRegNNoSp()
4883 %{
4884 constraint(ALLOC_IN_RC(no_special_reg32));
4885 match(RegN);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891 // Float Register
4892 // Float register operands
4893 operand vRegF()
4894 %{
4895 constraint(ALLOC_IN_RC(float_reg));
4896 match(RegF);
4897
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Double Register
4904 // Double register operands
4905 operand vRegD()
4906 %{
4907 constraint(ALLOC_IN_RC(double_reg));
4908 match(RegD);
4909
4910 op_cost(0);
4911 format %{ %}
4912 interface(REG_INTER);
4913 %}
4914
4915 // Generic vector class. This will be used for
4916 // all vector operands, including NEON and SVE.
4917 operand vReg()
4918 %{
4919 constraint(ALLOC_IN_RC(dynamic));
4920 match(VecA);
4921 match(VecD);
4922 match(VecX);
4923
4924 op_cost(0);
4925 format %{ %}
4926 interface(REG_INTER);
4927 %}
4928
4929 operand vecA()
4930 %{
4931 constraint(ALLOC_IN_RC(vectora_reg));
4932 match(VecA);
4933
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 operand vecD()
4940 %{
4941 constraint(ALLOC_IN_RC(vectord_reg));
4942 match(VecD);
4943
4944 op_cost(0);
4945 format %{ %}
4946 interface(REG_INTER);
4947 %}
4948
4949 operand vecX()
4950 %{
4951 constraint(ALLOC_IN_RC(vectorx_reg));
4952 match(VecX);
4953
4954 op_cost(0);
4955 format %{ %}
4956 interface(REG_INTER);
4957 %}
4958
4959 operand vRegD_V0()
4960 %{
4961 constraint(ALLOC_IN_RC(v0_reg));
4962 match(RegD);
4963 op_cost(0);
4964 format %{ %}
4965 interface(REG_INTER);
4966 %}
4967
4968 operand vRegD_V1()
4969 %{
4970 constraint(ALLOC_IN_RC(v1_reg));
4971 match(RegD);
4972 op_cost(0);
4973 format %{ %}
4974 interface(REG_INTER);
4975 %}
4976
4977 operand vRegD_V2()
4978 %{
4979 constraint(ALLOC_IN_RC(v2_reg));
4980 match(RegD);
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 operand vRegD_V3()
4987 %{
4988 constraint(ALLOC_IN_RC(v3_reg));
4989 match(RegD);
4990 op_cost(0);
4991 format %{ %}
4992 interface(REG_INTER);
4993 %}
4994
4995 operand vRegD_V4()
4996 %{
4997 constraint(ALLOC_IN_RC(v4_reg));
4998 match(RegD);
4999 op_cost(0);
5000 format %{ %}
5001 interface(REG_INTER);
5002 %}
5003
5004 operand vRegD_V5()
5005 %{
5006 constraint(ALLOC_IN_RC(v5_reg));
5007 match(RegD);
5008 op_cost(0);
5009 format %{ %}
5010 interface(REG_INTER);
5011 %}
5012
5013 operand vRegD_V6()
5014 %{
5015 constraint(ALLOC_IN_RC(v6_reg));
5016 match(RegD);
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vRegD_V7()
5023 %{
5024 constraint(ALLOC_IN_RC(v7_reg));
5025 match(RegD);
5026 op_cost(0);
5027 format %{ %}
5028 interface(REG_INTER);
5029 %}
5030
5031 operand vRegD_V12()
5032 %{
5033 constraint(ALLOC_IN_RC(v12_reg));
5034 match(RegD);
5035 op_cost(0);
5036 format %{ %}
5037 interface(REG_INTER);
5038 %}
5039
5040 operand vRegD_V13()
5041 %{
5042 constraint(ALLOC_IN_RC(v13_reg));
5043 match(RegD);
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 operand pReg()
5050 %{
5051 constraint(ALLOC_IN_RC(pr_reg));
5052 match(RegVectMask);
5053 match(pRegGov);
5054 op_cost(0);
5055 format %{ %}
5056 interface(REG_INTER);
5057 %}
5058
5059 operand pRegGov()
5060 %{
5061 constraint(ALLOC_IN_RC(gov_pr));
5062 match(RegVectMask);
5063 match(pReg);
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand pRegGov_P0()
5070 %{
5071 constraint(ALLOC_IN_RC(p0_reg));
5072 match(RegVectMask);
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand pRegGov_P1()
5079 %{
5080 constraint(ALLOC_IN_RC(p1_reg));
5081 match(RegVectMask);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 // Flags register, used as output of signed compare instructions
5088
5089 // note that on AArch64 we also use this register as the output for
5090 // for floating point compare instructions (CmpF CmpD). this ensures
5091 // that ordered inequality tests use GT, GE, LT or LE none of which
5092 // pass through cases where the result is unordered i.e. one or both
5093 // inputs to the compare is a NaN. this means that the ideal code can
5094 // replace e.g. a GT with an LE and not end up capturing the NaN case
5095 // (where the comparison should always fail). EQ and NE tests are
5096 // always generated in ideal code so that unordered folds into the NE
5097 // case, matching the behaviour of AArch64 NE.
5098 //
5099 // This differs from x86 where the outputs of FP compares use a
5100 // special FP flags registers and where compares based on this
5101 // register are distinguished into ordered inequalities (cmpOpUCF) and
5102 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5103 // to explicitly handle the unordered case in branches. x86 also has
5104 // to include extra CMoveX rules to accept a cmpOpUCF input.
5105
5106 operand rFlagsReg()
5107 %{
5108 constraint(ALLOC_IN_RC(int_flags));
5109 match(RegFlags);
5110
5111 op_cost(0);
5112 format %{ "RFLAGS" %}
5113 interface(REG_INTER);
5114 %}
5115
5116 // Flags register, used as output of unsigned compare instructions
5117 operand rFlagsRegU()
5118 %{
5119 constraint(ALLOC_IN_RC(int_flags));
5120 match(RegFlags);
5121
5122 op_cost(0);
5123 format %{ "RFLAGSU" %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Special Registers
5128
5129 // Method Register
5130 operand inline_cache_RegP(iRegP reg)
5131 %{
5132 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5133 match(reg);
5134 match(iRegPNoSp);
5135 op_cost(0);
5136 format %{ %}
5137 interface(REG_INTER);
5138 %}
5139
5140 // Thread Register
5141 operand thread_RegP(iRegP reg)
5142 %{
5143 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5144 match(reg);
5145 op_cost(0);
5146 format %{ %}
5147 interface(REG_INTER);
5148 %}
5149
5150 //----------Memory Operands----------------------------------------------------
5151
5152 operand indirect(iRegP reg)
5153 %{
5154 constraint(ALLOC_IN_RC(ptr_reg));
5155 match(reg);
5156 op_cost(0);
5157 format %{ "[$reg]" %}
5158 interface(MEMORY_INTER) %{
5159 base($reg);
5160 index(0xffffffff);
5161 scale(0x0);
5162 disp(0x0);
5163 %}
5164 %}
5165
5166 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5167 %{
5168 constraint(ALLOC_IN_RC(ptr_reg));
5169 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5170 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5171 op_cost(0);
5172 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5173 interface(MEMORY_INTER) %{
5174 base($reg);
5175 index($ireg);
5176 scale($scale);
5177 disp(0x0);
5178 %}
5179 %}
5180
5181 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5182 %{
5183 constraint(ALLOC_IN_RC(ptr_reg));
5184 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5185 match(AddP reg (LShiftL lreg scale));
5186 op_cost(0);
5187 format %{ "$reg, $lreg lsl($scale)" %}
5188 interface(MEMORY_INTER) %{
5189 base($reg);
5190 index($lreg);
5191 scale($scale);
5192 disp(0x0);
5193 %}
5194 %}
5195
5196 operand indIndexI2L(iRegP reg, iRegI ireg)
5197 %{
5198 constraint(ALLOC_IN_RC(ptr_reg));
5199 match(AddP reg (ConvI2L ireg));
5200 op_cost(0);
5201 format %{ "$reg, $ireg, 0, I2L" %}
5202 interface(MEMORY_INTER) %{
5203 base($reg);
5204 index($ireg);
5205 scale(0x0);
5206 disp(0x0);
5207 %}
5208 %}
5209
5210 operand indIndex(iRegP reg, iRegL lreg)
5211 %{
5212 constraint(ALLOC_IN_RC(ptr_reg));
5213 match(AddP reg lreg);
5214 op_cost(0);
5215 format %{ "$reg, $lreg" %}
5216 interface(MEMORY_INTER) %{
5217 base($reg);
5218 index($lreg);
5219 scale(0x0);
5220 disp(0x0);
5221 %}
5222 %}
5223
5224 operand indOffI1(iRegP reg, immIOffset1 off)
5225 %{
5226 constraint(ALLOC_IN_RC(ptr_reg));
5227 match(AddP reg off);
5228 op_cost(0);
5229 format %{ "[$reg, $off]" %}
5230 interface(MEMORY_INTER) %{
5231 base($reg);
5232 index(0xffffffff);
5233 scale(0x0);
5234 disp($off);
5235 %}
5236 %}
5237
5238 operand indOffI2(iRegP reg, immIOffset2 off)
5239 %{
5240 constraint(ALLOC_IN_RC(ptr_reg));
5241 match(AddP reg off);
5242 op_cost(0);
5243 format %{ "[$reg, $off]" %}
5244 interface(MEMORY_INTER) %{
5245 base($reg);
5246 index(0xffffffff);
5247 scale(0x0);
5248 disp($off);
5249 %}
5250 %}
5251
5252 operand indOffI4(iRegP reg, immIOffset4 off)
5253 %{
5254 constraint(ALLOC_IN_RC(ptr_reg));
5255 match(AddP reg off);
5256 op_cost(0);
5257 format %{ "[$reg, $off]" %}
5258 interface(MEMORY_INTER) %{
5259 base($reg);
5260 index(0xffffffff);
5261 scale(0x0);
5262 disp($off);
5263 %}
5264 %}
5265
5266 operand indOffI8(iRegP reg, immIOffset8 off)
5267 %{
5268 constraint(ALLOC_IN_RC(ptr_reg));
5269 match(AddP reg off);
5270 op_cost(0);
5271 format %{ "[$reg, $off]" %}
5272 interface(MEMORY_INTER) %{
5273 base($reg);
5274 index(0xffffffff);
5275 scale(0x0);
5276 disp($off);
5277 %}
5278 %}
5279
5280 operand indOffI16(iRegP reg, immIOffset16 off)
5281 %{
5282 constraint(ALLOC_IN_RC(ptr_reg));
5283 match(AddP reg off);
5284 op_cost(0);
5285 format %{ "[$reg, $off]" %}
5286 interface(MEMORY_INTER) %{
5287 base($reg);
5288 index(0xffffffff);
5289 scale(0x0);
5290 disp($off);
5291 %}
5292 %}
5293
5294 operand indOffL1(iRegP reg, immLoffset1 off)
5295 %{
5296 constraint(ALLOC_IN_RC(ptr_reg));
5297 match(AddP reg off);
5298 op_cost(0);
5299 format %{ "[$reg, $off]" %}
5300 interface(MEMORY_INTER) %{
5301 base($reg);
5302 index(0xffffffff);
5303 scale(0x0);
5304 disp($off);
5305 %}
5306 %}
5307
5308 operand indOffL2(iRegP reg, immLoffset2 off)
5309 %{
5310 constraint(ALLOC_IN_RC(ptr_reg));
5311 match(AddP reg off);
5312 op_cost(0);
5313 format %{ "[$reg, $off]" %}
5314 interface(MEMORY_INTER) %{
5315 base($reg);
5316 index(0xffffffff);
5317 scale(0x0);
5318 disp($off);
5319 %}
5320 %}
5321
5322 operand indOffL4(iRegP reg, immLoffset4 off)
5323 %{
5324 constraint(ALLOC_IN_RC(ptr_reg));
5325 match(AddP reg off);
5326 op_cost(0);
5327 format %{ "[$reg, $off]" %}
5328 interface(MEMORY_INTER) %{
5329 base($reg);
5330 index(0xffffffff);
5331 scale(0x0);
5332 disp($off);
5333 %}
5334 %}
5335
5336 operand indOffL8(iRegP reg, immLoffset8 off)
5337 %{
5338 constraint(ALLOC_IN_RC(ptr_reg));
5339 match(AddP reg off);
5340 op_cost(0);
5341 format %{ "[$reg, $off]" %}
5342 interface(MEMORY_INTER) %{
5343 base($reg);
5344 index(0xffffffff);
5345 scale(0x0);
5346 disp($off);
5347 %}
5348 %}
5349
5350 operand indOffL16(iRegP reg, immLoffset16 off)
5351 %{
5352 constraint(ALLOC_IN_RC(ptr_reg));
5353 match(AddP reg off);
5354 op_cost(0);
5355 format %{ "[$reg, $off]" %}
5356 interface(MEMORY_INTER) %{
5357 base($reg);
5358 index(0xffffffff);
5359 scale(0x0);
5360 disp($off);
5361 %}
5362 %}
5363
5364 operand indirectX2P(iRegL reg)
5365 %{
5366 constraint(ALLOC_IN_RC(ptr_reg));
5367 match(CastX2P reg);
5368 op_cost(0);
5369 format %{ "[$reg]\t# long -> ptr" %}
5370 interface(MEMORY_INTER) %{
5371 base($reg);
5372 index(0xffffffff);
5373 scale(0x0);
5374 disp(0x0);
5375 %}
5376 %}
5377
5378 operand indOffX2P(iRegL reg, immLOffset off)
5379 %{
5380 constraint(ALLOC_IN_RC(ptr_reg));
5381 match(AddP (CastX2P reg) off);
5382 op_cost(0);
5383 format %{ "[$reg, $off]\t# long -> ptr" %}
5384 interface(MEMORY_INTER) %{
5385 base($reg);
5386 index(0xffffffff);
5387 scale(0x0);
5388 disp($off);
5389 %}
5390 %}
5391
5392 operand indirectN(iRegN reg)
5393 %{
5394 predicate(CompressedOops::shift() == 0);
5395 constraint(ALLOC_IN_RC(ptr_reg));
5396 match(DecodeN reg);
5397 op_cost(0);
5398 format %{ "[$reg]\t# narrow" %}
5399 interface(MEMORY_INTER) %{
5400 base($reg);
5401 index(0xffffffff);
5402 scale(0x0);
5403 disp(0x0);
5404 %}
5405 %}
5406
5407 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5408 %{
5409 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5410 constraint(ALLOC_IN_RC(ptr_reg));
5411 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5412 op_cost(0);
5413 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5414 interface(MEMORY_INTER) %{
5415 base($reg);
5416 index($ireg);
5417 scale($scale);
5418 disp(0x0);
5419 %}
5420 %}
5421
5422 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5423 %{
5424 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP (DecodeN reg) (LShiftL lreg scale));
5427 op_cost(0);
5428 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index($lreg);
5432 scale($scale);
5433 disp(0x0);
5434 %}
5435 %}
5436
5437 operand indIndexI2LN(iRegN reg, iRegI ireg)
5438 %{
5439 predicate(CompressedOops::shift() == 0);
5440 constraint(ALLOC_IN_RC(ptr_reg));
5441 match(AddP (DecodeN reg) (ConvI2L ireg));
5442 op_cost(0);
5443 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5444 interface(MEMORY_INTER) %{
5445 base($reg);
5446 index($ireg);
5447 scale(0x0);
5448 disp(0x0);
5449 %}
5450 %}
5451
5452 operand indIndexN(iRegN reg, iRegL lreg)
5453 %{
5454 predicate(CompressedOops::shift() == 0);
5455 constraint(ALLOC_IN_RC(ptr_reg));
5456 match(AddP (DecodeN reg) lreg);
5457 op_cost(0);
5458 format %{ "$reg, $lreg\t# narrow" %}
5459 interface(MEMORY_INTER) %{
5460 base($reg);
5461 index($lreg);
5462 scale(0x0);
5463 disp(0x0);
5464 %}
5465 %}
5466
5467 operand indOffIN(iRegN reg, immIOffset off)
5468 %{
5469 predicate(CompressedOops::shift() == 0);
5470 constraint(ALLOC_IN_RC(ptr_reg));
5471 match(AddP (DecodeN reg) off);
5472 op_cost(0);
5473 format %{ "[$reg, $off]\t# narrow" %}
5474 interface(MEMORY_INTER) %{
5475 base($reg);
5476 index(0xffffffff);
5477 scale(0x0);
5478 disp($off);
5479 %}
5480 %}
5481
5482 operand indOffLN(iRegN reg, immLOffset off)
5483 %{
5484 predicate(CompressedOops::shift() == 0);
5485 constraint(ALLOC_IN_RC(ptr_reg));
5486 match(AddP (DecodeN reg) off);
5487 op_cost(0);
5488 format %{ "[$reg, $off]\t# narrow" %}
5489 interface(MEMORY_INTER) %{
5490 base($reg);
5491 index(0xffffffff);
5492 scale(0x0);
5493 disp($off);
5494 %}
5495 %}
5496
5497
5498 //----------Special Memory Operands--------------------------------------------
5499 // Stack Slot Operand - This operand is used for loading and storing temporary
5500 // values on the stack where a match requires a value to
5501 // flow through memory.
5502 operand stackSlotP(sRegP reg)
5503 %{
5504 constraint(ALLOC_IN_RC(stack_slots));
5505 op_cost(100);
5506 // No match rule because this operand is only generated in matching
5507 // match(RegP);
5508 format %{ "[$reg]" %}
5509 interface(MEMORY_INTER) %{
5510 base(0x1e); // RSP
5511 index(0x0); // No Index
5512 scale(0x0); // No Scale
5513 disp($reg); // Stack Offset
5514 %}
5515 %}
5516
5517 operand stackSlotI(sRegI reg)
5518 %{
5519 constraint(ALLOC_IN_RC(stack_slots));
5520 // No match rule because this operand is only generated in matching
5521 // match(RegI);
5522 format %{ "[$reg]" %}
5523 interface(MEMORY_INTER) %{
5524 base(0x1e); // RSP
5525 index(0x0); // No Index
5526 scale(0x0); // No Scale
5527 disp($reg); // Stack Offset
5528 %}
5529 %}
5530
5531 operand stackSlotF(sRegF reg)
5532 %{
5533 constraint(ALLOC_IN_RC(stack_slots));
5534 // No match rule because this operand is only generated in matching
5535 // match(RegF);
5536 format %{ "[$reg]" %}
5537 interface(MEMORY_INTER) %{
5538 base(0x1e); // RSP
5539 index(0x0); // No Index
5540 scale(0x0); // No Scale
5541 disp($reg); // Stack Offset
5542 %}
5543 %}
5544
5545 operand stackSlotD(sRegD reg)
5546 %{
5547 constraint(ALLOC_IN_RC(stack_slots));
5548 // No match rule because this operand is only generated in matching
5549 // match(RegD);
5550 format %{ "[$reg]" %}
5551 interface(MEMORY_INTER) %{
5552 base(0x1e); // RSP
5553 index(0x0); // No Index
5554 scale(0x0); // No Scale
5555 disp($reg); // Stack Offset
5556 %}
5557 %}
5558
5559 operand stackSlotL(sRegL reg)
5560 %{
5561 constraint(ALLOC_IN_RC(stack_slots));
5562 // No match rule because this operand is only generated in matching
5563 // match(RegL);
5564 format %{ "[$reg]" %}
5565 interface(MEMORY_INTER) %{
5566 base(0x1e); // RSP
5567 index(0x0); // No Index
5568 scale(0x0); // No Scale
5569 disp($reg); // Stack Offset
5570 %}
5571 %}
5572
5573 // Operands for expressing Control Flow
5574 // NOTE: Label is a predefined operand which should not be redefined in
5575 // the AD file. It is generically handled within the ADLC.
5576
5577 //----------Conditional Branch Operands----------------------------------------
5578 // Comparison Op - This is the operation of the comparison, and is limited to
5579 // the following set of codes:
5580 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5581 //
5582 // Other attributes of the comparison, such as unsignedness, are specified
5583 // by the comparison instruction that sets a condition code flags register.
5584 // That result is represented by a flags operand whose subtype is appropriate
5585 // to the unsignedness (etc.) of the comparison.
5586 //
5587 // Later, the instruction which matches both the Comparison Op (a Bool) and
5588 // the flags (produced by the Cmp) specifies the coding of the comparison op
5589 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5590
5591 // used for signed integral comparisons and fp comparisons
5592
5593 operand cmpOp()
5594 %{
5595 match(Bool);
5596
5597 format %{ "" %}
5598 interface(COND_INTER) %{
5599 equal(0x0, "eq");
5600 not_equal(0x1, "ne");
5601 less(0xb, "lt");
5602 greater_equal(0xa, "ge");
5603 less_equal(0xd, "le");
5604 greater(0xc, "gt");
5605 overflow(0x6, "vs");
5606 no_overflow(0x7, "vc");
5607 %}
5608 %}
5609
5610 // used for unsigned integral comparisons
5611
5612 operand cmpOpU()
5613 %{
5614 match(Bool);
5615
5616 format %{ "" %}
5617 interface(COND_INTER) %{
5618 equal(0x0, "eq");
5619 not_equal(0x1, "ne");
5620 less(0x3, "lo");
5621 greater_equal(0x2, "hs");
5622 less_equal(0x9, "ls");
5623 greater(0x8, "hi");
5624 overflow(0x6, "vs");
5625 no_overflow(0x7, "vc");
5626 %}
5627 %}
5628
5629 // used for certain integral comparisons which can be
5630 // converted to cbxx or tbxx instructions
5631
5632 operand cmpOpEqNe()
5633 %{
5634 match(Bool);
5635 op_cost(0);
5636 predicate(n->as_Bool()->_test._test == BoolTest::ne
5637 || n->as_Bool()->_test._test == BoolTest::eq);
5638
5639 format %{ "" %}
5640 interface(COND_INTER) %{
5641 equal(0x0, "eq");
5642 not_equal(0x1, "ne");
5643 less(0xb, "lt");
5644 greater_equal(0xa, "ge");
5645 less_equal(0xd, "le");
5646 greater(0xc, "gt");
5647 overflow(0x6, "vs");
5648 no_overflow(0x7, "vc");
5649 %}
5650 %}
5651
5652 // used for certain integral comparisons which can be
5653 // converted to cbxx or tbxx instructions
5654
5655 operand cmpOpLtGe()
5656 %{
5657 match(Bool);
5658 op_cost(0);
5659
5660 predicate(n->as_Bool()->_test._test == BoolTest::lt
5661 || n->as_Bool()->_test._test == BoolTest::ge);
5662
5663 format %{ "" %}
5664 interface(COND_INTER) %{
5665 equal(0x0, "eq");
5666 not_equal(0x1, "ne");
5667 less(0xb, "lt");
5668 greater_equal(0xa, "ge");
5669 less_equal(0xd, "le");
5670 greater(0xc, "gt");
5671 overflow(0x6, "vs");
5672 no_overflow(0x7, "vc");
5673 %}
5674 %}
5675
5676 // used for certain unsigned integral comparisons which can be
5677 // converted to cbxx or tbxx instructions
5678
5679 operand cmpOpUEqNeLeGt()
5680 %{
5681 match(Bool);
5682 op_cost(0);
5683
5684 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5685 n->as_Bool()->_test._test == BoolTest::ne ||
5686 n->as_Bool()->_test._test == BoolTest::le ||
5687 n->as_Bool()->_test._test == BoolTest::gt);
5688
5689 format %{ "" %}
5690 interface(COND_INTER) %{
5691 equal(0x0, "eq");
5692 not_equal(0x1, "ne");
5693 less(0x3, "lo");
5694 greater_equal(0x2, "hs");
5695 less_equal(0x9, "ls");
5696 greater(0x8, "hi");
5697 overflow(0x6, "vs");
5698 no_overflow(0x7, "vc");
5699 %}
5700 %}
5701
5702 // Special operand allowing long args to int ops to be truncated for free
5703
5704 operand iRegL2I(iRegL reg) %{
5705
5706 op_cost(0);
5707
5708 match(ConvL2I reg);
5709
5710 format %{ "l2i($reg)" %}
5711
5712 interface(REG_INTER)
5713 %}
5714
5715 operand iRegL2P(iRegL reg) %{
5716
5717 op_cost(0);
5718
5719 match(CastX2P reg);
5720
5721 format %{ "l2p($reg)" %}
5722
5723 interface(REG_INTER)
5724 %}
5725
5726 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5727 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5728 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5729 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5730
5731 //----------OPERAND CLASSES----------------------------------------------------
5732 // Operand Classes are groups of operands that are used as to simplify
5733 // instruction definitions by not requiring the AD writer to specify
5734 // separate instructions for every form of operand when the
5735 // instruction accepts multiple operand types with the same basic
5736 // encoding and format. The classic case of this is memory operands.
5737
5738 // memory is used to define read/write location for load/store
5739 // instruction defs. we can turn a memory op into an Address
5740
5741 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5742 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5743
5744 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5745 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5746
5747 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5748 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5749
5750 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5751 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5752
5753 // All of the memory operands. For the pipeline description.
5754 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5755 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5756 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5757
5758
5759 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5760 // operations. it allows the src to be either an iRegI or a (ConvL2I
5761 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5762 // can be elided because the 32-bit instruction will just employ the
5763 // lower 32 bits anyway.
5764 //
5765 // n.b. this does not elide all L2I conversions. if the truncated
5766 // value is consumed by more than one operation then the ConvL2I
5767 // cannot be bundled into the consuming nodes so an l2i gets planted
5768 // (actually a movw $dst $src) and the downstream instructions consume
5769 // the result of the l2i as an iRegI input. That's a shame since the
5770 // movw is actually redundant but its not too costly.
5771
5772 opclass iRegIorL2I(iRegI, iRegL2I);
5773 opclass iRegPorL2P(iRegP, iRegL2P);
5774
5775 //----------PIPELINE-----------------------------------------------------------
5776 // Rules which define the behavior of the target architectures pipeline.
5777
5778 // For specific pipelines, eg A53, define the stages of that pipeline
5779 //pipe_desc(ISS, EX1, EX2, WR);
5780 #define ISS S0
5781 #define EX1 S1
5782 #define EX2 S2
5783 #define WR S3
5784
5785 // Integer ALU reg operation
5786 pipeline %{
5787
5788 attributes %{
5789 // ARM instructions are of fixed length
5790 fixed_size_instructions; // Fixed size instructions TODO does
5791 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5792 // ARM instructions come in 32-bit word units
5793 instruction_unit_size = 4; // An instruction is 4 bytes long
5794 instruction_fetch_unit_size = 64; // The processor fetches one line
5795 instruction_fetch_units = 1; // of 64 bytes
5796
5797 // List of nop instructions
5798 nops( MachNop );
5799 %}
5800
5801 // We don't use an actual pipeline model so don't care about resources
5802 // or description. we do use pipeline classes to introduce fixed
5803 // latencies
5804
5805 //----------RESOURCES----------------------------------------------------------
5806 // Resources are the functional units available to the machine
5807
5808 resources( INS0, INS1, INS01 = INS0 | INS1,
5809 ALU0, ALU1, ALU = ALU0 | ALU1,
5810 MAC,
5811 DIV,
5812 BRANCH,
5813 LDST,
5814 NEON_FP);
5815
5816 //----------PIPELINE DESCRIPTION-----------------------------------------------
5817 // Pipeline Description specifies the stages in the machine's pipeline
5818
5819 // Define the pipeline as a generic 6 stage pipeline
5820 pipe_desc(S0, S1, S2, S3, S4, S5);
5821
5822 //----------PIPELINE CLASSES---------------------------------------------------
5823 // Pipeline Classes describe the stages in which input and output are
5824 // referenced by the hardware pipeline.
5825
5826 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5827 %{
5828 single_instruction;
5829 src1 : S1(read);
5830 src2 : S2(read);
5831 dst : S5(write);
5832 INS01 : ISS;
5833 NEON_FP : S5;
5834 %}
5835
5836 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5837 %{
5838 single_instruction;
5839 src1 : S1(read);
5840 src2 : S2(read);
5841 dst : S5(write);
5842 INS01 : ISS;
5843 NEON_FP : S5;
5844 %}
5845
5846 pipe_class fp_uop_s(vRegF dst, vRegF src)
5847 %{
5848 single_instruction;
5849 src : S1(read);
5850 dst : S5(write);
5851 INS01 : ISS;
5852 NEON_FP : S5;
5853 %}
5854
5855 pipe_class fp_uop_d(vRegD dst, vRegD src)
5856 %{
5857 single_instruction;
5858 src : S1(read);
5859 dst : S5(write);
5860 INS01 : ISS;
5861 NEON_FP : S5;
5862 %}
5863
5864 pipe_class fp_d2f(vRegF dst, vRegD src)
5865 %{
5866 single_instruction;
5867 src : S1(read);
5868 dst : S5(write);
5869 INS01 : ISS;
5870 NEON_FP : S5;
5871 %}
5872
5873 pipe_class fp_f2d(vRegD dst, vRegF src)
5874 %{
5875 single_instruction;
5876 src : S1(read);
5877 dst : S5(write);
5878 INS01 : ISS;
5879 NEON_FP : S5;
5880 %}
5881
5882 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5883 %{
5884 single_instruction;
5885 src : S1(read);
5886 dst : S5(write);
5887 INS01 : ISS;
5888 NEON_FP : S5;
5889 %}
5890
5891 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5892 %{
5893 single_instruction;
5894 src : S1(read);
5895 dst : S5(write);
5896 INS01 : ISS;
5897 NEON_FP : S5;
5898 %}
5899
5900 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5901 %{
5902 single_instruction;
5903 src : S1(read);
5904 dst : S5(write);
5905 INS01 : ISS;
5906 NEON_FP : S5;
5907 %}
5908
5909 pipe_class fp_l2f(vRegF dst, iRegL src)
5910 %{
5911 single_instruction;
5912 src : S1(read);
5913 dst : S5(write);
5914 INS01 : ISS;
5915 NEON_FP : S5;
5916 %}
5917
5918 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5919 %{
5920 single_instruction;
5921 src : S1(read);
5922 dst : S5(write);
5923 INS01 : ISS;
5924 NEON_FP : S5;
5925 %}
5926
5927 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5928 %{
5929 single_instruction;
5930 src : S1(read);
5931 dst : S5(write);
5932 INS01 : ISS;
5933 NEON_FP : S5;
5934 %}
5935
5936 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5937 %{
5938 single_instruction;
5939 src : S1(read);
5940 dst : S5(write);
5941 INS01 : ISS;
5942 NEON_FP : S5;
5943 %}
5944
5945 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5946 %{
5947 single_instruction;
5948 src : S1(read);
5949 dst : S5(write);
5950 INS01 : ISS;
5951 NEON_FP : S5;
5952 %}
5953
5954 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5955 %{
5956 single_instruction;
5957 src1 : S1(read);
5958 src2 : S2(read);
5959 dst : S5(write);
5960 INS0 : ISS;
5961 NEON_FP : S5;
5962 %}
5963
5964 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5965 %{
5966 single_instruction;
5967 src1 : S1(read);
5968 src2 : S2(read);
5969 dst : S5(write);
5970 INS0 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5975 %{
5976 single_instruction;
5977 cr : S1(read);
5978 src1 : S1(read);
5979 src2 : S1(read);
5980 dst : S3(write);
5981 INS01 : ISS;
5982 NEON_FP : S3;
5983 %}
5984
5985 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5986 %{
5987 single_instruction;
5988 cr : S1(read);
5989 src1 : S1(read);
5990 src2 : S1(read);
5991 dst : S3(write);
5992 INS01 : ISS;
5993 NEON_FP : S3;
5994 %}
5995
5996 pipe_class fp_imm_s(vRegF dst)
5997 %{
5998 single_instruction;
5999 dst : S3(write);
6000 INS01 : ISS;
6001 NEON_FP : S3;
6002 %}
6003
6004 pipe_class fp_imm_d(vRegD dst)
6005 %{
6006 single_instruction;
6007 dst : S3(write);
6008 INS01 : ISS;
6009 NEON_FP : S3;
6010 %}
6011
6012 pipe_class fp_load_constant_s(vRegF dst)
6013 %{
6014 single_instruction;
6015 dst : S4(write);
6016 INS01 : ISS;
6017 NEON_FP : S4;
6018 %}
6019
6020 pipe_class fp_load_constant_d(vRegD dst)
6021 %{
6022 single_instruction;
6023 dst : S4(write);
6024 INS01 : ISS;
6025 NEON_FP : S4;
6026 %}
6027
6028 //------- Integer ALU operations --------------------------
6029
6030 // Integer ALU reg-reg operation
6031 // Operands needed in EX1, result generated in EX2
6032 // Eg. ADD x0, x1, x2
6033 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6034 %{
6035 single_instruction;
6036 dst : EX2(write);
6037 src1 : EX1(read);
6038 src2 : EX1(read);
6039 INS01 : ISS; // Dual issue as instruction 0 or 1
6040 ALU : EX2;
6041 %}
6042
6043 // Integer ALU reg-reg operation with constant shift
6044 // Shifted register must be available in LATE_ISS instead of EX1
6045 // Eg. ADD x0, x1, x2, LSL #2
6046 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6047 %{
6048 single_instruction;
6049 dst : EX2(write);
6050 src1 : EX1(read);
6051 src2 : ISS(read);
6052 INS01 : ISS;
6053 ALU : EX2;
6054 %}
6055
6056 // Integer ALU reg operation with constant shift
6057 // Eg. LSL x0, x1, #shift
6058 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6059 %{
6060 single_instruction;
6061 dst : EX2(write);
6062 src1 : ISS(read);
6063 INS01 : ISS;
6064 ALU : EX2;
6065 %}
6066
6067 // Integer ALU reg-reg operation with variable shift
6068 // Both operands must be available in LATE_ISS instead of EX1
6069 // Result is available in EX1 instead of EX2
6070 // Eg. LSLV x0, x1, x2
6071 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6072 %{
6073 single_instruction;
6074 dst : EX1(write);
6075 src1 : ISS(read);
6076 src2 : ISS(read);
6077 INS01 : ISS;
6078 ALU : EX1;
6079 %}
6080
6081 // Integer ALU reg-reg operation with extract
6082 // As for _vshift above, but result generated in EX2
6083 // Eg. EXTR x0, x1, x2, #N
6084 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6085 %{
6086 single_instruction;
6087 dst : EX2(write);
6088 src1 : ISS(read);
6089 src2 : ISS(read);
6090 INS1 : ISS; // Can only dual issue as Instruction 1
6091 ALU : EX1;
6092 %}
6093
6094 // Integer ALU reg operation
6095 // Eg. NEG x0, x1
6096 pipe_class ialu_reg(iRegI dst, iRegI src)
6097 %{
6098 single_instruction;
6099 dst : EX2(write);
6100 src : EX1(read);
6101 INS01 : ISS;
6102 ALU : EX2;
6103 %}
6104
6105 // Integer ALU reg mmediate operation
6106 // Eg. ADD x0, x1, #N
6107 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6108 %{
6109 single_instruction;
6110 dst : EX2(write);
6111 src1 : EX1(read);
6112 INS01 : ISS;
6113 ALU : EX2;
6114 %}
6115
6116 // Integer ALU immediate operation (no source operands)
6117 // Eg. MOV x0, #N
6118 pipe_class ialu_imm(iRegI dst)
6119 %{
6120 single_instruction;
6121 dst : EX1(write);
6122 INS01 : ISS;
6123 ALU : EX1;
6124 %}
6125
6126 //------- Compare operation -------------------------------
6127
6128 // Compare reg-reg
6129 // Eg. CMP x0, x1
6130 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6131 %{
6132 single_instruction;
6133 // fixed_latency(16);
6134 cr : EX2(write);
6135 op1 : EX1(read);
6136 op2 : EX1(read);
6137 INS01 : ISS;
6138 ALU : EX2;
6139 %}
6140
6141 // Compare reg-reg
6142 // Eg. CMP x0, #N
6143 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6144 %{
6145 single_instruction;
6146 // fixed_latency(16);
6147 cr : EX2(write);
6148 op1 : EX1(read);
6149 INS01 : ISS;
6150 ALU : EX2;
6151 %}
6152
6153 //------- Conditional instructions ------------------------
6154
6155 // Conditional no operands
6156 // Eg. CSINC x0, zr, zr, <cond>
6157 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6158 %{
6159 single_instruction;
6160 cr : EX1(read);
6161 dst : EX2(write);
6162 INS01 : ISS;
6163 ALU : EX2;
6164 %}
6165
6166 // Conditional 2 operand
6167 // EG. CSEL X0, X1, X2, <cond>
6168 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6169 %{
6170 single_instruction;
6171 cr : EX1(read);
6172 src1 : EX1(read);
6173 src2 : EX1(read);
6174 dst : EX2(write);
6175 INS01 : ISS;
6176 ALU : EX2;
6177 %}
6178
6179 // Conditional 2 operand
6180 // EG. CSEL X0, X1, X2, <cond>
6181 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6182 %{
6183 single_instruction;
6184 cr : EX1(read);
6185 src : EX1(read);
6186 dst : EX2(write);
6187 INS01 : ISS;
6188 ALU : EX2;
6189 %}
6190
6191 //------- Multiply pipeline operations --------------------
6192
6193 // Multiply reg-reg
6194 // Eg. MUL w0, w1, w2
6195 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6196 %{
6197 single_instruction;
6198 dst : WR(write);
6199 src1 : ISS(read);
6200 src2 : ISS(read);
6201 INS01 : ISS;
6202 MAC : WR;
6203 %}
6204
6205 // Multiply accumulate
6206 // Eg. MADD w0, w1, w2, w3
6207 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6208 %{
6209 single_instruction;
6210 dst : WR(write);
6211 src1 : ISS(read);
6212 src2 : ISS(read);
6213 src3 : ISS(read);
6214 INS01 : ISS;
6215 MAC : WR;
6216 %}
6217
6218 // Eg. MUL w0, w1, w2
6219 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6220 %{
6221 single_instruction;
6222 fixed_latency(3); // Maximum latency for 64 bit mul
6223 dst : WR(write);
6224 src1 : ISS(read);
6225 src2 : ISS(read);
6226 INS01 : ISS;
6227 MAC : WR;
6228 %}
6229
6230 // Multiply accumulate
6231 // Eg. MADD w0, w1, w2, w3
6232 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6233 %{
6234 single_instruction;
6235 fixed_latency(3); // Maximum latency for 64 bit mul
6236 dst : WR(write);
6237 src1 : ISS(read);
6238 src2 : ISS(read);
6239 src3 : ISS(read);
6240 INS01 : ISS;
6241 MAC : WR;
6242 %}
6243
6244 //------- Divide pipeline operations --------------------
6245
6246 // Eg. SDIV w0, w1, w2
6247 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6248 %{
6249 single_instruction;
6250 fixed_latency(8); // Maximum latency for 32 bit divide
6251 dst : WR(write);
6252 src1 : ISS(read);
6253 src2 : ISS(read);
6254 INS0 : ISS; // Can only dual issue as instruction 0
6255 DIV : WR;
6256 %}
6257
6258 // Eg. SDIV x0, x1, x2
6259 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6260 %{
6261 single_instruction;
6262 fixed_latency(16); // Maximum latency for 64 bit divide
6263 dst : WR(write);
6264 src1 : ISS(read);
6265 src2 : ISS(read);
6266 INS0 : ISS; // Can only dual issue as instruction 0
6267 DIV : WR;
6268 %}
6269
6270 //------- Load pipeline operations ------------------------
6271
6272 // Load - prefetch
6273 // Eg. PFRM <mem>
6274 pipe_class iload_prefetch(memory mem)
6275 %{
6276 single_instruction;
6277 mem : ISS(read);
6278 INS01 : ISS;
6279 LDST : WR;
6280 %}
6281
6282 // Load - reg, mem
6283 // Eg. LDR x0, <mem>
6284 pipe_class iload_reg_mem(iRegI dst, memory mem)
6285 %{
6286 single_instruction;
6287 dst : WR(write);
6288 mem : ISS(read);
6289 INS01 : ISS;
6290 LDST : WR;
6291 %}
6292
6293 // Load - reg, reg
6294 // Eg. LDR x0, [sp, x1]
6295 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6296 %{
6297 single_instruction;
6298 dst : WR(write);
6299 src : ISS(read);
6300 INS01 : ISS;
6301 LDST : WR;
6302 %}
6303
6304 //------- Store pipeline operations -----------------------
6305
6306 // Store - zr, mem
6307 // Eg. STR zr, <mem>
6308 pipe_class istore_mem(memory mem)
6309 %{
6310 single_instruction;
6311 mem : ISS(read);
6312 INS01 : ISS;
6313 LDST : WR;
6314 %}
6315
6316 // Store - reg, mem
6317 // Eg. STR x0, <mem>
6318 pipe_class istore_reg_mem(iRegI src, memory mem)
6319 %{
6320 single_instruction;
6321 mem : ISS(read);
6322 src : EX2(read);
6323 INS01 : ISS;
6324 LDST : WR;
6325 %}
6326
6327 // Store - reg, reg
6328 // Eg. STR x0, [sp, x1]
6329 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6330 %{
6331 single_instruction;
6332 dst : ISS(read);
6333 src : EX2(read);
6334 INS01 : ISS;
6335 LDST : WR;
6336 %}
6337
6338 //------- Store pipeline operations -----------------------
6339
6340 // Branch
6341 pipe_class pipe_branch()
6342 %{
6343 single_instruction;
6344 INS01 : ISS;
6345 BRANCH : EX1;
6346 %}
6347
6348 // Conditional branch
6349 pipe_class pipe_branch_cond(rFlagsReg cr)
6350 %{
6351 single_instruction;
6352 cr : EX1(read);
6353 INS01 : ISS;
6354 BRANCH : EX1;
6355 %}
6356
6357 // Compare & Branch
6358 // EG. CBZ/CBNZ
6359 pipe_class pipe_cmp_branch(iRegI op1)
6360 %{
6361 single_instruction;
6362 op1 : EX1(read);
6363 INS01 : ISS;
6364 BRANCH : EX1;
6365 %}
6366
6367 //------- Synchronisation operations ----------------------
6368
6369 // Any operation requiring serialization.
6370 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6371 pipe_class pipe_serial()
6372 %{
6373 single_instruction;
6374 force_serialization;
6375 fixed_latency(16);
6376 INS01 : ISS(2); // Cannot dual issue with any other instruction
6377 LDST : WR;
6378 %}
6379
6380 // Generic big/slow expanded idiom - also serialized
6381 pipe_class pipe_slow()
6382 %{
6383 instruction_count(10);
6384 multiple_bundles;
6385 force_serialization;
6386 fixed_latency(16);
6387 INS01 : ISS(2); // Cannot dual issue with any other instruction
6388 LDST : WR;
6389 %}
6390
6391 // Empty pipeline class
6392 pipe_class pipe_class_empty()
6393 %{
6394 single_instruction;
6395 fixed_latency(0);
6396 %}
6397
6398 // Default pipeline class.
6399 pipe_class pipe_class_default()
6400 %{
6401 single_instruction;
6402 fixed_latency(2);
6403 %}
6404
6405 // Pipeline class for compares.
6406 pipe_class pipe_class_compare()
6407 %{
6408 single_instruction;
6409 fixed_latency(16);
6410 %}
6411
6412 // Pipeline class for memory operations.
6413 pipe_class pipe_class_memory()
6414 %{
6415 single_instruction;
6416 fixed_latency(16);
6417 %}
6418
6419 // Pipeline class for call.
6420 pipe_class pipe_class_call()
6421 %{
6422 single_instruction;
6423 fixed_latency(100);
6424 %}
6425
6426 // Define the class for the Nop node.
6427 define %{
6428 MachNop = pipe_class_empty;
6429 %}
6430
6431 %}
6432 //----------INSTRUCTIONS-------------------------------------------------------
6433 //
6434 // match -- States which machine-independent subtree may be replaced
6435 // by this instruction.
6436 // ins_cost -- The estimated cost of this instruction is used by instruction
6437 // selection to identify a minimum cost tree of machine
6438 // instructions that matches a tree of machine-independent
6439 // instructions.
6440 // format -- A string providing the disassembly for this instruction.
6441 // The value of an instruction's operand may be inserted
6442 // by referring to it with a '$' prefix.
6443 // opcode -- Three instruction opcodes may be provided. These are referred
6444 // to within an encode class as $primary, $secondary, and $tertiary
6445 // rrspectively. The primary opcode is commonly used to
6446 // indicate the type of machine instruction, while secondary
6447 // and tertiary are often used for prefix options or addressing
6448 // modes.
6449 // ins_encode -- A list of encode classes with parameters. The encode class
6450 // name must have been defined in an 'enc_class' specification
6451 // in the encode section of the architecture description.
6452
6453 // ============================================================================
6454 // Memory (Load/Store) Instructions
6455
6456 // Load Instructions
6457
6458 // Load Byte (8 bit signed)
6459 instruct loadB(iRegINoSp dst, memory1 mem)
6460 %{
6461 match(Set dst (LoadB mem));
6462 predicate(!needs_acquiring_load(n));
6463
6464 ins_cost(4 * INSN_COST);
6465 format %{ "ldrsbw $dst, $mem\t# byte" %}
6466
6467 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6468
6469 ins_pipe(iload_reg_mem);
6470 %}
6471
6472 // Load Byte (8 bit signed) into long
6473 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6474 %{
6475 match(Set dst (ConvI2L (LoadB mem)));
6476 predicate(!needs_acquiring_load(n->in(1)));
6477
6478 ins_cost(4 * INSN_COST);
6479 format %{ "ldrsb $dst, $mem\t# byte" %}
6480
6481 ins_encode(aarch64_enc_ldrsb(dst, mem));
6482
6483 ins_pipe(iload_reg_mem);
6484 %}
6485
6486 // Load Byte (8 bit unsigned)
6487 instruct loadUB(iRegINoSp dst, memory1 mem)
6488 %{
6489 match(Set dst (LoadUB mem));
6490 predicate(!needs_acquiring_load(n));
6491
6492 ins_cost(4 * INSN_COST);
6493 format %{ "ldrbw $dst, $mem\t# byte" %}
6494
6495 ins_encode(aarch64_enc_ldrb(dst, mem));
6496
6497 ins_pipe(iload_reg_mem);
6498 %}
6499
6500 // Load Byte (8 bit unsigned) into long
6501 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6502 %{
6503 match(Set dst (ConvI2L (LoadUB mem)));
6504 predicate(!needs_acquiring_load(n->in(1)));
6505
6506 ins_cost(4 * INSN_COST);
6507 format %{ "ldrb $dst, $mem\t# byte" %}
6508
6509 ins_encode(aarch64_enc_ldrb(dst, mem));
6510
6511 ins_pipe(iload_reg_mem);
6512 %}
6513
6514 // Load Short (16 bit signed)
6515 instruct loadS(iRegINoSp dst, memory2 mem)
6516 %{
6517 match(Set dst (LoadS mem));
6518 predicate(!needs_acquiring_load(n));
6519
6520 ins_cost(4 * INSN_COST);
6521 format %{ "ldrshw $dst, $mem\t# short" %}
6522
6523 ins_encode(aarch64_enc_ldrshw(dst, mem));
6524
6525 ins_pipe(iload_reg_mem);
6526 %}
6527
6528 // Load Short (16 bit signed) into long
6529 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6530 %{
6531 match(Set dst (ConvI2L (LoadS mem)));
6532 predicate(!needs_acquiring_load(n->in(1)));
6533
6534 ins_cost(4 * INSN_COST);
6535 format %{ "ldrsh $dst, $mem\t# short" %}
6536
6537 ins_encode(aarch64_enc_ldrsh(dst, mem));
6538
6539 ins_pipe(iload_reg_mem);
6540 %}
6541
6542 // Load Char (16 bit unsigned)
6543 instruct loadUS(iRegINoSp dst, memory2 mem)
6544 %{
6545 match(Set dst (LoadUS mem));
6546 predicate(!needs_acquiring_load(n));
6547
6548 ins_cost(4 * INSN_COST);
6549 format %{ "ldrh $dst, $mem\t# short" %}
6550
6551 ins_encode(aarch64_enc_ldrh(dst, mem));
6552
6553 ins_pipe(iload_reg_mem);
6554 %}
6555
6556 // Load Short/Char (16 bit unsigned) into long
6557 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6558 %{
6559 match(Set dst (ConvI2L (LoadUS mem)));
6560 predicate(!needs_acquiring_load(n->in(1)));
6561
6562 ins_cost(4 * INSN_COST);
6563 format %{ "ldrh $dst, $mem\t# short" %}
6564
6565 ins_encode(aarch64_enc_ldrh(dst, mem));
6566
6567 ins_pipe(iload_reg_mem);
6568 %}
6569
6570 // Load Integer (32 bit signed)
6571 instruct loadI(iRegINoSp dst, memory4 mem)
6572 %{
6573 match(Set dst (LoadI mem));
6574 predicate(!needs_acquiring_load(n));
6575
6576 ins_cost(4 * INSN_COST);
6577 format %{ "ldrw $dst, $mem\t# int" %}
6578
6579 ins_encode(aarch64_enc_ldrw(dst, mem));
6580
6581 ins_pipe(iload_reg_mem);
6582 %}
6583
6584 // Load Integer (32 bit signed) into long
6585 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6586 %{
6587 match(Set dst (ConvI2L (LoadI mem)));
6588 predicate(!needs_acquiring_load(n->in(1)));
6589
6590 ins_cost(4 * INSN_COST);
6591 format %{ "ldrsw $dst, $mem\t# int" %}
6592
6593 ins_encode(aarch64_enc_ldrsw(dst, mem));
6594
6595 ins_pipe(iload_reg_mem);
6596 %}
6597
6598 // Load Integer (32 bit unsigned) into long
6599 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6600 %{
6601 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6602 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6603
6604 ins_cost(4 * INSN_COST);
6605 format %{ "ldrw $dst, $mem\t# int" %}
6606
6607 ins_encode(aarch64_enc_ldrw(dst, mem));
6608
6609 ins_pipe(iload_reg_mem);
6610 %}
6611
6612 // Load Long (64 bit signed)
6613 instruct loadL(iRegLNoSp dst, memory8 mem)
6614 %{
6615 match(Set dst (LoadL mem));
6616 predicate(!needs_acquiring_load(n));
6617
6618 ins_cost(4 * INSN_COST);
6619 format %{ "ldr $dst, $mem\t# int" %}
6620
6621 ins_encode(aarch64_enc_ldr(dst, mem));
6622
6623 ins_pipe(iload_reg_mem);
6624 %}
6625
6626 // Load Range
6627 instruct loadRange(iRegINoSp dst, memory4 mem)
6628 %{
6629 match(Set dst (LoadRange mem));
6630
6631 ins_cost(4 * INSN_COST);
6632 format %{ "ldrw $dst, $mem\t# range" %}
6633
6634 ins_encode(aarch64_enc_ldrw(dst, mem));
6635
6636 ins_pipe(iload_reg_mem);
6637 %}
6638
6639 // Load Pointer
6640 instruct loadP(iRegPNoSp dst, memory8 mem)
6641 %{
6642 match(Set dst (LoadP mem));
6643 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6644
6645 ins_cost(4 * INSN_COST);
6646 format %{ "ldr $dst, $mem\t# ptr" %}
6647
6648 ins_encode(aarch64_enc_ldr(dst, mem));
6649
6650 ins_pipe(iload_reg_mem);
6651 %}
6652
6653 // Load Compressed Pointer
6654 instruct loadN(iRegNNoSp dst, memory4 mem)
6655 %{
6656 match(Set dst (LoadN mem));
6657 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6658
6659 ins_cost(4 * INSN_COST);
6660 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6661
6662 ins_encode(aarch64_enc_ldrw(dst, mem));
6663
6664 ins_pipe(iload_reg_mem);
6665 %}
6666
6667 // Load Klass Pointer
6668 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6669 %{
6670 match(Set dst (LoadKlass mem));
6671 predicate(!needs_acquiring_load(n));
6672
6673 ins_cost(4 * INSN_COST);
6674 format %{ "ldr $dst, $mem\t# class" %}
6675
6676 ins_encode(aarch64_enc_ldr(dst, mem));
6677
6678 ins_pipe(iload_reg_mem);
6679 %}
6680
6681 // Load Narrow Klass Pointer
6682 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6683 %{
6684 match(Set dst (LoadNKlass mem));
6685 predicate(!needs_acquiring_load(n));
6686
6687 ins_cost(4 * INSN_COST);
6688 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6689
6690 ins_encode(aarch64_enc_ldrw(dst, mem));
6691
6692 ins_pipe(iload_reg_mem);
6693 %}
6694
6695 // Load Float
6696 instruct loadF(vRegF dst, memory4 mem)
6697 %{
6698 match(Set dst (LoadF mem));
6699 predicate(!needs_acquiring_load(n));
6700
6701 ins_cost(4 * INSN_COST);
6702 format %{ "ldrs $dst, $mem\t# float" %}
6703
6704 ins_encode( aarch64_enc_ldrs(dst, mem) );
6705
6706 ins_pipe(pipe_class_memory);
6707 %}
6708
6709 // Load Double
6710 instruct loadD(vRegD dst, memory8 mem)
6711 %{
6712 match(Set dst (LoadD mem));
6713 predicate(!needs_acquiring_load(n));
6714
6715 ins_cost(4 * INSN_COST);
6716 format %{ "ldrd $dst, $mem\t# double" %}
6717
6718 ins_encode( aarch64_enc_ldrd(dst, mem) );
6719
6720 ins_pipe(pipe_class_memory);
6721 %}
6722
6723
6724 // Load Int Constant
6725 instruct loadConI(iRegINoSp dst, immI src)
6726 %{
6727 match(Set dst src);
6728
6729 ins_cost(INSN_COST);
6730 format %{ "mov $dst, $src\t# int" %}
6731
6732 ins_encode( aarch64_enc_movw_imm(dst, src) );
6733
6734 ins_pipe(ialu_imm);
6735 %}
6736
6737 // Load Long Constant
6738 instruct loadConL(iRegLNoSp dst, immL src)
6739 %{
6740 match(Set dst src);
6741
6742 ins_cost(INSN_COST);
6743 format %{ "mov $dst, $src\t# long" %}
6744
6745 ins_encode( aarch64_enc_mov_imm(dst, src) );
6746
6747 ins_pipe(ialu_imm);
6748 %}
6749
6750 // Load Pointer Constant
6751
6752 instruct loadConP(iRegPNoSp dst, immP con)
6753 %{
6754 match(Set dst con);
6755
6756 ins_cost(INSN_COST * 4);
6757 format %{
6758 "mov $dst, $con\t# ptr\n\t"
6759 %}
6760
6761 ins_encode(aarch64_enc_mov_p(dst, con));
6762
6763 ins_pipe(ialu_imm);
6764 %}
6765
6766 // Load Null Pointer Constant
6767
6768 instruct loadConP0(iRegPNoSp dst, immP0 con)
6769 %{
6770 match(Set dst con);
6771
6772 ins_cost(INSN_COST);
6773 format %{ "mov $dst, $con\t# nullptr ptr" %}
6774
6775 ins_encode(aarch64_enc_mov_p0(dst, con));
6776
6777 ins_pipe(ialu_imm);
6778 %}
6779
6780 // Load Pointer Constant One
6781
6782 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6783 %{
6784 match(Set dst con);
6785
6786 ins_cost(INSN_COST);
6787 format %{ "mov $dst, $con\t# nullptr ptr" %}
6788
6789 ins_encode(aarch64_enc_mov_p1(dst, con));
6790
6791 ins_pipe(ialu_imm);
6792 %}
6793
6794 // Load Byte Map Base Constant
6795
6796 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6797 %{
6798 match(Set dst con);
6799
6800 ins_cost(INSN_COST);
6801 format %{ "adr $dst, $con\t# Byte Map Base" %}
6802
6803 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6804
6805 ins_pipe(ialu_imm);
6806 %}
6807
6808 // Load Narrow Pointer Constant
6809
6810 instruct loadConN(iRegNNoSp dst, immN con)
6811 %{
6812 match(Set dst con);
6813
6814 ins_cost(INSN_COST * 4);
6815 format %{ "mov $dst, $con\t# compressed ptr" %}
6816
6817 ins_encode(aarch64_enc_mov_n(dst, con));
6818
6819 ins_pipe(ialu_imm);
6820 %}
6821
6822 // Load Narrow Null Pointer Constant
6823
6824 instruct loadConN0(iRegNNoSp dst, immN0 con)
6825 %{
6826 match(Set dst con);
6827
6828 ins_cost(INSN_COST);
6829 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6830
6831 ins_encode(aarch64_enc_mov_n0(dst, con));
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 // Load Narrow Klass Constant
6837
6838 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6839 %{
6840 match(Set dst con);
6841
6842 ins_cost(INSN_COST);
6843 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6844
6845 ins_encode(aarch64_enc_mov_nk(dst, con));
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Packed Float Constant
6851
6852 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6853 match(Set dst con);
6854 ins_cost(INSN_COST * 4);
6855 format %{ "fmovs $dst, $con"%}
6856 ins_encode %{
6857 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6858 %}
6859
6860 ins_pipe(fp_imm_s);
6861 %}
6862
6863 // Load Float Constant
6864
6865 instruct loadConF(vRegF dst, immF con) %{
6866 match(Set dst con);
6867
6868 ins_cost(INSN_COST * 4);
6869
6870 format %{
6871 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6872 %}
6873
6874 ins_encode %{
6875 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6876 %}
6877
6878 ins_pipe(fp_load_constant_s);
6879 %}
6880
6881 // Load Packed Double Constant
6882
6883 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6884 match(Set dst con);
6885 ins_cost(INSN_COST);
6886 format %{ "fmovd $dst, $con"%}
6887 ins_encode %{
6888 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6889 %}
6890
6891 ins_pipe(fp_imm_d);
6892 %}
6893
6894 // Load Double Constant
6895
6896 instruct loadConD(vRegD dst, immD con) %{
6897 match(Set dst con);
6898
6899 ins_cost(INSN_COST * 5);
6900 format %{
6901 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6902 %}
6903
6904 ins_encode %{
6905 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6906 %}
6907
6908 ins_pipe(fp_load_constant_d);
6909 %}
6910
6911 // Store Instructions
6912
6913 // Store Byte
6914 instruct storeB(iRegIorL2I src, memory1 mem)
6915 %{
6916 match(Set mem (StoreB mem src));
6917 predicate(!needs_releasing_store(n));
6918
6919 ins_cost(INSN_COST);
6920 format %{ "strb $src, $mem\t# byte" %}
6921
6922 ins_encode(aarch64_enc_strb(src, mem));
6923
6924 ins_pipe(istore_reg_mem);
6925 %}
6926
6927
6928 instruct storeimmB0(immI0 zero, memory1 mem)
6929 %{
6930 match(Set mem (StoreB mem zero));
6931 predicate(!needs_releasing_store(n));
6932
6933 ins_cost(INSN_COST);
6934 format %{ "strb rscractch2, $mem\t# byte" %}
6935
6936 ins_encode(aarch64_enc_strb0(mem));
6937
6938 ins_pipe(istore_mem);
6939 %}
6940
6941 // Store Char/Short
6942 instruct storeC(iRegIorL2I src, memory2 mem)
6943 %{
6944 match(Set mem (StoreC mem src));
6945 predicate(!needs_releasing_store(n));
6946
6947 ins_cost(INSN_COST);
6948 format %{ "strh $src, $mem\t# short" %}
6949
6950 ins_encode(aarch64_enc_strh(src, mem));
6951
6952 ins_pipe(istore_reg_mem);
6953 %}
6954
6955 instruct storeimmC0(immI0 zero, memory2 mem)
6956 %{
6957 match(Set mem (StoreC mem zero));
6958 predicate(!needs_releasing_store(n));
6959
6960 ins_cost(INSN_COST);
6961 format %{ "strh zr, $mem\t# short" %}
6962
6963 ins_encode(aarch64_enc_strh0(mem));
6964
6965 ins_pipe(istore_mem);
6966 %}
6967
6968 // Store Integer
6969
6970 instruct storeI(iRegIorL2I src, memory4 mem)
6971 %{
6972 match(Set mem(StoreI mem src));
6973 predicate(!needs_releasing_store(n));
6974
6975 ins_cost(INSN_COST);
6976 format %{ "strw $src, $mem\t# int" %}
6977
6978 ins_encode(aarch64_enc_strw(src, mem));
6979
6980 ins_pipe(istore_reg_mem);
6981 %}
6982
6983 instruct storeimmI0(immI0 zero, memory4 mem)
6984 %{
6985 match(Set mem(StoreI mem zero));
6986 predicate(!needs_releasing_store(n));
6987
6988 ins_cost(INSN_COST);
6989 format %{ "strw zr, $mem\t# int" %}
6990
6991 ins_encode(aarch64_enc_strw0(mem));
6992
6993 ins_pipe(istore_mem);
6994 %}
6995
6996 // Store Long (64 bit signed)
6997 instruct storeL(iRegL src, memory8 mem)
6998 %{
6999 match(Set mem (StoreL mem src));
7000 predicate(!needs_releasing_store(n));
7001
7002 ins_cost(INSN_COST);
7003 format %{ "str $src, $mem\t# int" %}
7004
7005 ins_encode(aarch64_enc_str(src, mem));
7006
7007 ins_pipe(istore_reg_mem);
7008 %}
7009
7010 // Store Long (64 bit signed)
7011 instruct storeimmL0(immL0 zero, memory8 mem)
7012 %{
7013 match(Set mem (StoreL mem zero));
7014 predicate(!needs_releasing_store(n));
7015
7016 ins_cost(INSN_COST);
7017 format %{ "str zr, $mem\t# int" %}
7018
7019 ins_encode(aarch64_enc_str0(mem));
7020
7021 ins_pipe(istore_mem);
7022 %}
7023
7024 // Store Pointer
7025 instruct storeP(iRegP src, memory8 mem)
7026 %{
7027 match(Set mem (StoreP mem src));
7028 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7029
7030 ins_cost(INSN_COST);
7031 format %{ "str $src, $mem\t# ptr" %}
7032
7033 ins_encode(aarch64_enc_str(src, mem));
7034
7035 ins_pipe(istore_reg_mem);
7036 %}
7037
7038 // Store Pointer
7039 instruct storeimmP0(immP0 zero, memory8 mem)
7040 %{
7041 match(Set mem (StoreP mem zero));
7042 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7043
7044 ins_cost(INSN_COST);
7045 format %{ "str zr, $mem\t# ptr" %}
7046
7047 ins_encode(aarch64_enc_str0(mem));
7048
7049 ins_pipe(istore_mem);
7050 %}
7051
7052 // Store Compressed Pointer
7053 instruct storeN(iRegN src, memory4 mem)
7054 %{
7055 match(Set mem (StoreN mem src));
7056 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7057
7058 ins_cost(INSN_COST);
7059 format %{ "strw $src, $mem\t# compressed ptr" %}
7060
7061 ins_encode(aarch64_enc_strw(src, mem));
7062
7063 ins_pipe(istore_reg_mem);
7064 %}
7065
7066 instruct storeImmN0(immN0 zero, memory4 mem)
7067 %{
7068 match(Set mem (StoreN mem zero));
7069 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7070
7071 ins_cost(INSN_COST);
7072 format %{ "strw zr, $mem\t# compressed ptr" %}
7073
7074 ins_encode(aarch64_enc_strw0(mem));
7075
7076 ins_pipe(istore_mem);
7077 %}
7078
7079 // Store Float
7080 instruct storeF(vRegF src, memory4 mem)
7081 %{
7082 match(Set mem (StoreF mem src));
7083 predicate(!needs_releasing_store(n));
7084
7085 ins_cost(INSN_COST);
7086 format %{ "strs $src, $mem\t# float" %}
7087
7088 ins_encode( aarch64_enc_strs(src, mem) );
7089
7090 ins_pipe(pipe_class_memory);
7091 %}
7092
7093 // TODO
7094 // implement storeImmF0 and storeFImmPacked
7095
7096 // Store Double
7097 instruct storeD(vRegD src, memory8 mem)
7098 %{
7099 match(Set mem (StoreD mem src));
7100 predicate(!needs_releasing_store(n));
7101
7102 ins_cost(INSN_COST);
7103 format %{ "strd $src, $mem\t# double" %}
7104
7105 ins_encode( aarch64_enc_strd(src, mem) );
7106
7107 ins_pipe(pipe_class_memory);
7108 %}
7109
7110 // Store Compressed Klass Pointer
7111 instruct storeNKlass(iRegN src, memory4 mem)
7112 %{
7113 predicate(!needs_releasing_store(n));
7114 match(Set mem (StoreNKlass mem src));
7115
7116 ins_cost(INSN_COST);
7117 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7118
7119 ins_encode(aarch64_enc_strw(src, mem));
7120
7121 ins_pipe(istore_reg_mem);
7122 %}
7123
7124 // TODO
7125 // implement storeImmD0 and storeDImmPacked
7126
7127 // prefetch instructions
7128 // Must be safe to execute with invalid address (cannot fault).
7129
7130 instruct prefetchalloc( memory8 mem ) %{
7131 match(PrefetchAllocation mem);
7132
7133 ins_cost(INSN_COST);
7134 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7135
7136 ins_encode( aarch64_enc_prefetchw(mem) );
7137
7138 ins_pipe(iload_prefetch);
7139 %}
7140
7141 // ---------------- volatile loads and stores ----------------
7142
7143 // Load Byte (8 bit signed)
7144 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7145 %{
7146 match(Set dst (LoadB mem));
7147
7148 ins_cost(VOLATILE_REF_COST);
7149 format %{ "ldarsb $dst, $mem\t# byte" %}
7150
7151 ins_encode(aarch64_enc_ldarsb(dst, mem));
7152
7153 ins_pipe(pipe_serial);
7154 %}
7155
7156 // Load Byte (8 bit signed) into long
7157 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7158 %{
7159 match(Set dst (ConvI2L (LoadB mem)));
7160
7161 ins_cost(VOLATILE_REF_COST);
7162 format %{ "ldarsb $dst, $mem\t# byte" %}
7163
7164 ins_encode(aarch64_enc_ldarsb(dst, mem));
7165
7166 ins_pipe(pipe_serial);
7167 %}
7168
7169 // Load Byte (8 bit unsigned)
7170 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7171 %{
7172 match(Set dst (LoadUB mem));
7173
7174 ins_cost(VOLATILE_REF_COST);
7175 format %{ "ldarb $dst, $mem\t# byte" %}
7176
7177 ins_encode(aarch64_enc_ldarb(dst, mem));
7178
7179 ins_pipe(pipe_serial);
7180 %}
7181
7182 // Load Byte (8 bit unsigned) into long
7183 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7184 %{
7185 match(Set dst (ConvI2L (LoadUB mem)));
7186
7187 ins_cost(VOLATILE_REF_COST);
7188 format %{ "ldarb $dst, $mem\t# byte" %}
7189
7190 ins_encode(aarch64_enc_ldarb(dst, mem));
7191
7192 ins_pipe(pipe_serial);
7193 %}
7194
7195 // Load Short (16 bit signed)
7196 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7197 %{
7198 match(Set dst (LoadS mem));
7199
7200 ins_cost(VOLATILE_REF_COST);
7201 format %{ "ldarshw $dst, $mem\t# short" %}
7202
7203 ins_encode(aarch64_enc_ldarshw(dst, mem));
7204
7205 ins_pipe(pipe_serial);
7206 %}
7207
7208 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7209 %{
7210 match(Set dst (LoadUS mem));
7211
7212 ins_cost(VOLATILE_REF_COST);
7213 format %{ "ldarhw $dst, $mem\t# short" %}
7214
7215 ins_encode(aarch64_enc_ldarhw(dst, mem));
7216
7217 ins_pipe(pipe_serial);
7218 %}
7219
7220 // Load Short/Char (16 bit unsigned) into long
7221 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7222 %{
7223 match(Set dst (ConvI2L (LoadUS mem)));
7224
7225 ins_cost(VOLATILE_REF_COST);
7226 format %{ "ldarh $dst, $mem\t# short" %}
7227
7228 ins_encode(aarch64_enc_ldarh(dst, mem));
7229
7230 ins_pipe(pipe_serial);
7231 %}
7232
7233 // Load Short/Char (16 bit signed) into long
7234 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7235 %{
7236 match(Set dst (ConvI2L (LoadS mem)));
7237
7238 ins_cost(VOLATILE_REF_COST);
7239 format %{ "ldarh $dst, $mem\t# short" %}
7240
7241 ins_encode(aarch64_enc_ldarsh(dst, mem));
7242
7243 ins_pipe(pipe_serial);
7244 %}
7245
7246 // Load Integer (32 bit signed)
7247 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7248 %{
7249 match(Set dst (LoadI mem));
7250
7251 ins_cost(VOLATILE_REF_COST);
7252 format %{ "ldarw $dst, $mem\t# int" %}
7253
7254 ins_encode(aarch64_enc_ldarw(dst, mem));
7255
7256 ins_pipe(pipe_serial);
7257 %}
7258
7259 // Load Integer (32 bit unsigned) into long
7260 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7261 %{
7262 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarw $dst, $mem\t# int" %}
7266
7267 ins_encode(aarch64_enc_ldarw(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Long (64 bit signed)
7273 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (LoadL mem));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldar $dst, $mem\t# int" %}
7279
7280 ins_encode(aarch64_enc_ldar(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Pointer
7286 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadP mem));
7289 predicate(n->as_Load()->barrier_data() == 0);
7290
7291 ins_cost(VOLATILE_REF_COST);
7292 format %{ "ldar $dst, $mem\t# ptr" %}
7293
7294 ins_encode(aarch64_enc_ldar(dst, mem));
7295
7296 ins_pipe(pipe_serial);
7297 %}
7298
7299 // Load Compressed Pointer
7300 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7301 %{
7302 match(Set dst (LoadN mem));
7303 predicate(n->as_Load()->barrier_data() == 0);
7304
7305 ins_cost(VOLATILE_REF_COST);
7306 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7307
7308 ins_encode(aarch64_enc_ldarw(dst, mem));
7309
7310 ins_pipe(pipe_serial);
7311 %}
7312
7313 // Load Float
7314 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadF mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldars $dst, $mem\t# float" %}
7320
7321 ins_encode( aarch64_enc_fldars(dst, mem) );
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Double
7327 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (LoadD mem));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldard $dst, $mem\t# double" %}
7333
7334 ins_encode( aarch64_enc_fldard(dst, mem) );
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Store Byte
7340 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set mem (StoreB mem src));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "stlrb $src, $mem\t# byte" %}
7346
7347 ins_encode(aarch64_enc_stlrb(src, mem));
7348
7349 ins_pipe(pipe_class_memory);
7350 %}
7351
7352 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7353 %{
7354 match(Set mem (StoreB mem zero));
7355
7356 ins_cost(VOLATILE_REF_COST);
7357 format %{ "stlrb zr, $mem\t# byte" %}
7358
7359 ins_encode(aarch64_enc_stlrb0(mem));
7360
7361 ins_pipe(pipe_class_memory);
7362 %}
7363
7364 // Store Char/Short
7365 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7366 %{
7367 match(Set mem (StoreC mem src));
7368
7369 ins_cost(VOLATILE_REF_COST);
7370 format %{ "stlrh $src, $mem\t# short" %}
7371
7372 ins_encode(aarch64_enc_stlrh(src, mem));
7373
7374 ins_pipe(pipe_class_memory);
7375 %}
7376
7377 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7378 %{
7379 match(Set mem (StoreC mem zero));
7380
7381 ins_cost(VOLATILE_REF_COST);
7382 format %{ "stlrh zr, $mem\t# short" %}
7383
7384 ins_encode(aarch64_enc_stlrh0(mem));
7385
7386 ins_pipe(pipe_class_memory);
7387 %}
7388
7389 // Store Integer
7390
7391 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set mem(StoreI mem src));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "stlrw $src, $mem\t# int" %}
7397
7398 ins_encode(aarch64_enc_stlrw(src, mem));
7399
7400 ins_pipe(pipe_class_memory);
7401 %}
7402
7403 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7404 %{
7405 match(Set mem(StoreI mem zero));
7406
7407 ins_cost(VOLATILE_REF_COST);
7408 format %{ "stlrw zr, $mem\t# int" %}
7409
7410 ins_encode(aarch64_enc_stlrw0(mem));
7411
7412 ins_pipe(pipe_class_memory);
7413 %}
7414
7415 // Store Long (64 bit signed)
7416 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7417 %{
7418 match(Set mem (StoreL mem src));
7419
7420 ins_cost(VOLATILE_REF_COST);
7421 format %{ "stlr $src, $mem\t# int" %}
7422
7423 ins_encode(aarch64_enc_stlr(src, mem));
7424
7425 ins_pipe(pipe_class_memory);
7426 %}
7427
7428 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7429 %{
7430 match(Set mem (StoreL mem zero));
7431
7432 ins_cost(VOLATILE_REF_COST);
7433 format %{ "stlr zr, $mem\t# int" %}
7434
7435 ins_encode(aarch64_enc_stlr0(mem));
7436
7437 ins_pipe(pipe_class_memory);
7438 %}
7439
7440 // Store Pointer
7441 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7442 %{
7443 match(Set mem (StoreP mem src));
7444 predicate(n->as_Store()->barrier_data() == 0);
7445
7446 ins_cost(VOLATILE_REF_COST);
7447 format %{ "stlr $src, $mem\t# ptr" %}
7448
7449 ins_encode(aarch64_enc_stlr(src, mem));
7450
7451 ins_pipe(pipe_class_memory);
7452 %}
7453
7454 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7455 %{
7456 match(Set mem (StoreP mem zero));
7457 predicate(n->as_Store()->barrier_data() == 0);
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlr zr, $mem\t# ptr" %}
7461
7462 ins_encode(aarch64_enc_stlr0(mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 // Store Compressed Pointer
7468 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreN mem src));
7471 predicate(n->as_Store()->barrier_data() == 0);
7472
7473 ins_cost(VOLATILE_REF_COST);
7474 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7475
7476 ins_encode(aarch64_enc_stlrw(src, mem));
7477
7478 ins_pipe(pipe_class_memory);
7479 %}
7480
7481 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set mem (StoreN mem zero));
7484 predicate(n->as_Store()->barrier_data() == 0);
7485
7486 ins_cost(VOLATILE_REF_COST);
7487 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7488
7489 ins_encode(aarch64_enc_stlrw0(mem));
7490
7491 ins_pipe(pipe_class_memory);
7492 %}
7493
7494 // Store Float
7495 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7496 %{
7497 match(Set mem (StoreF mem src));
7498
7499 ins_cost(VOLATILE_REF_COST);
7500 format %{ "stlrs $src, $mem\t# float" %}
7501
7502 ins_encode( aarch64_enc_fstlrs(src, mem) );
7503
7504 ins_pipe(pipe_class_memory);
7505 %}
7506
7507 // TODO
7508 // implement storeImmF0 and storeFImmPacked
7509
7510 // Store Double
7511 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7512 %{
7513 match(Set mem (StoreD mem src));
7514
7515 ins_cost(VOLATILE_REF_COST);
7516 format %{ "stlrd $src, $mem\t# double" %}
7517
7518 ins_encode( aarch64_enc_fstlrd(src, mem) );
7519
7520 ins_pipe(pipe_class_memory);
7521 %}
7522
7523 // ---------------- end of volatile loads and stores ----------------
7524
7525 instruct cacheWB(indirect addr)
7526 %{
7527 predicate(VM_Version::supports_data_cache_line_flush());
7528 match(CacheWB addr);
7529
7530 ins_cost(100);
7531 format %{"cache wb $addr" %}
7532 ins_encode %{
7533 assert($addr->index_position() < 0, "should be");
7534 assert($addr$$disp == 0, "should be");
7535 __ cache_wb(Address($addr$$base$$Register, 0));
7536 %}
7537 ins_pipe(pipe_slow); // XXX
7538 %}
7539
7540 instruct cacheWBPreSync()
7541 %{
7542 predicate(VM_Version::supports_data_cache_line_flush());
7543 match(CacheWBPreSync);
7544
7545 ins_cost(100);
7546 format %{"cache wb presync" %}
7547 ins_encode %{
7548 __ cache_wbsync(true);
7549 %}
7550 ins_pipe(pipe_slow); // XXX
7551 %}
7552
7553 instruct cacheWBPostSync()
7554 %{
7555 predicate(VM_Version::supports_data_cache_line_flush());
7556 match(CacheWBPostSync);
7557
7558 ins_cost(100);
7559 format %{"cache wb postsync" %}
7560 ins_encode %{
7561 __ cache_wbsync(false);
7562 %}
7563 ins_pipe(pipe_slow); // XXX
7564 %}
7565
7566 // ============================================================================
7567 // BSWAP Instructions
7568
7569 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7570 match(Set dst (ReverseBytesI src));
7571
7572 ins_cost(INSN_COST);
7573 format %{ "revw $dst, $src" %}
7574
7575 ins_encode %{
7576 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7577 %}
7578
7579 ins_pipe(ialu_reg);
7580 %}
7581
7582 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7583 match(Set dst (ReverseBytesL src));
7584
7585 ins_cost(INSN_COST);
7586 format %{ "rev $dst, $src" %}
7587
7588 ins_encode %{
7589 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7590 %}
7591
7592 ins_pipe(ialu_reg);
7593 %}
7594
7595 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7596 match(Set dst (ReverseBytesUS src));
7597
7598 ins_cost(INSN_COST);
7599 format %{ "rev16w $dst, $src" %}
7600
7601 ins_encode %{
7602 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7603 %}
7604
7605 ins_pipe(ialu_reg);
7606 %}
7607
7608 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7609 match(Set dst (ReverseBytesS src));
7610
7611 ins_cost(INSN_COST);
7612 format %{ "rev16w $dst, $src\n\t"
7613 "sbfmw $dst, $dst, #0, #15" %}
7614
7615 ins_encode %{
7616 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7617 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7618 %}
7619
7620 ins_pipe(ialu_reg);
7621 %}
7622
7623 // ============================================================================
7624 // Zero Count Instructions
7625
7626 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7627 match(Set dst (CountLeadingZerosI src));
7628
7629 ins_cost(INSN_COST);
7630 format %{ "clzw $dst, $src" %}
7631 ins_encode %{
7632 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7633 %}
7634
7635 ins_pipe(ialu_reg);
7636 %}
7637
7638 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7639 match(Set dst (CountLeadingZerosL src));
7640
7641 ins_cost(INSN_COST);
7642 format %{ "clz $dst, $src" %}
7643 ins_encode %{
7644 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7645 %}
7646
7647 ins_pipe(ialu_reg);
7648 %}
7649
7650 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7651 match(Set dst (CountTrailingZerosI src));
7652
7653 ins_cost(INSN_COST * 2);
7654 format %{ "rbitw $dst, $src\n\t"
7655 "clzw $dst, $dst" %}
7656 ins_encode %{
7657 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7658 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7659 %}
7660
7661 ins_pipe(ialu_reg);
7662 %}
7663
7664 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7665 match(Set dst (CountTrailingZerosL src));
7666
7667 ins_cost(INSN_COST * 2);
7668 format %{ "rbit $dst, $src\n\t"
7669 "clz $dst, $dst" %}
7670 ins_encode %{
7671 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7672 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7673 %}
7674
7675 ins_pipe(ialu_reg);
7676 %}
7677
7678 //---------- Population Count Instructions -------------------------------------
7679 //
7680
7681 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7682 match(Set dst (PopCountI src));
7683 effect(TEMP tmp);
7684 ins_cost(INSN_COST * 13);
7685
7686 format %{ "movw $src, $src\n\t"
7687 "mov $tmp, $src\t# vector (1D)\n\t"
7688 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7689 "addv $tmp, $tmp\t# vector (8B)\n\t"
7690 "mov $dst, $tmp\t# vector (1D)" %}
7691 ins_encode %{
7692 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7693 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7694 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7695 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7696 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7697 %}
7698
7699 ins_pipe(pipe_class_default);
7700 %}
7701
7702 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7703 match(Set dst (PopCountI (LoadI mem)));
7704 effect(TEMP tmp);
7705 ins_cost(INSN_COST * 13);
7706
7707 format %{ "ldrs $tmp, $mem\n\t"
7708 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7709 "addv $tmp, $tmp\t# vector (8B)\n\t"
7710 "mov $dst, $tmp\t# vector (1D)" %}
7711 ins_encode %{
7712 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7713 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7714 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7715 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7716 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7717 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7718 %}
7719
7720 ins_pipe(pipe_class_default);
7721 %}
7722
7723 // Note: Long.bitCount(long) returns an int.
7724 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7725 match(Set dst (PopCountL src));
7726 effect(TEMP tmp);
7727 ins_cost(INSN_COST * 13);
7728
7729 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7730 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7731 "addv $tmp, $tmp\t# vector (8B)\n\t"
7732 "mov $dst, $tmp\t# vector (1D)" %}
7733 ins_encode %{
7734 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7735 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7736 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7737 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7738 %}
7739
7740 ins_pipe(pipe_class_default);
7741 %}
7742
7743 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7744 match(Set dst (PopCountL (LoadL mem)));
7745 effect(TEMP tmp);
7746 ins_cost(INSN_COST * 13);
7747
7748 format %{ "ldrd $tmp, $mem\n\t"
7749 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7750 "addv $tmp, $tmp\t# vector (8B)\n\t"
7751 "mov $dst, $tmp\t# vector (1D)" %}
7752 ins_encode %{
7753 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7754 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7755 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7756 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7757 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7758 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7759 %}
7760
7761 ins_pipe(pipe_class_default);
7762 %}
7763
7764 // ============================================================================
7765 // VerifyVectorAlignment Instruction
7766
7767 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7768 match(Set addr (VerifyVectorAlignment addr mask));
7769 effect(KILL cr);
7770 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7771 ins_encode %{
7772 Label Lskip;
7773 // check if masked bits of addr are zero
7774 __ tst($addr$$Register, $mask$$constant);
7775 __ br(Assembler::EQ, Lskip);
7776 __ stop("verify_vector_alignment found a misaligned vector memory access");
7777 __ bind(Lskip);
7778 %}
7779 ins_pipe(pipe_slow);
7780 %}
7781
7782 // ============================================================================
7783 // MemBar Instruction
7784
7785 instruct load_fence() %{
7786 match(LoadFence);
7787 ins_cost(VOLATILE_REF_COST);
7788
7789 format %{ "load_fence" %}
7790
7791 ins_encode %{
7792 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7793 %}
7794 ins_pipe(pipe_serial);
7795 %}
7796
7797 instruct unnecessary_membar_acquire() %{
7798 predicate(unnecessary_acquire(n));
7799 match(MemBarAcquire);
7800 ins_cost(0);
7801
7802 format %{ "membar_acquire (elided)" %}
7803
7804 ins_encode %{
7805 __ block_comment("membar_acquire (elided)");
7806 %}
7807
7808 ins_pipe(pipe_class_empty);
7809 %}
7810
7811 instruct membar_acquire() %{
7812 match(MemBarAcquire);
7813 ins_cost(VOLATILE_REF_COST);
7814
7815 format %{ "membar_acquire\n\t"
7816 "dmb ishld" %}
7817
7818 ins_encode %{
7819 __ block_comment("membar_acquire");
7820 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7821 %}
7822
7823 ins_pipe(pipe_serial);
7824 %}
7825
7826
7827 instruct membar_acquire_lock() %{
7828 match(MemBarAcquireLock);
7829 ins_cost(VOLATILE_REF_COST);
7830
7831 format %{ "membar_acquire_lock (elided)" %}
7832
7833 ins_encode %{
7834 __ block_comment("membar_acquire_lock (elided)");
7835 %}
7836
7837 ins_pipe(pipe_serial);
7838 %}
7839
7840 instruct store_fence() %{
7841 match(StoreFence);
7842 ins_cost(VOLATILE_REF_COST);
7843
7844 format %{ "store_fence" %}
7845
7846 ins_encode %{
7847 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7848 %}
7849 ins_pipe(pipe_serial);
7850 %}
7851
7852 instruct unnecessary_membar_release() %{
7853 predicate(unnecessary_release(n));
7854 match(MemBarRelease);
7855 ins_cost(0);
7856
7857 format %{ "membar_release (elided)" %}
7858
7859 ins_encode %{
7860 __ block_comment("membar_release (elided)");
7861 %}
7862 ins_pipe(pipe_serial);
7863 %}
7864
7865 instruct membar_release() %{
7866 match(MemBarRelease);
7867 ins_cost(VOLATILE_REF_COST);
7868
7869 format %{ "membar_release\n\t"
7870 "dmb ishst\n\tdmb ishld" %}
7871
7872 ins_encode %{
7873 __ block_comment("membar_release");
7874 // These will be merged if AlwaysMergeDMB is enabled.
7875 __ membar(Assembler::StoreStore);
7876 __ membar(Assembler::LoadStore);
7877 %}
7878 ins_pipe(pipe_serial);
7879 %}
7880
7881 instruct membar_storestore() %{
7882 match(MemBarStoreStore);
7883 match(StoreStoreFence);
7884 ins_cost(VOLATILE_REF_COST);
7885
7886 format %{ "MEMBAR-store-store" %}
7887
7888 ins_encode %{
7889 __ membar(Assembler::StoreStore);
7890 %}
7891 ins_pipe(pipe_serial);
7892 %}
7893
7894 instruct membar_release_lock() %{
7895 match(MemBarReleaseLock);
7896 ins_cost(VOLATILE_REF_COST);
7897
7898 format %{ "membar_release_lock (elided)" %}
7899
7900 ins_encode %{
7901 __ block_comment("membar_release_lock (elided)");
7902 %}
7903
7904 ins_pipe(pipe_serial);
7905 %}
7906
7907 instruct unnecessary_membar_volatile() %{
7908 predicate(unnecessary_volatile(n));
7909 match(MemBarVolatile);
7910 ins_cost(0);
7911
7912 format %{ "membar_volatile (elided)" %}
7913
7914 ins_encode %{
7915 __ block_comment("membar_volatile (elided)");
7916 %}
7917
7918 ins_pipe(pipe_serial);
7919 %}
7920
7921 instruct membar_volatile() %{
7922 match(MemBarVolatile);
7923 ins_cost(VOLATILE_REF_COST*100);
7924
7925 format %{ "membar_volatile\n\t"
7926 "dmb ish"%}
7927
7928 ins_encode %{
7929 __ block_comment("membar_volatile");
7930 __ membar(Assembler::StoreLoad);
7931 %}
7932
7933 ins_pipe(pipe_serial);
7934 %}
7935
7936 // ============================================================================
7937 // Cast/Convert Instructions
7938
7939 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7940 match(Set dst (CastX2P src));
7941
7942 ins_cost(INSN_COST);
7943 format %{ "mov $dst, $src\t# long -> ptr" %}
7944
7945 ins_encode %{
7946 if ($dst$$reg != $src$$reg) {
7947 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7948 }
7949 %}
7950
7951 ins_pipe(ialu_reg);
7952 %}
7953
7954 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7955 match(Set dst (CastP2X src));
7956
7957 ins_cost(INSN_COST);
7958 format %{ "mov $dst, $src\t# ptr -> long" %}
7959
7960 ins_encode %{
7961 if ($dst$$reg != $src$$reg) {
7962 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7963 }
7964 %}
7965
7966 ins_pipe(ialu_reg);
7967 %}
7968
7969 // Convert oop into int for vectors alignment masking
7970 instruct convP2I(iRegINoSp dst, iRegP src) %{
7971 match(Set dst (ConvL2I (CastP2X src)));
7972
7973 ins_cost(INSN_COST);
7974 format %{ "movw $dst, $src\t# ptr -> int" %}
7975 ins_encode %{
7976 __ movw($dst$$Register, $src$$Register);
7977 %}
7978
7979 ins_pipe(ialu_reg);
7980 %}
7981
7982 // Convert compressed oop into int for vectors alignment masking
7983 // in case of 32bit oops (heap < 4Gb).
7984 instruct convN2I(iRegINoSp dst, iRegN src)
7985 %{
7986 predicate(CompressedOops::shift() == 0);
7987 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7988
7989 ins_cost(INSN_COST);
7990 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7991 ins_encode %{
7992 __ movw($dst$$Register, $src$$Register);
7993 %}
7994
7995 ins_pipe(ialu_reg);
7996 %}
7997
7998
7999 // Convert oop pointer into compressed form
8000 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8001 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8002 match(Set dst (EncodeP src));
8003 effect(KILL cr);
8004 ins_cost(INSN_COST * 3);
8005 format %{ "encode_heap_oop $dst, $src" %}
8006 ins_encode %{
8007 Register s = $src$$Register;
8008 Register d = $dst$$Register;
8009 __ encode_heap_oop(d, s);
8010 %}
8011 ins_pipe(ialu_reg);
8012 %}
8013
8014 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8015 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8016 match(Set dst (EncodeP src));
8017 ins_cost(INSN_COST * 3);
8018 format %{ "encode_heap_oop_not_null $dst, $src" %}
8019 ins_encode %{
8020 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8021 %}
8022 ins_pipe(ialu_reg);
8023 %}
8024
8025 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8026 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8027 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8028 match(Set dst (DecodeN src));
8029 ins_cost(INSN_COST * 3);
8030 format %{ "decode_heap_oop $dst, $src" %}
8031 ins_encode %{
8032 Register s = $src$$Register;
8033 Register d = $dst$$Register;
8034 __ decode_heap_oop(d, s);
8035 %}
8036 ins_pipe(ialu_reg);
8037 %}
8038
8039 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8040 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8041 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8042 match(Set dst (DecodeN src));
8043 ins_cost(INSN_COST * 3);
8044 format %{ "decode_heap_oop_not_null $dst, $src" %}
8045 ins_encode %{
8046 Register s = $src$$Register;
8047 Register d = $dst$$Register;
8048 __ decode_heap_oop_not_null(d, s);
8049 %}
8050 ins_pipe(ialu_reg);
8051 %}
8052
8053 // n.b. AArch64 implementations of encode_klass_not_null and
8054 // decode_klass_not_null do not modify the flags register so, unlike
8055 // Intel, we don't kill CR as a side effect here
8056
8057 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8058 match(Set dst (EncodePKlass src));
8059
8060 ins_cost(INSN_COST * 3);
8061 format %{ "encode_klass_not_null $dst,$src" %}
8062
8063 ins_encode %{
8064 Register src_reg = as_Register($src$$reg);
8065 Register dst_reg = as_Register($dst$$reg);
8066 __ encode_klass_not_null(dst_reg, src_reg);
8067 %}
8068
8069 ins_pipe(ialu_reg);
8070 %}
8071
8072 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8073 match(Set dst (DecodeNKlass src));
8074
8075 ins_cost(INSN_COST * 3);
8076 format %{ "decode_klass_not_null $dst,$src" %}
8077
8078 ins_encode %{
8079 Register src_reg = as_Register($src$$reg);
8080 Register dst_reg = as_Register($dst$$reg);
8081 if (dst_reg != src_reg) {
8082 __ decode_klass_not_null(dst_reg, src_reg);
8083 } else {
8084 __ decode_klass_not_null(dst_reg);
8085 }
8086 %}
8087
8088 ins_pipe(ialu_reg);
8089 %}
8090
8091 instruct checkCastPP(iRegPNoSp dst)
8092 %{
8093 match(Set dst (CheckCastPP dst));
8094
8095 size(0);
8096 format %{ "# checkcastPP of $dst" %}
8097 ins_encode(/* empty encoding */);
8098 ins_pipe(pipe_class_empty);
8099 %}
8100
8101 instruct castPP(iRegPNoSp dst)
8102 %{
8103 match(Set dst (CastPP dst));
8104
8105 size(0);
8106 format %{ "# castPP of $dst" %}
8107 ins_encode(/* empty encoding */);
8108 ins_pipe(pipe_class_empty);
8109 %}
8110
8111 instruct castII(iRegI dst)
8112 %{
8113 match(Set dst (CastII dst));
8114
8115 size(0);
8116 format %{ "# castII of $dst" %}
8117 ins_encode(/* empty encoding */);
8118 ins_cost(0);
8119 ins_pipe(pipe_class_empty);
8120 %}
8121
8122 instruct castLL(iRegL dst)
8123 %{
8124 match(Set dst (CastLL dst));
8125
8126 size(0);
8127 format %{ "# castLL of $dst" %}
8128 ins_encode(/* empty encoding */);
8129 ins_cost(0);
8130 ins_pipe(pipe_class_empty);
8131 %}
8132
8133 instruct castFF(vRegF dst)
8134 %{
8135 match(Set dst (CastFF dst));
8136
8137 size(0);
8138 format %{ "# castFF of $dst" %}
8139 ins_encode(/* empty encoding */);
8140 ins_cost(0);
8141 ins_pipe(pipe_class_empty);
8142 %}
8143
8144 instruct castDD(vRegD dst)
8145 %{
8146 match(Set dst (CastDD dst));
8147
8148 size(0);
8149 format %{ "# castDD of $dst" %}
8150 ins_encode(/* empty encoding */);
8151 ins_cost(0);
8152 ins_pipe(pipe_class_empty);
8153 %}
8154
8155 instruct castVV(vReg dst)
8156 %{
8157 match(Set dst (CastVV dst));
8158
8159 size(0);
8160 format %{ "# castVV of $dst" %}
8161 ins_encode(/* empty encoding */);
8162 ins_cost(0);
8163 ins_pipe(pipe_class_empty);
8164 %}
8165
8166 instruct castVVMask(pRegGov dst)
8167 %{
8168 match(Set dst (CastVV dst));
8169
8170 size(0);
8171 format %{ "# castVV of $dst" %}
8172 ins_encode(/* empty encoding */);
8173 ins_cost(0);
8174 ins_pipe(pipe_class_empty);
8175 %}
8176
8177 // ============================================================================
8178 // Atomic operation instructions
8179 //
8180
8181 // standard CompareAndSwapX when we are using barriers
8182 // these have higher priority than the rules selected by a predicate
8183
8184 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8185 // can't match them
8186
8187 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8188
8189 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8190 ins_cost(2 * VOLATILE_REF_COST);
8191
8192 effect(KILL cr);
8193
8194 format %{
8195 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8196 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8197 %}
8198
8199 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8200 aarch64_enc_cset_eq(res));
8201
8202 ins_pipe(pipe_slow);
8203 %}
8204
8205 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8206
8207 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8208 ins_cost(2 * VOLATILE_REF_COST);
8209
8210 effect(KILL cr);
8211
8212 format %{
8213 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8214 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8215 %}
8216
8217 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8218 aarch64_enc_cset_eq(res));
8219
8220 ins_pipe(pipe_slow);
8221 %}
8222
8223 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8224
8225 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8226 ins_cost(2 * VOLATILE_REF_COST);
8227
8228 effect(KILL cr);
8229
8230 format %{
8231 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8232 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8233 %}
8234
8235 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8236 aarch64_enc_cset_eq(res));
8237
8238 ins_pipe(pipe_slow);
8239 %}
8240
8241 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8242
8243 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8244 ins_cost(2 * VOLATILE_REF_COST);
8245
8246 effect(KILL cr);
8247
8248 format %{
8249 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8250 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8251 %}
8252
8253 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8254 aarch64_enc_cset_eq(res));
8255
8256 ins_pipe(pipe_slow);
8257 %}
8258
8259 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8260
8261 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8262 predicate(n->as_LoadStore()->barrier_data() == 0);
8263 ins_cost(2 * VOLATILE_REF_COST);
8264
8265 effect(KILL cr);
8266
8267 format %{
8268 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8269 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8270 %}
8271
8272 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8273 aarch64_enc_cset_eq(res));
8274
8275 ins_pipe(pipe_slow);
8276 %}
8277
8278 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8279
8280 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8281 predicate(n->as_LoadStore()->barrier_data() == 0);
8282 ins_cost(2 * VOLATILE_REF_COST);
8283
8284 effect(KILL cr);
8285
8286 format %{
8287 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8288 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8289 %}
8290
8291 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8292 aarch64_enc_cset_eq(res));
8293
8294 ins_pipe(pipe_slow);
8295 %}
8296
8297 // alternative CompareAndSwapX when we are eliding barriers
8298
8299 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8300
8301 predicate(needs_acquiring_load_exclusive(n));
8302 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8303 ins_cost(VOLATILE_REF_COST);
8304
8305 effect(KILL cr);
8306
8307 format %{
8308 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8309 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8310 %}
8311
8312 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8313 aarch64_enc_cset_eq(res));
8314
8315 ins_pipe(pipe_slow);
8316 %}
8317
8318 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8319
8320 predicate(needs_acquiring_load_exclusive(n));
8321 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8322 ins_cost(VOLATILE_REF_COST);
8323
8324 effect(KILL cr);
8325
8326 format %{
8327 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8328 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8329 %}
8330
8331 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8332 aarch64_enc_cset_eq(res));
8333
8334 ins_pipe(pipe_slow);
8335 %}
8336
8337 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8338
8339 predicate(needs_acquiring_load_exclusive(n));
8340 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8341 ins_cost(VOLATILE_REF_COST);
8342
8343 effect(KILL cr);
8344
8345 format %{
8346 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8347 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8348 %}
8349
8350 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8351 aarch64_enc_cset_eq(res));
8352
8353 ins_pipe(pipe_slow);
8354 %}
8355
8356 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8357
8358 predicate(needs_acquiring_load_exclusive(n));
8359 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8360 ins_cost(VOLATILE_REF_COST);
8361
8362 effect(KILL cr);
8363
8364 format %{
8365 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8366 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8367 %}
8368
8369 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8370 aarch64_enc_cset_eq(res));
8371
8372 ins_pipe(pipe_slow);
8373 %}
8374
8375 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8376
8377 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8378 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8379 ins_cost(VOLATILE_REF_COST);
8380
8381 effect(KILL cr);
8382
8383 format %{
8384 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8385 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8386 %}
8387
8388 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8389 aarch64_enc_cset_eq(res));
8390
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8395
8396 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8397 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8398 ins_cost(VOLATILE_REF_COST);
8399
8400 effect(KILL cr);
8401
8402 format %{
8403 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8404 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8405 %}
8406
8407 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8408 aarch64_enc_cset_eq(res));
8409
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413
8414 // ---------------------------------------------------------------------
8415
8416 // BEGIN This section of the file is automatically generated. Do not edit --------------
8417
8418 // Sundry CAS operations. Note that release is always true,
8419 // regardless of the memory ordering of the CAS. This is because we
8420 // need the volatile case to be sequentially consistent but there is
8421 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8422 // can't check the type of memory ordering here, so we always emit a
8423 // STLXR.
8424
8425 // This section is generated from cas.m4
8426
8427
8428 // This pattern is generated automatically from cas.m4.
8429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8430 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8431 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8432 ins_cost(2 * VOLATILE_REF_COST);
8433 effect(TEMP_DEF res, KILL cr);
8434 format %{
8435 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8436 %}
8437 ins_encode %{
8438 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8439 Assembler::byte, /*acquire*/ false, /*release*/ true,
8440 /*weak*/ false, $res$$Register);
8441 __ sxtbw($res$$Register, $res$$Register);
8442 %}
8443 ins_pipe(pipe_slow);
8444 %}
8445
8446 // This pattern is generated automatically from cas.m4.
8447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8448 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8449 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8450 ins_cost(2 * VOLATILE_REF_COST);
8451 effect(TEMP_DEF res, KILL cr);
8452 format %{
8453 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8454 %}
8455 ins_encode %{
8456 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8457 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8458 /*weak*/ false, $res$$Register);
8459 __ sxthw($res$$Register, $res$$Register);
8460 %}
8461 ins_pipe(pipe_slow);
8462 %}
8463
8464 // This pattern is generated automatically from cas.m4.
8465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8466 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8467 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8468 ins_cost(2 * VOLATILE_REF_COST);
8469 effect(TEMP_DEF res, KILL cr);
8470 format %{
8471 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8472 %}
8473 ins_encode %{
8474 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8475 Assembler::word, /*acquire*/ false, /*release*/ true,
8476 /*weak*/ false, $res$$Register);
8477 %}
8478 ins_pipe(pipe_slow);
8479 %}
8480
8481 // This pattern is generated automatically from cas.m4.
8482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8483 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8484 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8485 ins_cost(2 * VOLATILE_REF_COST);
8486 effect(TEMP_DEF res, KILL cr);
8487 format %{
8488 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8489 %}
8490 ins_encode %{
8491 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8492 Assembler::xword, /*acquire*/ false, /*release*/ true,
8493 /*weak*/ false, $res$$Register);
8494 %}
8495 ins_pipe(pipe_slow);
8496 %}
8497
8498 // This pattern is generated automatically from cas.m4.
8499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8500 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8501 predicate(n->as_LoadStore()->barrier_data() == 0);
8502 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8503 ins_cost(2 * VOLATILE_REF_COST);
8504 effect(TEMP_DEF res, KILL cr);
8505 format %{
8506 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8507 %}
8508 ins_encode %{
8509 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8510 Assembler::word, /*acquire*/ false, /*release*/ true,
8511 /*weak*/ false, $res$$Register);
8512 %}
8513 ins_pipe(pipe_slow);
8514 %}
8515
8516 // This pattern is generated automatically from cas.m4.
8517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8518 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8519 predicate(n->as_LoadStore()->barrier_data() == 0);
8520 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8521 ins_cost(2 * VOLATILE_REF_COST);
8522 effect(TEMP_DEF res, KILL cr);
8523 format %{
8524 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8525 %}
8526 ins_encode %{
8527 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8528 Assembler::xword, /*acquire*/ false, /*release*/ true,
8529 /*weak*/ false, $res$$Register);
8530 %}
8531 ins_pipe(pipe_slow);
8532 %}
8533
8534 // This pattern is generated automatically from cas.m4.
8535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8536 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8537 predicate(needs_acquiring_load_exclusive(n));
8538 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8539 ins_cost(VOLATILE_REF_COST);
8540 effect(TEMP_DEF res, KILL cr);
8541 format %{
8542 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8543 %}
8544 ins_encode %{
8545 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8546 Assembler::byte, /*acquire*/ true, /*release*/ true,
8547 /*weak*/ false, $res$$Register);
8548 __ sxtbw($res$$Register, $res$$Register);
8549 %}
8550 ins_pipe(pipe_slow);
8551 %}
8552
8553 // This pattern is generated automatically from cas.m4.
8554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8555 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8556 predicate(needs_acquiring_load_exclusive(n));
8557 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8558 ins_cost(VOLATILE_REF_COST);
8559 effect(TEMP_DEF res, KILL cr);
8560 format %{
8561 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8562 %}
8563 ins_encode %{
8564 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8565 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8566 /*weak*/ false, $res$$Register);
8567 __ sxthw($res$$Register, $res$$Register);
8568 %}
8569 ins_pipe(pipe_slow);
8570 %}
8571
8572 // This pattern is generated automatically from cas.m4.
8573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8574 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8575 predicate(needs_acquiring_load_exclusive(n));
8576 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8577 ins_cost(VOLATILE_REF_COST);
8578 effect(TEMP_DEF res, KILL cr);
8579 format %{
8580 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8581 %}
8582 ins_encode %{
8583 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8584 Assembler::word, /*acquire*/ true, /*release*/ true,
8585 /*weak*/ false, $res$$Register);
8586 %}
8587 ins_pipe(pipe_slow);
8588 %}
8589
8590 // This pattern is generated automatically from cas.m4.
8591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8592 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8593 predicate(needs_acquiring_load_exclusive(n));
8594 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8595 ins_cost(VOLATILE_REF_COST);
8596 effect(TEMP_DEF res, KILL cr);
8597 format %{
8598 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8599 %}
8600 ins_encode %{
8601 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8602 Assembler::xword, /*acquire*/ true, /*release*/ true,
8603 /*weak*/ false, $res$$Register);
8604 %}
8605 ins_pipe(pipe_slow);
8606 %}
8607
8608 // This pattern is generated automatically from cas.m4.
8609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8610 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8611 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8612 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8613 ins_cost(VOLATILE_REF_COST);
8614 effect(TEMP_DEF res, KILL cr);
8615 format %{
8616 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8617 %}
8618 ins_encode %{
8619 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8620 Assembler::word, /*acquire*/ true, /*release*/ true,
8621 /*weak*/ false, $res$$Register);
8622 %}
8623 ins_pipe(pipe_slow);
8624 %}
8625
8626 // This pattern is generated automatically from cas.m4.
8627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8628 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8629 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8630 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8631 ins_cost(VOLATILE_REF_COST);
8632 effect(TEMP_DEF res, KILL cr);
8633 format %{
8634 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8635 %}
8636 ins_encode %{
8637 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8638 Assembler::xword, /*acquire*/ true, /*release*/ true,
8639 /*weak*/ false, $res$$Register);
8640 %}
8641 ins_pipe(pipe_slow);
8642 %}
8643
8644 // This pattern is generated automatically from cas.m4.
8645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8646 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8647 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8648 ins_cost(2 * VOLATILE_REF_COST);
8649 effect(KILL cr);
8650 format %{
8651 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8652 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8653 %}
8654 ins_encode %{
8655 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8656 Assembler::byte, /*acquire*/ false, /*release*/ true,
8657 /*weak*/ true, noreg);
8658 __ csetw($res$$Register, Assembler::EQ);
8659 %}
8660 ins_pipe(pipe_slow);
8661 %}
8662
8663 // This pattern is generated automatically from cas.m4.
8664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8665 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8666 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8667 ins_cost(2 * VOLATILE_REF_COST);
8668 effect(KILL cr);
8669 format %{
8670 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8671 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8672 %}
8673 ins_encode %{
8674 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8675 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8676 /*weak*/ true, noreg);
8677 __ csetw($res$$Register, Assembler::EQ);
8678 %}
8679 ins_pipe(pipe_slow);
8680 %}
8681
8682 // This pattern is generated automatically from cas.m4.
8683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8684 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8685 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8686 ins_cost(2 * VOLATILE_REF_COST);
8687 effect(KILL cr);
8688 format %{
8689 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8690 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::word, /*acquire*/ false, /*release*/ true,
8695 /*weak*/ true, noreg);
8696 __ csetw($res$$Register, Assembler::EQ);
8697 %}
8698 ins_pipe(pipe_slow);
8699 %}
8700
8701 // This pattern is generated automatically from cas.m4.
8702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8703 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8704 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8705 ins_cost(2 * VOLATILE_REF_COST);
8706 effect(KILL cr);
8707 format %{
8708 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8709 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8710 %}
8711 ins_encode %{
8712 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8713 Assembler::xword, /*acquire*/ false, /*release*/ true,
8714 /*weak*/ true, noreg);
8715 __ csetw($res$$Register, Assembler::EQ);
8716 %}
8717 ins_pipe(pipe_slow);
8718 %}
8719
8720 // This pattern is generated automatically from cas.m4.
8721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8722 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8723 predicate(n->as_LoadStore()->barrier_data() == 0);
8724 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8725 ins_cost(2 * VOLATILE_REF_COST);
8726 effect(KILL cr);
8727 format %{
8728 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8729 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8730 %}
8731 ins_encode %{
8732 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8733 Assembler::word, /*acquire*/ false, /*release*/ true,
8734 /*weak*/ true, noreg);
8735 __ csetw($res$$Register, Assembler::EQ);
8736 %}
8737 ins_pipe(pipe_slow);
8738 %}
8739
8740 // This pattern is generated automatically from cas.m4.
8741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8742 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8743 predicate(n->as_LoadStore()->barrier_data() == 0);
8744 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8745 ins_cost(2 * VOLATILE_REF_COST);
8746 effect(KILL cr);
8747 format %{
8748 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8749 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8750 %}
8751 ins_encode %{
8752 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8753 Assembler::xword, /*acquire*/ false, /*release*/ true,
8754 /*weak*/ true, noreg);
8755 __ csetw($res$$Register, Assembler::EQ);
8756 %}
8757 ins_pipe(pipe_slow);
8758 %}
8759
8760 // This pattern is generated automatically from cas.m4.
8761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8762 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8763 predicate(needs_acquiring_load_exclusive(n));
8764 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8765 ins_cost(VOLATILE_REF_COST);
8766 effect(KILL cr);
8767 format %{
8768 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8769 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8770 %}
8771 ins_encode %{
8772 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8773 Assembler::byte, /*acquire*/ true, /*release*/ true,
8774 /*weak*/ true, noreg);
8775 __ csetw($res$$Register, Assembler::EQ);
8776 %}
8777 ins_pipe(pipe_slow);
8778 %}
8779
8780 // This pattern is generated automatically from cas.m4.
8781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8782 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8783 predicate(needs_acquiring_load_exclusive(n));
8784 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8785 ins_cost(VOLATILE_REF_COST);
8786 effect(KILL cr);
8787 format %{
8788 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8789 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8790 %}
8791 ins_encode %{
8792 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8793 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8794 /*weak*/ true, noreg);
8795 __ csetw($res$$Register, Assembler::EQ);
8796 %}
8797 ins_pipe(pipe_slow);
8798 %}
8799
8800 // This pattern is generated automatically from cas.m4.
8801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8802 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8803 predicate(needs_acquiring_load_exclusive(n));
8804 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8805 ins_cost(VOLATILE_REF_COST);
8806 effect(KILL cr);
8807 format %{
8808 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8809 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8810 %}
8811 ins_encode %{
8812 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8813 Assembler::word, /*acquire*/ true, /*release*/ true,
8814 /*weak*/ true, noreg);
8815 __ csetw($res$$Register, Assembler::EQ);
8816 %}
8817 ins_pipe(pipe_slow);
8818 %}
8819
8820 // This pattern is generated automatically from cas.m4.
8821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8822 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8823 predicate(needs_acquiring_load_exclusive(n));
8824 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8825 ins_cost(VOLATILE_REF_COST);
8826 effect(KILL cr);
8827 format %{
8828 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8829 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8830 %}
8831 ins_encode %{
8832 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8833 Assembler::xword, /*acquire*/ true, /*release*/ true,
8834 /*weak*/ true, noreg);
8835 __ csetw($res$$Register, Assembler::EQ);
8836 %}
8837 ins_pipe(pipe_slow);
8838 %}
8839
8840 // This pattern is generated automatically from cas.m4.
8841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8842 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8843 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8844 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8845 ins_cost(VOLATILE_REF_COST);
8846 effect(KILL cr);
8847 format %{
8848 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8849 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8850 %}
8851 ins_encode %{
8852 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8853 Assembler::word, /*acquire*/ true, /*release*/ true,
8854 /*weak*/ true, noreg);
8855 __ csetw($res$$Register, Assembler::EQ);
8856 %}
8857 ins_pipe(pipe_slow);
8858 %}
8859
8860 // This pattern is generated automatically from cas.m4.
8861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8862 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8863 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8864 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8865 ins_cost(VOLATILE_REF_COST);
8866 effect(KILL cr);
8867 format %{
8868 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8869 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8870 %}
8871 ins_encode %{
8872 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8873 Assembler::xword, /*acquire*/ true, /*release*/ true,
8874 /*weak*/ true, noreg);
8875 __ csetw($res$$Register, Assembler::EQ);
8876 %}
8877 ins_pipe(pipe_slow);
8878 %}
8879
8880 // END This section of the file is automatically generated. Do not edit --------------
8881 // ---------------------------------------------------------------------
8882
8883 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8884 match(Set prev (GetAndSetI mem newv));
8885 ins_cost(2 * VOLATILE_REF_COST);
8886 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8887 ins_encode %{
8888 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8889 %}
8890 ins_pipe(pipe_serial);
8891 %}
8892
8893 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8894 match(Set prev (GetAndSetL mem newv));
8895 ins_cost(2 * VOLATILE_REF_COST);
8896 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8897 ins_encode %{
8898 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8899 %}
8900 ins_pipe(pipe_serial);
8901 %}
8902
8903 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8904 predicate(n->as_LoadStore()->barrier_data() == 0);
8905 match(Set prev (GetAndSetN mem newv));
8906 ins_cost(2 * VOLATILE_REF_COST);
8907 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8908 ins_encode %{
8909 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8910 %}
8911 ins_pipe(pipe_serial);
8912 %}
8913
8914 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8915 predicate(n->as_LoadStore()->barrier_data() == 0);
8916 match(Set prev (GetAndSetP mem newv));
8917 ins_cost(2 * VOLATILE_REF_COST);
8918 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8919 ins_encode %{
8920 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8921 %}
8922 ins_pipe(pipe_serial);
8923 %}
8924
8925 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8926 predicate(needs_acquiring_load_exclusive(n));
8927 match(Set prev (GetAndSetI mem newv));
8928 ins_cost(VOLATILE_REF_COST);
8929 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8930 ins_encode %{
8931 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8932 %}
8933 ins_pipe(pipe_serial);
8934 %}
8935
8936 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8937 predicate(needs_acquiring_load_exclusive(n));
8938 match(Set prev (GetAndSetL mem newv));
8939 ins_cost(VOLATILE_REF_COST);
8940 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8941 ins_encode %{
8942 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8943 %}
8944 ins_pipe(pipe_serial);
8945 %}
8946
8947 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8948 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8949 match(Set prev (GetAndSetN mem newv));
8950 ins_cost(VOLATILE_REF_COST);
8951 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8952 ins_encode %{
8953 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8954 %}
8955 ins_pipe(pipe_serial);
8956 %}
8957
8958 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8959 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8960 match(Set prev (GetAndSetP mem newv));
8961 ins_cost(VOLATILE_REF_COST);
8962 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8963 ins_encode %{
8964 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8965 %}
8966 ins_pipe(pipe_serial);
8967 %}
8968
8969
8970 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8971 match(Set newval (GetAndAddL mem incr));
8972 ins_cost(2 * VOLATILE_REF_COST + 1);
8973 format %{ "get_and_addL $newval, [$mem], $incr" %}
8974 ins_encode %{
8975 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8976 %}
8977 ins_pipe(pipe_serial);
8978 %}
8979
8980 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8981 predicate(n->as_LoadStore()->result_not_used());
8982 match(Set dummy (GetAndAddL mem incr));
8983 ins_cost(2 * VOLATILE_REF_COST);
8984 format %{ "get_and_addL [$mem], $incr" %}
8985 ins_encode %{
8986 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8987 %}
8988 ins_pipe(pipe_serial);
8989 %}
8990
8991 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8992 match(Set newval (GetAndAddL mem incr));
8993 ins_cost(2 * VOLATILE_REF_COST + 1);
8994 format %{ "get_and_addL $newval, [$mem], $incr" %}
8995 ins_encode %{
8996 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8997 %}
8998 ins_pipe(pipe_serial);
8999 %}
9000
9001 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9002 predicate(n->as_LoadStore()->result_not_used());
9003 match(Set dummy (GetAndAddL mem incr));
9004 ins_cost(2 * VOLATILE_REF_COST);
9005 format %{ "get_and_addL [$mem], $incr" %}
9006 ins_encode %{
9007 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9008 %}
9009 ins_pipe(pipe_serial);
9010 %}
9011
9012 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9013 match(Set newval (GetAndAddI mem incr));
9014 ins_cost(2 * VOLATILE_REF_COST + 1);
9015 format %{ "get_and_addI $newval, [$mem], $incr" %}
9016 ins_encode %{
9017 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9018 %}
9019 ins_pipe(pipe_serial);
9020 %}
9021
9022 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9023 predicate(n->as_LoadStore()->result_not_used());
9024 match(Set dummy (GetAndAddI mem incr));
9025 ins_cost(2 * VOLATILE_REF_COST);
9026 format %{ "get_and_addI [$mem], $incr" %}
9027 ins_encode %{
9028 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9029 %}
9030 ins_pipe(pipe_serial);
9031 %}
9032
9033 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9034 match(Set newval (GetAndAddI mem incr));
9035 ins_cost(2 * VOLATILE_REF_COST + 1);
9036 format %{ "get_and_addI $newval, [$mem], $incr" %}
9037 ins_encode %{
9038 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9039 %}
9040 ins_pipe(pipe_serial);
9041 %}
9042
9043 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9044 predicate(n->as_LoadStore()->result_not_used());
9045 match(Set dummy (GetAndAddI mem incr));
9046 ins_cost(2 * VOLATILE_REF_COST);
9047 format %{ "get_and_addI [$mem], $incr" %}
9048 ins_encode %{
9049 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9050 %}
9051 ins_pipe(pipe_serial);
9052 %}
9053
9054 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9055 predicate(needs_acquiring_load_exclusive(n));
9056 match(Set newval (GetAndAddL mem incr));
9057 ins_cost(VOLATILE_REF_COST + 1);
9058 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9059 ins_encode %{
9060 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9061 %}
9062 ins_pipe(pipe_serial);
9063 %}
9064
9065 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9066 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9067 match(Set dummy (GetAndAddL mem incr));
9068 ins_cost(VOLATILE_REF_COST);
9069 format %{ "get_and_addL_acq [$mem], $incr" %}
9070 ins_encode %{
9071 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9072 %}
9073 ins_pipe(pipe_serial);
9074 %}
9075
9076 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9077 predicate(needs_acquiring_load_exclusive(n));
9078 match(Set newval (GetAndAddL mem incr));
9079 ins_cost(VOLATILE_REF_COST + 1);
9080 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9081 ins_encode %{
9082 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9083 %}
9084 ins_pipe(pipe_serial);
9085 %}
9086
9087 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9088 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9089 match(Set dummy (GetAndAddL mem incr));
9090 ins_cost(VOLATILE_REF_COST);
9091 format %{ "get_and_addL_acq [$mem], $incr" %}
9092 ins_encode %{
9093 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9094 %}
9095 ins_pipe(pipe_serial);
9096 %}
9097
9098 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9099 predicate(needs_acquiring_load_exclusive(n));
9100 match(Set newval (GetAndAddI mem incr));
9101 ins_cost(VOLATILE_REF_COST + 1);
9102 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9103 ins_encode %{
9104 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9105 %}
9106 ins_pipe(pipe_serial);
9107 %}
9108
9109 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9110 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9111 match(Set dummy (GetAndAddI mem incr));
9112 ins_cost(VOLATILE_REF_COST);
9113 format %{ "get_and_addI_acq [$mem], $incr" %}
9114 ins_encode %{
9115 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9116 %}
9117 ins_pipe(pipe_serial);
9118 %}
9119
9120 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9121 predicate(needs_acquiring_load_exclusive(n));
9122 match(Set newval (GetAndAddI mem incr));
9123 ins_cost(VOLATILE_REF_COST + 1);
9124 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9125 ins_encode %{
9126 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9127 %}
9128 ins_pipe(pipe_serial);
9129 %}
9130
9131 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9132 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9133 match(Set dummy (GetAndAddI mem incr));
9134 ins_cost(VOLATILE_REF_COST);
9135 format %{ "get_and_addI_acq [$mem], $incr" %}
9136 ins_encode %{
9137 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9138 %}
9139 ins_pipe(pipe_serial);
9140 %}
9141
9142 // Manifest a CmpU result in an integer register.
9143 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9144 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9145 %{
9146 match(Set dst (CmpU3 src1 src2));
9147 effect(KILL flags);
9148
9149 ins_cost(INSN_COST * 3);
9150 format %{
9151 "cmpw $src1, $src2\n\t"
9152 "csetw $dst, ne\n\t"
9153 "cnegw $dst, lo\t# CmpU3(reg)"
9154 %}
9155 ins_encode %{
9156 __ cmpw($src1$$Register, $src2$$Register);
9157 __ csetw($dst$$Register, Assembler::NE);
9158 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9159 %}
9160
9161 ins_pipe(pipe_class_default);
9162 %}
9163
9164 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9165 %{
9166 match(Set dst (CmpU3 src1 src2));
9167 effect(KILL flags);
9168
9169 ins_cost(INSN_COST * 3);
9170 format %{
9171 "subsw zr, $src1, $src2\n\t"
9172 "csetw $dst, ne\n\t"
9173 "cnegw $dst, lo\t# CmpU3(imm)"
9174 %}
9175 ins_encode %{
9176 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9177 __ csetw($dst$$Register, Assembler::NE);
9178 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9179 %}
9180
9181 ins_pipe(pipe_class_default);
9182 %}
9183
9184 // Manifest a CmpUL result in an integer register.
9185 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9186 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9187 %{
9188 match(Set dst (CmpUL3 src1 src2));
9189 effect(KILL flags);
9190
9191 ins_cost(INSN_COST * 3);
9192 format %{
9193 "cmp $src1, $src2\n\t"
9194 "csetw $dst, ne\n\t"
9195 "cnegw $dst, lo\t# CmpUL3(reg)"
9196 %}
9197 ins_encode %{
9198 __ cmp($src1$$Register, $src2$$Register);
9199 __ csetw($dst$$Register, Assembler::NE);
9200 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9201 %}
9202
9203 ins_pipe(pipe_class_default);
9204 %}
9205
9206 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9207 %{
9208 match(Set dst (CmpUL3 src1 src2));
9209 effect(KILL flags);
9210
9211 ins_cost(INSN_COST * 3);
9212 format %{
9213 "subs zr, $src1, $src2\n\t"
9214 "csetw $dst, ne\n\t"
9215 "cnegw $dst, lo\t# CmpUL3(imm)"
9216 %}
9217 ins_encode %{
9218 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9219 __ csetw($dst$$Register, Assembler::NE);
9220 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9221 %}
9222
9223 ins_pipe(pipe_class_default);
9224 %}
9225
9226 // Manifest a CmpL result in an integer register.
9227 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9228 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9229 %{
9230 match(Set dst (CmpL3 src1 src2));
9231 effect(KILL flags);
9232
9233 ins_cost(INSN_COST * 3);
9234 format %{
9235 "cmp $src1, $src2\n\t"
9236 "csetw $dst, ne\n\t"
9237 "cnegw $dst, lt\t# CmpL3(reg)"
9238 %}
9239 ins_encode %{
9240 __ cmp($src1$$Register, $src2$$Register);
9241 __ csetw($dst$$Register, Assembler::NE);
9242 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9243 %}
9244
9245 ins_pipe(pipe_class_default);
9246 %}
9247
9248 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9249 %{
9250 match(Set dst (CmpL3 src1 src2));
9251 effect(KILL flags);
9252
9253 ins_cost(INSN_COST * 3);
9254 format %{
9255 "subs zr, $src1, $src2\n\t"
9256 "csetw $dst, ne\n\t"
9257 "cnegw $dst, lt\t# CmpL3(imm)"
9258 %}
9259 ins_encode %{
9260 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9261 __ csetw($dst$$Register, Assembler::NE);
9262 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9263 %}
9264
9265 ins_pipe(pipe_class_default);
9266 %}
9267
9268 // ============================================================================
9269 // Conditional Move Instructions
9270
9271 // n.b. we have identical rules for both a signed compare op (cmpOp)
9272 // and an unsigned compare op (cmpOpU). it would be nice if we could
9273 // define an op class which merged both inputs and use it to type the
9274 // argument to a single rule. unfortunatelyt his fails because the
9275 // opclass does not live up to the COND_INTER interface of its
9276 // component operands. When the generic code tries to negate the
9277 // operand it ends up running the generci Machoper::negate method
9278 // which throws a ShouldNotHappen. So, we have to provide two flavours
9279 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9280
9281 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9282 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9283
9284 ins_cost(INSN_COST * 2);
9285 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9286
9287 ins_encode %{
9288 __ cselw(as_Register($dst$$reg),
9289 as_Register($src2$$reg),
9290 as_Register($src1$$reg),
9291 (Assembler::Condition)$cmp$$cmpcode);
9292 %}
9293
9294 ins_pipe(icond_reg_reg);
9295 %}
9296
9297 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9298 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9299
9300 ins_cost(INSN_COST * 2);
9301 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9302
9303 ins_encode %{
9304 __ cselw(as_Register($dst$$reg),
9305 as_Register($src2$$reg),
9306 as_Register($src1$$reg),
9307 (Assembler::Condition)$cmp$$cmpcode);
9308 %}
9309
9310 ins_pipe(icond_reg_reg);
9311 %}
9312
9313 // special cases where one arg is zero
9314
9315 // n.b. this is selected in preference to the rule above because it
9316 // avoids loading constant 0 into a source register
9317
9318 // TODO
9319 // we ought only to be able to cull one of these variants as the ideal
9320 // transforms ought always to order the zero consistently (to left/right?)
9321
9322 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9323 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9324
9325 ins_cost(INSN_COST * 2);
9326 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9327
9328 ins_encode %{
9329 __ cselw(as_Register($dst$$reg),
9330 as_Register($src$$reg),
9331 zr,
9332 (Assembler::Condition)$cmp$$cmpcode);
9333 %}
9334
9335 ins_pipe(icond_reg);
9336 %}
9337
9338 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9339 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9340
9341 ins_cost(INSN_COST * 2);
9342 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9343
9344 ins_encode %{
9345 __ cselw(as_Register($dst$$reg),
9346 as_Register($src$$reg),
9347 zr,
9348 (Assembler::Condition)$cmp$$cmpcode);
9349 %}
9350
9351 ins_pipe(icond_reg);
9352 %}
9353
9354 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9355 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9356
9357 ins_cost(INSN_COST * 2);
9358 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9359
9360 ins_encode %{
9361 __ cselw(as_Register($dst$$reg),
9362 zr,
9363 as_Register($src$$reg),
9364 (Assembler::Condition)$cmp$$cmpcode);
9365 %}
9366
9367 ins_pipe(icond_reg);
9368 %}
9369
9370 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9371 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9372
9373 ins_cost(INSN_COST * 2);
9374 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9375
9376 ins_encode %{
9377 __ cselw(as_Register($dst$$reg),
9378 zr,
9379 as_Register($src$$reg),
9380 (Assembler::Condition)$cmp$$cmpcode);
9381 %}
9382
9383 ins_pipe(icond_reg);
9384 %}
9385
9386 // special case for creating a boolean 0 or 1
9387
9388 // n.b. this is selected in preference to the rule above because it
9389 // avoids loading constants 0 and 1 into a source register
9390
9391 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9392 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9393
9394 ins_cost(INSN_COST * 2);
9395 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9396
9397 ins_encode %{
9398 // equivalently
9399 // cset(as_Register($dst$$reg),
9400 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9401 __ csincw(as_Register($dst$$reg),
9402 zr,
9403 zr,
9404 (Assembler::Condition)$cmp$$cmpcode);
9405 %}
9406
9407 ins_pipe(icond_none);
9408 %}
9409
9410 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9411 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9412
9413 ins_cost(INSN_COST * 2);
9414 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9415
9416 ins_encode %{
9417 // equivalently
9418 // cset(as_Register($dst$$reg),
9419 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9420 __ csincw(as_Register($dst$$reg),
9421 zr,
9422 zr,
9423 (Assembler::Condition)$cmp$$cmpcode);
9424 %}
9425
9426 ins_pipe(icond_none);
9427 %}
9428
9429 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9430 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9431
9432 ins_cost(INSN_COST * 2);
9433 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9434
9435 ins_encode %{
9436 __ csel(as_Register($dst$$reg),
9437 as_Register($src2$$reg),
9438 as_Register($src1$$reg),
9439 (Assembler::Condition)$cmp$$cmpcode);
9440 %}
9441
9442 ins_pipe(icond_reg_reg);
9443 %}
9444
9445 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9446 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9447
9448 ins_cost(INSN_COST * 2);
9449 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9450
9451 ins_encode %{
9452 __ csel(as_Register($dst$$reg),
9453 as_Register($src2$$reg),
9454 as_Register($src1$$reg),
9455 (Assembler::Condition)$cmp$$cmpcode);
9456 %}
9457
9458 ins_pipe(icond_reg_reg);
9459 %}
9460
9461 // special cases where one arg is zero
9462
9463 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9464 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9465
9466 ins_cost(INSN_COST * 2);
9467 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9468
9469 ins_encode %{
9470 __ csel(as_Register($dst$$reg),
9471 zr,
9472 as_Register($src$$reg),
9473 (Assembler::Condition)$cmp$$cmpcode);
9474 %}
9475
9476 ins_pipe(icond_reg);
9477 %}
9478
9479 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9480 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9481
9482 ins_cost(INSN_COST * 2);
9483 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9484
9485 ins_encode %{
9486 __ csel(as_Register($dst$$reg),
9487 zr,
9488 as_Register($src$$reg),
9489 (Assembler::Condition)$cmp$$cmpcode);
9490 %}
9491
9492 ins_pipe(icond_reg);
9493 %}
9494
9495 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9496 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9497
9498 ins_cost(INSN_COST * 2);
9499 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9500
9501 ins_encode %{
9502 __ csel(as_Register($dst$$reg),
9503 as_Register($src$$reg),
9504 zr,
9505 (Assembler::Condition)$cmp$$cmpcode);
9506 %}
9507
9508 ins_pipe(icond_reg);
9509 %}
9510
9511 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9512 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9513
9514 ins_cost(INSN_COST * 2);
9515 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9516
9517 ins_encode %{
9518 __ csel(as_Register($dst$$reg),
9519 as_Register($src$$reg),
9520 zr,
9521 (Assembler::Condition)$cmp$$cmpcode);
9522 %}
9523
9524 ins_pipe(icond_reg);
9525 %}
9526
9527 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9528 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9529
9530 ins_cost(INSN_COST * 2);
9531 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9532
9533 ins_encode %{
9534 __ csel(as_Register($dst$$reg),
9535 as_Register($src2$$reg),
9536 as_Register($src1$$reg),
9537 (Assembler::Condition)$cmp$$cmpcode);
9538 %}
9539
9540 ins_pipe(icond_reg_reg);
9541 %}
9542
9543 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9544 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9545
9546 ins_cost(INSN_COST * 2);
9547 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9548
9549 ins_encode %{
9550 __ csel(as_Register($dst$$reg),
9551 as_Register($src2$$reg),
9552 as_Register($src1$$reg),
9553 (Assembler::Condition)$cmp$$cmpcode);
9554 %}
9555
9556 ins_pipe(icond_reg_reg);
9557 %}
9558
9559 // special cases where one arg is zero
9560
9561 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9562 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9563
9564 ins_cost(INSN_COST * 2);
9565 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9566
9567 ins_encode %{
9568 __ csel(as_Register($dst$$reg),
9569 zr,
9570 as_Register($src$$reg),
9571 (Assembler::Condition)$cmp$$cmpcode);
9572 %}
9573
9574 ins_pipe(icond_reg);
9575 %}
9576
9577 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9578 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9579
9580 ins_cost(INSN_COST * 2);
9581 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9582
9583 ins_encode %{
9584 __ csel(as_Register($dst$$reg),
9585 zr,
9586 as_Register($src$$reg),
9587 (Assembler::Condition)$cmp$$cmpcode);
9588 %}
9589
9590 ins_pipe(icond_reg);
9591 %}
9592
9593 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9594 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9595
9596 ins_cost(INSN_COST * 2);
9597 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9598
9599 ins_encode %{
9600 __ csel(as_Register($dst$$reg),
9601 as_Register($src$$reg),
9602 zr,
9603 (Assembler::Condition)$cmp$$cmpcode);
9604 %}
9605
9606 ins_pipe(icond_reg);
9607 %}
9608
9609 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9610 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9611
9612 ins_cost(INSN_COST * 2);
9613 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9614
9615 ins_encode %{
9616 __ csel(as_Register($dst$$reg),
9617 as_Register($src$$reg),
9618 zr,
9619 (Assembler::Condition)$cmp$$cmpcode);
9620 %}
9621
9622 ins_pipe(icond_reg);
9623 %}
9624
9625 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9626 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9627
9628 ins_cost(INSN_COST * 2);
9629 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9630
9631 ins_encode %{
9632 __ cselw(as_Register($dst$$reg),
9633 as_Register($src2$$reg),
9634 as_Register($src1$$reg),
9635 (Assembler::Condition)$cmp$$cmpcode);
9636 %}
9637
9638 ins_pipe(icond_reg_reg);
9639 %}
9640
9641 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9642 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9643
9644 ins_cost(INSN_COST * 2);
9645 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9646
9647 ins_encode %{
9648 __ cselw(as_Register($dst$$reg),
9649 as_Register($src2$$reg),
9650 as_Register($src1$$reg),
9651 (Assembler::Condition)$cmp$$cmpcode);
9652 %}
9653
9654 ins_pipe(icond_reg_reg);
9655 %}
9656
9657 // special cases where one arg is zero
9658
9659 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9660 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9661
9662 ins_cost(INSN_COST * 2);
9663 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9664
9665 ins_encode %{
9666 __ cselw(as_Register($dst$$reg),
9667 zr,
9668 as_Register($src$$reg),
9669 (Assembler::Condition)$cmp$$cmpcode);
9670 %}
9671
9672 ins_pipe(icond_reg);
9673 %}
9674
9675 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9676 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9677
9678 ins_cost(INSN_COST * 2);
9679 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9680
9681 ins_encode %{
9682 __ cselw(as_Register($dst$$reg),
9683 zr,
9684 as_Register($src$$reg),
9685 (Assembler::Condition)$cmp$$cmpcode);
9686 %}
9687
9688 ins_pipe(icond_reg);
9689 %}
9690
9691 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9692 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9693
9694 ins_cost(INSN_COST * 2);
9695 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9696
9697 ins_encode %{
9698 __ cselw(as_Register($dst$$reg),
9699 as_Register($src$$reg),
9700 zr,
9701 (Assembler::Condition)$cmp$$cmpcode);
9702 %}
9703
9704 ins_pipe(icond_reg);
9705 %}
9706
9707 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9708 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9709
9710 ins_cost(INSN_COST * 2);
9711 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9712
9713 ins_encode %{
9714 __ cselw(as_Register($dst$$reg),
9715 as_Register($src$$reg),
9716 zr,
9717 (Assembler::Condition)$cmp$$cmpcode);
9718 %}
9719
9720 ins_pipe(icond_reg);
9721 %}
9722
9723 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9724 %{
9725 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9726
9727 ins_cost(INSN_COST * 3);
9728
9729 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9730 ins_encode %{
9731 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9732 __ fcsels(as_FloatRegister($dst$$reg),
9733 as_FloatRegister($src2$$reg),
9734 as_FloatRegister($src1$$reg),
9735 cond);
9736 %}
9737
9738 ins_pipe(fp_cond_reg_reg_s);
9739 %}
9740
9741 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9742 %{
9743 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9744
9745 ins_cost(INSN_COST * 3);
9746
9747 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9748 ins_encode %{
9749 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9750 __ fcsels(as_FloatRegister($dst$$reg),
9751 as_FloatRegister($src2$$reg),
9752 as_FloatRegister($src1$$reg),
9753 cond);
9754 %}
9755
9756 ins_pipe(fp_cond_reg_reg_s);
9757 %}
9758
9759 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9760 %{
9761 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9762
9763 ins_cost(INSN_COST * 3);
9764
9765 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9766 ins_encode %{
9767 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9768 __ fcseld(as_FloatRegister($dst$$reg),
9769 as_FloatRegister($src2$$reg),
9770 as_FloatRegister($src1$$reg),
9771 cond);
9772 %}
9773
9774 ins_pipe(fp_cond_reg_reg_d);
9775 %}
9776
9777 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9778 %{
9779 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9780
9781 ins_cost(INSN_COST * 3);
9782
9783 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9784 ins_encode %{
9785 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9786 __ fcseld(as_FloatRegister($dst$$reg),
9787 as_FloatRegister($src2$$reg),
9788 as_FloatRegister($src1$$reg),
9789 cond);
9790 %}
9791
9792 ins_pipe(fp_cond_reg_reg_d);
9793 %}
9794
9795 // ============================================================================
9796 // Arithmetic Instructions
9797 //
9798
9799 // Integer Addition
9800
9801 // TODO
9802 // these currently employ operations which do not set CR and hence are
9803 // not flagged as killing CR but we would like to isolate the cases
9804 // where we want to set flags from those where we don't. need to work
9805 // out how to do that.
9806
9807 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9808 match(Set dst (AddI src1 src2));
9809
9810 ins_cost(INSN_COST);
9811 format %{ "addw $dst, $src1, $src2" %}
9812
9813 ins_encode %{
9814 __ addw(as_Register($dst$$reg),
9815 as_Register($src1$$reg),
9816 as_Register($src2$$reg));
9817 %}
9818
9819 ins_pipe(ialu_reg_reg);
9820 %}
9821
9822 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9823 match(Set dst (AddI src1 src2));
9824
9825 ins_cost(INSN_COST);
9826 format %{ "addw $dst, $src1, $src2" %}
9827
9828 // use opcode to indicate that this is an add not a sub
9829 opcode(0x0);
9830
9831 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9832
9833 ins_pipe(ialu_reg_imm);
9834 %}
9835
9836 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9837 match(Set dst (AddI (ConvL2I src1) src2));
9838
9839 ins_cost(INSN_COST);
9840 format %{ "addw $dst, $src1, $src2" %}
9841
9842 // use opcode to indicate that this is an add not a sub
9843 opcode(0x0);
9844
9845 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9846
9847 ins_pipe(ialu_reg_imm);
9848 %}
9849
9850 // Pointer Addition
9851 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9852 match(Set dst (AddP src1 src2));
9853
9854 ins_cost(INSN_COST);
9855 format %{ "add $dst, $src1, $src2\t# ptr" %}
9856
9857 ins_encode %{
9858 __ add(as_Register($dst$$reg),
9859 as_Register($src1$$reg),
9860 as_Register($src2$$reg));
9861 %}
9862
9863 ins_pipe(ialu_reg_reg);
9864 %}
9865
9866 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9867 match(Set dst (AddP src1 (ConvI2L src2)));
9868
9869 ins_cost(1.9 * INSN_COST);
9870 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9871
9872 ins_encode %{
9873 __ add(as_Register($dst$$reg),
9874 as_Register($src1$$reg),
9875 as_Register($src2$$reg), ext::sxtw);
9876 %}
9877
9878 ins_pipe(ialu_reg_reg);
9879 %}
9880
9881 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9882 match(Set dst (AddP src1 (LShiftL src2 scale)));
9883
9884 ins_cost(1.9 * INSN_COST);
9885 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9886
9887 ins_encode %{
9888 __ lea(as_Register($dst$$reg),
9889 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9890 Address::lsl($scale$$constant)));
9891 %}
9892
9893 ins_pipe(ialu_reg_reg_shift);
9894 %}
9895
9896 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9897 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9898
9899 ins_cost(1.9 * INSN_COST);
9900 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9901
9902 ins_encode %{
9903 __ lea(as_Register($dst$$reg),
9904 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9905 Address::sxtw($scale$$constant)));
9906 %}
9907
9908 ins_pipe(ialu_reg_reg_shift);
9909 %}
9910
9911 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9912 match(Set dst (LShiftL (ConvI2L src) scale));
9913
9914 ins_cost(INSN_COST);
9915 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9916
9917 ins_encode %{
9918 __ sbfiz(as_Register($dst$$reg),
9919 as_Register($src$$reg),
9920 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9921 %}
9922
9923 ins_pipe(ialu_reg_shift);
9924 %}
9925
9926 // Pointer Immediate Addition
9927 // n.b. this needs to be more expensive than using an indirect memory
9928 // operand
9929 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9930 match(Set dst (AddP src1 src2));
9931
9932 ins_cost(INSN_COST);
9933 format %{ "add $dst, $src1, $src2\t# ptr" %}
9934
9935 // use opcode to indicate that this is an add not a sub
9936 opcode(0x0);
9937
9938 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9939
9940 ins_pipe(ialu_reg_imm);
9941 %}
9942
9943 // Long Addition
9944 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9945
9946 match(Set dst (AddL src1 src2));
9947
9948 ins_cost(INSN_COST);
9949 format %{ "add $dst, $src1, $src2" %}
9950
9951 ins_encode %{
9952 __ add(as_Register($dst$$reg),
9953 as_Register($src1$$reg),
9954 as_Register($src2$$reg));
9955 %}
9956
9957 ins_pipe(ialu_reg_reg);
9958 %}
9959
9960 // No constant pool entries requiredLong Immediate Addition.
9961 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9962 match(Set dst (AddL src1 src2));
9963
9964 ins_cost(INSN_COST);
9965 format %{ "add $dst, $src1, $src2" %}
9966
9967 // use opcode to indicate that this is an add not a sub
9968 opcode(0x0);
9969
9970 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9971
9972 ins_pipe(ialu_reg_imm);
9973 %}
9974
9975 // Integer Subtraction
9976 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9977 match(Set dst (SubI src1 src2));
9978
9979 ins_cost(INSN_COST);
9980 format %{ "subw $dst, $src1, $src2" %}
9981
9982 ins_encode %{
9983 __ subw(as_Register($dst$$reg),
9984 as_Register($src1$$reg),
9985 as_Register($src2$$reg));
9986 %}
9987
9988 ins_pipe(ialu_reg_reg);
9989 %}
9990
9991 // Immediate Subtraction
9992 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9993 match(Set dst (SubI src1 src2));
9994
9995 ins_cost(INSN_COST);
9996 format %{ "subw $dst, $src1, $src2" %}
9997
9998 // use opcode to indicate that this is a sub not an add
9999 opcode(0x1);
10000
10001 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10002
10003 ins_pipe(ialu_reg_imm);
10004 %}
10005
10006 // Long Subtraction
10007 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10008
10009 match(Set dst (SubL src1 src2));
10010
10011 ins_cost(INSN_COST);
10012 format %{ "sub $dst, $src1, $src2" %}
10013
10014 ins_encode %{
10015 __ sub(as_Register($dst$$reg),
10016 as_Register($src1$$reg),
10017 as_Register($src2$$reg));
10018 %}
10019
10020 ins_pipe(ialu_reg_reg);
10021 %}
10022
10023 // No constant pool entries requiredLong Immediate Subtraction.
10024 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10025 match(Set dst (SubL src1 src2));
10026
10027 ins_cost(INSN_COST);
10028 format %{ "sub$dst, $src1, $src2" %}
10029
10030 // use opcode to indicate that this is a sub not an add
10031 opcode(0x1);
10032
10033 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10034
10035 ins_pipe(ialu_reg_imm);
10036 %}
10037
10038 // Integer Negation (special case for sub)
10039
10040 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10041 match(Set dst (SubI zero src));
10042
10043 ins_cost(INSN_COST);
10044 format %{ "negw $dst, $src\t# int" %}
10045
10046 ins_encode %{
10047 __ negw(as_Register($dst$$reg),
10048 as_Register($src$$reg));
10049 %}
10050
10051 ins_pipe(ialu_reg);
10052 %}
10053
10054 // Long Negation
10055
10056 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10057 match(Set dst (SubL zero src));
10058
10059 ins_cost(INSN_COST);
10060 format %{ "neg $dst, $src\t# long" %}
10061
10062 ins_encode %{
10063 __ neg(as_Register($dst$$reg),
10064 as_Register($src$$reg));
10065 %}
10066
10067 ins_pipe(ialu_reg);
10068 %}
10069
10070 // Integer Multiply
10071
10072 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10073 match(Set dst (MulI src1 src2));
10074
10075 ins_cost(INSN_COST * 3);
10076 format %{ "mulw $dst, $src1, $src2" %}
10077
10078 ins_encode %{
10079 __ mulw(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg));
10082 %}
10083
10084 ins_pipe(imul_reg_reg);
10085 %}
10086
10087 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10088 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10089
10090 ins_cost(INSN_COST * 3);
10091 format %{ "smull $dst, $src1, $src2" %}
10092
10093 ins_encode %{
10094 __ smull(as_Register($dst$$reg),
10095 as_Register($src1$$reg),
10096 as_Register($src2$$reg));
10097 %}
10098
10099 ins_pipe(imul_reg_reg);
10100 %}
10101
10102 // Long Multiply
10103
10104 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10105 match(Set dst (MulL src1 src2));
10106
10107 ins_cost(INSN_COST * 5);
10108 format %{ "mul $dst, $src1, $src2" %}
10109
10110 ins_encode %{
10111 __ mul(as_Register($dst$$reg),
10112 as_Register($src1$$reg),
10113 as_Register($src2$$reg));
10114 %}
10115
10116 ins_pipe(lmul_reg_reg);
10117 %}
10118
10119 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10120 %{
10121 match(Set dst (MulHiL src1 src2));
10122
10123 ins_cost(INSN_COST * 7);
10124 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10125
10126 ins_encode %{
10127 __ smulh(as_Register($dst$$reg),
10128 as_Register($src1$$reg),
10129 as_Register($src2$$reg));
10130 %}
10131
10132 ins_pipe(lmul_reg_reg);
10133 %}
10134
10135 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10136 %{
10137 match(Set dst (UMulHiL src1 src2));
10138
10139 ins_cost(INSN_COST * 7);
10140 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10141
10142 ins_encode %{
10143 __ umulh(as_Register($dst$$reg),
10144 as_Register($src1$$reg),
10145 as_Register($src2$$reg));
10146 %}
10147
10148 ins_pipe(lmul_reg_reg);
10149 %}
10150
10151 // Combined Integer Multiply & Add/Sub
10152
10153 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10154 match(Set dst (AddI src3 (MulI src1 src2)));
10155
10156 ins_cost(INSN_COST * 3);
10157 format %{ "madd $dst, $src1, $src2, $src3" %}
10158
10159 ins_encode %{
10160 __ maddw(as_Register($dst$$reg),
10161 as_Register($src1$$reg),
10162 as_Register($src2$$reg),
10163 as_Register($src3$$reg));
10164 %}
10165
10166 ins_pipe(imac_reg_reg);
10167 %}
10168
10169 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10170 match(Set dst (SubI src3 (MulI src1 src2)));
10171
10172 ins_cost(INSN_COST * 3);
10173 format %{ "msub $dst, $src1, $src2, $src3" %}
10174
10175 ins_encode %{
10176 __ msubw(as_Register($dst$$reg),
10177 as_Register($src1$$reg),
10178 as_Register($src2$$reg),
10179 as_Register($src3$$reg));
10180 %}
10181
10182 ins_pipe(imac_reg_reg);
10183 %}
10184
10185 // Combined Integer Multiply & Neg
10186
10187 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10188 match(Set dst (MulI (SubI zero src1) src2));
10189
10190 ins_cost(INSN_COST * 3);
10191 format %{ "mneg $dst, $src1, $src2" %}
10192
10193 ins_encode %{
10194 __ mnegw(as_Register($dst$$reg),
10195 as_Register($src1$$reg),
10196 as_Register($src2$$reg));
10197 %}
10198
10199 ins_pipe(imac_reg_reg);
10200 %}
10201
10202 // Combined Long Multiply & Add/Sub
10203
10204 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10205 match(Set dst (AddL src3 (MulL src1 src2)));
10206
10207 ins_cost(INSN_COST * 5);
10208 format %{ "madd $dst, $src1, $src2, $src3" %}
10209
10210 ins_encode %{
10211 __ madd(as_Register($dst$$reg),
10212 as_Register($src1$$reg),
10213 as_Register($src2$$reg),
10214 as_Register($src3$$reg));
10215 %}
10216
10217 ins_pipe(lmac_reg_reg);
10218 %}
10219
10220 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10221 match(Set dst (SubL src3 (MulL src1 src2)));
10222
10223 ins_cost(INSN_COST * 5);
10224 format %{ "msub $dst, $src1, $src2, $src3" %}
10225
10226 ins_encode %{
10227 __ msub(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 as_Register($src3$$reg));
10231 %}
10232
10233 ins_pipe(lmac_reg_reg);
10234 %}
10235
10236 // Combined Long Multiply & Neg
10237
10238 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10239 match(Set dst (MulL (SubL zero src1) src2));
10240
10241 ins_cost(INSN_COST * 5);
10242 format %{ "mneg $dst, $src1, $src2" %}
10243
10244 ins_encode %{
10245 __ mneg(as_Register($dst$$reg),
10246 as_Register($src1$$reg),
10247 as_Register($src2$$reg));
10248 %}
10249
10250 ins_pipe(lmac_reg_reg);
10251 %}
10252
10253 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10254
10255 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10256 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10257
10258 ins_cost(INSN_COST * 3);
10259 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10260
10261 ins_encode %{
10262 __ smaddl(as_Register($dst$$reg),
10263 as_Register($src1$$reg),
10264 as_Register($src2$$reg),
10265 as_Register($src3$$reg));
10266 %}
10267
10268 ins_pipe(imac_reg_reg);
10269 %}
10270
10271 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10272 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10273
10274 ins_cost(INSN_COST * 3);
10275 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10276
10277 ins_encode %{
10278 __ smsubl(as_Register($dst$$reg),
10279 as_Register($src1$$reg),
10280 as_Register($src2$$reg),
10281 as_Register($src3$$reg));
10282 %}
10283
10284 ins_pipe(imac_reg_reg);
10285 %}
10286
10287 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10288 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10289
10290 ins_cost(INSN_COST * 3);
10291 format %{ "smnegl $dst, $src1, $src2" %}
10292
10293 ins_encode %{
10294 __ smnegl(as_Register($dst$$reg),
10295 as_Register($src1$$reg),
10296 as_Register($src2$$reg));
10297 %}
10298
10299 ins_pipe(imac_reg_reg);
10300 %}
10301
10302 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10303
10304 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10305 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10306
10307 ins_cost(INSN_COST * 5);
10308 format %{ "mulw rscratch1, $src1, $src2\n\t"
10309 "maddw $dst, $src3, $src4, rscratch1" %}
10310
10311 ins_encode %{
10312 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10313 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10314
10315 ins_pipe(imac_reg_reg);
10316 %}
10317
10318 // Integer Divide
10319
10320 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10321 match(Set dst (DivI src1 src2));
10322
10323 ins_cost(INSN_COST * 19);
10324 format %{ "sdivw $dst, $src1, $src2" %}
10325
10326 ins_encode(aarch64_enc_divw(dst, src1, src2));
10327 ins_pipe(idiv_reg_reg);
10328 %}
10329
10330 // Long Divide
10331
10332 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10333 match(Set dst (DivL src1 src2));
10334
10335 ins_cost(INSN_COST * 35);
10336 format %{ "sdiv $dst, $src1, $src2" %}
10337
10338 ins_encode(aarch64_enc_div(dst, src1, src2));
10339 ins_pipe(ldiv_reg_reg);
10340 %}
10341
10342 // Integer Remainder
10343
10344 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10345 match(Set dst (ModI src1 src2));
10346
10347 ins_cost(INSN_COST * 22);
10348 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10349 "msubw $dst, rscratch1, $src2, $src1" %}
10350
10351 ins_encode(aarch64_enc_modw(dst, src1, src2));
10352 ins_pipe(idiv_reg_reg);
10353 %}
10354
10355 // Long Remainder
10356
10357 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10358 match(Set dst (ModL src1 src2));
10359
10360 ins_cost(INSN_COST * 38);
10361 format %{ "sdiv rscratch1, $src1, $src2\n"
10362 "msub $dst, rscratch1, $src2, $src1" %}
10363
10364 ins_encode(aarch64_enc_mod(dst, src1, src2));
10365 ins_pipe(ldiv_reg_reg);
10366 %}
10367
10368 // Unsigned Integer Divide
10369
10370 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10371 match(Set dst (UDivI src1 src2));
10372
10373 ins_cost(INSN_COST * 19);
10374 format %{ "udivw $dst, $src1, $src2" %}
10375
10376 ins_encode %{
10377 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10378 %}
10379
10380 ins_pipe(idiv_reg_reg);
10381 %}
10382
10383 // Unsigned Long Divide
10384
10385 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10386 match(Set dst (UDivL src1 src2));
10387
10388 ins_cost(INSN_COST * 35);
10389 format %{ "udiv $dst, $src1, $src2" %}
10390
10391 ins_encode %{
10392 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10393 %}
10394
10395 ins_pipe(ldiv_reg_reg);
10396 %}
10397
10398 // Unsigned Integer Remainder
10399
10400 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10401 match(Set dst (UModI src1 src2));
10402
10403 ins_cost(INSN_COST * 22);
10404 format %{ "udivw rscratch1, $src1, $src2\n\t"
10405 "msubw $dst, rscratch1, $src2, $src1" %}
10406
10407 ins_encode %{
10408 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10409 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10410 %}
10411
10412 ins_pipe(idiv_reg_reg);
10413 %}
10414
10415 // Unsigned Long Remainder
10416
10417 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10418 match(Set dst (UModL src1 src2));
10419
10420 ins_cost(INSN_COST * 38);
10421 format %{ "udiv rscratch1, $src1, $src2\n"
10422 "msub $dst, rscratch1, $src2, $src1" %}
10423
10424 ins_encode %{
10425 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10426 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10427 %}
10428
10429 ins_pipe(ldiv_reg_reg);
10430 %}
10431
10432 // Integer Shifts
10433
10434 // Shift Left Register
10435 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10436 match(Set dst (LShiftI src1 src2));
10437
10438 ins_cost(INSN_COST * 2);
10439 format %{ "lslvw $dst, $src1, $src2" %}
10440
10441 ins_encode %{
10442 __ lslvw(as_Register($dst$$reg),
10443 as_Register($src1$$reg),
10444 as_Register($src2$$reg));
10445 %}
10446
10447 ins_pipe(ialu_reg_reg_vshift);
10448 %}
10449
10450 // Shift Left Immediate
10451 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10452 match(Set dst (LShiftI src1 src2));
10453
10454 ins_cost(INSN_COST);
10455 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10456
10457 ins_encode %{
10458 __ lslw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 $src2$$constant & 0x1f);
10461 %}
10462
10463 ins_pipe(ialu_reg_shift);
10464 %}
10465
10466 // Shift Right Logical Register
10467 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10468 match(Set dst (URShiftI src1 src2));
10469
10470 ins_cost(INSN_COST * 2);
10471 format %{ "lsrvw $dst, $src1, $src2" %}
10472
10473 ins_encode %{
10474 __ lsrvw(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 as_Register($src2$$reg));
10477 %}
10478
10479 ins_pipe(ialu_reg_reg_vshift);
10480 %}
10481
10482 // Shift Right Logical Immediate
10483 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10484 match(Set dst (URShiftI src1 src2));
10485
10486 ins_cost(INSN_COST);
10487 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10488
10489 ins_encode %{
10490 __ lsrw(as_Register($dst$$reg),
10491 as_Register($src1$$reg),
10492 $src2$$constant & 0x1f);
10493 %}
10494
10495 ins_pipe(ialu_reg_shift);
10496 %}
10497
10498 // Shift Right Arithmetic Register
10499 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10500 match(Set dst (RShiftI src1 src2));
10501
10502 ins_cost(INSN_COST * 2);
10503 format %{ "asrvw $dst, $src1, $src2" %}
10504
10505 ins_encode %{
10506 __ asrvw(as_Register($dst$$reg),
10507 as_Register($src1$$reg),
10508 as_Register($src2$$reg));
10509 %}
10510
10511 ins_pipe(ialu_reg_reg_vshift);
10512 %}
10513
10514 // Shift Right Arithmetic Immediate
10515 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10516 match(Set dst (RShiftI src1 src2));
10517
10518 ins_cost(INSN_COST);
10519 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10520
10521 ins_encode %{
10522 __ asrw(as_Register($dst$$reg),
10523 as_Register($src1$$reg),
10524 $src2$$constant & 0x1f);
10525 %}
10526
10527 ins_pipe(ialu_reg_shift);
10528 %}
10529
10530 // Combined Int Mask and Right Shift (using UBFM)
10531 // TODO
10532
10533 // Long Shifts
10534
10535 // Shift Left Register
10536 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10537 match(Set dst (LShiftL src1 src2));
10538
10539 ins_cost(INSN_COST * 2);
10540 format %{ "lslv $dst, $src1, $src2" %}
10541
10542 ins_encode %{
10543 __ lslv(as_Register($dst$$reg),
10544 as_Register($src1$$reg),
10545 as_Register($src2$$reg));
10546 %}
10547
10548 ins_pipe(ialu_reg_reg_vshift);
10549 %}
10550
10551 // Shift Left Immediate
10552 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10553 match(Set dst (LShiftL src1 src2));
10554
10555 ins_cost(INSN_COST);
10556 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10557
10558 ins_encode %{
10559 __ lsl(as_Register($dst$$reg),
10560 as_Register($src1$$reg),
10561 $src2$$constant & 0x3f);
10562 %}
10563
10564 ins_pipe(ialu_reg_shift);
10565 %}
10566
10567 // Shift Right Logical Register
10568 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10569 match(Set dst (URShiftL src1 src2));
10570
10571 ins_cost(INSN_COST * 2);
10572 format %{ "lsrv $dst, $src1, $src2" %}
10573
10574 ins_encode %{
10575 __ lsrv(as_Register($dst$$reg),
10576 as_Register($src1$$reg),
10577 as_Register($src2$$reg));
10578 %}
10579
10580 ins_pipe(ialu_reg_reg_vshift);
10581 %}
10582
10583 // Shift Right Logical Immediate
10584 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10585 match(Set dst (URShiftL src1 src2));
10586
10587 ins_cost(INSN_COST);
10588 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10589
10590 ins_encode %{
10591 __ lsr(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 $src2$$constant & 0x3f);
10594 %}
10595
10596 ins_pipe(ialu_reg_shift);
10597 %}
10598
10599 // A special-case pattern for card table stores.
10600 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10601 match(Set dst (URShiftL (CastP2X src1) src2));
10602
10603 ins_cost(INSN_COST);
10604 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10605
10606 ins_encode %{
10607 __ lsr(as_Register($dst$$reg),
10608 as_Register($src1$$reg),
10609 $src2$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_shift);
10613 %}
10614
10615 // Shift Right Arithmetic Register
10616 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10617 match(Set dst (RShiftL src1 src2));
10618
10619 ins_cost(INSN_COST * 2);
10620 format %{ "asrv $dst, $src1, $src2" %}
10621
10622 ins_encode %{
10623 __ asrv(as_Register($dst$$reg),
10624 as_Register($src1$$reg),
10625 as_Register($src2$$reg));
10626 %}
10627
10628 ins_pipe(ialu_reg_reg_vshift);
10629 %}
10630
10631 // Shift Right Arithmetic Immediate
10632 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10633 match(Set dst (RShiftL src1 src2));
10634
10635 ins_cost(INSN_COST);
10636 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10637
10638 ins_encode %{
10639 __ asr(as_Register($dst$$reg),
10640 as_Register($src1$$reg),
10641 $src2$$constant & 0x3f);
10642 %}
10643
10644 ins_pipe(ialu_reg_shift);
10645 %}
10646
10647 // BEGIN This section of the file is automatically generated. Do not edit --------------
10648 // This section is generated from aarch64_ad.m4
10649
10650 // This pattern is automatically generated from aarch64_ad.m4
10651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10652 instruct regL_not_reg(iRegLNoSp dst,
10653 iRegL src1, immL_M1 m1,
10654 rFlagsReg cr) %{
10655 match(Set dst (XorL src1 m1));
10656 ins_cost(INSN_COST);
10657 format %{ "eon $dst, $src1, zr" %}
10658
10659 ins_encode %{
10660 __ eon(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 zr,
10663 Assembler::LSL, 0);
10664 %}
10665
10666 ins_pipe(ialu_reg);
10667 %}
10668
10669 // This pattern is automatically generated from aarch64_ad.m4
10670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10671 instruct regI_not_reg(iRegINoSp dst,
10672 iRegIorL2I src1, immI_M1 m1,
10673 rFlagsReg cr) %{
10674 match(Set dst (XorI src1 m1));
10675 ins_cost(INSN_COST);
10676 format %{ "eonw $dst, $src1, zr" %}
10677
10678 ins_encode %{
10679 __ eonw(as_Register($dst$$reg),
10680 as_Register($src1$$reg),
10681 zr,
10682 Assembler::LSL, 0);
10683 %}
10684
10685 ins_pipe(ialu_reg);
10686 %}
10687
10688 // This pattern is automatically generated from aarch64_ad.m4
10689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10690 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10691 immI0 zero, iRegIorL2I src1, immI src2) %{
10692 match(Set dst (SubI zero (URShiftI src1 src2)));
10693
10694 ins_cost(1.9 * INSN_COST);
10695 format %{ "negw $dst, $src1, LSR $src2" %}
10696
10697 ins_encode %{
10698 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10699 Assembler::LSR, $src2$$constant & 0x1f);
10700 %}
10701
10702 ins_pipe(ialu_reg_shift);
10703 %}
10704
10705 // This pattern is automatically generated from aarch64_ad.m4
10706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10707 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10708 immI0 zero, iRegIorL2I src1, immI src2) %{
10709 match(Set dst (SubI zero (RShiftI src1 src2)));
10710
10711 ins_cost(1.9 * INSN_COST);
10712 format %{ "negw $dst, $src1, ASR $src2" %}
10713
10714 ins_encode %{
10715 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10716 Assembler::ASR, $src2$$constant & 0x1f);
10717 %}
10718
10719 ins_pipe(ialu_reg_shift);
10720 %}
10721
10722 // This pattern is automatically generated from aarch64_ad.m4
10723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10724 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10725 immI0 zero, iRegIorL2I src1, immI src2) %{
10726 match(Set dst (SubI zero (LShiftI src1 src2)));
10727
10728 ins_cost(1.9 * INSN_COST);
10729 format %{ "negw $dst, $src1, LSL $src2" %}
10730
10731 ins_encode %{
10732 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10733 Assembler::LSL, $src2$$constant & 0x1f);
10734 %}
10735
10736 ins_pipe(ialu_reg_shift);
10737 %}
10738
10739 // This pattern is automatically generated from aarch64_ad.m4
10740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10741 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10742 immL0 zero, iRegL src1, immI src2) %{
10743 match(Set dst (SubL zero (URShiftL src1 src2)));
10744
10745 ins_cost(1.9 * INSN_COST);
10746 format %{ "neg $dst, $src1, LSR $src2" %}
10747
10748 ins_encode %{
10749 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10750 Assembler::LSR, $src2$$constant & 0x3f);
10751 %}
10752
10753 ins_pipe(ialu_reg_shift);
10754 %}
10755
10756 // This pattern is automatically generated from aarch64_ad.m4
10757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10758 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10759 immL0 zero, iRegL src1, immI src2) %{
10760 match(Set dst (SubL zero (RShiftL src1 src2)));
10761
10762 ins_cost(1.9 * INSN_COST);
10763 format %{ "neg $dst, $src1, ASR $src2" %}
10764
10765 ins_encode %{
10766 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10767 Assembler::ASR, $src2$$constant & 0x3f);
10768 %}
10769
10770 ins_pipe(ialu_reg_shift);
10771 %}
10772
10773 // This pattern is automatically generated from aarch64_ad.m4
10774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10775 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10776 immL0 zero, iRegL src1, immI src2) %{
10777 match(Set dst (SubL zero (LShiftL src1 src2)));
10778
10779 ins_cost(1.9 * INSN_COST);
10780 format %{ "neg $dst, $src1, LSL $src2" %}
10781
10782 ins_encode %{
10783 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10784 Assembler::LSL, $src2$$constant & 0x3f);
10785 %}
10786
10787 ins_pipe(ialu_reg_shift);
10788 %}
10789
10790 // This pattern is automatically generated from aarch64_ad.m4
10791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10792 instruct AndI_reg_not_reg(iRegINoSp dst,
10793 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10794 match(Set dst (AndI src1 (XorI src2 m1)));
10795 ins_cost(INSN_COST);
10796 format %{ "bicw $dst, $src1, $src2" %}
10797
10798 ins_encode %{
10799 __ bicw(as_Register($dst$$reg),
10800 as_Register($src1$$reg),
10801 as_Register($src2$$reg),
10802 Assembler::LSL, 0);
10803 %}
10804
10805 ins_pipe(ialu_reg_reg);
10806 %}
10807
10808 // This pattern is automatically generated from aarch64_ad.m4
10809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10810 instruct AndL_reg_not_reg(iRegLNoSp dst,
10811 iRegL src1, iRegL src2, immL_M1 m1) %{
10812 match(Set dst (AndL src1 (XorL src2 m1)));
10813 ins_cost(INSN_COST);
10814 format %{ "bic $dst, $src1, $src2" %}
10815
10816 ins_encode %{
10817 __ bic(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg),
10820 Assembler::LSL, 0);
10821 %}
10822
10823 ins_pipe(ialu_reg_reg);
10824 %}
10825
10826 // This pattern is automatically generated from aarch64_ad.m4
10827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10828 instruct OrI_reg_not_reg(iRegINoSp dst,
10829 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10830 match(Set dst (OrI src1 (XorI src2 m1)));
10831 ins_cost(INSN_COST);
10832 format %{ "ornw $dst, $src1, $src2" %}
10833
10834 ins_encode %{
10835 __ ornw(as_Register($dst$$reg),
10836 as_Register($src1$$reg),
10837 as_Register($src2$$reg),
10838 Assembler::LSL, 0);
10839 %}
10840
10841 ins_pipe(ialu_reg_reg);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct OrL_reg_not_reg(iRegLNoSp dst,
10847 iRegL src1, iRegL src2, immL_M1 m1) %{
10848 match(Set dst (OrL src1 (XorL src2 m1)));
10849 ins_cost(INSN_COST);
10850 format %{ "orn $dst, $src1, $src2" %}
10851
10852 ins_encode %{
10853 __ orn(as_Register($dst$$reg),
10854 as_Register($src1$$reg),
10855 as_Register($src2$$reg),
10856 Assembler::LSL, 0);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct XorI_reg_not_reg(iRegINoSp dst,
10865 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10866 match(Set dst (XorI m1 (XorI src2 src1)));
10867 ins_cost(INSN_COST);
10868 format %{ "eonw $dst, $src1, $src2" %}
10869
10870 ins_encode %{
10871 __ eonw(as_Register($dst$$reg),
10872 as_Register($src1$$reg),
10873 as_Register($src2$$reg),
10874 Assembler::LSL, 0);
10875 %}
10876
10877 ins_pipe(ialu_reg_reg);
10878 %}
10879
10880 // This pattern is automatically generated from aarch64_ad.m4
10881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10882 instruct XorL_reg_not_reg(iRegLNoSp dst,
10883 iRegL src1, iRegL src2, immL_M1 m1) %{
10884 match(Set dst (XorL m1 (XorL src2 src1)));
10885 ins_cost(INSN_COST);
10886 format %{ "eon $dst, $src1, $src2" %}
10887
10888 ins_encode %{
10889 __ eon(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg),
10892 Assembler::LSL, 0);
10893 %}
10894
10895 ins_pipe(ialu_reg_reg);
10896 %}
10897
10898 // This pattern is automatically generated from aarch64_ad.m4
10899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10900 // val & (-1 ^ (val >>> shift)) ==> bicw
10901 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10902 iRegIorL2I src1, iRegIorL2I src2,
10903 immI src3, immI_M1 src4) %{
10904 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10905 ins_cost(1.9 * INSN_COST);
10906 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10907
10908 ins_encode %{
10909 __ bicw(as_Register($dst$$reg),
10910 as_Register($src1$$reg),
10911 as_Register($src2$$reg),
10912 Assembler::LSR,
10913 $src3$$constant & 0x1f);
10914 %}
10915
10916 ins_pipe(ialu_reg_reg_shift);
10917 %}
10918
10919 // This pattern is automatically generated from aarch64_ad.m4
10920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10921 // val & (-1 ^ (val >>> shift)) ==> bic
10922 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10923 iRegL src1, iRegL src2,
10924 immI src3, immL_M1 src4) %{
10925 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10926 ins_cost(1.9 * INSN_COST);
10927 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10928
10929 ins_encode %{
10930 __ bic(as_Register($dst$$reg),
10931 as_Register($src1$$reg),
10932 as_Register($src2$$reg),
10933 Assembler::LSR,
10934 $src3$$constant & 0x3f);
10935 %}
10936
10937 ins_pipe(ialu_reg_reg_shift);
10938 %}
10939
10940 // This pattern is automatically generated from aarch64_ad.m4
10941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10942 // val & (-1 ^ (val >> shift)) ==> bicw
10943 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10944 iRegIorL2I src1, iRegIorL2I src2,
10945 immI src3, immI_M1 src4) %{
10946 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10947 ins_cost(1.9 * INSN_COST);
10948 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10949
10950 ins_encode %{
10951 __ bicw(as_Register($dst$$reg),
10952 as_Register($src1$$reg),
10953 as_Register($src2$$reg),
10954 Assembler::ASR,
10955 $src3$$constant & 0x1f);
10956 %}
10957
10958 ins_pipe(ialu_reg_reg_shift);
10959 %}
10960
10961 // This pattern is automatically generated from aarch64_ad.m4
10962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10963 // val & (-1 ^ (val >> shift)) ==> bic
10964 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10965 iRegL src1, iRegL src2,
10966 immI src3, immL_M1 src4) %{
10967 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10968 ins_cost(1.9 * INSN_COST);
10969 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10970
10971 ins_encode %{
10972 __ bic(as_Register($dst$$reg),
10973 as_Register($src1$$reg),
10974 as_Register($src2$$reg),
10975 Assembler::ASR,
10976 $src3$$constant & 0x3f);
10977 %}
10978
10979 ins_pipe(ialu_reg_reg_shift);
10980 %}
10981
10982 // This pattern is automatically generated from aarch64_ad.m4
10983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10984 // val & (-1 ^ (val ror shift)) ==> bicw
10985 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10986 iRegIorL2I src1, iRegIorL2I src2,
10987 immI src3, immI_M1 src4) %{
10988 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10989 ins_cost(1.9 * INSN_COST);
10990 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10991
10992 ins_encode %{
10993 __ bicw(as_Register($dst$$reg),
10994 as_Register($src1$$reg),
10995 as_Register($src2$$reg),
10996 Assembler::ROR,
10997 $src3$$constant & 0x1f);
10998 %}
10999
11000 ins_pipe(ialu_reg_reg_shift);
11001 %}
11002
11003 // This pattern is automatically generated from aarch64_ad.m4
11004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11005 // val & (-1 ^ (val ror shift)) ==> bic
11006 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11007 iRegL src1, iRegL src2,
11008 immI src3, immL_M1 src4) %{
11009 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11010 ins_cost(1.9 * INSN_COST);
11011 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11012
11013 ins_encode %{
11014 __ bic(as_Register($dst$$reg),
11015 as_Register($src1$$reg),
11016 as_Register($src2$$reg),
11017 Assembler::ROR,
11018 $src3$$constant & 0x3f);
11019 %}
11020
11021 ins_pipe(ialu_reg_reg_shift);
11022 %}
11023
11024 // This pattern is automatically generated from aarch64_ad.m4
11025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11026 // val & (-1 ^ (val << shift)) ==> bicw
11027 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11028 iRegIorL2I src1, iRegIorL2I src2,
11029 immI src3, immI_M1 src4) %{
11030 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11031 ins_cost(1.9 * INSN_COST);
11032 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11033
11034 ins_encode %{
11035 __ bicw(as_Register($dst$$reg),
11036 as_Register($src1$$reg),
11037 as_Register($src2$$reg),
11038 Assembler::LSL,
11039 $src3$$constant & 0x1f);
11040 %}
11041
11042 ins_pipe(ialu_reg_reg_shift);
11043 %}
11044
11045 // This pattern is automatically generated from aarch64_ad.m4
11046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11047 // val & (-1 ^ (val << shift)) ==> bic
11048 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11049 iRegL src1, iRegL src2,
11050 immI src3, immL_M1 src4) %{
11051 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11052 ins_cost(1.9 * INSN_COST);
11053 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11054
11055 ins_encode %{
11056 __ bic(as_Register($dst$$reg),
11057 as_Register($src1$$reg),
11058 as_Register($src2$$reg),
11059 Assembler::LSL,
11060 $src3$$constant & 0x3f);
11061 %}
11062
11063 ins_pipe(ialu_reg_reg_shift);
11064 %}
11065
11066 // This pattern is automatically generated from aarch64_ad.m4
11067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11068 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11069 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11070 iRegIorL2I src1, iRegIorL2I src2,
11071 immI src3, immI_M1 src4) %{
11072 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11073 ins_cost(1.9 * INSN_COST);
11074 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11075
11076 ins_encode %{
11077 __ eonw(as_Register($dst$$reg),
11078 as_Register($src1$$reg),
11079 as_Register($src2$$reg),
11080 Assembler::LSR,
11081 $src3$$constant & 0x1f);
11082 %}
11083
11084 ins_pipe(ialu_reg_reg_shift);
11085 %}
11086
11087 // This pattern is automatically generated from aarch64_ad.m4
11088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11089 // val ^ (-1 ^ (val >>> shift)) ==> eon
11090 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11091 iRegL src1, iRegL src2,
11092 immI src3, immL_M1 src4) %{
11093 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11094 ins_cost(1.9 * INSN_COST);
11095 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11096
11097 ins_encode %{
11098 __ eon(as_Register($dst$$reg),
11099 as_Register($src1$$reg),
11100 as_Register($src2$$reg),
11101 Assembler::LSR,
11102 $src3$$constant & 0x3f);
11103 %}
11104
11105 ins_pipe(ialu_reg_reg_shift);
11106 %}
11107
11108 // This pattern is automatically generated from aarch64_ad.m4
11109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11110 // val ^ (-1 ^ (val >> shift)) ==> eonw
11111 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11112 iRegIorL2I src1, iRegIorL2I src2,
11113 immI src3, immI_M1 src4) %{
11114 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11115 ins_cost(1.9 * INSN_COST);
11116 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11117
11118 ins_encode %{
11119 __ eonw(as_Register($dst$$reg),
11120 as_Register($src1$$reg),
11121 as_Register($src2$$reg),
11122 Assembler::ASR,
11123 $src3$$constant & 0x1f);
11124 %}
11125
11126 ins_pipe(ialu_reg_reg_shift);
11127 %}
11128
11129 // This pattern is automatically generated from aarch64_ad.m4
11130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11131 // val ^ (-1 ^ (val >> shift)) ==> eon
11132 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11133 iRegL src1, iRegL src2,
11134 immI src3, immL_M1 src4) %{
11135 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11136 ins_cost(1.9 * INSN_COST);
11137 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11138
11139 ins_encode %{
11140 __ eon(as_Register($dst$$reg),
11141 as_Register($src1$$reg),
11142 as_Register($src2$$reg),
11143 Assembler::ASR,
11144 $src3$$constant & 0x3f);
11145 %}
11146
11147 ins_pipe(ialu_reg_reg_shift);
11148 %}
11149
11150 // This pattern is automatically generated from aarch64_ad.m4
11151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11152 // val ^ (-1 ^ (val ror shift)) ==> eonw
11153 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11154 iRegIorL2I src1, iRegIorL2I src2,
11155 immI src3, immI_M1 src4) %{
11156 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11157 ins_cost(1.9 * INSN_COST);
11158 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11159
11160 ins_encode %{
11161 __ eonw(as_Register($dst$$reg),
11162 as_Register($src1$$reg),
11163 as_Register($src2$$reg),
11164 Assembler::ROR,
11165 $src3$$constant & 0x1f);
11166 %}
11167
11168 ins_pipe(ialu_reg_reg_shift);
11169 %}
11170
11171 // This pattern is automatically generated from aarch64_ad.m4
11172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11173 // val ^ (-1 ^ (val ror shift)) ==> eon
11174 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11175 iRegL src1, iRegL src2,
11176 immI src3, immL_M1 src4) %{
11177 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11178 ins_cost(1.9 * INSN_COST);
11179 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11180
11181 ins_encode %{
11182 __ eon(as_Register($dst$$reg),
11183 as_Register($src1$$reg),
11184 as_Register($src2$$reg),
11185 Assembler::ROR,
11186 $src3$$constant & 0x3f);
11187 %}
11188
11189 ins_pipe(ialu_reg_reg_shift);
11190 %}
11191
11192 // This pattern is automatically generated from aarch64_ad.m4
11193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11194 // val ^ (-1 ^ (val << shift)) ==> eonw
11195 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11196 iRegIorL2I src1, iRegIorL2I src2,
11197 immI src3, immI_M1 src4) %{
11198 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11199 ins_cost(1.9 * INSN_COST);
11200 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11201
11202 ins_encode %{
11203 __ eonw(as_Register($dst$$reg),
11204 as_Register($src1$$reg),
11205 as_Register($src2$$reg),
11206 Assembler::LSL,
11207 $src3$$constant & 0x1f);
11208 %}
11209
11210 ins_pipe(ialu_reg_reg_shift);
11211 %}
11212
11213 // This pattern is automatically generated from aarch64_ad.m4
11214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11215 // val ^ (-1 ^ (val << shift)) ==> eon
11216 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11217 iRegL src1, iRegL src2,
11218 immI src3, immL_M1 src4) %{
11219 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11220 ins_cost(1.9 * INSN_COST);
11221 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11222
11223 ins_encode %{
11224 __ eon(as_Register($dst$$reg),
11225 as_Register($src1$$reg),
11226 as_Register($src2$$reg),
11227 Assembler::LSL,
11228 $src3$$constant & 0x3f);
11229 %}
11230
11231 ins_pipe(ialu_reg_reg_shift);
11232 %}
11233
11234 // This pattern is automatically generated from aarch64_ad.m4
11235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11236 // val | (-1 ^ (val >>> shift)) ==> ornw
11237 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11238 iRegIorL2I src1, iRegIorL2I src2,
11239 immI src3, immI_M1 src4) %{
11240 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11241 ins_cost(1.9 * INSN_COST);
11242 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11243
11244 ins_encode %{
11245 __ ornw(as_Register($dst$$reg),
11246 as_Register($src1$$reg),
11247 as_Register($src2$$reg),
11248 Assembler::LSR,
11249 $src3$$constant & 0x1f);
11250 %}
11251
11252 ins_pipe(ialu_reg_reg_shift);
11253 %}
11254
11255 // This pattern is automatically generated from aarch64_ad.m4
11256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11257 // val | (-1 ^ (val >>> shift)) ==> orn
11258 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11259 iRegL src1, iRegL src2,
11260 immI src3, immL_M1 src4) %{
11261 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11262 ins_cost(1.9 * INSN_COST);
11263 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11264
11265 ins_encode %{
11266 __ orn(as_Register($dst$$reg),
11267 as_Register($src1$$reg),
11268 as_Register($src2$$reg),
11269 Assembler::LSR,
11270 $src3$$constant & 0x3f);
11271 %}
11272
11273 ins_pipe(ialu_reg_reg_shift);
11274 %}
11275
11276 // This pattern is automatically generated from aarch64_ad.m4
11277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11278 // val | (-1 ^ (val >> shift)) ==> ornw
11279 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11280 iRegIorL2I src1, iRegIorL2I src2,
11281 immI src3, immI_M1 src4) %{
11282 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11283 ins_cost(1.9 * INSN_COST);
11284 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11285
11286 ins_encode %{
11287 __ ornw(as_Register($dst$$reg),
11288 as_Register($src1$$reg),
11289 as_Register($src2$$reg),
11290 Assembler::ASR,
11291 $src3$$constant & 0x1f);
11292 %}
11293
11294 ins_pipe(ialu_reg_reg_shift);
11295 %}
11296
11297 // This pattern is automatically generated from aarch64_ad.m4
11298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11299 // val | (-1 ^ (val >> shift)) ==> orn
11300 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11301 iRegL src1, iRegL src2,
11302 immI src3, immL_M1 src4) %{
11303 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11304 ins_cost(1.9 * INSN_COST);
11305 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11306
11307 ins_encode %{
11308 __ orn(as_Register($dst$$reg),
11309 as_Register($src1$$reg),
11310 as_Register($src2$$reg),
11311 Assembler::ASR,
11312 $src3$$constant & 0x3f);
11313 %}
11314
11315 ins_pipe(ialu_reg_reg_shift);
11316 %}
11317
11318 // This pattern is automatically generated from aarch64_ad.m4
11319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11320 // val | (-1 ^ (val ror shift)) ==> ornw
11321 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11322 iRegIorL2I src1, iRegIorL2I src2,
11323 immI src3, immI_M1 src4) %{
11324 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11325 ins_cost(1.9 * INSN_COST);
11326 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11327
11328 ins_encode %{
11329 __ ornw(as_Register($dst$$reg),
11330 as_Register($src1$$reg),
11331 as_Register($src2$$reg),
11332 Assembler::ROR,
11333 $src3$$constant & 0x1f);
11334 %}
11335
11336 ins_pipe(ialu_reg_reg_shift);
11337 %}
11338
11339 // This pattern is automatically generated from aarch64_ad.m4
11340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11341 // val | (-1 ^ (val ror shift)) ==> orn
11342 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11343 iRegL src1, iRegL src2,
11344 immI src3, immL_M1 src4) %{
11345 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11346 ins_cost(1.9 * INSN_COST);
11347 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11348
11349 ins_encode %{
11350 __ orn(as_Register($dst$$reg),
11351 as_Register($src1$$reg),
11352 as_Register($src2$$reg),
11353 Assembler::ROR,
11354 $src3$$constant & 0x3f);
11355 %}
11356
11357 ins_pipe(ialu_reg_reg_shift);
11358 %}
11359
11360 // This pattern is automatically generated from aarch64_ad.m4
11361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11362 // val | (-1 ^ (val << shift)) ==> ornw
11363 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11364 iRegIorL2I src1, iRegIorL2I src2,
11365 immI src3, immI_M1 src4) %{
11366 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11367 ins_cost(1.9 * INSN_COST);
11368 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11369
11370 ins_encode %{
11371 __ ornw(as_Register($dst$$reg),
11372 as_Register($src1$$reg),
11373 as_Register($src2$$reg),
11374 Assembler::LSL,
11375 $src3$$constant & 0x1f);
11376 %}
11377
11378 ins_pipe(ialu_reg_reg_shift);
11379 %}
11380
11381 // This pattern is automatically generated from aarch64_ad.m4
11382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11383 // val | (-1 ^ (val << shift)) ==> orn
11384 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11385 iRegL src1, iRegL src2,
11386 immI src3, immL_M1 src4) %{
11387 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11388 ins_cost(1.9 * INSN_COST);
11389 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11390
11391 ins_encode %{
11392 __ orn(as_Register($dst$$reg),
11393 as_Register($src1$$reg),
11394 as_Register($src2$$reg),
11395 Assembler::LSL,
11396 $src3$$constant & 0x3f);
11397 %}
11398
11399 ins_pipe(ialu_reg_reg_shift);
11400 %}
11401
11402 // This pattern is automatically generated from aarch64_ad.m4
11403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11404 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11405 iRegIorL2I src1, iRegIorL2I src2,
11406 immI src3) %{
11407 match(Set dst (AndI src1 (URShiftI src2 src3)));
11408
11409 ins_cost(1.9 * INSN_COST);
11410 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11411
11412 ins_encode %{
11413 __ andw(as_Register($dst$$reg),
11414 as_Register($src1$$reg),
11415 as_Register($src2$$reg),
11416 Assembler::LSR,
11417 $src3$$constant & 0x1f);
11418 %}
11419
11420 ins_pipe(ialu_reg_reg_shift);
11421 %}
11422
11423 // This pattern is automatically generated from aarch64_ad.m4
11424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11425 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11426 iRegL src1, iRegL src2,
11427 immI src3) %{
11428 match(Set dst (AndL src1 (URShiftL src2 src3)));
11429
11430 ins_cost(1.9 * INSN_COST);
11431 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11432
11433 ins_encode %{
11434 __ andr(as_Register($dst$$reg),
11435 as_Register($src1$$reg),
11436 as_Register($src2$$reg),
11437 Assembler::LSR,
11438 $src3$$constant & 0x3f);
11439 %}
11440
11441 ins_pipe(ialu_reg_reg_shift);
11442 %}
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11447 iRegIorL2I src1, iRegIorL2I src2,
11448 immI src3) %{
11449 match(Set dst (AndI src1 (RShiftI src2 src3)));
11450
11451 ins_cost(1.9 * INSN_COST);
11452 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11453
11454 ins_encode %{
11455 __ andw(as_Register($dst$$reg),
11456 as_Register($src1$$reg),
11457 as_Register($src2$$reg),
11458 Assembler::ASR,
11459 $src3$$constant & 0x1f);
11460 %}
11461
11462 ins_pipe(ialu_reg_reg_shift);
11463 %}
11464
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11468 iRegL src1, iRegL src2,
11469 immI src3) %{
11470 match(Set dst (AndL src1 (RShiftL src2 src3)));
11471
11472 ins_cost(1.9 * INSN_COST);
11473 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11474
11475 ins_encode %{
11476 __ andr(as_Register($dst$$reg),
11477 as_Register($src1$$reg),
11478 as_Register($src2$$reg),
11479 Assembler::ASR,
11480 $src3$$constant & 0x3f);
11481 %}
11482
11483 ins_pipe(ialu_reg_reg_shift);
11484 %}
11485
11486 // This pattern is automatically generated from aarch64_ad.m4
11487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11488 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11489 iRegIorL2I src1, iRegIorL2I src2,
11490 immI src3) %{
11491 match(Set dst (AndI src1 (LShiftI src2 src3)));
11492
11493 ins_cost(1.9 * INSN_COST);
11494 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11495
11496 ins_encode %{
11497 __ andw(as_Register($dst$$reg),
11498 as_Register($src1$$reg),
11499 as_Register($src2$$reg),
11500 Assembler::LSL,
11501 $src3$$constant & 0x1f);
11502 %}
11503
11504 ins_pipe(ialu_reg_reg_shift);
11505 %}
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11510 iRegL src1, iRegL src2,
11511 immI src3) %{
11512 match(Set dst (AndL src1 (LShiftL src2 src3)));
11513
11514 ins_cost(1.9 * INSN_COST);
11515 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11516
11517 ins_encode %{
11518 __ andr(as_Register($dst$$reg),
11519 as_Register($src1$$reg),
11520 as_Register($src2$$reg),
11521 Assembler::LSL,
11522 $src3$$constant & 0x3f);
11523 %}
11524
11525 ins_pipe(ialu_reg_reg_shift);
11526 %}
11527
11528 // This pattern is automatically generated from aarch64_ad.m4
11529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11530 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11531 iRegIorL2I src1, iRegIorL2I src2,
11532 immI src3) %{
11533 match(Set dst (AndI src1 (RotateRight src2 src3)));
11534
11535 ins_cost(1.9 * INSN_COST);
11536 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11537
11538 ins_encode %{
11539 __ andw(as_Register($dst$$reg),
11540 as_Register($src1$$reg),
11541 as_Register($src2$$reg),
11542 Assembler::ROR,
11543 $src3$$constant & 0x1f);
11544 %}
11545
11546 ins_pipe(ialu_reg_reg_shift);
11547 %}
11548
11549 // This pattern is automatically generated from aarch64_ad.m4
11550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11551 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11552 iRegL src1, iRegL src2,
11553 immI src3) %{
11554 match(Set dst (AndL src1 (RotateRight src2 src3)));
11555
11556 ins_cost(1.9 * INSN_COST);
11557 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11558
11559 ins_encode %{
11560 __ andr(as_Register($dst$$reg),
11561 as_Register($src1$$reg),
11562 as_Register($src2$$reg),
11563 Assembler::ROR,
11564 $src3$$constant & 0x3f);
11565 %}
11566
11567 ins_pipe(ialu_reg_reg_shift);
11568 %}
11569
11570 // This pattern is automatically generated from aarch64_ad.m4
11571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11572 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11573 iRegIorL2I src1, iRegIorL2I src2,
11574 immI src3) %{
11575 match(Set dst (XorI src1 (URShiftI src2 src3)));
11576
11577 ins_cost(1.9 * INSN_COST);
11578 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11579
11580 ins_encode %{
11581 __ eorw(as_Register($dst$$reg),
11582 as_Register($src1$$reg),
11583 as_Register($src2$$reg),
11584 Assembler::LSR,
11585 $src3$$constant & 0x1f);
11586 %}
11587
11588 ins_pipe(ialu_reg_reg_shift);
11589 %}
11590
11591 // This pattern is automatically generated from aarch64_ad.m4
11592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11593 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11594 iRegL src1, iRegL src2,
11595 immI src3) %{
11596 match(Set dst (XorL src1 (URShiftL src2 src3)));
11597
11598 ins_cost(1.9 * INSN_COST);
11599 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11600
11601 ins_encode %{
11602 __ eor(as_Register($dst$$reg),
11603 as_Register($src1$$reg),
11604 as_Register($src2$$reg),
11605 Assembler::LSR,
11606 $src3$$constant & 0x3f);
11607 %}
11608
11609 ins_pipe(ialu_reg_reg_shift);
11610 %}
11611
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11615 iRegIorL2I src1, iRegIorL2I src2,
11616 immI src3) %{
11617 match(Set dst (XorI src1 (RShiftI src2 src3)));
11618
11619 ins_cost(1.9 * INSN_COST);
11620 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11621
11622 ins_encode %{
11623 __ eorw(as_Register($dst$$reg),
11624 as_Register($src1$$reg),
11625 as_Register($src2$$reg),
11626 Assembler::ASR,
11627 $src3$$constant & 0x1f);
11628 %}
11629
11630 ins_pipe(ialu_reg_reg_shift);
11631 %}
11632
11633 // This pattern is automatically generated from aarch64_ad.m4
11634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11635 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11636 iRegL src1, iRegL src2,
11637 immI src3) %{
11638 match(Set dst (XorL src1 (RShiftL src2 src3)));
11639
11640 ins_cost(1.9 * INSN_COST);
11641 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11642
11643 ins_encode %{
11644 __ eor(as_Register($dst$$reg),
11645 as_Register($src1$$reg),
11646 as_Register($src2$$reg),
11647 Assembler::ASR,
11648 $src3$$constant & 0x3f);
11649 %}
11650
11651 ins_pipe(ialu_reg_reg_shift);
11652 %}
11653
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11657 iRegIorL2I src1, iRegIorL2I src2,
11658 immI src3) %{
11659 match(Set dst (XorI src1 (LShiftI src2 src3)));
11660
11661 ins_cost(1.9 * INSN_COST);
11662 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11663
11664 ins_encode %{
11665 __ eorw(as_Register($dst$$reg),
11666 as_Register($src1$$reg),
11667 as_Register($src2$$reg),
11668 Assembler::LSL,
11669 $src3$$constant & 0x1f);
11670 %}
11671
11672 ins_pipe(ialu_reg_reg_shift);
11673 %}
11674
11675 // This pattern is automatically generated from aarch64_ad.m4
11676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11677 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11678 iRegL src1, iRegL src2,
11679 immI src3) %{
11680 match(Set dst (XorL src1 (LShiftL src2 src3)));
11681
11682 ins_cost(1.9 * INSN_COST);
11683 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11684
11685 ins_encode %{
11686 __ eor(as_Register($dst$$reg),
11687 as_Register($src1$$reg),
11688 as_Register($src2$$reg),
11689 Assembler::LSL,
11690 $src3$$constant & 0x3f);
11691 %}
11692
11693 ins_pipe(ialu_reg_reg_shift);
11694 %}
11695
11696 // This pattern is automatically generated from aarch64_ad.m4
11697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11698 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11699 iRegIorL2I src1, iRegIorL2I src2,
11700 immI src3) %{
11701 match(Set dst (XorI src1 (RotateRight src2 src3)));
11702
11703 ins_cost(1.9 * INSN_COST);
11704 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11705
11706 ins_encode %{
11707 __ eorw(as_Register($dst$$reg),
11708 as_Register($src1$$reg),
11709 as_Register($src2$$reg),
11710 Assembler::ROR,
11711 $src3$$constant & 0x1f);
11712 %}
11713
11714 ins_pipe(ialu_reg_reg_shift);
11715 %}
11716
11717 // This pattern is automatically generated from aarch64_ad.m4
11718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11719 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11720 iRegL src1, iRegL src2,
11721 immI src3) %{
11722 match(Set dst (XorL src1 (RotateRight src2 src3)));
11723
11724 ins_cost(1.9 * INSN_COST);
11725 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11726
11727 ins_encode %{
11728 __ eor(as_Register($dst$$reg),
11729 as_Register($src1$$reg),
11730 as_Register($src2$$reg),
11731 Assembler::ROR,
11732 $src3$$constant & 0x3f);
11733 %}
11734
11735 ins_pipe(ialu_reg_reg_shift);
11736 %}
11737
11738 // This pattern is automatically generated from aarch64_ad.m4
11739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11740 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11741 iRegIorL2I src1, iRegIorL2I src2,
11742 immI src3) %{
11743 match(Set dst (OrI src1 (URShiftI src2 src3)));
11744
11745 ins_cost(1.9 * INSN_COST);
11746 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11747
11748 ins_encode %{
11749 __ orrw(as_Register($dst$$reg),
11750 as_Register($src1$$reg),
11751 as_Register($src2$$reg),
11752 Assembler::LSR,
11753 $src3$$constant & 0x1f);
11754 %}
11755
11756 ins_pipe(ialu_reg_reg_shift);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11762 iRegL src1, iRegL src2,
11763 immI src3) %{
11764 match(Set dst (OrL src1 (URShiftL src2 src3)));
11765
11766 ins_cost(1.9 * INSN_COST);
11767 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11768
11769 ins_encode %{
11770 __ orr(as_Register($dst$$reg),
11771 as_Register($src1$$reg),
11772 as_Register($src2$$reg),
11773 Assembler::LSR,
11774 $src3$$constant & 0x3f);
11775 %}
11776
11777 ins_pipe(ialu_reg_reg_shift);
11778 %}
11779
11780 // This pattern is automatically generated from aarch64_ad.m4
11781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11782 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11783 iRegIorL2I src1, iRegIorL2I src2,
11784 immI src3) %{
11785 match(Set dst (OrI src1 (RShiftI src2 src3)));
11786
11787 ins_cost(1.9 * INSN_COST);
11788 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11789
11790 ins_encode %{
11791 __ orrw(as_Register($dst$$reg),
11792 as_Register($src1$$reg),
11793 as_Register($src2$$reg),
11794 Assembler::ASR,
11795 $src3$$constant & 0x1f);
11796 %}
11797
11798 ins_pipe(ialu_reg_reg_shift);
11799 %}
11800
11801 // This pattern is automatically generated from aarch64_ad.m4
11802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11803 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11804 iRegL src1, iRegL src2,
11805 immI src3) %{
11806 match(Set dst (OrL src1 (RShiftL src2 src3)));
11807
11808 ins_cost(1.9 * INSN_COST);
11809 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11810
11811 ins_encode %{
11812 __ orr(as_Register($dst$$reg),
11813 as_Register($src1$$reg),
11814 as_Register($src2$$reg),
11815 Assembler::ASR,
11816 $src3$$constant & 0x3f);
11817 %}
11818
11819 ins_pipe(ialu_reg_reg_shift);
11820 %}
11821
11822 // This pattern is automatically generated from aarch64_ad.m4
11823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11824 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11825 iRegIorL2I src1, iRegIorL2I src2,
11826 immI src3) %{
11827 match(Set dst (OrI src1 (LShiftI src2 src3)));
11828
11829 ins_cost(1.9 * INSN_COST);
11830 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11831
11832 ins_encode %{
11833 __ orrw(as_Register($dst$$reg),
11834 as_Register($src1$$reg),
11835 as_Register($src2$$reg),
11836 Assembler::LSL,
11837 $src3$$constant & 0x1f);
11838 %}
11839
11840 ins_pipe(ialu_reg_reg_shift);
11841 %}
11842
11843 // This pattern is automatically generated from aarch64_ad.m4
11844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11845 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11846 iRegL src1, iRegL src2,
11847 immI src3) %{
11848 match(Set dst (OrL src1 (LShiftL src2 src3)));
11849
11850 ins_cost(1.9 * INSN_COST);
11851 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11852
11853 ins_encode %{
11854 __ orr(as_Register($dst$$reg),
11855 as_Register($src1$$reg),
11856 as_Register($src2$$reg),
11857 Assembler::LSL,
11858 $src3$$constant & 0x3f);
11859 %}
11860
11861 ins_pipe(ialu_reg_reg_shift);
11862 %}
11863
11864 // This pattern is automatically generated from aarch64_ad.m4
11865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11866 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11867 iRegIorL2I src1, iRegIorL2I src2,
11868 immI src3) %{
11869 match(Set dst (OrI src1 (RotateRight src2 src3)));
11870
11871 ins_cost(1.9 * INSN_COST);
11872 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11873
11874 ins_encode %{
11875 __ orrw(as_Register($dst$$reg),
11876 as_Register($src1$$reg),
11877 as_Register($src2$$reg),
11878 Assembler::ROR,
11879 $src3$$constant & 0x1f);
11880 %}
11881
11882 ins_pipe(ialu_reg_reg_shift);
11883 %}
11884
11885 // This pattern is automatically generated from aarch64_ad.m4
11886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11887 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11888 iRegL src1, iRegL src2,
11889 immI src3) %{
11890 match(Set dst (OrL src1 (RotateRight src2 src3)));
11891
11892 ins_cost(1.9 * INSN_COST);
11893 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11894
11895 ins_encode %{
11896 __ orr(as_Register($dst$$reg),
11897 as_Register($src1$$reg),
11898 as_Register($src2$$reg),
11899 Assembler::ROR,
11900 $src3$$constant & 0x3f);
11901 %}
11902
11903 ins_pipe(ialu_reg_reg_shift);
11904 %}
11905
11906 // This pattern is automatically generated from aarch64_ad.m4
11907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11908 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11909 iRegIorL2I src1, iRegIorL2I src2,
11910 immI src3) %{
11911 match(Set dst (AddI src1 (URShiftI src2 src3)));
11912
11913 ins_cost(1.9 * INSN_COST);
11914 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11915
11916 ins_encode %{
11917 __ addw(as_Register($dst$$reg),
11918 as_Register($src1$$reg),
11919 as_Register($src2$$reg),
11920 Assembler::LSR,
11921 $src3$$constant & 0x1f);
11922 %}
11923
11924 ins_pipe(ialu_reg_reg_shift);
11925 %}
11926
11927 // This pattern is automatically generated from aarch64_ad.m4
11928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11929 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11930 iRegL src1, iRegL src2,
11931 immI src3) %{
11932 match(Set dst (AddL src1 (URShiftL src2 src3)));
11933
11934 ins_cost(1.9 * INSN_COST);
11935 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11936
11937 ins_encode %{
11938 __ add(as_Register($dst$$reg),
11939 as_Register($src1$$reg),
11940 as_Register($src2$$reg),
11941 Assembler::LSR,
11942 $src3$$constant & 0x3f);
11943 %}
11944
11945 ins_pipe(ialu_reg_reg_shift);
11946 %}
11947
11948 // This pattern is automatically generated from aarch64_ad.m4
11949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11950 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11951 iRegIorL2I src1, iRegIorL2I src2,
11952 immI src3) %{
11953 match(Set dst (AddI src1 (RShiftI src2 src3)));
11954
11955 ins_cost(1.9 * INSN_COST);
11956 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11957
11958 ins_encode %{
11959 __ addw(as_Register($dst$$reg),
11960 as_Register($src1$$reg),
11961 as_Register($src2$$reg),
11962 Assembler::ASR,
11963 $src3$$constant & 0x1f);
11964 %}
11965
11966 ins_pipe(ialu_reg_reg_shift);
11967 %}
11968
11969 // This pattern is automatically generated from aarch64_ad.m4
11970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11971 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11972 iRegL src1, iRegL src2,
11973 immI src3) %{
11974 match(Set dst (AddL src1 (RShiftL src2 src3)));
11975
11976 ins_cost(1.9 * INSN_COST);
11977 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11978
11979 ins_encode %{
11980 __ add(as_Register($dst$$reg),
11981 as_Register($src1$$reg),
11982 as_Register($src2$$reg),
11983 Assembler::ASR,
11984 $src3$$constant & 0x3f);
11985 %}
11986
11987 ins_pipe(ialu_reg_reg_shift);
11988 %}
11989
11990 // This pattern is automatically generated from aarch64_ad.m4
11991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11992 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11993 iRegIorL2I src1, iRegIorL2I src2,
11994 immI src3) %{
11995 match(Set dst (AddI src1 (LShiftI src2 src3)));
11996
11997 ins_cost(1.9 * INSN_COST);
11998 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11999
12000 ins_encode %{
12001 __ addw(as_Register($dst$$reg),
12002 as_Register($src1$$reg),
12003 as_Register($src2$$reg),
12004 Assembler::LSL,
12005 $src3$$constant & 0x1f);
12006 %}
12007
12008 ins_pipe(ialu_reg_reg_shift);
12009 %}
12010
12011 // This pattern is automatically generated from aarch64_ad.m4
12012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12013 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12014 iRegL src1, iRegL src2,
12015 immI src3) %{
12016 match(Set dst (AddL src1 (LShiftL src2 src3)));
12017
12018 ins_cost(1.9 * INSN_COST);
12019 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12020
12021 ins_encode %{
12022 __ add(as_Register($dst$$reg),
12023 as_Register($src1$$reg),
12024 as_Register($src2$$reg),
12025 Assembler::LSL,
12026 $src3$$constant & 0x3f);
12027 %}
12028
12029 ins_pipe(ialu_reg_reg_shift);
12030 %}
12031
12032 // This pattern is automatically generated from aarch64_ad.m4
12033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12034 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12035 iRegIorL2I src1, iRegIorL2I src2,
12036 immI src3) %{
12037 match(Set dst (SubI src1 (URShiftI src2 src3)));
12038
12039 ins_cost(1.9 * INSN_COST);
12040 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12041
12042 ins_encode %{
12043 __ subw(as_Register($dst$$reg),
12044 as_Register($src1$$reg),
12045 as_Register($src2$$reg),
12046 Assembler::LSR,
12047 $src3$$constant & 0x1f);
12048 %}
12049
12050 ins_pipe(ialu_reg_reg_shift);
12051 %}
12052
12053 // This pattern is automatically generated from aarch64_ad.m4
12054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12055 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12056 iRegL src1, iRegL src2,
12057 immI src3) %{
12058 match(Set dst (SubL src1 (URShiftL src2 src3)));
12059
12060 ins_cost(1.9 * INSN_COST);
12061 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12062
12063 ins_encode %{
12064 __ sub(as_Register($dst$$reg),
12065 as_Register($src1$$reg),
12066 as_Register($src2$$reg),
12067 Assembler::LSR,
12068 $src3$$constant & 0x3f);
12069 %}
12070
12071 ins_pipe(ialu_reg_reg_shift);
12072 %}
12073
12074 // This pattern is automatically generated from aarch64_ad.m4
12075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12076 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12077 iRegIorL2I src1, iRegIorL2I src2,
12078 immI src3) %{
12079 match(Set dst (SubI src1 (RShiftI src2 src3)));
12080
12081 ins_cost(1.9 * INSN_COST);
12082 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12083
12084 ins_encode %{
12085 __ subw(as_Register($dst$$reg),
12086 as_Register($src1$$reg),
12087 as_Register($src2$$reg),
12088 Assembler::ASR,
12089 $src3$$constant & 0x1f);
12090 %}
12091
12092 ins_pipe(ialu_reg_reg_shift);
12093 %}
12094
12095 // This pattern is automatically generated from aarch64_ad.m4
12096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12097 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12098 iRegL src1, iRegL src2,
12099 immI src3) %{
12100 match(Set dst (SubL src1 (RShiftL src2 src3)));
12101
12102 ins_cost(1.9 * INSN_COST);
12103 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12104
12105 ins_encode %{
12106 __ sub(as_Register($dst$$reg),
12107 as_Register($src1$$reg),
12108 as_Register($src2$$reg),
12109 Assembler::ASR,
12110 $src3$$constant & 0x3f);
12111 %}
12112
12113 ins_pipe(ialu_reg_reg_shift);
12114 %}
12115
12116 // This pattern is automatically generated from aarch64_ad.m4
12117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12118 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12119 iRegIorL2I src1, iRegIorL2I src2,
12120 immI src3) %{
12121 match(Set dst (SubI src1 (LShiftI src2 src3)));
12122
12123 ins_cost(1.9 * INSN_COST);
12124 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12125
12126 ins_encode %{
12127 __ subw(as_Register($dst$$reg),
12128 as_Register($src1$$reg),
12129 as_Register($src2$$reg),
12130 Assembler::LSL,
12131 $src3$$constant & 0x1f);
12132 %}
12133
12134 ins_pipe(ialu_reg_reg_shift);
12135 %}
12136
12137 // This pattern is automatically generated from aarch64_ad.m4
12138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12139 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12140 iRegL src1, iRegL src2,
12141 immI src3) %{
12142 match(Set dst (SubL src1 (LShiftL src2 src3)));
12143
12144 ins_cost(1.9 * INSN_COST);
12145 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12146
12147 ins_encode %{
12148 __ sub(as_Register($dst$$reg),
12149 as_Register($src1$$reg),
12150 as_Register($src2$$reg),
12151 Assembler::LSL,
12152 $src3$$constant & 0x3f);
12153 %}
12154
12155 ins_pipe(ialu_reg_reg_shift);
12156 %}
12157
12158 // This pattern is automatically generated from aarch64_ad.m4
12159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12160
12161 // Shift Left followed by Shift Right.
12162 // This idiom is used by the compiler for the i2b bytecode etc.
12163 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12164 %{
12165 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12166 ins_cost(INSN_COST * 2);
12167 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12168 ins_encode %{
12169 int lshift = $lshift_count$$constant & 63;
12170 int rshift = $rshift_count$$constant & 63;
12171 int s = 63 - lshift;
12172 int r = (rshift - lshift) & 63;
12173 __ sbfm(as_Register($dst$$reg),
12174 as_Register($src$$reg),
12175 r, s);
12176 %}
12177
12178 ins_pipe(ialu_reg_shift);
12179 %}
12180
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183
12184 // Shift Left followed by Shift Right.
12185 // This idiom is used by the compiler for the i2b bytecode etc.
12186 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12187 %{
12188 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12189 ins_cost(INSN_COST * 2);
12190 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12191 ins_encode %{
12192 int lshift = $lshift_count$$constant & 31;
12193 int rshift = $rshift_count$$constant & 31;
12194 int s = 31 - lshift;
12195 int r = (rshift - lshift) & 31;
12196 __ sbfmw(as_Register($dst$$reg),
12197 as_Register($src$$reg),
12198 r, s);
12199 %}
12200
12201 ins_pipe(ialu_reg_shift);
12202 %}
12203
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206
12207 // Shift Left followed by Shift Right.
12208 // This idiom is used by the compiler for the i2b bytecode etc.
12209 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12210 %{
12211 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12212 ins_cost(INSN_COST * 2);
12213 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12214 ins_encode %{
12215 int lshift = $lshift_count$$constant & 63;
12216 int rshift = $rshift_count$$constant & 63;
12217 int s = 63 - lshift;
12218 int r = (rshift - lshift) & 63;
12219 __ ubfm(as_Register($dst$$reg),
12220 as_Register($src$$reg),
12221 r, s);
12222 %}
12223
12224 ins_pipe(ialu_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229
12230 // Shift Left followed by Shift Right.
12231 // This idiom is used by the compiler for the i2b bytecode etc.
12232 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12233 %{
12234 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12235 ins_cost(INSN_COST * 2);
12236 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12237 ins_encode %{
12238 int lshift = $lshift_count$$constant & 31;
12239 int rshift = $rshift_count$$constant & 31;
12240 int s = 31 - lshift;
12241 int r = (rshift - lshift) & 31;
12242 __ ubfmw(as_Register($dst$$reg),
12243 as_Register($src$$reg),
12244 r, s);
12245 %}
12246
12247 ins_pipe(ialu_reg_shift);
12248 %}
12249
12250 // Bitfield extract with shift & mask
12251
12252 // This pattern is automatically generated from aarch64_ad.m4
12253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12254 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12255 %{
12256 match(Set dst (AndI (URShiftI src rshift) mask));
12257 // Make sure we are not going to exceed what ubfxw can do.
12258 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12259
12260 ins_cost(INSN_COST);
12261 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12262 ins_encode %{
12263 int rshift = $rshift$$constant & 31;
12264 intptr_t mask = $mask$$constant;
12265 int width = exact_log2(mask+1);
12266 __ ubfxw(as_Register($dst$$reg),
12267 as_Register($src$$reg), rshift, width);
12268 %}
12269 ins_pipe(ialu_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12275 %{
12276 match(Set dst (AndL (URShiftL src rshift) mask));
12277 // Make sure we are not going to exceed what ubfx can do.
12278 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12279
12280 ins_cost(INSN_COST);
12281 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12282 ins_encode %{
12283 int rshift = $rshift$$constant & 63;
12284 intptr_t mask = $mask$$constant;
12285 int width = exact_log2_long(mask+1);
12286 __ ubfx(as_Register($dst$$reg),
12287 as_Register($src$$reg), rshift, width);
12288 %}
12289 ins_pipe(ialu_reg_shift);
12290 %}
12291
12292
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295
12296 // We can use ubfx when extending an And with a mask when we know mask
12297 // is positive. We know that because immI_bitmask guarantees it.
12298 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12299 %{
12300 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12301 // Make sure we are not going to exceed what ubfxw can do.
12302 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12303
12304 ins_cost(INSN_COST * 2);
12305 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12306 ins_encode %{
12307 int rshift = $rshift$$constant & 31;
12308 intptr_t mask = $mask$$constant;
12309 int width = exact_log2(mask+1);
12310 __ ubfx(as_Register($dst$$reg),
12311 as_Register($src$$reg), rshift, width);
12312 %}
12313 ins_pipe(ialu_reg_shift);
12314 %}
12315
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319
12320 // We can use ubfiz when masking by a positive number and then left shifting the result.
12321 // We know that the mask is positive because immI_bitmask guarantees it.
12322 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12323 %{
12324 match(Set dst (LShiftI (AndI src mask) lshift));
12325 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12326
12327 ins_cost(INSN_COST);
12328 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12329 ins_encode %{
12330 int lshift = $lshift$$constant & 31;
12331 intptr_t mask = $mask$$constant;
12332 int width = exact_log2(mask+1);
12333 __ ubfizw(as_Register($dst$$reg),
12334 as_Register($src$$reg), lshift, width);
12335 %}
12336 ins_pipe(ialu_reg_shift);
12337 %}
12338
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341
12342 // We can use ubfiz when masking by a positive number and then left shifting the result.
12343 // We know that the mask is positive because immL_bitmask guarantees it.
12344 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12345 %{
12346 match(Set dst (LShiftL (AndL src mask) lshift));
12347 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12348
12349 ins_cost(INSN_COST);
12350 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12351 ins_encode %{
12352 int lshift = $lshift$$constant & 63;
12353 intptr_t mask = $mask$$constant;
12354 int width = exact_log2_long(mask+1);
12355 __ ubfiz(as_Register($dst$$reg),
12356 as_Register($src$$reg), lshift, width);
12357 %}
12358 ins_pipe(ialu_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363
12364 // We can use ubfiz when masking by a positive number and then left shifting the result.
12365 // We know that the mask is positive because immI_bitmask guarantees it.
12366 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12367 %{
12368 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12369 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12370
12371 ins_cost(INSN_COST);
12372 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12373 ins_encode %{
12374 int lshift = $lshift$$constant & 31;
12375 intptr_t mask = $mask$$constant;
12376 int width = exact_log2(mask+1);
12377 __ ubfizw(as_Register($dst$$reg),
12378 as_Register($src$$reg), lshift, width);
12379 %}
12380 ins_pipe(ialu_reg_shift);
12381 %}
12382
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385
12386 // We can use ubfiz when masking by a positive number and then left shifting the result.
12387 // We know that the mask is positive because immL_bitmask guarantees it.
12388 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12389 %{
12390 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12391 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12392
12393 ins_cost(INSN_COST);
12394 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12395 ins_encode %{
12396 int lshift = $lshift$$constant & 63;
12397 intptr_t mask = $mask$$constant;
12398 int width = exact_log2_long(mask+1);
12399 __ ubfiz(as_Register($dst$$reg),
12400 as_Register($src$$reg), lshift, width);
12401 %}
12402 ins_pipe(ialu_reg_shift);
12403 %}
12404
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408
12409 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12410 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12411 %{
12412 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12413 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12414
12415 ins_cost(INSN_COST);
12416 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12417 ins_encode %{
12418 int lshift = $lshift$$constant & 63;
12419 intptr_t mask = $mask$$constant;
12420 int width = exact_log2(mask+1);
12421 __ ubfiz(as_Register($dst$$reg),
12422 as_Register($src$$reg), lshift, width);
12423 %}
12424 ins_pipe(ialu_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429
12430 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12431 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12432 %{
12433 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12434 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12435
12436 ins_cost(INSN_COST);
12437 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12438 ins_encode %{
12439 int lshift = $lshift$$constant & 31;
12440 intptr_t mask = $mask$$constant;
12441 int width = exact_log2(mask+1);
12442 __ ubfiz(as_Register($dst$$reg),
12443 as_Register($src$$reg), lshift, width);
12444 %}
12445 ins_pipe(ialu_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450
12451 // Can skip int2long conversions after AND with small bitmask
12452 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12453 %{
12454 match(Set dst (ConvI2L (AndI src msk)));
12455 ins_cost(INSN_COST);
12456 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12457 ins_encode %{
12458 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12459 %}
12460 ins_pipe(ialu_reg_shift);
12461 %}
12462
12463
12464 // Rotations
12465
12466 // This pattern is automatically generated from aarch64_ad.m4
12467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12468 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12469 %{
12470 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12471 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12472
12473 ins_cost(INSN_COST);
12474 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12475
12476 ins_encode %{
12477 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12478 $rshift$$constant & 63);
12479 %}
12480 ins_pipe(ialu_reg_reg_extr);
12481 %}
12482
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12487 %{
12488 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12489 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12490
12491 ins_cost(INSN_COST);
12492 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12493
12494 ins_encode %{
12495 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12496 $rshift$$constant & 31);
12497 %}
12498 ins_pipe(ialu_reg_reg_extr);
12499 %}
12500
12501
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12505 %{
12506 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12507 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12508
12509 ins_cost(INSN_COST);
12510 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12511
12512 ins_encode %{
12513 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12514 $rshift$$constant & 63);
12515 %}
12516 ins_pipe(ialu_reg_reg_extr);
12517 %}
12518
12519
12520 // This pattern is automatically generated from aarch64_ad.m4
12521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12522 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12523 %{
12524 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12525 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12526
12527 ins_cost(INSN_COST);
12528 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12529
12530 ins_encode %{
12531 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12532 $rshift$$constant & 31);
12533 %}
12534 ins_pipe(ialu_reg_reg_extr);
12535 %}
12536
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12540 %{
12541 match(Set dst (RotateRight src shift));
12542
12543 ins_cost(INSN_COST);
12544 format %{ "ror $dst, $src, $shift" %}
12545
12546 ins_encode %{
12547 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12548 $shift$$constant & 0x1f);
12549 %}
12550 ins_pipe(ialu_reg_reg_vshift);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12556 %{
12557 match(Set dst (RotateRight src shift));
12558
12559 ins_cost(INSN_COST);
12560 format %{ "ror $dst, $src, $shift" %}
12561
12562 ins_encode %{
12563 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12564 $shift$$constant & 0x3f);
12565 %}
12566 ins_pipe(ialu_reg_reg_vshift);
12567 %}
12568
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12572 %{
12573 match(Set dst (RotateRight src shift));
12574
12575 ins_cost(INSN_COST);
12576 format %{ "ror $dst, $src, $shift" %}
12577
12578 ins_encode %{
12579 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12580 %}
12581 ins_pipe(ialu_reg_reg_vshift);
12582 %}
12583
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12587 %{
12588 match(Set dst (RotateRight src shift));
12589
12590 ins_cost(INSN_COST);
12591 format %{ "ror $dst, $src, $shift" %}
12592
12593 ins_encode %{
12594 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12595 %}
12596 ins_pipe(ialu_reg_reg_vshift);
12597 %}
12598
12599 // This pattern is automatically generated from aarch64_ad.m4
12600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12601 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12602 %{
12603 match(Set dst (RotateLeft src shift));
12604
12605 ins_cost(INSN_COST);
12606 format %{ "rol $dst, $src, $shift" %}
12607
12608 ins_encode %{
12609 __ subw(rscratch1, zr, as_Register($shift$$reg));
12610 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12611 %}
12612 ins_pipe(ialu_reg_reg_vshift);
12613 %}
12614
12615 // This pattern is automatically generated from aarch64_ad.m4
12616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12617 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12618 %{
12619 match(Set dst (RotateLeft src shift));
12620
12621 ins_cost(INSN_COST);
12622 format %{ "rol $dst, $src, $shift" %}
12623
12624 ins_encode %{
12625 __ subw(rscratch1, zr, as_Register($shift$$reg));
12626 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12627 %}
12628 ins_pipe(ialu_reg_reg_vshift);
12629 %}
12630
12631
12632 // Add/subtract (extended)
12633
12634 // This pattern is automatically generated from aarch64_ad.m4
12635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12636 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12637 %{
12638 match(Set dst (AddL src1 (ConvI2L src2)));
12639 ins_cost(INSN_COST);
12640 format %{ "add $dst, $src1, $src2, sxtw" %}
12641
12642 ins_encode %{
12643 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12644 as_Register($src2$$reg), ext::sxtw);
12645 %}
12646 ins_pipe(ialu_reg_reg);
12647 %}
12648
12649 // This pattern is automatically generated from aarch64_ad.m4
12650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12651 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12652 %{
12653 match(Set dst (SubL src1 (ConvI2L src2)));
12654 ins_cost(INSN_COST);
12655 format %{ "sub $dst, $src1, $src2, sxtw" %}
12656
12657 ins_encode %{
12658 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12659 as_Register($src2$$reg), ext::sxtw);
12660 %}
12661 ins_pipe(ialu_reg_reg);
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12667 %{
12668 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12669 ins_cost(INSN_COST);
12670 format %{ "add $dst, $src1, $src2, sxth" %}
12671
12672 ins_encode %{
12673 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12674 as_Register($src2$$reg), ext::sxth);
12675 %}
12676 ins_pipe(ialu_reg_reg);
12677 %}
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12682 %{
12683 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12684 ins_cost(INSN_COST);
12685 format %{ "add $dst, $src1, $src2, sxtb" %}
12686
12687 ins_encode %{
12688 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12689 as_Register($src2$$reg), ext::sxtb);
12690 %}
12691 ins_pipe(ialu_reg_reg);
12692 %}
12693
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12697 %{
12698 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12699 ins_cost(INSN_COST);
12700 format %{ "add $dst, $src1, $src2, uxtb" %}
12701
12702 ins_encode %{
12703 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12704 as_Register($src2$$reg), ext::uxtb);
12705 %}
12706 ins_pipe(ialu_reg_reg);
12707 %}
12708
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12712 %{
12713 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12714 ins_cost(INSN_COST);
12715 format %{ "add $dst, $src1, $src2, sxth" %}
12716
12717 ins_encode %{
12718 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12719 as_Register($src2$$reg), ext::sxth);
12720 %}
12721 ins_pipe(ialu_reg_reg);
12722 %}
12723
12724 // This pattern is automatically generated from aarch64_ad.m4
12725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12726 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12727 %{
12728 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12729 ins_cost(INSN_COST);
12730 format %{ "add $dst, $src1, $src2, sxtw" %}
12731
12732 ins_encode %{
12733 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12734 as_Register($src2$$reg), ext::sxtw);
12735 %}
12736 ins_pipe(ialu_reg_reg);
12737 %}
12738
12739 // This pattern is automatically generated from aarch64_ad.m4
12740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12741 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12742 %{
12743 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12744 ins_cost(INSN_COST);
12745 format %{ "add $dst, $src1, $src2, sxtb" %}
12746
12747 ins_encode %{
12748 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12749 as_Register($src2$$reg), ext::sxtb);
12750 %}
12751 ins_pipe(ialu_reg_reg);
12752 %}
12753
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12757 %{
12758 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12759 ins_cost(INSN_COST);
12760 format %{ "add $dst, $src1, $src2, uxtb" %}
12761
12762 ins_encode %{
12763 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12764 as_Register($src2$$reg), ext::uxtb);
12765 %}
12766 ins_pipe(ialu_reg_reg);
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12772 %{
12773 match(Set dst (AddI src1 (AndI src2 mask)));
12774 ins_cost(INSN_COST);
12775 format %{ "addw $dst, $src1, $src2, uxtb" %}
12776
12777 ins_encode %{
12778 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12779 as_Register($src2$$reg), ext::uxtb);
12780 %}
12781 ins_pipe(ialu_reg_reg);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12787 %{
12788 match(Set dst (AddI src1 (AndI src2 mask)));
12789 ins_cost(INSN_COST);
12790 format %{ "addw $dst, $src1, $src2, uxth" %}
12791
12792 ins_encode %{
12793 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12794 as_Register($src2$$reg), ext::uxth);
12795 %}
12796 ins_pipe(ialu_reg_reg);
12797 %}
12798
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12802 %{
12803 match(Set dst (AddL src1 (AndL src2 mask)));
12804 ins_cost(INSN_COST);
12805 format %{ "add $dst, $src1, $src2, uxtb" %}
12806
12807 ins_encode %{
12808 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12809 as_Register($src2$$reg), ext::uxtb);
12810 %}
12811 ins_pipe(ialu_reg_reg);
12812 %}
12813
12814 // This pattern is automatically generated from aarch64_ad.m4
12815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12816 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12817 %{
12818 match(Set dst (AddL src1 (AndL src2 mask)));
12819 ins_cost(INSN_COST);
12820 format %{ "add $dst, $src1, $src2, uxth" %}
12821
12822 ins_encode %{
12823 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12824 as_Register($src2$$reg), ext::uxth);
12825 %}
12826 ins_pipe(ialu_reg_reg);
12827 %}
12828
12829 // This pattern is automatically generated from aarch64_ad.m4
12830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12831 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12832 %{
12833 match(Set dst (AddL src1 (AndL src2 mask)));
12834 ins_cost(INSN_COST);
12835 format %{ "add $dst, $src1, $src2, uxtw" %}
12836
12837 ins_encode %{
12838 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12839 as_Register($src2$$reg), ext::uxtw);
12840 %}
12841 ins_pipe(ialu_reg_reg);
12842 %}
12843
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12847 %{
12848 match(Set dst (SubI src1 (AndI src2 mask)));
12849 ins_cost(INSN_COST);
12850 format %{ "subw $dst, $src1, $src2, uxtb" %}
12851
12852 ins_encode %{
12853 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12854 as_Register($src2$$reg), ext::uxtb);
12855 %}
12856 ins_pipe(ialu_reg_reg);
12857 %}
12858
12859 // This pattern is automatically generated from aarch64_ad.m4
12860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12861 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12862 %{
12863 match(Set dst (SubI src1 (AndI src2 mask)));
12864 ins_cost(INSN_COST);
12865 format %{ "subw $dst, $src1, $src2, uxth" %}
12866
12867 ins_encode %{
12868 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12869 as_Register($src2$$reg), ext::uxth);
12870 %}
12871 ins_pipe(ialu_reg_reg);
12872 %}
12873
12874 // This pattern is automatically generated from aarch64_ad.m4
12875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12876 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12877 %{
12878 match(Set dst (SubL src1 (AndL src2 mask)));
12879 ins_cost(INSN_COST);
12880 format %{ "sub $dst, $src1, $src2, uxtb" %}
12881
12882 ins_encode %{
12883 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12884 as_Register($src2$$reg), ext::uxtb);
12885 %}
12886 ins_pipe(ialu_reg_reg);
12887 %}
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12892 %{
12893 match(Set dst (SubL src1 (AndL src2 mask)));
12894 ins_cost(INSN_COST);
12895 format %{ "sub $dst, $src1, $src2, uxth" %}
12896
12897 ins_encode %{
12898 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12899 as_Register($src2$$reg), ext::uxth);
12900 %}
12901 ins_pipe(ialu_reg_reg);
12902 %}
12903
12904 // This pattern is automatically generated from aarch64_ad.m4
12905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12906 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12907 %{
12908 match(Set dst (SubL src1 (AndL src2 mask)));
12909 ins_cost(INSN_COST);
12910 format %{ "sub $dst, $src1, $src2, uxtw" %}
12911
12912 ins_encode %{
12913 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12914 as_Register($src2$$reg), ext::uxtw);
12915 %}
12916 ins_pipe(ialu_reg_reg);
12917 %}
12918
12919
12920 // This pattern is automatically generated from aarch64_ad.m4
12921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12922 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12923 %{
12924 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12925 ins_cost(1.9 * INSN_COST);
12926 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12927
12928 ins_encode %{
12929 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12930 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12931 %}
12932 ins_pipe(ialu_reg_reg_shift);
12933 %}
12934
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12938 %{
12939 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12940 ins_cost(1.9 * INSN_COST);
12941 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12942
12943 ins_encode %{
12944 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12945 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12946 %}
12947 ins_pipe(ialu_reg_reg_shift);
12948 %}
12949
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12953 %{
12954 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12955 ins_cost(1.9 * INSN_COST);
12956 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12957
12958 ins_encode %{
12959 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12960 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12961 %}
12962 ins_pipe(ialu_reg_reg_shift);
12963 %}
12964
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12968 %{
12969 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12970 ins_cost(1.9 * INSN_COST);
12971 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12972
12973 ins_encode %{
12974 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12975 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12976 %}
12977 ins_pipe(ialu_reg_reg_shift);
12978 %}
12979
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12983 %{
12984 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12985 ins_cost(1.9 * INSN_COST);
12986 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12987
12988 ins_encode %{
12989 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12990 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12991 %}
12992 ins_pipe(ialu_reg_reg_shift);
12993 %}
12994
12995 // This pattern is automatically generated from aarch64_ad.m4
12996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12997 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12998 %{
12999 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13000 ins_cost(1.9 * INSN_COST);
13001 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13002
13003 ins_encode %{
13004 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13005 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13006 %}
13007 ins_pipe(ialu_reg_reg_shift);
13008 %}
13009
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13013 %{
13014 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13015 ins_cost(1.9 * INSN_COST);
13016 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13017
13018 ins_encode %{
13019 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13020 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13021 %}
13022 ins_pipe(ialu_reg_reg_shift);
13023 %}
13024
13025 // This pattern is automatically generated from aarch64_ad.m4
13026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13027 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13028 %{
13029 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13030 ins_cost(1.9 * INSN_COST);
13031 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13032
13033 ins_encode %{
13034 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13035 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13036 %}
13037 ins_pipe(ialu_reg_reg_shift);
13038 %}
13039
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13043 %{
13044 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13045 ins_cost(1.9 * INSN_COST);
13046 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13047
13048 ins_encode %{
13049 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13050 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13051 %}
13052 ins_pipe(ialu_reg_reg_shift);
13053 %}
13054
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13058 %{
13059 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13060 ins_cost(1.9 * INSN_COST);
13061 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13062
13063 ins_encode %{
13064 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13065 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13066 %}
13067 ins_pipe(ialu_reg_reg_shift);
13068 %}
13069
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13073 %{
13074 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13075 ins_cost(1.9 * INSN_COST);
13076 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13077
13078 ins_encode %{
13079 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13080 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13081 %}
13082 ins_pipe(ialu_reg_reg_shift);
13083 %}
13084
13085 // This pattern is automatically generated from aarch64_ad.m4
13086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13087 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13088 %{
13089 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13090 ins_cost(1.9 * INSN_COST);
13091 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13092
13093 ins_encode %{
13094 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13095 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13096 %}
13097 ins_pipe(ialu_reg_reg_shift);
13098 %}
13099
13100 // This pattern is automatically generated from aarch64_ad.m4
13101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13102 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13103 %{
13104 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13105 ins_cost(1.9 * INSN_COST);
13106 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13107
13108 ins_encode %{
13109 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13110 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13111 %}
13112 ins_pipe(ialu_reg_reg_shift);
13113 %}
13114
13115 // This pattern is automatically generated from aarch64_ad.m4
13116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13117 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13118 %{
13119 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13120 ins_cost(1.9 * INSN_COST);
13121 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13122
13123 ins_encode %{
13124 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13125 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13126 %}
13127 ins_pipe(ialu_reg_reg_shift);
13128 %}
13129
13130 // This pattern is automatically generated from aarch64_ad.m4
13131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13132 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13133 %{
13134 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13135 ins_cost(1.9 * INSN_COST);
13136 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13137
13138 ins_encode %{
13139 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13140 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13141 %}
13142 ins_pipe(ialu_reg_reg_shift);
13143 %}
13144
13145 // This pattern is automatically generated from aarch64_ad.m4
13146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13147 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13148 %{
13149 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13150 ins_cost(1.9 * INSN_COST);
13151 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13152
13153 ins_encode %{
13154 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13155 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13156 %}
13157 ins_pipe(ialu_reg_reg_shift);
13158 %}
13159
13160 // This pattern is automatically generated from aarch64_ad.m4
13161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13162 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13163 %{
13164 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13165 ins_cost(1.9 * INSN_COST);
13166 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13167
13168 ins_encode %{
13169 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13170 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13171 %}
13172 ins_pipe(ialu_reg_reg_shift);
13173 %}
13174
13175 // This pattern is automatically generated from aarch64_ad.m4
13176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13177 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13178 %{
13179 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13180 ins_cost(1.9 * INSN_COST);
13181 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13182
13183 ins_encode %{
13184 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13185 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13186 %}
13187 ins_pipe(ialu_reg_reg_shift);
13188 %}
13189
13190 // This pattern is automatically generated from aarch64_ad.m4
13191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13192 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13193 %{
13194 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13195 ins_cost(1.9 * INSN_COST);
13196 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13197
13198 ins_encode %{
13199 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13200 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13201 %}
13202 ins_pipe(ialu_reg_reg_shift);
13203 %}
13204
13205 // This pattern is automatically generated from aarch64_ad.m4
13206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13207 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13208 %{
13209 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13210 ins_cost(1.9 * INSN_COST);
13211 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13212
13213 ins_encode %{
13214 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13215 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13216 %}
13217 ins_pipe(ialu_reg_reg_shift);
13218 %}
13219
13220 // This pattern is automatically generated from aarch64_ad.m4
13221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13222 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13223 %{
13224 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13225 ins_cost(1.9 * INSN_COST);
13226 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13227
13228 ins_encode %{
13229 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13230 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13231 %}
13232 ins_pipe(ialu_reg_reg_shift);
13233 %}
13234
13235 // This pattern is automatically generated from aarch64_ad.m4
13236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13237 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13238 %{
13239 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13240 ins_cost(1.9 * INSN_COST);
13241 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13242
13243 ins_encode %{
13244 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13245 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13246 %}
13247 ins_pipe(ialu_reg_reg_shift);
13248 %}
13249
13250 // This pattern is automatically generated from aarch64_ad.m4
13251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13252 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13253 %{
13254 effect(DEF dst, USE src1, USE src2, USE cr);
13255 ins_cost(INSN_COST * 2);
13256 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13257
13258 ins_encode %{
13259 __ cselw($dst$$Register,
13260 $src1$$Register,
13261 $src2$$Register,
13262 Assembler::LT);
13263 %}
13264 ins_pipe(icond_reg_reg);
13265 %}
13266
13267 // This pattern is automatically generated from aarch64_ad.m4
13268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13269 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13270 %{
13271 effect(DEF dst, USE src1, USE src2, USE cr);
13272 ins_cost(INSN_COST * 2);
13273 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13274
13275 ins_encode %{
13276 __ cselw($dst$$Register,
13277 $src1$$Register,
13278 $src2$$Register,
13279 Assembler::GT);
13280 %}
13281 ins_pipe(icond_reg_reg);
13282 %}
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13287 %{
13288 effect(DEF dst, USE src1, USE cr);
13289 ins_cost(INSN_COST * 2);
13290 format %{ "cselw $dst, $src1, zr lt\t" %}
13291
13292 ins_encode %{
13293 __ cselw($dst$$Register,
13294 $src1$$Register,
13295 zr,
13296 Assembler::LT);
13297 %}
13298 ins_pipe(icond_reg);
13299 %}
13300
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13304 %{
13305 effect(DEF dst, USE src1, USE cr);
13306 ins_cost(INSN_COST * 2);
13307 format %{ "cselw $dst, $src1, zr gt\t" %}
13308
13309 ins_encode %{
13310 __ cselw($dst$$Register,
13311 $src1$$Register,
13312 zr,
13313 Assembler::GT);
13314 %}
13315 ins_pipe(icond_reg);
13316 %}
13317
13318 // This pattern is automatically generated from aarch64_ad.m4
13319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13320 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13321 %{
13322 effect(DEF dst, USE src1, USE cr);
13323 ins_cost(INSN_COST * 2);
13324 format %{ "csincw $dst, $src1, zr le\t" %}
13325
13326 ins_encode %{
13327 __ csincw($dst$$Register,
13328 $src1$$Register,
13329 zr,
13330 Assembler::LE);
13331 %}
13332 ins_pipe(icond_reg);
13333 %}
13334
13335 // This pattern is automatically generated from aarch64_ad.m4
13336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13337 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13338 %{
13339 effect(DEF dst, USE src1, USE cr);
13340 ins_cost(INSN_COST * 2);
13341 format %{ "csincw $dst, $src1, zr gt\t" %}
13342
13343 ins_encode %{
13344 __ csincw($dst$$Register,
13345 $src1$$Register,
13346 zr,
13347 Assembler::GT);
13348 %}
13349 ins_pipe(icond_reg);
13350 %}
13351
13352 // This pattern is automatically generated from aarch64_ad.m4
13353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13354 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13355 %{
13356 effect(DEF dst, USE src1, USE cr);
13357 ins_cost(INSN_COST * 2);
13358 format %{ "csinvw $dst, $src1, zr lt\t" %}
13359
13360 ins_encode %{
13361 __ csinvw($dst$$Register,
13362 $src1$$Register,
13363 zr,
13364 Assembler::LT);
13365 %}
13366 ins_pipe(icond_reg);
13367 %}
13368
13369 // This pattern is automatically generated from aarch64_ad.m4
13370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13371 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13372 %{
13373 effect(DEF dst, USE src1, USE cr);
13374 ins_cost(INSN_COST * 2);
13375 format %{ "csinvw $dst, $src1, zr ge\t" %}
13376
13377 ins_encode %{
13378 __ csinvw($dst$$Register,
13379 $src1$$Register,
13380 zr,
13381 Assembler::GE);
13382 %}
13383 ins_pipe(icond_reg);
13384 %}
13385
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13389 %{
13390 match(Set dst (MinI src imm));
13391 ins_cost(INSN_COST * 3);
13392 expand %{
13393 rFlagsReg cr;
13394 compI_reg_imm0(cr, src);
13395 cmovI_reg_imm0_lt(dst, src, cr);
13396 %}
13397 %}
13398
13399 // This pattern is automatically generated from aarch64_ad.m4
13400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13401 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13402 %{
13403 match(Set dst (MinI imm src));
13404 ins_cost(INSN_COST * 3);
13405 expand %{
13406 rFlagsReg cr;
13407 compI_reg_imm0(cr, src);
13408 cmovI_reg_imm0_lt(dst, src, cr);
13409 %}
13410 %}
13411
13412 // This pattern is automatically generated from aarch64_ad.m4
13413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13414 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13415 %{
13416 match(Set dst (MinI src imm));
13417 ins_cost(INSN_COST * 3);
13418 expand %{
13419 rFlagsReg cr;
13420 compI_reg_imm0(cr, src);
13421 cmovI_reg_imm1_le(dst, src, cr);
13422 %}
13423 %}
13424
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13428 %{
13429 match(Set dst (MinI imm src));
13430 ins_cost(INSN_COST * 3);
13431 expand %{
13432 rFlagsReg cr;
13433 compI_reg_imm0(cr, src);
13434 cmovI_reg_imm1_le(dst, src, cr);
13435 %}
13436 %}
13437
13438 // This pattern is automatically generated from aarch64_ad.m4
13439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13440 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13441 %{
13442 match(Set dst (MinI src imm));
13443 ins_cost(INSN_COST * 3);
13444 expand %{
13445 rFlagsReg cr;
13446 compI_reg_imm0(cr, src);
13447 cmovI_reg_immM1_lt(dst, src, cr);
13448 %}
13449 %}
13450
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13454 %{
13455 match(Set dst (MinI imm src));
13456 ins_cost(INSN_COST * 3);
13457 expand %{
13458 rFlagsReg cr;
13459 compI_reg_imm0(cr, src);
13460 cmovI_reg_immM1_lt(dst, src, cr);
13461 %}
13462 %}
13463
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13467 %{
13468 match(Set dst (MaxI src imm));
13469 ins_cost(INSN_COST * 3);
13470 expand %{
13471 rFlagsReg cr;
13472 compI_reg_imm0(cr, src);
13473 cmovI_reg_imm0_gt(dst, src, cr);
13474 %}
13475 %}
13476
13477 // This pattern is automatically generated from aarch64_ad.m4
13478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13479 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13480 %{
13481 match(Set dst (MaxI imm src));
13482 ins_cost(INSN_COST * 3);
13483 expand %{
13484 rFlagsReg cr;
13485 compI_reg_imm0(cr, src);
13486 cmovI_reg_imm0_gt(dst, src, cr);
13487 %}
13488 %}
13489
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13493 %{
13494 match(Set dst (MaxI src imm));
13495 ins_cost(INSN_COST * 3);
13496 expand %{
13497 rFlagsReg cr;
13498 compI_reg_imm0(cr, src);
13499 cmovI_reg_imm1_gt(dst, src, cr);
13500 %}
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13506 %{
13507 match(Set dst (MaxI imm src));
13508 ins_cost(INSN_COST * 3);
13509 expand %{
13510 rFlagsReg cr;
13511 compI_reg_imm0(cr, src);
13512 cmovI_reg_imm1_gt(dst, src, cr);
13513 %}
13514 %}
13515
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13519 %{
13520 match(Set dst (MaxI src imm));
13521 ins_cost(INSN_COST * 3);
13522 expand %{
13523 rFlagsReg cr;
13524 compI_reg_imm0(cr, src);
13525 cmovI_reg_immM1_ge(dst, src, cr);
13526 %}
13527 %}
13528
13529 // This pattern is automatically generated from aarch64_ad.m4
13530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13531 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13532 %{
13533 match(Set dst (MaxI imm src));
13534 ins_cost(INSN_COST * 3);
13535 expand %{
13536 rFlagsReg cr;
13537 compI_reg_imm0(cr, src);
13538 cmovI_reg_immM1_ge(dst, src, cr);
13539 %}
13540 %}
13541
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13545 %{
13546 match(Set dst (ReverseI src));
13547 ins_cost(INSN_COST);
13548 format %{ "rbitw $dst, $src" %}
13549 ins_encode %{
13550 __ rbitw($dst$$Register, $src$$Register);
13551 %}
13552 ins_pipe(ialu_reg);
13553 %}
13554
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13558 %{
13559 match(Set dst (ReverseL src));
13560 ins_cost(INSN_COST);
13561 format %{ "rbit $dst, $src" %}
13562 ins_encode %{
13563 __ rbit($dst$$Register, $src$$Register);
13564 %}
13565 ins_pipe(ialu_reg);
13566 %}
13567
13568
13569 // END This section of the file is automatically generated. Do not edit --------------
13570
13571
13572 // ============================================================================
13573 // Floating Point Arithmetic Instructions
13574
13575 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13576 match(Set dst (AddF src1 src2));
13577
13578 ins_cost(INSN_COST * 5);
13579 format %{ "fadds $dst, $src1, $src2" %}
13580
13581 ins_encode %{
13582 __ fadds(as_FloatRegister($dst$$reg),
13583 as_FloatRegister($src1$$reg),
13584 as_FloatRegister($src2$$reg));
13585 %}
13586
13587 ins_pipe(fp_dop_reg_reg_s);
13588 %}
13589
13590 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13591 match(Set dst (AddD src1 src2));
13592
13593 ins_cost(INSN_COST * 5);
13594 format %{ "faddd $dst, $src1, $src2" %}
13595
13596 ins_encode %{
13597 __ faddd(as_FloatRegister($dst$$reg),
13598 as_FloatRegister($src1$$reg),
13599 as_FloatRegister($src2$$reg));
13600 %}
13601
13602 ins_pipe(fp_dop_reg_reg_d);
13603 %}
13604
13605 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13606 match(Set dst (SubF src1 src2));
13607
13608 ins_cost(INSN_COST * 5);
13609 format %{ "fsubs $dst, $src1, $src2" %}
13610
13611 ins_encode %{
13612 __ fsubs(as_FloatRegister($dst$$reg),
13613 as_FloatRegister($src1$$reg),
13614 as_FloatRegister($src2$$reg));
13615 %}
13616
13617 ins_pipe(fp_dop_reg_reg_s);
13618 %}
13619
13620 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13621 match(Set dst (SubD src1 src2));
13622
13623 ins_cost(INSN_COST * 5);
13624 format %{ "fsubd $dst, $src1, $src2" %}
13625
13626 ins_encode %{
13627 __ fsubd(as_FloatRegister($dst$$reg),
13628 as_FloatRegister($src1$$reg),
13629 as_FloatRegister($src2$$reg));
13630 %}
13631
13632 ins_pipe(fp_dop_reg_reg_d);
13633 %}
13634
13635 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13636 match(Set dst (MulF src1 src2));
13637
13638 ins_cost(INSN_COST * 6);
13639 format %{ "fmuls $dst, $src1, $src2" %}
13640
13641 ins_encode %{
13642 __ fmuls(as_FloatRegister($dst$$reg),
13643 as_FloatRegister($src1$$reg),
13644 as_FloatRegister($src2$$reg));
13645 %}
13646
13647 ins_pipe(fp_dop_reg_reg_s);
13648 %}
13649
13650 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13651 match(Set dst (MulD src1 src2));
13652
13653 ins_cost(INSN_COST * 6);
13654 format %{ "fmuld $dst, $src1, $src2" %}
13655
13656 ins_encode %{
13657 __ fmuld(as_FloatRegister($dst$$reg),
13658 as_FloatRegister($src1$$reg),
13659 as_FloatRegister($src2$$reg));
13660 %}
13661
13662 ins_pipe(fp_dop_reg_reg_d);
13663 %}
13664
13665 // src1 * src2 + src3
13666 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13667 match(Set dst (FmaF src3 (Binary src1 src2)));
13668
13669 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13670
13671 ins_encode %{
13672 assert(UseFMA, "Needs FMA instructions support.");
13673 __ fmadds(as_FloatRegister($dst$$reg),
13674 as_FloatRegister($src1$$reg),
13675 as_FloatRegister($src2$$reg),
13676 as_FloatRegister($src3$$reg));
13677 %}
13678
13679 ins_pipe(pipe_class_default);
13680 %}
13681
13682 // src1 * src2 + src3
13683 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13684 match(Set dst (FmaD src3 (Binary src1 src2)));
13685
13686 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13687
13688 ins_encode %{
13689 assert(UseFMA, "Needs FMA instructions support.");
13690 __ fmaddd(as_FloatRegister($dst$$reg),
13691 as_FloatRegister($src1$$reg),
13692 as_FloatRegister($src2$$reg),
13693 as_FloatRegister($src3$$reg));
13694 %}
13695
13696 ins_pipe(pipe_class_default);
13697 %}
13698
13699 // src1 * (-src2) + src3
13700 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13701 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13702 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13703
13704 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13705
13706 ins_encode %{
13707 assert(UseFMA, "Needs FMA instructions support.");
13708 __ fmsubs(as_FloatRegister($dst$$reg),
13709 as_FloatRegister($src1$$reg),
13710 as_FloatRegister($src2$$reg),
13711 as_FloatRegister($src3$$reg));
13712 %}
13713
13714 ins_pipe(pipe_class_default);
13715 %}
13716
13717 // src1 * (-src2) + src3
13718 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13719 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13720 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13721
13722 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13723
13724 ins_encode %{
13725 assert(UseFMA, "Needs FMA instructions support.");
13726 __ fmsubd(as_FloatRegister($dst$$reg),
13727 as_FloatRegister($src1$$reg),
13728 as_FloatRegister($src2$$reg),
13729 as_FloatRegister($src3$$reg));
13730 %}
13731
13732 ins_pipe(pipe_class_default);
13733 %}
13734
13735 // src1 * (-src2) - src3
13736 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13737 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13738 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13739
13740 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13741
13742 ins_encode %{
13743 assert(UseFMA, "Needs FMA instructions support.");
13744 __ fnmadds(as_FloatRegister($dst$$reg),
13745 as_FloatRegister($src1$$reg),
13746 as_FloatRegister($src2$$reg),
13747 as_FloatRegister($src3$$reg));
13748 %}
13749
13750 ins_pipe(pipe_class_default);
13751 %}
13752
13753 // src1 * (-src2) - src3
13754 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13755 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13756 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13757
13758 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13759
13760 ins_encode %{
13761 assert(UseFMA, "Needs FMA instructions support.");
13762 __ fnmaddd(as_FloatRegister($dst$$reg),
13763 as_FloatRegister($src1$$reg),
13764 as_FloatRegister($src2$$reg),
13765 as_FloatRegister($src3$$reg));
13766 %}
13767
13768 ins_pipe(pipe_class_default);
13769 %}
13770
13771 // src1 * src2 - src3
13772 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13773 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13774
13775 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13776
13777 ins_encode %{
13778 assert(UseFMA, "Needs FMA instructions support.");
13779 __ fnmsubs(as_FloatRegister($dst$$reg),
13780 as_FloatRegister($src1$$reg),
13781 as_FloatRegister($src2$$reg),
13782 as_FloatRegister($src3$$reg));
13783 %}
13784
13785 ins_pipe(pipe_class_default);
13786 %}
13787
13788 // src1 * src2 - src3
13789 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13790 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13791
13792 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13793
13794 ins_encode %{
13795 assert(UseFMA, "Needs FMA instructions support.");
13796 // n.b. insn name should be fnmsubd
13797 __ fnmsub(as_FloatRegister($dst$$reg),
13798 as_FloatRegister($src1$$reg),
13799 as_FloatRegister($src2$$reg),
13800 as_FloatRegister($src3$$reg));
13801 %}
13802
13803 ins_pipe(pipe_class_default);
13804 %}
13805
13806
13807 // Math.max(FF)F
13808 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13809 match(Set dst (MaxF src1 src2));
13810
13811 format %{ "fmaxs $dst, $src1, $src2" %}
13812 ins_encode %{
13813 __ fmaxs(as_FloatRegister($dst$$reg),
13814 as_FloatRegister($src1$$reg),
13815 as_FloatRegister($src2$$reg));
13816 %}
13817
13818 ins_pipe(fp_dop_reg_reg_s);
13819 %}
13820
13821 // Math.min(FF)F
13822 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13823 match(Set dst (MinF src1 src2));
13824
13825 format %{ "fmins $dst, $src1, $src2" %}
13826 ins_encode %{
13827 __ fmins(as_FloatRegister($dst$$reg),
13828 as_FloatRegister($src1$$reg),
13829 as_FloatRegister($src2$$reg));
13830 %}
13831
13832 ins_pipe(fp_dop_reg_reg_s);
13833 %}
13834
13835 // Math.max(DD)D
13836 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13837 match(Set dst (MaxD src1 src2));
13838
13839 format %{ "fmaxd $dst, $src1, $src2" %}
13840 ins_encode %{
13841 __ fmaxd(as_FloatRegister($dst$$reg),
13842 as_FloatRegister($src1$$reg),
13843 as_FloatRegister($src2$$reg));
13844 %}
13845
13846 ins_pipe(fp_dop_reg_reg_d);
13847 %}
13848
13849 // Math.min(DD)D
13850 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13851 match(Set dst (MinD src1 src2));
13852
13853 format %{ "fmind $dst, $src1, $src2" %}
13854 ins_encode %{
13855 __ fmind(as_FloatRegister($dst$$reg),
13856 as_FloatRegister($src1$$reg),
13857 as_FloatRegister($src2$$reg));
13858 %}
13859
13860 ins_pipe(fp_dop_reg_reg_d);
13861 %}
13862
13863
13864 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13865 match(Set dst (DivF src1 src2));
13866
13867 ins_cost(INSN_COST * 18);
13868 format %{ "fdivs $dst, $src1, $src2" %}
13869
13870 ins_encode %{
13871 __ fdivs(as_FloatRegister($dst$$reg),
13872 as_FloatRegister($src1$$reg),
13873 as_FloatRegister($src2$$reg));
13874 %}
13875
13876 ins_pipe(fp_div_s);
13877 %}
13878
13879 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13880 match(Set dst (DivD src1 src2));
13881
13882 ins_cost(INSN_COST * 32);
13883 format %{ "fdivd $dst, $src1, $src2" %}
13884
13885 ins_encode %{
13886 __ fdivd(as_FloatRegister($dst$$reg),
13887 as_FloatRegister($src1$$reg),
13888 as_FloatRegister($src2$$reg));
13889 %}
13890
13891 ins_pipe(fp_div_d);
13892 %}
13893
13894 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13895 match(Set dst (NegF src));
13896
13897 ins_cost(INSN_COST * 3);
13898 format %{ "fneg $dst, $src" %}
13899
13900 ins_encode %{
13901 __ fnegs(as_FloatRegister($dst$$reg),
13902 as_FloatRegister($src$$reg));
13903 %}
13904
13905 ins_pipe(fp_uop_s);
13906 %}
13907
13908 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13909 match(Set dst (NegD src));
13910
13911 ins_cost(INSN_COST * 3);
13912 format %{ "fnegd $dst, $src" %}
13913
13914 ins_encode %{
13915 __ fnegd(as_FloatRegister($dst$$reg),
13916 as_FloatRegister($src$$reg));
13917 %}
13918
13919 ins_pipe(fp_uop_d);
13920 %}
13921
13922 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13923 %{
13924 match(Set dst (AbsI src));
13925
13926 effect(KILL cr);
13927 ins_cost(INSN_COST * 2);
13928 format %{ "cmpw $src, zr\n\t"
13929 "cnegw $dst, $src, Assembler::LT\t# int abs"
13930 %}
13931
13932 ins_encode %{
13933 __ cmpw(as_Register($src$$reg), zr);
13934 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13935 %}
13936 ins_pipe(pipe_class_default);
13937 %}
13938
13939 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13940 %{
13941 match(Set dst (AbsL src));
13942
13943 effect(KILL cr);
13944 ins_cost(INSN_COST * 2);
13945 format %{ "cmp $src, zr\n\t"
13946 "cneg $dst, $src, Assembler::LT\t# long abs"
13947 %}
13948
13949 ins_encode %{
13950 __ cmp(as_Register($src$$reg), zr);
13951 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13952 %}
13953 ins_pipe(pipe_class_default);
13954 %}
13955
13956 instruct absF_reg(vRegF dst, vRegF src) %{
13957 match(Set dst (AbsF src));
13958
13959 ins_cost(INSN_COST * 3);
13960 format %{ "fabss $dst, $src" %}
13961 ins_encode %{
13962 __ fabss(as_FloatRegister($dst$$reg),
13963 as_FloatRegister($src$$reg));
13964 %}
13965
13966 ins_pipe(fp_uop_s);
13967 %}
13968
13969 instruct absD_reg(vRegD dst, vRegD src) %{
13970 match(Set dst (AbsD src));
13971
13972 ins_cost(INSN_COST * 3);
13973 format %{ "fabsd $dst, $src" %}
13974 ins_encode %{
13975 __ fabsd(as_FloatRegister($dst$$reg),
13976 as_FloatRegister($src$$reg));
13977 %}
13978
13979 ins_pipe(fp_uop_d);
13980 %}
13981
13982 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13983 match(Set dst (AbsF (SubF src1 src2)));
13984
13985 ins_cost(INSN_COST * 3);
13986 format %{ "fabds $dst, $src1, $src2" %}
13987 ins_encode %{
13988 __ fabds(as_FloatRegister($dst$$reg),
13989 as_FloatRegister($src1$$reg),
13990 as_FloatRegister($src2$$reg));
13991 %}
13992
13993 ins_pipe(fp_uop_s);
13994 %}
13995
13996 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13997 match(Set dst (AbsD (SubD src1 src2)));
13998
13999 ins_cost(INSN_COST * 3);
14000 format %{ "fabdd $dst, $src1, $src2" %}
14001 ins_encode %{
14002 __ fabdd(as_FloatRegister($dst$$reg),
14003 as_FloatRegister($src1$$reg),
14004 as_FloatRegister($src2$$reg));
14005 %}
14006
14007 ins_pipe(fp_uop_d);
14008 %}
14009
14010 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14011 match(Set dst (SqrtD src));
14012
14013 ins_cost(INSN_COST * 50);
14014 format %{ "fsqrtd $dst, $src" %}
14015 ins_encode %{
14016 __ fsqrtd(as_FloatRegister($dst$$reg),
14017 as_FloatRegister($src$$reg));
14018 %}
14019
14020 ins_pipe(fp_div_s);
14021 %}
14022
14023 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14024 match(Set dst (SqrtF src));
14025
14026 ins_cost(INSN_COST * 50);
14027 format %{ "fsqrts $dst, $src" %}
14028 ins_encode %{
14029 __ fsqrts(as_FloatRegister($dst$$reg),
14030 as_FloatRegister($src$$reg));
14031 %}
14032
14033 ins_pipe(fp_div_d);
14034 %}
14035
14036 // Math.rint, floor, ceil
14037 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14038 match(Set dst (RoundDoubleMode src rmode));
14039 format %{ "frint $dst, $src, $rmode" %}
14040 ins_encode %{
14041 switch ($rmode$$constant) {
14042 case RoundDoubleModeNode::rmode_rint:
14043 __ frintnd(as_FloatRegister($dst$$reg),
14044 as_FloatRegister($src$$reg));
14045 break;
14046 case RoundDoubleModeNode::rmode_floor:
14047 __ frintmd(as_FloatRegister($dst$$reg),
14048 as_FloatRegister($src$$reg));
14049 break;
14050 case RoundDoubleModeNode::rmode_ceil:
14051 __ frintpd(as_FloatRegister($dst$$reg),
14052 as_FloatRegister($src$$reg));
14053 break;
14054 }
14055 %}
14056 ins_pipe(fp_uop_d);
14057 %}
14058
14059 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14060 match(Set dst (CopySignD src1 (Binary src2 zero)));
14061 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14062 format %{ "CopySignD $dst $src1 $src2" %}
14063 ins_encode %{
14064 FloatRegister dst = as_FloatRegister($dst$$reg),
14065 src1 = as_FloatRegister($src1$$reg),
14066 src2 = as_FloatRegister($src2$$reg),
14067 zero = as_FloatRegister($zero$$reg);
14068 __ fnegd(dst, zero);
14069 __ bsl(dst, __ T8B, src2, src1);
14070 %}
14071 ins_pipe(fp_uop_d);
14072 %}
14073
14074 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14075 match(Set dst (CopySignF src1 src2));
14076 effect(TEMP_DEF dst, USE src1, USE src2);
14077 format %{ "CopySignF $dst $src1 $src2" %}
14078 ins_encode %{
14079 FloatRegister dst = as_FloatRegister($dst$$reg),
14080 src1 = as_FloatRegister($src1$$reg),
14081 src2 = as_FloatRegister($src2$$reg);
14082 __ movi(dst, __ T2S, 0x80, 24);
14083 __ bsl(dst, __ T8B, src2, src1);
14084 %}
14085 ins_pipe(fp_uop_d);
14086 %}
14087
14088 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14089 match(Set dst (SignumD src (Binary zero one)));
14090 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14091 format %{ "signumD $dst, $src" %}
14092 ins_encode %{
14093 FloatRegister src = as_FloatRegister($src$$reg),
14094 dst = as_FloatRegister($dst$$reg),
14095 zero = as_FloatRegister($zero$$reg),
14096 one = as_FloatRegister($one$$reg);
14097 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14098 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14099 // Bit selection instruction gets bit from "one" for each enabled bit in
14100 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14101 // NaN the whole "src" will be copied because "dst" is zero. For all other
14102 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14103 // from "src", and all other bits are copied from 1.0.
14104 __ bsl(dst, __ T8B, one, src);
14105 %}
14106 ins_pipe(fp_uop_d);
14107 %}
14108
14109 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14110 match(Set dst (SignumF src (Binary zero one)));
14111 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14112 format %{ "signumF $dst, $src" %}
14113 ins_encode %{
14114 FloatRegister src = as_FloatRegister($src$$reg),
14115 dst = as_FloatRegister($dst$$reg),
14116 zero = as_FloatRegister($zero$$reg),
14117 one = as_FloatRegister($one$$reg);
14118 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14119 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14120 // Bit selection instruction gets bit from "one" for each enabled bit in
14121 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14122 // NaN the whole "src" will be copied because "dst" is zero. For all other
14123 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14124 // from "src", and all other bits are copied from 1.0.
14125 __ bsl(dst, __ T8B, one, src);
14126 %}
14127 ins_pipe(fp_uop_d);
14128 %}
14129
14130 instruct onspinwait() %{
14131 match(OnSpinWait);
14132 ins_cost(INSN_COST);
14133
14134 format %{ "onspinwait" %}
14135
14136 ins_encode %{
14137 __ spin_wait();
14138 %}
14139 ins_pipe(pipe_class_empty);
14140 %}
14141
14142 // ============================================================================
14143 // Logical Instructions
14144
14145 // Integer Logical Instructions
14146
14147 // And Instructions
14148
14149
14150 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14151 match(Set dst (AndI src1 src2));
14152
14153 format %{ "andw $dst, $src1, $src2\t# int" %}
14154
14155 ins_cost(INSN_COST);
14156 ins_encode %{
14157 __ andw(as_Register($dst$$reg),
14158 as_Register($src1$$reg),
14159 as_Register($src2$$reg));
14160 %}
14161
14162 ins_pipe(ialu_reg_reg);
14163 %}
14164
14165 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14166 match(Set dst (AndI src1 src2));
14167
14168 format %{ "andsw $dst, $src1, $src2\t# int" %}
14169
14170 ins_cost(INSN_COST);
14171 ins_encode %{
14172 __ andw(as_Register($dst$$reg),
14173 as_Register($src1$$reg),
14174 (uint64_t)($src2$$constant));
14175 %}
14176
14177 ins_pipe(ialu_reg_imm);
14178 %}
14179
14180 // Or Instructions
14181
14182 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14183 match(Set dst (OrI src1 src2));
14184
14185 format %{ "orrw $dst, $src1, $src2\t# int" %}
14186
14187 ins_cost(INSN_COST);
14188 ins_encode %{
14189 __ orrw(as_Register($dst$$reg),
14190 as_Register($src1$$reg),
14191 as_Register($src2$$reg));
14192 %}
14193
14194 ins_pipe(ialu_reg_reg);
14195 %}
14196
14197 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14198 match(Set dst (OrI src1 src2));
14199
14200 format %{ "orrw $dst, $src1, $src2\t# int" %}
14201
14202 ins_cost(INSN_COST);
14203 ins_encode %{
14204 __ orrw(as_Register($dst$$reg),
14205 as_Register($src1$$reg),
14206 (uint64_t)($src2$$constant));
14207 %}
14208
14209 ins_pipe(ialu_reg_imm);
14210 %}
14211
14212 // Xor Instructions
14213
14214 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14215 match(Set dst (XorI src1 src2));
14216
14217 format %{ "eorw $dst, $src1, $src2\t# int" %}
14218
14219 ins_cost(INSN_COST);
14220 ins_encode %{
14221 __ eorw(as_Register($dst$$reg),
14222 as_Register($src1$$reg),
14223 as_Register($src2$$reg));
14224 %}
14225
14226 ins_pipe(ialu_reg_reg);
14227 %}
14228
14229 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14230 match(Set dst (XorI src1 src2));
14231
14232 format %{ "eorw $dst, $src1, $src2\t# int" %}
14233
14234 ins_cost(INSN_COST);
14235 ins_encode %{
14236 __ eorw(as_Register($dst$$reg),
14237 as_Register($src1$$reg),
14238 (uint64_t)($src2$$constant));
14239 %}
14240
14241 ins_pipe(ialu_reg_imm);
14242 %}
14243
14244 // Long Logical Instructions
14245 // TODO
14246
14247 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14248 match(Set dst (AndL src1 src2));
14249
14250 format %{ "and $dst, $src1, $src2\t# int" %}
14251
14252 ins_cost(INSN_COST);
14253 ins_encode %{
14254 __ andr(as_Register($dst$$reg),
14255 as_Register($src1$$reg),
14256 as_Register($src2$$reg));
14257 %}
14258
14259 ins_pipe(ialu_reg_reg);
14260 %}
14261
14262 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14263 match(Set dst (AndL src1 src2));
14264
14265 format %{ "and $dst, $src1, $src2\t# int" %}
14266
14267 ins_cost(INSN_COST);
14268 ins_encode %{
14269 __ andr(as_Register($dst$$reg),
14270 as_Register($src1$$reg),
14271 (uint64_t)($src2$$constant));
14272 %}
14273
14274 ins_pipe(ialu_reg_imm);
14275 %}
14276
14277 // Or Instructions
14278
14279 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14280 match(Set dst (OrL src1 src2));
14281
14282 format %{ "orr $dst, $src1, $src2\t# int" %}
14283
14284 ins_cost(INSN_COST);
14285 ins_encode %{
14286 __ orr(as_Register($dst$$reg),
14287 as_Register($src1$$reg),
14288 as_Register($src2$$reg));
14289 %}
14290
14291 ins_pipe(ialu_reg_reg);
14292 %}
14293
14294 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14295 match(Set dst (OrL src1 src2));
14296
14297 format %{ "orr $dst, $src1, $src2\t# int" %}
14298
14299 ins_cost(INSN_COST);
14300 ins_encode %{
14301 __ orr(as_Register($dst$$reg),
14302 as_Register($src1$$reg),
14303 (uint64_t)($src2$$constant));
14304 %}
14305
14306 ins_pipe(ialu_reg_imm);
14307 %}
14308
14309 // Xor Instructions
14310
14311 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14312 match(Set dst (XorL src1 src2));
14313
14314 format %{ "eor $dst, $src1, $src2\t# int" %}
14315
14316 ins_cost(INSN_COST);
14317 ins_encode %{
14318 __ eor(as_Register($dst$$reg),
14319 as_Register($src1$$reg),
14320 as_Register($src2$$reg));
14321 %}
14322
14323 ins_pipe(ialu_reg_reg);
14324 %}
14325
14326 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14327 match(Set dst (XorL src1 src2));
14328
14329 ins_cost(INSN_COST);
14330 format %{ "eor $dst, $src1, $src2\t# int" %}
14331
14332 ins_encode %{
14333 __ eor(as_Register($dst$$reg),
14334 as_Register($src1$$reg),
14335 (uint64_t)($src2$$constant));
14336 %}
14337
14338 ins_pipe(ialu_reg_imm);
14339 %}
14340
14341 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14342 %{
14343 match(Set dst (ConvI2L src));
14344
14345 ins_cost(INSN_COST);
14346 format %{ "sxtw $dst, $src\t# i2l" %}
14347 ins_encode %{
14348 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14349 %}
14350 ins_pipe(ialu_reg_shift);
14351 %}
14352
14353 // this pattern occurs in bigmath arithmetic
14354 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14355 %{
14356 match(Set dst (AndL (ConvI2L src) mask));
14357
14358 ins_cost(INSN_COST);
14359 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14360 ins_encode %{
14361 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14362 %}
14363
14364 ins_pipe(ialu_reg_shift);
14365 %}
14366
14367 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14368 match(Set dst (ConvL2I src));
14369
14370 ins_cost(INSN_COST);
14371 format %{ "movw $dst, $src \t// l2i" %}
14372
14373 ins_encode %{
14374 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14375 %}
14376
14377 ins_pipe(ialu_reg);
14378 %}
14379
14380 instruct convD2F_reg(vRegF dst, vRegD src) %{
14381 match(Set dst (ConvD2F src));
14382
14383 ins_cost(INSN_COST * 5);
14384 format %{ "fcvtd $dst, $src \t// d2f" %}
14385
14386 ins_encode %{
14387 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14388 %}
14389
14390 ins_pipe(fp_d2f);
14391 %}
14392
14393 instruct convF2D_reg(vRegD dst, vRegF src) %{
14394 match(Set dst (ConvF2D src));
14395
14396 ins_cost(INSN_COST * 5);
14397 format %{ "fcvts $dst, $src \t// f2d" %}
14398
14399 ins_encode %{
14400 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14401 %}
14402
14403 ins_pipe(fp_f2d);
14404 %}
14405
14406 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14407 match(Set dst (ConvF2I src));
14408
14409 ins_cost(INSN_COST * 5);
14410 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14411
14412 ins_encode %{
14413 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14414 %}
14415
14416 ins_pipe(fp_f2i);
14417 %}
14418
14419 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14420 match(Set dst (ConvF2L src));
14421
14422 ins_cost(INSN_COST * 5);
14423 format %{ "fcvtzs $dst, $src \t// f2l" %}
14424
14425 ins_encode %{
14426 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14427 %}
14428
14429 ins_pipe(fp_f2l);
14430 %}
14431
14432 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14433 match(Set dst (ConvF2HF src));
14434 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14435 "smov $dst, $tmp\t# move result from $tmp to $dst"
14436 %}
14437 effect(TEMP tmp);
14438 ins_encode %{
14439 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14440 %}
14441 ins_pipe(pipe_slow);
14442 %}
14443
14444 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14445 match(Set dst (ConvHF2F src));
14446 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14447 "fcvt $dst, $tmp\t# convert half to single precision"
14448 %}
14449 effect(TEMP tmp);
14450 ins_encode %{
14451 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14452 %}
14453 ins_pipe(pipe_slow);
14454 %}
14455
14456 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14457 match(Set dst (ConvI2F src));
14458
14459 ins_cost(INSN_COST * 5);
14460 format %{ "scvtfws $dst, $src \t// i2f" %}
14461
14462 ins_encode %{
14463 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14464 %}
14465
14466 ins_pipe(fp_i2f);
14467 %}
14468
14469 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14470 match(Set dst (ConvL2F src));
14471
14472 ins_cost(INSN_COST * 5);
14473 format %{ "scvtfs $dst, $src \t// l2f" %}
14474
14475 ins_encode %{
14476 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14477 %}
14478
14479 ins_pipe(fp_l2f);
14480 %}
14481
14482 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14483 match(Set dst (ConvD2I src));
14484
14485 ins_cost(INSN_COST * 5);
14486 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14487
14488 ins_encode %{
14489 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14490 %}
14491
14492 ins_pipe(fp_d2i);
14493 %}
14494
14495 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14496 match(Set dst (ConvD2L src));
14497
14498 ins_cost(INSN_COST * 5);
14499 format %{ "fcvtzd $dst, $src \t// d2l" %}
14500
14501 ins_encode %{
14502 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14503 %}
14504
14505 ins_pipe(fp_d2l);
14506 %}
14507
14508 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14509 match(Set dst (ConvI2D src));
14510
14511 ins_cost(INSN_COST * 5);
14512 format %{ "scvtfwd $dst, $src \t// i2d" %}
14513
14514 ins_encode %{
14515 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14516 %}
14517
14518 ins_pipe(fp_i2d);
14519 %}
14520
14521 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14522 match(Set dst (ConvL2D src));
14523
14524 ins_cost(INSN_COST * 5);
14525 format %{ "scvtfd $dst, $src \t// l2d" %}
14526
14527 ins_encode %{
14528 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14529 %}
14530
14531 ins_pipe(fp_l2d);
14532 %}
14533
14534 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14535 %{
14536 match(Set dst (RoundD src));
14537 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14538 format %{ "java_round_double $dst,$src"%}
14539 ins_encode %{
14540 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14541 as_FloatRegister($ftmp$$reg));
14542 %}
14543 ins_pipe(pipe_slow);
14544 %}
14545
14546 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14547 %{
14548 match(Set dst (RoundF src));
14549 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14550 format %{ "java_round_float $dst,$src"%}
14551 ins_encode %{
14552 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14553 as_FloatRegister($ftmp$$reg));
14554 %}
14555 ins_pipe(pipe_slow);
14556 %}
14557
14558 // stack <-> reg and reg <-> reg shuffles with no conversion
14559
14560 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14561
14562 match(Set dst (MoveF2I src));
14563
14564 effect(DEF dst, USE src);
14565
14566 ins_cost(4 * INSN_COST);
14567
14568 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14569
14570 ins_encode %{
14571 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14572 %}
14573
14574 ins_pipe(iload_reg_reg);
14575
14576 %}
14577
14578 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14579
14580 match(Set dst (MoveI2F src));
14581
14582 effect(DEF dst, USE src);
14583
14584 ins_cost(4 * INSN_COST);
14585
14586 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14587
14588 ins_encode %{
14589 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14590 %}
14591
14592 ins_pipe(pipe_class_memory);
14593
14594 %}
14595
14596 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14597
14598 match(Set dst (MoveD2L src));
14599
14600 effect(DEF dst, USE src);
14601
14602 ins_cost(4 * INSN_COST);
14603
14604 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14605
14606 ins_encode %{
14607 __ ldr($dst$$Register, Address(sp, $src$$disp));
14608 %}
14609
14610 ins_pipe(iload_reg_reg);
14611
14612 %}
14613
14614 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14615
14616 match(Set dst (MoveL2D src));
14617
14618 effect(DEF dst, USE src);
14619
14620 ins_cost(4 * INSN_COST);
14621
14622 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14623
14624 ins_encode %{
14625 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14626 %}
14627
14628 ins_pipe(pipe_class_memory);
14629
14630 %}
14631
14632 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14633
14634 match(Set dst (MoveF2I src));
14635
14636 effect(DEF dst, USE src);
14637
14638 ins_cost(INSN_COST);
14639
14640 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14641
14642 ins_encode %{
14643 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14644 %}
14645
14646 ins_pipe(pipe_class_memory);
14647
14648 %}
14649
14650 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14651
14652 match(Set dst (MoveI2F src));
14653
14654 effect(DEF dst, USE src);
14655
14656 ins_cost(INSN_COST);
14657
14658 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14659
14660 ins_encode %{
14661 __ strw($src$$Register, Address(sp, $dst$$disp));
14662 %}
14663
14664 ins_pipe(istore_reg_reg);
14665
14666 %}
14667
14668 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14669
14670 match(Set dst (MoveD2L src));
14671
14672 effect(DEF dst, USE src);
14673
14674 ins_cost(INSN_COST);
14675
14676 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14677
14678 ins_encode %{
14679 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14680 %}
14681
14682 ins_pipe(pipe_class_memory);
14683
14684 %}
14685
14686 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14687
14688 match(Set dst (MoveL2D src));
14689
14690 effect(DEF dst, USE src);
14691
14692 ins_cost(INSN_COST);
14693
14694 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14695
14696 ins_encode %{
14697 __ str($src$$Register, Address(sp, $dst$$disp));
14698 %}
14699
14700 ins_pipe(istore_reg_reg);
14701
14702 %}
14703
14704 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14705
14706 match(Set dst (MoveF2I src));
14707
14708 effect(DEF dst, USE src);
14709
14710 ins_cost(INSN_COST);
14711
14712 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14713
14714 ins_encode %{
14715 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14716 %}
14717
14718 ins_pipe(fp_f2i);
14719
14720 %}
14721
14722 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14723
14724 match(Set dst (MoveI2F src));
14725
14726 effect(DEF dst, USE src);
14727
14728 ins_cost(INSN_COST);
14729
14730 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14731
14732 ins_encode %{
14733 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14734 %}
14735
14736 ins_pipe(fp_i2f);
14737
14738 %}
14739
14740 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14741
14742 match(Set dst (MoveD2L src));
14743
14744 effect(DEF dst, USE src);
14745
14746 ins_cost(INSN_COST);
14747
14748 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14749
14750 ins_encode %{
14751 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14752 %}
14753
14754 ins_pipe(fp_d2l);
14755
14756 %}
14757
14758 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14759
14760 match(Set dst (MoveL2D src));
14761
14762 effect(DEF dst, USE src);
14763
14764 ins_cost(INSN_COST);
14765
14766 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14767
14768 ins_encode %{
14769 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14770 %}
14771
14772 ins_pipe(fp_l2d);
14773
14774 %}
14775
14776 // ============================================================================
14777 // clearing of an array
14778
14779 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14780 %{
14781 match(Set dummy (ClearArray cnt base));
14782 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14783
14784 ins_cost(4 * INSN_COST);
14785 format %{ "ClearArray $cnt, $base" %}
14786
14787 ins_encode %{
14788 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14789 if (tpc == nullptr) {
14790 ciEnv::current()->record_failure("CodeCache is full");
14791 return;
14792 }
14793 %}
14794
14795 ins_pipe(pipe_class_memory);
14796 %}
14797
14798 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14799 %{
14800 predicate((uint64_t)n->in(2)->get_long()
14801 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14802 match(Set dummy (ClearArray cnt base));
14803 effect(TEMP temp, USE_KILL base, KILL cr);
14804
14805 ins_cost(4 * INSN_COST);
14806 format %{ "ClearArray $cnt, $base" %}
14807
14808 ins_encode %{
14809 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14810 if (tpc == nullptr) {
14811 ciEnv::current()->record_failure("CodeCache is full");
14812 return;
14813 }
14814 %}
14815
14816 ins_pipe(pipe_class_memory);
14817 %}
14818
14819 // ============================================================================
14820 // Overflow Math Instructions
14821
14822 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14823 %{
14824 match(Set cr (OverflowAddI op1 op2));
14825
14826 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14827 ins_cost(INSN_COST);
14828 ins_encode %{
14829 __ cmnw($op1$$Register, $op2$$Register);
14830 %}
14831
14832 ins_pipe(icmp_reg_reg);
14833 %}
14834
14835 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14836 %{
14837 match(Set cr (OverflowAddI op1 op2));
14838
14839 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14840 ins_cost(INSN_COST);
14841 ins_encode %{
14842 __ cmnw($op1$$Register, $op2$$constant);
14843 %}
14844
14845 ins_pipe(icmp_reg_imm);
14846 %}
14847
14848 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14849 %{
14850 match(Set cr (OverflowAddL op1 op2));
14851
14852 format %{ "cmn $op1, $op2\t# overflow check long" %}
14853 ins_cost(INSN_COST);
14854 ins_encode %{
14855 __ cmn($op1$$Register, $op2$$Register);
14856 %}
14857
14858 ins_pipe(icmp_reg_reg);
14859 %}
14860
14861 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14862 %{
14863 match(Set cr (OverflowAddL op1 op2));
14864
14865 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14866 ins_cost(INSN_COST);
14867 ins_encode %{
14868 __ adds(zr, $op1$$Register, $op2$$constant);
14869 %}
14870
14871 ins_pipe(icmp_reg_imm);
14872 %}
14873
14874 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14875 %{
14876 match(Set cr (OverflowSubI op1 op2));
14877
14878 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14879 ins_cost(INSN_COST);
14880 ins_encode %{
14881 __ cmpw($op1$$Register, $op2$$Register);
14882 %}
14883
14884 ins_pipe(icmp_reg_reg);
14885 %}
14886
14887 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14888 %{
14889 match(Set cr (OverflowSubI op1 op2));
14890
14891 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14892 ins_cost(INSN_COST);
14893 ins_encode %{
14894 __ cmpw($op1$$Register, $op2$$constant);
14895 %}
14896
14897 ins_pipe(icmp_reg_imm);
14898 %}
14899
14900 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14901 %{
14902 match(Set cr (OverflowSubL op1 op2));
14903
14904 format %{ "cmp $op1, $op2\t# overflow check long" %}
14905 ins_cost(INSN_COST);
14906 ins_encode %{
14907 __ cmp($op1$$Register, $op2$$Register);
14908 %}
14909
14910 ins_pipe(icmp_reg_reg);
14911 %}
14912
14913 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14914 %{
14915 match(Set cr (OverflowSubL op1 op2));
14916
14917 format %{ "cmp $op1, $op2\t# overflow check long" %}
14918 ins_cost(INSN_COST);
14919 ins_encode %{
14920 __ subs(zr, $op1$$Register, $op2$$constant);
14921 %}
14922
14923 ins_pipe(icmp_reg_imm);
14924 %}
14925
14926 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14927 %{
14928 match(Set cr (OverflowSubI zero op1));
14929
14930 format %{ "cmpw zr, $op1\t# overflow check int" %}
14931 ins_cost(INSN_COST);
14932 ins_encode %{
14933 __ cmpw(zr, $op1$$Register);
14934 %}
14935
14936 ins_pipe(icmp_reg_imm);
14937 %}
14938
14939 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14940 %{
14941 match(Set cr (OverflowSubL zero op1));
14942
14943 format %{ "cmp zr, $op1\t# overflow check long" %}
14944 ins_cost(INSN_COST);
14945 ins_encode %{
14946 __ cmp(zr, $op1$$Register);
14947 %}
14948
14949 ins_pipe(icmp_reg_imm);
14950 %}
14951
14952 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14953 %{
14954 match(Set cr (OverflowMulI op1 op2));
14955
14956 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14957 "cmp rscratch1, rscratch1, sxtw\n\t"
14958 "movw rscratch1, #0x80000000\n\t"
14959 "cselw rscratch1, rscratch1, zr, NE\n\t"
14960 "cmpw rscratch1, #1" %}
14961 ins_cost(5 * INSN_COST);
14962 ins_encode %{
14963 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14964 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14965 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14966 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14967 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14968 %}
14969
14970 ins_pipe(pipe_slow);
14971 %}
14972
14973 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14974 %{
14975 match(If cmp (OverflowMulI op1 op2));
14976 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14977 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14978 effect(USE labl, KILL cr);
14979
14980 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14981 "cmp rscratch1, rscratch1, sxtw\n\t"
14982 "b$cmp $labl" %}
14983 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14984 ins_encode %{
14985 Label* L = $labl$$label;
14986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14987 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14988 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14989 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14990 %}
14991
14992 ins_pipe(pipe_serial);
14993 %}
14994
14995 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14996 %{
14997 match(Set cr (OverflowMulL op1 op2));
14998
14999 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15000 "smulh rscratch2, $op1, $op2\n\t"
15001 "cmp rscratch2, rscratch1, ASR #63\n\t"
15002 "movw rscratch1, #0x80000000\n\t"
15003 "cselw rscratch1, rscratch1, zr, NE\n\t"
15004 "cmpw rscratch1, #1" %}
15005 ins_cost(6 * INSN_COST);
15006 ins_encode %{
15007 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15008 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15009 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15010 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15011 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15012 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15013 %}
15014
15015 ins_pipe(pipe_slow);
15016 %}
15017
15018 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15019 %{
15020 match(If cmp (OverflowMulL op1 op2));
15021 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15022 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15023 effect(USE labl, KILL cr);
15024
15025 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15026 "smulh rscratch2, $op1, $op2\n\t"
15027 "cmp rscratch2, rscratch1, ASR #63\n\t"
15028 "b$cmp $labl" %}
15029 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15030 ins_encode %{
15031 Label* L = $labl$$label;
15032 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15033 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15034 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15035 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15036 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15037 %}
15038
15039 ins_pipe(pipe_serial);
15040 %}
15041
15042 // ============================================================================
15043 // Compare Instructions
15044
15045 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15046 %{
15047 match(Set cr (CmpI op1 op2));
15048
15049 effect(DEF cr, USE op1, USE op2);
15050
15051 ins_cost(INSN_COST);
15052 format %{ "cmpw $op1, $op2" %}
15053
15054 ins_encode(aarch64_enc_cmpw(op1, op2));
15055
15056 ins_pipe(icmp_reg_reg);
15057 %}
15058
15059 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15060 %{
15061 match(Set cr (CmpI op1 zero));
15062
15063 effect(DEF cr, USE op1);
15064
15065 ins_cost(INSN_COST);
15066 format %{ "cmpw $op1, 0" %}
15067
15068 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15069
15070 ins_pipe(icmp_reg_imm);
15071 %}
15072
15073 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15074 %{
15075 match(Set cr (CmpI op1 op2));
15076
15077 effect(DEF cr, USE op1);
15078
15079 ins_cost(INSN_COST);
15080 format %{ "cmpw $op1, $op2" %}
15081
15082 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15083
15084 ins_pipe(icmp_reg_imm);
15085 %}
15086
15087 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15088 %{
15089 match(Set cr (CmpI op1 op2));
15090
15091 effect(DEF cr, USE op1);
15092
15093 ins_cost(INSN_COST * 2);
15094 format %{ "cmpw $op1, $op2" %}
15095
15096 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15097
15098 ins_pipe(icmp_reg_imm);
15099 %}
15100
15101 // Unsigned compare Instructions; really, same as signed compare
15102 // except it should only be used to feed an If or a CMovI which takes a
15103 // cmpOpU.
15104
15105 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15106 %{
15107 match(Set cr (CmpU op1 op2));
15108
15109 effect(DEF cr, USE op1, USE op2);
15110
15111 ins_cost(INSN_COST);
15112 format %{ "cmpw $op1, $op2\t# unsigned" %}
15113
15114 ins_encode(aarch64_enc_cmpw(op1, op2));
15115
15116 ins_pipe(icmp_reg_reg);
15117 %}
15118
15119 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15120 %{
15121 match(Set cr (CmpU op1 zero));
15122
15123 effect(DEF cr, USE op1);
15124
15125 ins_cost(INSN_COST);
15126 format %{ "cmpw $op1, #0\t# unsigned" %}
15127
15128 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15129
15130 ins_pipe(icmp_reg_imm);
15131 %}
15132
15133 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15134 %{
15135 match(Set cr (CmpU op1 op2));
15136
15137 effect(DEF cr, USE op1);
15138
15139 ins_cost(INSN_COST);
15140 format %{ "cmpw $op1, $op2\t# unsigned" %}
15141
15142 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15143
15144 ins_pipe(icmp_reg_imm);
15145 %}
15146
15147 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15148 %{
15149 match(Set cr (CmpU op1 op2));
15150
15151 effect(DEF cr, USE op1);
15152
15153 ins_cost(INSN_COST * 2);
15154 format %{ "cmpw $op1, $op2\t# unsigned" %}
15155
15156 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15157
15158 ins_pipe(icmp_reg_imm);
15159 %}
15160
15161 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15162 %{
15163 match(Set cr (CmpL op1 op2));
15164
15165 effect(DEF cr, USE op1, USE op2);
15166
15167 ins_cost(INSN_COST);
15168 format %{ "cmp $op1, $op2" %}
15169
15170 ins_encode(aarch64_enc_cmp(op1, op2));
15171
15172 ins_pipe(icmp_reg_reg);
15173 %}
15174
15175 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15176 %{
15177 match(Set cr (CmpL op1 zero));
15178
15179 effect(DEF cr, USE op1);
15180
15181 ins_cost(INSN_COST);
15182 format %{ "tst $op1" %}
15183
15184 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15185
15186 ins_pipe(icmp_reg_imm);
15187 %}
15188
15189 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15190 %{
15191 match(Set cr (CmpL op1 op2));
15192
15193 effect(DEF cr, USE op1);
15194
15195 ins_cost(INSN_COST);
15196 format %{ "cmp $op1, $op2" %}
15197
15198 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15199
15200 ins_pipe(icmp_reg_imm);
15201 %}
15202
15203 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15204 %{
15205 match(Set cr (CmpL op1 op2));
15206
15207 effect(DEF cr, USE op1);
15208
15209 ins_cost(INSN_COST * 2);
15210 format %{ "cmp $op1, $op2" %}
15211
15212 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15213
15214 ins_pipe(icmp_reg_imm);
15215 %}
15216
15217 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15218 %{
15219 match(Set cr (CmpUL op1 op2));
15220
15221 effect(DEF cr, USE op1, USE op2);
15222
15223 ins_cost(INSN_COST);
15224 format %{ "cmp $op1, $op2" %}
15225
15226 ins_encode(aarch64_enc_cmp(op1, op2));
15227
15228 ins_pipe(icmp_reg_reg);
15229 %}
15230
15231 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15232 %{
15233 match(Set cr (CmpUL op1 zero));
15234
15235 effect(DEF cr, USE op1);
15236
15237 ins_cost(INSN_COST);
15238 format %{ "tst $op1" %}
15239
15240 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15241
15242 ins_pipe(icmp_reg_imm);
15243 %}
15244
15245 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15246 %{
15247 match(Set cr (CmpUL op1 op2));
15248
15249 effect(DEF cr, USE op1);
15250
15251 ins_cost(INSN_COST);
15252 format %{ "cmp $op1, $op2" %}
15253
15254 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15255
15256 ins_pipe(icmp_reg_imm);
15257 %}
15258
15259 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15260 %{
15261 match(Set cr (CmpUL op1 op2));
15262
15263 effect(DEF cr, USE op1);
15264
15265 ins_cost(INSN_COST * 2);
15266 format %{ "cmp $op1, $op2" %}
15267
15268 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15269
15270 ins_pipe(icmp_reg_imm);
15271 %}
15272
15273 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15274 %{
15275 match(Set cr (CmpP op1 op2));
15276
15277 effect(DEF cr, USE op1, USE op2);
15278
15279 ins_cost(INSN_COST);
15280 format %{ "cmp $op1, $op2\t // ptr" %}
15281
15282 ins_encode(aarch64_enc_cmpp(op1, op2));
15283
15284 ins_pipe(icmp_reg_reg);
15285 %}
15286
15287 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15288 %{
15289 match(Set cr (CmpN op1 op2));
15290
15291 effect(DEF cr, USE op1, USE op2);
15292
15293 ins_cost(INSN_COST);
15294 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15295
15296 ins_encode(aarch64_enc_cmpn(op1, op2));
15297
15298 ins_pipe(icmp_reg_reg);
15299 %}
15300
15301 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15302 %{
15303 match(Set cr (CmpP op1 zero));
15304
15305 effect(DEF cr, USE op1, USE zero);
15306
15307 ins_cost(INSN_COST);
15308 format %{ "cmp $op1, 0\t // ptr" %}
15309
15310 ins_encode(aarch64_enc_testp(op1));
15311
15312 ins_pipe(icmp_reg_imm);
15313 %}
15314
15315 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15316 %{
15317 match(Set cr (CmpN op1 zero));
15318
15319 effect(DEF cr, USE op1, USE zero);
15320
15321 ins_cost(INSN_COST);
15322 format %{ "cmp $op1, 0\t // compressed ptr" %}
15323
15324 ins_encode(aarch64_enc_testn(op1));
15325
15326 ins_pipe(icmp_reg_imm);
15327 %}
15328
15329 // FP comparisons
15330 //
15331 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15332 // using normal cmpOp. See declaration of rFlagsReg for details.
15333
15334 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15335 %{
15336 match(Set cr (CmpF src1 src2));
15337
15338 ins_cost(3 * INSN_COST);
15339 format %{ "fcmps $src1, $src2" %}
15340
15341 ins_encode %{
15342 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15343 %}
15344
15345 ins_pipe(pipe_class_compare);
15346 %}
15347
15348 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15349 %{
15350 match(Set cr (CmpF src1 src2));
15351
15352 ins_cost(3 * INSN_COST);
15353 format %{ "fcmps $src1, 0.0" %}
15354
15355 ins_encode %{
15356 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15357 %}
15358
15359 ins_pipe(pipe_class_compare);
15360 %}
15361 // FROM HERE
15362
15363 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15364 %{
15365 match(Set cr (CmpD src1 src2));
15366
15367 ins_cost(3 * INSN_COST);
15368 format %{ "fcmpd $src1, $src2" %}
15369
15370 ins_encode %{
15371 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15372 %}
15373
15374 ins_pipe(pipe_class_compare);
15375 %}
15376
15377 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15378 %{
15379 match(Set cr (CmpD src1 src2));
15380
15381 ins_cost(3 * INSN_COST);
15382 format %{ "fcmpd $src1, 0.0" %}
15383
15384 ins_encode %{
15385 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15386 %}
15387
15388 ins_pipe(pipe_class_compare);
15389 %}
15390
15391 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15392 %{
15393 match(Set dst (CmpF3 src1 src2));
15394 effect(KILL cr);
15395
15396 ins_cost(5 * INSN_COST);
15397 format %{ "fcmps $src1, $src2\n\t"
15398 "csinvw($dst, zr, zr, eq\n\t"
15399 "csnegw($dst, $dst, $dst, lt)"
15400 %}
15401
15402 ins_encode %{
15403 Label done;
15404 FloatRegister s1 = as_FloatRegister($src1$$reg);
15405 FloatRegister s2 = as_FloatRegister($src2$$reg);
15406 Register d = as_Register($dst$$reg);
15407 __ fcmps(s1, s2);
15408 // installs 0 if EQ else -1
15409 __ csinvw(d, zr, zr, Assembler::EQ);
15410 // keeps -1 if less or unordered else installs 1
15411 __ csnegw(d, d, d, Assembler::LT);
15412 __ bind(done);
15413 %}
15414
15415 ins_pipe(pipe_class_default);
15416
15417 %}
15418
15419 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15420 %{
15421 match(Set dst (CmpD3 src1 src2));
15422 effect(KILL cr);
15423
15424 ins_cost(5 * INSN_COST);
15425 format %{ "fcmpd $src1, $src2\n\t"
15426 "csinvw($dst, zr, zr, eq\n\t"
15427 "csnegw($dst, $dst, $dst, lt)"
15428 %}
15429
15430 ins_encode %{
15431 Label done;
15432 FloatRegister s1 = as_FloatRegister($src1$$reg);
15433 FloatRegister s2 = as_FloatRegister($src2$$reg);
15434 Register d = as_Register($dst$$reg);
15435 __ fcmpd(s1, s2);
15436 // installs 0 if EQ else -1
15437 __ csinvw(d, zr, zr, Assembler::EQ);
15438 // keeps -1 if less or unordered else installs 1
15439 __ csnegw(d, d, d, Assembler::LT);
15440 __ bind(done);
15441 %}
15442 ins_pipe(pipe_class_default);
15443
15444 %}
15445
15446 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15447 %{
15448 match(Set dst (CmpF3 src1 zero));
15449 effect(KILL cr);
15450
15451 ins_cost(5 * INSN_COST);
15452 format %{ "fcmps $src1, 0.0\n\t"
15453 "csinvw($dst, zr, zr, eq\n\t"
15454 "csnegw($dst, $dst, $dst, lt)"
15455 %}
15456
15457 ins_encode %{
15458 Label done;
15459 FloatRegister s1 = as_FloatRegister($src1$$reg);
15460 Register d = as_Register($dst$$reg);
15461 __ fcmps(s1, 0.0);
15462 // installs 0 if EQ else -1
15463 __ csinvw(d, zr, zr, Assembler::EQ);
15464 // keeps -1 if less or unordered else installs 1
15465 __ csnegw(d, d, d, Assembler::LT);
15466 __ bind(done);
15467 %}
15468
15469 ins_pipe(pipe_class_default);
15470
15471 %}
15472
15473 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15474 %{
15475 match(Set dst (CmpD3 src1 zero));
15476 effect(KILL cr);
15477
15478 ins_cost(5 * INSN_COST);
15479 format %{ "fcmpd $src1, 0.0\n\t"
15480 "csinvw($dst, zr, zr, eq\n\t"
15481 "csnegw($dst, $dst, $dst, lt)"
15482 %}
15483
15484 ins_encode %{
15485 Label done;
15486 FloatRegister s1 = as_FloatRegister($src1$$reg);
15487 Register d = as_Register($dst$$reg);
15488 __ fcmpd(s1, 0.0);
15489 // installs 0 if EQ else -1
15490 __ csinvw(d, zr, zr, Assembler::EQ);
15491 // keeps -1 if less or unordered else installs 1
15492 __ csnegw(d, d, d, Assembler::LT);
15493 __ bind(done);
15494 %}
15495 ins_pipe(pipe_class_default);
15496
15497 %}
15498
15499 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15500 %{
15501 match(Set dst (CmpLTMask p q));
15502 effect(KILL cr);
15503
15504 ins_cost(3 * INSN_COST);
15505
15506 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15507 "csetw $dst, lt\n\t"
15508 "subw $dst, zr, $dst"
15509 %}
15510
15511 ins_encode %{
15512 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15513 __ csetw(as_Register($dst$$reg), Assembler::LT);
15514 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15515 %}
15516
15517 ins_pipe(ialu_reg_reg);
15518 %}
15519
15520 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15521 %{
15522 match(Set dst (CmpLTMask src zero));
15523 effect(KILL cr);
15524
15525 ins_cost(INSN_COST);
15526
15527 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15528
15529 ins_encode %{
15530 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15531 %}
15532
15533 ins_pipe(ialu_reg_shift);
15534 %}
15535
15536 // ============================================================================
15537 // Max and Min
15538
15539 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15540
15541 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15542 %{
15543 effect(DEF cr, USE src);
15544 ins_cost(INSN_COST);
15545 format %{ "cmpw $src, 0" %}
15546
15547 ins_encode %{
15548 __ cmpw($src$$Register, 0);
15549 %}
15550 ins_pipe(icmp_reg_imm);
15551 %}
15552
15553 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15554 %{
15555 match(Set dst (MinI src1 src2));
15556 ins_cost(INSN_COST * 3);
15557
15558 expand %{
15559 rFlagsReg cr;
15560 compI_reg_reg(cr, src1, src2);
15561 cmovI_reg_reg_lt(dst, src1, src2, cr);
15562 %}
15563 %}
15564
15565 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15566 %{
15567 match(Set dst (MaxI src1 src2));
15568 ins_cost(INSN_COST * 3);
15569
15570 expand %{
15571 rFlagsReg cr;
15572 compI_reg_reg(cr, src1, src2);
15573 cmovI_reg_reg_gt(dst, src1, src2, cr);
15574 %}
15575 %}
15576
15577
15578 // ============================================================================
15579 // Branch Instructions
15580
15581 // Direct Branch.
15582 instruct branch(label lbl)
15583 %{
15584 match(Goto);
15585
15586 effect(USE lbl);
15587
15588 ins_cost(BRANCH_COST);
15589 format %{ "b $lbl" %}
15590
15591 ins_encode(aarch64_enc_b(lbl));
15592
15593 ins_pipe(pipe_branch);
15594 %}
15595
15596 // Conditional Near Branch
15597 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15598 %{
15599 // Same match rule as `branchConFar'.
15600 match(If cmp cr);
15601
15602 effect(USE lbl);
15603
15604 ins_cost(BRANCH_COST);
15605 // If set to 1 this indicates that the current instruction is a
15606 // short variant of a long branch. This avoids using this
15607 // instruction in first-pass matching. It will then only be used in
15608 // the `Shorten_branches' pass.
15609 // ins_short_branch(1);
15610 format %{ "b$cmp $lbl" %}
15611
15612 ins_encode(aarch64_enc_br_con(cmp, lbl));
15613
15614 ins_pipe(pipe_branch_cond);
15615 %}
15616
15617 // Conditional Near Branch Unsigned
15618 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15619 %{
15620 // Same match rule as `branchConFar'.
15621 match(If cmp cr);
15622
15623 effect(USE lbl);
15624
15625 ins_cost(BRANCH_COST);
15626 // If set to 1 this indicates that the current instruction is a
15627 // short variant of a long branch. This avoids using this
15628 // instruction in first-pass matching. It will then only be used in
15629 // the `Shorten_branches' pass.
15630 // ins_short_branch(1);
15631 format %{ "b$cmp $lbl\t# unsigned" %}
15632
15633 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15634
15635 ins_pipe(pipe_branch_cond);
15636 %}
15637
15638 // Make use of CBZ and CBNZ. These instructions, as well as being
15639 // shorter than (cmp; branch), have the additional benefit of not
15640 // killing the flags.
15641
15642 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15643 match(If cmp (CmpI op1 op2));
15644 effect(USE labl);
15645
15646 ins_cost(BRANCH_COST);
15647 format %{ "cbw$cmp $op1, $labl" %}
15648 ins_encode %{
15649 Label* L = $labl$$label;
15650 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15651 if (cond == Assembler::EQ)
15652 __ cbzw($op1$$Register, *L);
15653 else
15654 __ cbnzw($op1$$Register, *L);
15655 %}
15656 ins_pipe(pipe_cmp_branch);
15657 %}
15658
15659 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15660 match(If cmp (CmpL op1 op2));
15661 effect(USE labl);
15662
15663 ins_cost(BRANCH_COST);
15664 format %{ "cb$cmp $op1, $labl" %}
15665 ins_encode %{
15666 Label* L = $labl$$label;
15667 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15668 if (cond == Assembler::EQ)
15669 __ cbz($op1$$Register, *L);
15670 else
15671 __ cbnz($op1$$Register, *L);
15672 %}
15673 ins_pipe(pipe_cmp_branch);
15674 %}
15675
15676 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15677 match(If cmp (CmpP op1 op2));
15678 effect(USE labl);
15679
15680 ins_cost(BRANCH_COST);
15681 format %{ "cb$cmp $op1, $labl" %}
15682 ins_encode %{
15683 Label* L = $labl$$label;
15684 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15685 if (cond == Assembler::EQ)
15686 __ cbz($op1$$Register, *L);
15687 else
15688 __ cbnz($op1$$Register, *L);
15689 %}
15690 ins_pipe(pipe_cmp_branch);
15691 %}
15692
15693 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15694 match(If cmp (CmpN op1 op2));
15695 effect(USE labl);
15696
15697 ins_cost(BRANCH_COST);
15698 format %{ "cbw$cmp $op1, $labl" %}
15699 ins_encode %{
15700 Label* L = $labl$$label;
15701 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15702 if (cond == Assembler::EQ)
15703 __ cbzw($op1$$Register, *L);
15704 else
15705 __ cbnzw($op1$$Register, *L);
15706 %}
15707 ins_pipe(pipe_cmp_branch);
15708 %}
15709
15710 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15711 match(If cmp (CmpP (DecodeN oop) zero));
15712 effect(USE labl);
15713
15714 ins_cost(BRANCH_COST);
15715 format %{ "cb$cmp $oop, $labl" %}
15716 ins_encode %{
15717 Label* L = $labl$$label;
15718 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15719 if (cond == Assembler::EQ)
15720 __ cbzw($oop$$Register, *L);
15721 else
15722 __ cbnzw($oop$$Register, *L);
15723 %}
15724 ins_pipe(pipe_cmp_branch);
15725 %}
15726
15727 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15728 match(If cmp (CmpU op1 op2));
15729 effect(USE labl);
15730
15731 ins_cost(BRANCH_COST);
15732 format %{ "cbw$cmp $op1, $labl" %}
15733 ins_encode %{
15734 Label* L = $labl$$label;
15735 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15736 if (cond == Assembler::EQ || cond == Assembler::LS) {
15737 __ cbzw($op1$$Register, *L);
15738 } else {
15739 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15740 __ cbnzw($op1$$Register, *L);
15741 }
15742 %}
15743 ins_pipe(pipe_cmp_branch);
15744 %}
15745
15746 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15747 match(If cmp (CmpUL op1 op2));
15748 effect(USE labl);
15749
15750 ins_cost(BRANCH_COST);
15751 format %{ "cb$cmp $op1, $labl" %}
15752 ins_encode %{
15753 Label* L = $labl$$label;
15754 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15755 if (cond == Assembler::EQ || cond == Assembler::LS) {
15756 __ cbz($op1$$Register, *L);
15757 } else {
15758 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15759 __ cbnz($op1$$Register, *L);
15760 }
15761 %}
15762 ins_pipe(pipe_cmp_branch);
15763 %}
15764
15765 // Test bit and Branch
15766
15767 // Patterns for short (< 32KiB) variants
15768 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15769 match(If cmp (CmpL op1 op2));
15770 effect(USE labl);
15771
15772 ins_cost(BRANCH_COST);
15773 format %{ "cb$cmp $op1, $labl # long" %}
15774 ins_encode %{
15775 Label* L = $labl$$label;
15776 Assembler::Condition cond =
15777 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15778 __ tbr(cond, $op1$$Register, 63, *L);
15779 %}
15780 ins_pipe(pipe_cmp_branch);
15781 ins_short_branch(1);
15782 %}
15783
15784 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15785 match(If cmp (CmpI op1 op2));
15786 effect(USE labl);
15787
15788 ins_cost(BRANCH_COST);
15789 format %{ "cb$cmp $op1, $labl # int" %}
15790 ins_encode %{
15791 Label* L = $labl$$label;
15792 Assembler::Condition cond =
15793 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15794 __ tbr(cond, $op1$$Register, 31, *L);
15795 %}
15796 ins_pipe(pipe_cmp_branch);
15797 ins_short_branch(1);
15798 %}
15799
15800 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15801 match(If cmp (CmpL (AndL op1 op2) op3));
15802 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15803 effect(USE labl);
15804
15805 ins_cost(BRANCH_COST);
15806 format %{ "tb$cmp $op1, $op2, $labl" %}
15807 ins_encode %{
15808 Label* L = $labl$$label;
15809 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15810 int bit = exact_log2_long($op2$$constant);
15811 __ tbr(cond, $op1$$Register, bit, *L);
15812 %}
15813 ins_pipe(pipe_cmp_branch);
15814 ins_short_branch(1);
15815 %}
15816
15817 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15818 match(If cmp (CmpI (AndI op1 op2) op3));
15819 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15820 effect(USE labl);
15821
15822 ins_cost(BRANCH_COST);
15823 format %{ "tb$cmp $op1, $op2, $labl" %}
15824 ins_encode %{
15825 Label* L = $labl$$label;
15826 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15827 int bit = exact_log2((juint)$op2$$constant);
15828 __ tbr(cond, $op1$$Register, bit, *L);
15829 %}
15830 ins_pipe(pipe_cmp_branch);
15831 ins_short_branch(1);
15832 %}
15833
15834 // And far variants
15835 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15836 match(If cmp (CmpL op1 op2));
15837 effect(USE labl);
15838
15839 ins_cost(BRANCH_COST);
15840 format %{ "cb$cmp $op1, $labl # long" %}
15841 ins_encode %{
15842 Label* L = $labl$$label;
15843 Assembler::Condition cond =
15844 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15845 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15846 %}
15847 ins_pipe(pipe_cmp_branch);
15848 %}
15849
15850 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15851 match(If cmp (CmpI op1 op2));
15852 effect(USE labl);
15853
15854 ins_cost(BRANCH_COST);
15855 format %{ "cb$cmp $op1, $labl # int" %}
15856 ins_encode %{
15857 Label* L = $labl$$label;
15858 Assembler::Condition cond =
15859 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15860 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15861 %}
15862 ins_pipe(pipe_cmp_branch);
15863 %}
15864
15865 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15866 match(If cmp (CmpL (AndL op1 op2) op3));
15867 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15868 effect(USE labl);
15869
15870 ins_cost(BRANCH_COST);
15871 format %{ "tb$cmp $op1, $op2, $labl" %}
15872 ins_encode %{
15873 Label* L = $labl$$label;
15874 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15875 int bit = exact_log2_long($op2$$constant);
15876 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15877 %}
15878 ins_pipe(pipe_cmp_branch);
15879 %}
15880
15881 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15882 match(If cmp (CmpI (AndI op1 op2) op3));
15883 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15884 effect(USE labl);
15885
15886 ins_cost(BRANCH_COST);
15887 format %{ "tb$cmp $op1, $op2, $labl" %}
15888 ins_encode %{
15889 Label* L = $labl$$label;
15890 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15891 int bit = exact_log2((juint)$op2$$constant);
15892 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15893 %}
15894 ins_pipe(pipe_cmp_branch);
15895 %}
15896
15897 // Test bits
15898
15899 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15900 match(Set cr (CmpL (AndL op1 op2) op3));
15901 predicate(Assembler::operand_valid_for_logical_immediate
15902 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15903
15904 ins_cost(INSN_COST);
15905 format %{ "tst $op1, $op2 # long" %}
15906 ins_encode %{
15907 __ tst($op1$$Register, $op2$$constant);
15908 %}
15909 ins_pipe(ialu_reg_reg);
15910 %}
15911
15912 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15913 match(Set cr (CmpI (AndI op1 op2) op3));
15914 predicate(Assembler::operand_valid_for_logical_immediate
15915 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15916
15917 ins_cost(INSN_COST);
15918 format %{ "tst $op1, $op2 # int" %}
15919 ins_encode %{
15920 __ tstw($op1$$Register, $op2$$constant);
15921 %}
15922 ins_pipe(ialu_reg_reg);
15923 %}
15924
15925 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15926 match(Set cr (CmpL (AndL op1 op2) op3));
15927
15928 ins_cost(INSN_COST);
15929 format %{ "tst $op1, $op2 # long" %}
15930 ins_encode %{
15931 __ tst($op1$$Register, $op2$$Register);
15932 %}
15933 ins_pipe(ialu_reg_reg);
15934 %}
15935
15936 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15937 match(Set cr (CmpI (AndI op1 op2) op3));
15938
15939 ins_cost(INSN_COST);
15940 format %{ "tstw $op1, $op2 # int" %}
15941 ins_encode %{
15942 __ tstw($op1$$Register, $op2$$Register);
15943 %}
15944 ins_pipe(ialu_reg_reg);
15945 %}
15946
15947
15948 // Conditional Far Branch
15949 // Conditional Far Branch Unsigned
15950 // TODO: fixme
15951
15952 // counted loop end branch near
15953 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15954 %{
15955 match(CountedLoopEnd cmp cr);
15956
15957 effect(USE lbl);
15958
15959 ins_cost(BRANCH_COST);
15960 // short variant.
15961 // ins_short_branch(1);
15962 format %{ "b$cmp $lbl \t// counted loop end" %}
15963
15964 ins_encode(aarch64_enc_br_con(cmp, lbl));
15965
15966 ins_pipe(pipe_branch);
15967 %}
15968
15969 // counted loop end branch far
15970 // TODO: fixme
15971
15972 // ============================================================================
15973 // inlined locking and unlocking
15974
15975 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15976 %{
15977 predicate(LockingMode != LM_LIGHTWEIGHT);
15978 match(Set cr (FastLock object box));
15979 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15980
15981 ins_cost(5 * INSN_COST);
15982 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15983
15984 ins_encode %{
15985 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15986 %}
15987
15988 ins_pipe(pipe_serial);
15989 %}
15990
15991 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15992 %{
15993 predicate(LockingMode != LM_LIGHTWEIGHT);
15994 match(Set cr (FastUnlock object box));
15995 effect(TEMP tmp, TEMP tmp2);
15996
15997 ins_cost(5 * INSN_COST);
15998 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15999
16000 ins_encode %{
16001 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16002 %}
16003
16004 ins_pipe(pipe_serial);
16005 %}
16006
16007 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16008 %{
16009 predicate(LockingMode == LM_LIGHTWEIGHT);
16010 match(Set cr (FastLock object box));
16011 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16012
16013 ins_cost(5 * INSN_COST);
16014 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16015
16016 ins_encode %{
16017 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16018 %}
16019
16020 ins_pipe(pipe_serial);
16021 %}
16022
16023 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16024 %{
16025 predicate(LockingMode == LM_LIGHTWEIGHT);
16026 match(Set cr (FastUnlock object box));
16027 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16028
16029 ins_cost(5 * INSN_COST);
16030 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16031
16032 ins_encode %{
16033 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16034 %}
16035
16036 ins_pipe(pipe_serial);
16037 %}
16038
16039 // ============================================================================
16040 // Safepoint Instructions
16041
16042 // TODO
16043 // provide a near and far version of this code
16044
16045 instruct safePoint(rFlagsReg cr, iRegP poll)
16046 %{
16047 match(SafePoint poll);
16048 effect(KILL cr);
16049
16050 format %{
16051 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16052 %}
16053 ins_encode %{
16054 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16055 %}
16056 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16057 %}
16058
16059
16060 // ============================================================================
16061 // Procedure Call/Return Instructions
16062
16063 // Call Java Static Instruction
16064
16065 instruct CallStaticJavaDirect(method meth)
16066 %{
16067 match(CallStaticJava);
16068
16069 effect(USE meth);
16070
16071 ins_cost(CALL_COST);
16072
16073 format %{ "call,static $meth \t// ==> " %}
16074
16075 ins_encode(aarch64_enc_java_static_call(meth),
16076 aarch64_enc_call_epilog);
16077
16078 ins_pipe(pipe_class_call);
16079 %}
16080
16081 // TO HERE
16082
16083 // Call Java Dynamic Instruction
16084 instruct CallDynamicJavaDirect(method meth)
16085 %{
16086 match(CallDynamicJava);
16087
16088 effect(USE meth);
16089
16090 ins_cost(CALL_COST);
16091
16092 format %{ "CALL,dynamic $meth \t// ==> " %}
16093
16094 ins_encode(aarch64_enc_java_dynamic_call(meth),
16095 aarch64_enc_call_epilog);
16096
16097 ins_pipe(pipe_class_call);
16098 %}
16099
16100 // Call Runtime Instruction
16101
16102 instruct CallRuntimeDirect(method meth)
16103 %{
16104 match(CallRuntime);
16105
16106 effect(USE meth);
16107
16108 ins_cost(CALL_COST);
16109
16110 format %{ "CALL, runtime $meth" %}
16111
16112 ins_encode( aarch64_enc_java_to_runtime(meth) );
16113
16114 ins_pipe(pipe_class_call);
16115 %}
16116
16117 // Call Runtime Instruction
16118
16119 instruct CallLeafDirect(method meth)
16120 %{
16121 match(CallLeaf);
16122
16123 effect(USE meth);
16124
16125 ins_cost(CALL_COST);
16126
16127 format %{ "CALL, runtime leaf $meth" %}
16128
16129 ins_encode( aarch64_enc_java_to_runtime(meth) );
16130
16131 ins_pipe(pipe_class_call);
16132 %}
16133
16134 // Call Runtime Instruction without safepoint and with vector arguments
16135 instruct CallLeafDirectVector(method meth)
16136 %{
16137 match(CallLeafVector);
16138
16139 effect(USE meth);
16140
16141 ins_cost(CALL_COST);
16142
16143 format %{ "CALL, runtime leaf vector $meth" %}
16144
16145 ins_encode(aarch64_enc_java_to_runtime(meth));
16146
16147 ins_pipe(pipe_class_call);
16148 %}
16149
16150 // Call Runtime Instruction
16151
16152 instruct CallLeafNoFPDirect(method meth)
16153 %{
16154 match(CallLeafNoFP);
16155
16156 effect(USE meth);
16157
16158 ins_cost(CALL_COST);
16159
16160 format %{ "CALL, runtime leaf nofp $meth" %}
16161
16162 ins_encode( aarch64_enc_java_to_runtime(meth) );
16163
16164 ins_pipe(pipe_class_call);
16165 %}
16166
16167 // Tail Call; Jump from runtime stub to Java code.
16168 // Also known as an 'interprocedural jump'.
16169 // Target of jump will eventually return to caller.
16170 // TailJump below removes the return address.
16171 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16172 // emitted just above the TailCall which has reset rfp to the caller state.
16173 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16174 %{
16175 match(TailCall jump_target method_ptr);
16176
16177 ins_cost(CALL_COST);
16178
16179 format %{ "br $jump_target\t# $method_ptr holds method" %}
16180
16181 ins_encode(aarch64_enc_tail_call(jump_target));
16182
16183 ins_pipe(pipe_class_call);
16184 %}
16185
16186 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16187 %{
16188 match(TailJump jump_target ex_oop);
16189
16190 ins_cost(CALL_COST);
16191
16192 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16193
16194 ins_encode(aarch64_enc_tail_jmp(jump_target));
16195
16196 ins_pipe(pipe_class_call);
16197 %}
16198
16199 // Forward exception.
16200 instruct ForwardExceptionjmp()
16201 %{
16202 match(ForwardException);
16203 ins_cost(CALL_COST);
16204
16205 format %{ "b forward_exception_stub" %}
16206 ins_encode %{
16207 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16208 %}
16209 ins_pipe(pipe_class_call);
16210 %}
16211
16212 // Create exception oop: created by stack-crawling runtime code.
16213 // Created exception is now available to this handler, and is setup
16214 // just prior to jumping to this handler. No code emitted.
16215 // TODO check
16216 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16217 instruct CreateException(iRegP_R0 ex_oop)
16218 %{
16219 match(Set ex_oop (CreateEx));
16220
16221 format %{ " -- \t// exception oop; no code emitted" %}
16222
16223 size(0);
16224
16225 ins_encode( /*empty*/ );
16226
16227 ins_pipe(pipe_class_empty);
16228 %}
16229
16230 // Rethrow exception: The exception oop will come in the first
16231 // argument position. Then JUMP (not call) to the rethrow stub code.
16232 instruct RethrowException() %{
16233 match(Rethrow);
16234 ins_cost(CALL_COST);
16235
16236 format %{ "b rethrow_stub" %}
16237
16238 ins_encode( aarch64_enc_rethrow() );
16239
16240 ins_pipe(pipe_class_call);
16241 %}
16242
16243
16244 // Return Instruction
16245 // epilog node loads ret address into lr as part of frame pop
16246 instruct Ret()
16247 %{
16248 match(Return);
16249
16250 format %{ "ret\t// return register" %}
16251
16252 ins_encode( aarch64_enc_ret() );
16253
16254 ins_pipe(pipe_branch);
16255 %}
16256
16257 // Die now.
16258 instruct ShouldNotReachHere() %{
16259 match(Halt);
16260
16261 ins_cost(CALL_COST);
16262 format %{ "ShouldNotReachHere" %}
16263
16264 ins_encode %{
16265 if (is_reachable()) {
16266 __ stop(_halt_reason);
16267 }
16268 %}
16269
16270 ins_pipe(pipe_class_default);
16271 %}
16272
16273 // ============================================================================
16274 // Partial Subtype Check
16275 //
16276 // superklass array for an instance of the superklass. Set a hidden
16277 // internal cache on a hit (cache is checked with exposed code in
16278 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16279 // encoding ALSO sets flags.
16280
16281 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16282 %{
16283 match(Set result (PartialSubtypeCheck sub super));
16284 predicate(!UseSecondarySupersTable);
16285 effect(KILL cr, KILL temp);
16286
16287 ins_cost(20 * INSN_COST); // slightly larger than the next version
16288 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16289
16290 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16291
16292 opcode(0x1); // Force zero of result reg on hit
16293
16294 ins_pipe(pipe_class_memory);
16295 %}
16296
16297 // Two versions of partialSubtypeCheck, both used when we need to
16298 // search for a super class in the secondary supers array. The first
16299 // is used when we don't know _a priori_ the class being searched
16300 // for. The second, far more common, is used when we do know: this is
16301 // used for instanceof, checkcast, and any case where C2 can determine
16302 // it by constant propagation.
16303
16304 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16305 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16306 rFlagsReg cr)
16307 %{
16308 match(Set result (PartialSubtypeCheck sub super));
16309 predicate(UseSecondarySupersTable);
16310 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16311
16312 ins_cost(10 * INSN_COST); // slightly larger than the next version
16313 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16314
16315 ins_encode %{
16316 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16317 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16318 $vtemp$$FloatRegister,
16319 $result$$Register, /*L_success*/nullptr);
16320 %}
16321
16322 ins_pipe(pipe_class_memory);
16323 %}
16324
16325 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16326 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16327 rFlagsReg cr)
16328 %{
16329 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16330 predicate(UseSecondarySupersTable);
16331 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16332
16333 ins_cost(5 * INSN_COST); // smaller than the next version
16334 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16335
16336 ins_encode %{
16337 bool success = false;
16338 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16339 if (InlineSecondarySupersTest) {
16340 success =
16341 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16342 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16343 $vtemp$$FloatRegister,
16344 $result$$Register,
16345 super_klass_slot);
16346 } else {
16347 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16348 success = (call != nullptr);
16349 }
16350 if (!success) {
16351 ciEnv::current()->record_failure("CodeCache is full");
16352 return;
16353 }
16354 %}
16355
16356 ins_pipe(pipe_class_memory);
16357 %}
16358
16359 // Intrisics for String.compareTo()
16360
16361 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16362 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16363 %{
16364 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16365 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16366 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16367
16368 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16369 ins_encode %{
16370 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16371 __ string_compare($str1$$Register, $str2$$Register,
16372 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16373 $tmp1$$Register, $tmp2$$Register,
16374 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16375 %}
16376 ins_pipe(pipe_class_memory);
16377 %}
16378
16379 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16380 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16381 %{
16382 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16383 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16384 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16385
16386 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16387 ins_encode %{
16388 __ string_compare($str1$$Register, $str2$$Register,
16389 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16390 $tmp1$$Register, $tmp2$$Register,
16391 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16392 %}
16393 ins_pipe(pipe_class_memory);
16394 %}
16395
16396 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16397 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16398 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16399 %{
16400 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16401 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16402 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16403 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16404
16405 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16406 ins_encode %{
16407 __ string_compare($str1$$Register, $str2$$Register,
16408 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16409 $tmp1$$Register, $tmp2$$Register,
16410 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16411 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16412 %}
16413 ins_pipe(pipe_class_memory);
16414 %}
16415
16416 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16417 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16418 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16419 %{
16420 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16421 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16422 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16423 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16424
16425 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16426 ins_encode %{
16427 __ string_compare($str1$$Register, $str2$$Register,
16428 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16429 $tmp1$$Register, $tmp2$$Register,
16430 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16431 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16432 %}
16433 ins_pipe(pipe_class_memory);
16434 %}
16435
16436 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16437 // these string_compare variants as NEON register type for convenience so that the prototype of
16438 // string_compare can be shared with all variants.
16439
16440 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16441 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16442 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16443 pRegGov_P1 pgtmp2, rFlagsReg cr)
16444 %{
16445 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16446 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16447 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16448 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16449
16450 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16451 ins_encode %{
16452 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16453 __ string_compare($str1$$Register, $str2$$Register,
16454 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16455 $tmp1$$Register, $tmp2$$Register,
16456 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16457 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16458 StrIntrinsicNode::LL);
16459 %}
16460 ins_pipe(pipe_class_memory);
16461 %}
16462
16463 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16464 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16465 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16466 pRegGov_P1 pgtmp2, rFlagsReg cr)
16467 %{
16468 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16469 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16470 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16471 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16472
16473 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16474 ins_encode %{
16475 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16476 __ string_compare($str1$$Register, $str2$$Register,
16477 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16478 $tmp1$$Register, $tmp2$$Register,
16479 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16480 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16481 StrIntrinsicNode::LU);
16482 %}
16483 ins_pipe(pipe_class_memory);
16484 %}
16485
16486 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16487 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16488 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16489 pRegGov_P1 pgtmp2, rFlagsReg cr)
16490 %{
16491 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16492 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16493 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16494 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16495
16496 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16497 ins_encode %{
16498 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16499 __ string_compare($str1$$Register, $str2$$Register,
16500 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16501 $tmp1$$Register, $tmp2$$Register,
16502 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16503 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16504 StrIntrinsicNode::UL);
16505 %}
16506 ins_pipe(pipe_class_memory);
16507 %}
16508
16509 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16510 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16511 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16512 pRegGov_P1 pgtmp2, rFlagsReg cr)
16513 %{
16514 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16515 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16516 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16517 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16518
16519 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16520 ins_encode %{
16521 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16522 __ string_compare($str1$$Register, $str2$$Register,
16523 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16524 $tmp1$$Register, $tmp2$$Register,
16525 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16526 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16527 StrIntrinsicNode::UU);
16528 %}
16529 ins_pipe(pipe_class_memory);
16530 %}
16531
16532 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16533 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16534 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16535 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16536 %{
16537 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16538 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16539 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16540 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16541 TEMP vtmp0, TEMP vtmp1, KILL cr);
16542 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16543 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16544
16545 ins_encode %{
16546 __ string_indexof($str1$$Register, $str2$$Register,
16547 $cnt1$$Register, $cnt2$$Register,
16548 $tmp1$$Register, $tmp2$$Register,
16549 $tmp3$$Register, $tmp4$$Register,
16550 $tmp5$$Register, $tmp6$$Register,
16551 -1, $result$$Register, StrIntrinsicNode::UU);
16552 %}
16553 ins_pipe(pipe_class_memory);
16554 %}
16555
16556 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16557 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16558 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16559 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16560 %{
16561 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16562 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16563 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16564 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16565 TEMP vtmp0, TEMP vtmp1, KILL cr);
16566 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16567 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16568
16569 ins_encode %{
16570 __ string_indexof($str1$$Register, $str2$$Register,
16571 $cnt1$$Register, $cnt2$$Register,
16572 $tmp1$$Register, $tmp2$$Register,
16573 $tmp3$$Register, $tmp4$$Register,
16574 $tmp5$$Register, $tmp6$$Register,
16575 -1, $result$$Register, StrIntrinsicNode::LL);
16576 %}
16577 ins_pipe(pipe_class_memory);
16578 %}
16579
16580 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16581 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16582 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16583 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16584 %{
16585 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16586 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16587 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16588 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16589 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16590 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16591 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16592
16593 ins_encode %{
16594 __ string_indexof($str1$$Register, $str2$$Register,
16595 $cnt1$$Register, $cnt2$$Register,
16596 $tmp1$$Register, $tmp2$$Register,
16597 $tmp3$$Register, $tmp4$$Register,
16598 $tmp5$$Register, $tmp6$$Register,
16599 -1, $result$$Register, StrIntrinsicNode::UL);
16600 %}
16601 ins_pipe(pipe_class_memory);
16602 %}
16603
16604 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16605 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16606 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16607 %{
16608 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16609 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16610 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16611 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16612 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16613 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16614
16615 ins_encode %{
16616 int icnt2 = (int)$int_cnt2$$constant;
16617 __ string_indexof($str1$$Register, $str2$$Register,
16618 $cnt1$$Register, zr,
16619 $tmp1$$Register, $tmp2$$Register,
16620 $tmp3$$Register, $tmp4$$Register, zr, zr,
16621 icnt2, $result$$Register, StrIntrinsicNode::UU);
16622 %}
16623 ins_pipe(pipe_class_memory);
16624 %}
16625
16626 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16627 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16628 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16629 %{
16630 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16631 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16632 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16633 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16634 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16635 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16636
16637 ins_encode %{
16638 int icnt2 = (int)$int_cnt2$$constant;
16639 __ string_indexof($str1$$Register, $str2$$Register,
16640 $cnt1$$Register, zr,
16641 $tmp1$$Register, $tmp2$$Register,
16642 $tmp3$$Register, $tmp4$$Register, zr, zr,
16643 icnt2, $result$$Register, StrIntrinsicNode::LL);
16644 %}
16645 ins_pipe(pipe_class_memory);
16646 %}
16647
16648 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16649 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16650 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16651 %{
16652 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16653 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16654 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16655 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16656 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16657 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16658
16659 ins_encode %{
16660 int icnt2 = (int)$int_cnt2$$constant;
16661 __ string_indexof($str1$$Register, $str2$$Register,
16662 $cnt1$$Register, zr,
16663 $tmp1$$Register, $tmp2$$Register,
16664 $tmp3$$Register, $tmp4$$Register, zr, zr,
16665 icnt2, $result$$Register, StrIntrinsicNode::UL);
16666 %}
16667 ins_pipe(pipe_class_memory);
16668 %}
16669
16670 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16671 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16672 iRegINoSp tmp3, rFlagsReg cr)
16673 %{
16674 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16675 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16676 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16677 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16678
16679 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16680
16681 ins_encode %{
16682 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16683 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16684 $tmp3$$Register);
16685 %}
16686 ins_pipe(pipe_class_memory);
16687 %}
16688
16689 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16690 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16691 iRegINoSp tmp3, rFlagsReg cr)
16692 %{
16693 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16694 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16695 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16696 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16697
16698 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16699
16700 ins_encode %{
16701 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16702 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16703 $tmp3$$Register);
16704 %}
16705 ins_pipe(pipe_class_memory);
16706 %}
16707
16708 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16709 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16710 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16711 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16712 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16713 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16714 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16715 ins_encode %{
16716 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16717 $result$$Register, $ztmp1$$FloatRegister,
16718 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16719 $ptmp$$PRegister, true /* isL */);
16720 %}
16721 ins_pipe(pipe_class_memory);
16722 %}
16723
16724 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16725 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16726 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16727 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16728 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16729 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16730 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16731 ins_encode %{
16732 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16733 $result$$Register, $ztmp1$$FloatRegister,
16734 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16735 $ptmp$$PRegister, false /* isL */);
16736 %}
16737 ins_pipe(pipe_class_memory);
16738 %}
16739
16740 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16741 iRegI_R0 result, rFlagsReg cr)
16742 %{
16743 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16744 match(Set result (StrEquals (Binary str1 str2) cnt));
16745 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16746
16747 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16748 ins_encode %{
16749 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16750 __ string_equals($str1$$Register, $str2$$Register,
16751 $result$$Register, $cnt$$Register);
16752 %}
16753 ins_pipe(pipe_class_memory);
16754 %}
16755
16756 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16757 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16758 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16759 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16760 iRegP_R10 tmp, rFlagsReg cr)
16761 %{
16762 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16763 match(Set result (AryEq ary1 ary2));
16764 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16765 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16766 TEMP vtmp6, TEMP vtmp7, KILL cr);
16767
16768 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16769 ins_encode %{
16770 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16771 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16772 $result$$Register, $tmp$$Register, 1);
16773 if (tpc == nullptr) {
16774 ciEnv::current()->record_failure("CodeCache is full");
16775 return;
16776 }
16777 %}
16778 ins_pipe(pipe_class_memory);
16779 %}
16780
16781 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16782 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16783 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16784 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16785 iRegP_R10 tmp, rFlagsReg cr)
16786 %{
16787 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16788 match(Set result (AryEq ary1 ary2));
16789 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16790 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16791 TEMP vtmp6, TEMP vtmp7, KILL cr);
16792
16793 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16794 ins_encode %{
16795 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16796 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16797 $result$$Register, $tmp$$Register, 2);
16798 if (tpc == nullptr) {
16799 ciEnv::current()->record_failure("CodeCache is full");
16800 return;
16801 }
16802 %}
16803 ins_pipe(pipe_class_memory);
16804 %}
16805
16806 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16807 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16808 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16809 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16810 %{
16811 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16812 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16813 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16814
16815 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16816 ins_encode %{
16817 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16818 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16819 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16820 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16821 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16822 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16823 (BasicType)$basic_type$$constant);
16824 if (tpc == nullptr) {
16825 ciEnv::current()->record_failure("CodeCache is full");
16826 return;
16827 }
16828 %}
16829 ins_pipe(pipe_class_memory);
16830 %}
16831
16832 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16833 %{
16834 match(Set result (CountPositives ary1 len));
16835 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16836 format %{ "count positives byte[] $ary1,$len -> $result" %}
16837 ins_encode %{
16838 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16839 if (tpc == nullptr) {
16840 ciEnv::current()->record_failure("CodeCache is full");
16841 return;
16842 }
16843 %}
16844 ins_pipe( pipe_slow );
16845 %}
16846
16847 // fast char[] to byte[] compression
16848 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16849 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16850 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16851 iRegI_R0 result, rFlagsReg cr)
16852 %{
16853 match(Set result (StrCompressedCopy src (Binary dst len)));
16854 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16855 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16856
16857 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16858 ins_encode %{
16859 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16860 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16861 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16862 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16863 %}
16864 ins_pipe(pipe_slow);
16865 %}
16866
16867 // fast byte[] to char[] inflation
16868 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16869 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16870 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16871 %{
16872 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16873 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16874 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16875 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16876
16877 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16878 ins_encode %{
16879 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16880 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16881 $vtmp2$$FloatRegister, $tmp$$Register);
16882 if (tpc == nullptr) {
16883 ciEnv::current()->record_failure("CodeCache is full");
16884 return;
16885 }
16886 %}
16887 ins_pipe(pipe_class_memory);
16888 %}
16889
16890 // encode char[] to byte[] in ISO_8859_1
16891 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16892 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16893 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16894 iRegI_R0 result, rFlagsReg cr)
16895 %{
16896 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16897 match(Set result (EncodeISOArray src (Binary dst len)));
16898 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16899 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16900
16901 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16902 ins_encode %{
16903 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16904 $result$$Register, false,
16905 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16906 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16907 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16908 %}
16909 ins_pipe(pipe_class_memory);
16910 %}
16911
16912 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16913 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16914 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16915 iRegI_R0 result, rFlagsReg cr)
16916 %{
16917 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16918 match(Set result (EncodeISOArray src (Binary dst len)));
16919 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16920 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16921
16922 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16923 ins_encode %{
16924 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16925 $result$$Register, true,
16926 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16927 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16928 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16929 %}
16930 ins_pipe(pipe_class_memory);
16931 %}
16932
16933 //----------------------------- CompressBits/ExpandBits ------------------------
16934
16935 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16936 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16937 match(Set dst (CompressBits src mask));
16938 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16939 format %{ "mov $tsrc, $src\n\t"
16940 "mov $tmask, $mask\n\t"
16941 "bext $tdst, $tsrc, $tmask\n\t"
16942 "mov $dst, $tdst"
16943 %}
16944 ins_encode %{
16945 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16946 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16947 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16948 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16949 %}
16950 ins_pipe(pipe_slow);
16951 %}
16952
16953 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16954 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16955 match(Set dst (CompressBits (LoadI mem) mask));
16956 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16957 format %{ "ldrs $tsrc, $mem\n\t"
16958 "ldrs $tmask, $mask\n\t"
16959 "bext $tdst, $tsrc, $tmask\n\t"
16960 "mov $dst, $tdst"
16961 %}
16962 ins_encode %{
16963 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16965 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16966 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16967 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16968 %}
16969 ins_pipe(pipe_slow);
16970 %}
16971
16972 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16973 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16974 match(Set dst (CompressBits src mask));
16975 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16976 format %{ "mov $tsrc, $src\n\t"
16977 "mov $tmask, $mask\n\t"
16978 "bext $tdst, $tsrc, $tmask\n\t"
16979 "mov $dst, $tdst"
16980 %}
16981 ins_encode %{
16982 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16983 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16984 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16985 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16986 %}
16987 ins_pipe(pipe_slow);
16988 %}
16989
16990 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16991 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16992 match(Set dst (CompressBits (LoadL mem) mask));
16993 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16994 format %{ "ldrd $tsrc, $mem\n\t"
16995 "ldrd $tmask, $mask\n\t"
16996 "bext $tdst, $tsrc, $tmask\n\t"
16997 "mov $dst, $tdst"
16998 %}
16999 ins_encode %{
17000 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17002 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17003 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17004 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17005 %}
17006 ins_pipe(pipe_slow);
17007 %}
17008
17009 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17010 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17011 match(Set dst (ExpandBits src mask));
17012 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17013 format %{ "mov $tsrc, $src\n\t"
17014 "mov $tmask, $mask\n\t"
17015 "bdep $tdst, $tsrc, $tmask\n\t"
17016 "mov $dst, $tdst"
17017 %}
17018 ins_encode %{
17019 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17020 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17021 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17022 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17023 %}
17024 ins_pipe(pipe_slow);
17025 %}
17026
17027 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17028 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17029 match(Set dst (ExpandBits (LoadI mem) mask));
17030 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17031 format %{ "ldrs $tsrc, $mem\n\t"
17032 "ldrs $tmask, $mask\n\t"
17033 "bdep $tdst, $tsrc, $tmask\n\t"
17034 "mov $dst, $tdst"
17035 %}
17036 ins_encode %{
17037 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17039 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17040 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17041 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17042 %}
17043 ins_pipe(pipe_slow);
17044 %}
17045
17046 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17047 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17048 match(Set dst (ExpandBits src mask));
17049 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17050 format %{ "mov $tsrc, $src\n\t"
17051 "mov $tmask, $mask\n\t"
17052 "bdep $tdst, $tsrc, $tmask\n\t"
17053 "mov $dst, $tdst"
17054 %}
17055 ins_encode %{
17056 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17057 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17058 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17059 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17060 %}
17061 ins_pipe(pipe_slow);
17062 %}
17063
17064
17065 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17066 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17067 match(Set dst (ExpandBits (LoadL mem) mask));
17068 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17069 format %{ "ldrd $tsrc, $mem\n\t"
17070 "ldrd $tmask, $mask\n\t"
17071 "bdep $tdst, $tsrc, $tmask\n\t"
17072 "mov $dst, $tdst"
17073 %}
17074 ins_encode %{
17075 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17076 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17077 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17078 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17079 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17080 %}
17081 ins_pipe(pipe_slow);
17082 %}
17083
17084 // ============================================================================
17085 // This name is KNOWN by the ADLC and cannot be changed.
17086 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17087 // for this guy.
17088 instruct tlsLoadP(thread_RegP dst)
17089 %{
17090 match(Set dst (ThreadLocal));
17091
17092 ins_cost(0);
17093
17094 format %{ " -- \t// $dst=Thread::current(), empty" %}
17095
17096 size(0);
17097
17098 ins_encode( /*empty*/ );
17099
17100 ins_pipe(pipe_class_empty);
17101 %}
17102
17103 //----------PEEPHOLE RULES-----------------------------------------------------
17104 // These must follow all instruction definitions as they use the names
17105 // defined in the instructions definitions.
17106 //
17107 // peepmatch ( root_instr_name [preceding_instruction]* );
17108 //
17109 // peepconstraint %{
17110 // (instruction_number.operand_name relational_op instruction_number.operand_name
17111 // [, ...] );
17112 // // instruction numbers are zero-based using left to right order in peepmatch
17113 //
17114 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17115 // // provide an instruction_number.operand_name for each operand that appears
17116 // // in the replacement instruction's match rule
17117 //
17118 // ---------VM FLAGS---------------------------------------------------------
17119 //
17120 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17121 //
17122 // Each peephole rule is given an identifying number starting with zero and
17123 // increasing by one in the order seen by the parser. An individual peephole
17124 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17125 // on the command-line.
17126 //
17127 // ---------CURRENT LIMITATIONS----------------------------------------------
17128 //
17129 // Only match adjacent instructions in same basic block
17130 // Only equality constraints
17131 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17132 // Only one replacement instruction
17133 //
17134 // ---------EXAMPLE----------------------------------------------------------
17135 //
17136 // // pertinent parts of existing instructions in architecture description
17137 // instruct movI(iRegINoSp dst, iRegI src)
17138 // %{
17139 // match(Set dst (CopyI src));
17140 // %}
17141 //
17142 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17143 // %{
17144 // match(Set dst (AddI dst src));
17145 // effect(KILL cr);
17146 // %}
17147 //
17148 // // Change (inc mov) to lea
17149 // peephole %{
17150 // // increment preceded by register-register move
17151 // peepmatch ( incI_iReg movI );
17152 // // require that the destination register of the increment
17153 // // match the destination register of the move
17154 // peepconstraint ( 0.dst == 1.dst );
17155 // // construct a replacement instruction that sets
17156 // // the destination to ( move's source register + one )
17157 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17158 // %}
17159 //
17160
17161 // Implementation no longer uses movX instructions since
17162 // machine-independent system no longer uses CopyX nodes.
17163 //
17164 // peephole
17165 // %{
17166 // peepmatch (incI_iReg movI);
17167 // peepconstraint (0.dst == 1.dst);
17168 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17169 // %}
17170
17171 // peephole
17172 // %{
17173 // peepmatch (decI_iReg movI);
17174 // peepconstraint (0.dst == 1.dst);
17175 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17176 // %}
17177
17178 // peephole
17179 // %{
17180 // peepmatch (addI_iReg_imm movI);
17181 // peepconstraint (0.dst == 1.dst);
17182 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17183 // %}
17184
17185 // peephole
17186 // %{
17187 // peepmatch (incL_iReg movL);
17188 // peepconstraint (0.dst == 1.dst);
17189 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17190 // %}
17191
17192 // peephole
17193 // %{
17194 // peepmatch (decL_iReg movL);
17195 // peepconstraint (0.dst == 1.dst);
17196 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17197 // %}
17198
17199 // peephole
17200 // %{
17201 // peepmatch (addL_iReg_imm movL);
17202 // peepconstraint (0.dst == 1.dst);
17203 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17204 // %}
17205
17206 // peephole
17207 // %{
17208 // peepmatch (addP_iReg_imm movP);
17209 // peepconstraint (0.dst == 1.dst);
17210 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17211 // %}
17212
17213 // // Change load of spilled value to only a spill
17214 // instruct storeI(memory mem, iRegI src)
17215 // %{
17216 // match(Set mem (StoreI mem src));
17217 // %}
17218 //
17219 // instruct loadI(iRegINoSp dst, memory mem)
17220 // %{
17221 // match(Set dst (LoadI mem));
17222 // %}
17223 //
17224
17225 //----------SMARTSPILL RULES---------------------------------------------------
17226 // These must follow all instruction definitions as they use the names
17227 // defined in the instructions definitions.
17228
17229 // Local Variables:
17230 // mode: c++
17231 // End: