Bug Summary

File:out/../deps/v8/src/codegen/x64/assembler-x64.cc
Warning:line 2311, column 5
Value stored to 'size' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name assembler-x64.cc -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=all -relaxed-aliasing -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/home/maurizio/node-v18.6.0/out -resource-dir /usr/local/lib/clang/16.0.0 -D _GLIBCXX_USE_CXX11_ABI=1 -D NODE_OPENSSL_CONF_NAME=nodejs_conf -D NODE_OPENSSL_HAS_QUIC -D V8_GYP_BUILD -D V8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=64 -D __STDC_FORMAT_MACROS -D OPENSSL_NO_PINSHARED -D OPENSSL_THREADS -D V8_TARGET_ARCH_X64 -D V8_HAVE_TARGET_OS -D V8_TARGET_OS_LINUX -D V8_EMBEDDER_STRING="-node.8" -D ENABLE_DISASSEMBLER -D V8_PROMISE_INTERNAL_FIELD_COUNT=1 -D V8_SHORT_BUILTIN_CALLS -D OBJECT_PRINT -D V8_INTL_SUPPORT -D V8_ATOMIC_OBJECT_FIELD_WRITES -D V8_ENABLE_LAZY_SOURCE_POSITIONS -D V8_USE_SIPHASH -D V8_SHARED_RO_HEAP -D V8_WIN64_UNWINDING_INFO -D V8_ENABLE_REGEXP_INTERPRETER_THREADED_DISPATCH -D V8_SNAPSHOT_COMPRESSION -D V8_ENABLE_WEBASSEMBLY -D V8_ENABLE_JAVASCRIPT_PROMISE_HOOKS -D V8_ALLOCATION_FOLDING -D V8_ALLOCATION_SITE_TRACKING -D V8_SCRIPTORMODULE_LEGACY_LIFETIME -D V8_ADVANCED_BIGINT_ALGORITHMS -D ICU_UTIL_DATA_IMPL=ICU_UTIL_DATA_STATIC -D UCONFIG_NO_SERVICE=1 -D U_ENABLE_DYLOAD=0 -D U_STATIC_IMPLEMENTATION=1 -D U_HAVE_STD_STRING=1 -D UCONFIG_NO_BREAK_ITERATION=0 -I ../deps/v8 -I ../deps/v8/include -I /home/maurizio/node-v18.6.0/out/Release/obj/gen/inspector-generated-output-root -I ../deps/v8/third_party/inspector_protocol -I /home/maurizio/node-v18.6.0/out/Release/obj/gen -I /home/maurizio/node-v18.6.0/out/Release/obj/gen/generate-bytecode-output-root -I ../deps/icu-small/source/i18n -I ../deps/icu-small/source/common -I ../deps/v8/third_party/zlib -I ../deps/v8/third_party/zlib/google -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward -internal-isystem /usr/local/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-unused-parameter -Wno-return-type -std=gnu++17 -fdeprecated-macro -fdebug-compilation-dir=/home/maurizio/node-v18.6.0/out -ferror-limit 19 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-08-22-142216-507842-1 -x c++ ../deps/v8/src/codegen/x64/assembler-x64.cc
1// Copyright 2012 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/codegen/x64/assembler-x64.h"
6
7#include <cstring>
8
9#if V8_TARGET_ARCH_X641
10
11#if V8_LIBC_MSVCRT
12#include <intrin.h> // _xgetbv()
13#endif
14#if V8_OS_DARWIN
15#include <sys/sysctl.h>
16#endif
17
18#include "src/base/bits.h"
19#include "src/base/cpu.h"
20#include "src/base/platform/wrappers.h"
21#include "src/codegen/assembler-inl.h"
22#include "src/codegen/macro-assembler.h"
23#include "src/codegen/string-constants.h"
24#include "src/deoptimizer/deoptimizer.h"
25#include "src/init/v8.h"
26
27namespace v8 {
28namespace internal {
29
30// -----------------------------------------------------------------------------
31// Implementation of CpuFeatures
32
33namespace {
34
35#if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X641
36
37V8_INLINEinline __attribute__((always_inline)) uint64_t xgetbv(unsigned int xcr) {
38#if V8_LIBC_MSVCRT
39 return _xgetbv(xcr);
40#else
41 unsigned eax, edx;
42 // Check xgetbv; this uses a .byte sequence instead of the instruction
43 // directly because older assemblers do not include support for xgetbv and
44 // there is no easy way to conditionally compile based on the assembler
45 // used.
46 __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
47 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
48#endif
49}
50
51bool OSHasAVXSupport() {
52#if V8_OS_DARWIN
53 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
54 // caused by ISRs, so we detect that here and disable AVX in that case.
55 char buffer[128];
56 size_t buffer_size = arraysize(buffer)(sizeof(ArraySizeHelper(buffer)));
57 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
58 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
59 FATAL("V8 failed to get kernel version")V8_Fatal("V8 failed to get kernel version");
60 }
61 // The buffer now contains a string of the form XX.YY.ZZ, where
62 // XX is the major kernel version component.
63 char* period_pos = strchr(buffer, '.');
64 DCHECK_NOT_NULL(period_pos)((void) 0);
65 *period_pos = '\0';
66 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
67 if (kernel_version_major <= 13) return false;
68#endif // V8_OS_DARWIN
69 // Check whether OS claims to support AVX.
70 uint64_t feature_mask = xgetbv(0); // XCR_XFEATURE_ENABLED_MASK
71 return (feature_mask & 0x6) == 0x6;
72}
73
74#endif // V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
75
76} // namespace
77
78bool CpuFeatures::SupportsWasmSimd128() {
79#if V8_ENABLE_WEBASSEMBLY1
80 if (IsSupported(SSE4_1)) return true;
81 if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true;
82#endif // V8_ENABLE_WEBASSEMBLY
83 return false;
84}
85
86void CpuFeatures::ProbeImpl(bool cross_compile) {
87 // Only use statically determined features for cross compile (snapshot).
88 if (cross_compile) return;
89
90#if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X641
91 base::CPU cpu;
92 CHECK(cpu.has_sse2())do { if ((__builtin_expect(!!(!(cpu.has_sse2())), 0))) { V8_Fatal
("Check failed: %s.", "cpu.has_sse2()"); } } while (false)
; // SSE2 support is mandatory.
93 CHECK(cpu.has_cmov())do { if ((__builtin_expect(!!(!(cpu.has_cmov())), 0))) { V8_Fatal
("Check failed: %s.", "cpu.has_cmov()"); } } while (false)
; // CMOV support is mandatory.
94
95 if (cpu.has_sse42()) SetSupported(SSE4_2);
96 if (cpu.has_sse41()) SetSupported(SSE4_1);
97 if (cpu.has_ssse3()) SetSupported(SSSE3);
98 if (cpu.has_sse3()) SetSupported(SSE3);
99 if (cpu.has_avx() && cpu.has_osxsave() && OSHasAVXSupport()) {
100 SetSupported(AVX);
101 if (cpu.has_avx2()) SetSupported(AVX2);
102 if (cpu.has_fma3()) SetSupported(FMA3);
103 }
104
105 // SAHF is not generally available in long mode.
106 if (cpu.has_sahf() && FLAG_enable_sahf) SetSupported(SAHF);
107 if (cpu.has_bmi1() && FLAG_enable_bmi1) SetSupported(BMI1);
108 if (cpu.has_bmi2() && FLAG_enable_bmi2) SetSupported(BMI2);
109 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) SetSupported(LZCNT);
110 if (cpu.has_popcnt() && FLAG_enable_popcnt) SetSupported(POPCNT);
111 if (strcmp(FLAG_mcpu, "auto") == 0) {
112 if (cpu.is_atom()) SetSupported(INTEL_ATOM);
113 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
114 SetSupported(INTEL_ATOM);
115 }
116
117 // Ensure that supported cpu features make sense. E.g. it is wrong to support
118 // AVX but not SSE4_2, if we have --enable-avx and --no-enable-sse4-2, the
119 // code above would set AVX to supported, and SSE4_2 to unsupported, then the
120 // checks below will set AVX to unsupported.
121 if (!FLAG_enable_sse3) SetUnsupported(SSE3);
122 if (!FLAG_enable_ssse3 || !IsSupported(SSE3)) SetUnsupported(SSSE3);
123 if (!FLAG_enable_sse4_1 || !IsSupported(SSSE3)) SetUnsupported(SSE4_1);
124 if (!FLAG_enable_sse4_2 || !IsSupported(SSE4_1)) SetUnsupported(SSE4_2);
125 if (!FLAG_enable_avx || !IsSupported(SSE4_2)) SetUnsupported(AVX);
126 if (!FLAG_enable_avx2 || !IsSupported(AVX)) SetUnsupported(AVX2);
127 if (!FLAG_enable_fma3 || !IsSupported(AVX)) SetUnsupported(FMA3);
128
129 // Set a static value on whether Simd is supported.
130 // This variable is only used for certain archs to query SupportWasmSimd128()
131 // at runtime in builtins using an extern ref. Other callers should use
132 // CpuFeatures::SupportWasmSimd128().
133 CpuFeatures::supports_wasm_simd_128_ = CpuFeatures::SupportsWasmSimd128();
134
135 if (cpu.has_cetss()) SetSupported(CETSS);
136 // The static variable is used for codegen of certain CETSS instructions.
137 CpuFeatures::supports_cetss_ = IsSupported(CETSS);
138#endif // V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
139}
140
141void CpuFeatures::PrintTarget() {}
142void CpuFeatures::PrintFeatures() {
143 printf(
144 "SSE3=%d SSSE3=%d SSE4_1=%d SSE4_2=%d SAHF=%d AVX=%d AVX2=%d FMA3=%d "
145 "BMI1=%d "
146 "BMI2=%d "
147 "LZCNT=%d "
148 "POPCNT=%d ATOM=%d\n",
149 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
150 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SSE4_2),
151 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
152 CpuFeatures::IsSupported(AVX2), CpuFeatures::IsSupported(FMA3),
153 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
154 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
155 CpuFeatures::IsSupported(INTEL_ATOM));
156}
157
158// -----------------------------------------------------------------------------
159// Implementation of RelocInfo
160
161uint32_t RelocInfo::wasm_call_tag() const {
162 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL)((void) 0);
163 return ReadUnalignedValue<uint32_t>(pc_);
164}
165
166// -----------------------------------------------------------------------------
167// Implementation of Operand
168
169Operand::Operand(Operand operand, int32_t offset) {
170 DCHECK_GE(operand.data().len, 1)((void) 0);
171 // Operand encodes REX ModR/M [SIB] [Disp].
172 byte modrm = operand.data().buf[0];
173 DCHECK_LT(modrm, 0xC0)((void) 0); // Disallow mode 3 (register target).
174 bool has_sib = ((modrm & 0x07) == 0x04);
175 byte mode = modrm & 0xC0;
176 int disp_offset = has_sib ? 2 : 1;
177 int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
178 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
179 // displacement.
180 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
181 int32_t disp_value = 0;
182 if (mode == 0x80 || is_baseless) {
183 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
184 disp_value = ReadUnalignedValue<int32_t>(
185 reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
186 } else if (mode == 0x40) {
187 // Mode 1: Byte displacement.
188 disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
189 }
190
191 // Write new operand with same registers, but with modified displacement.
192 DCHECK(offset >= 0 ? disp_value + offset > disp_value((void) 0)
193 : disp_value + offset < disp_value)((void) 0); // No overflow.
194 disp_value += offset;
195 data_.rex = operand.data().rex;
196 if (!is_int8(disp_value) || is_baseless) {
197 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
198 data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
199 data_.len = disp_offset + 4;
200 WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
201 disp_value);
202 } else if (disp_value != 0 || (base_reg == 0x05)) {
203 // Need 8 bits of displacement.
204 data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
205 data_.len = disp_offset + 1;
206 data_.buf[disp_offset] = static_cast<byte>(disp_value);
207 } else {
208 // Need no displacement.
209 data_.buf[0] = (modrm & 0x3F); // Mode 0.
210 data_.len = disp_offset;
211 }
212 if (has_sib) {
213 data_.buf[1] = operand.data().buf[1];
214 }
215}
216
217bool Operand::AddressUsesRegister(Register reg) const {
218 int code = reg.code();
219 DCHECK_NE(data_.buf[0] & 0xC0, 0xC0)((void) 0); // Always a memory operand.
220 // Start with only low three bits of base register. Initial decoding
221 // doesn't distinguish on the REX.B bit.
222 int base_code = data_.buf[0] & 0x07;
223 if (base_code == rsp.code()) {
224 // SIB byte present in buf_[1].
225 // Check the index register from the SIB byte + REX.X prefix.
226 int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
227 // Index code (including REX.X) of 0x04 (rsp) means no index register.
228 if (index_code != rsp.code() && index_code == code) return true;
229 // Add REX.B to get the full base register code.
230 base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
231 // A base register of 0x05 (rbp) with mod = 0 means no base register.
232 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
233 return code == base_code;
234 } else {
235 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
236 // no base register.
237 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
238 base_code |= ((data_.rex & 0x01) << 3);
239 return code == base_code;
240 }
241}
242
243void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
244 DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty())((void) 0);
245 for (auto& request : heap_object_requests_) {
246 Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
247 switch (request.kind()) {
248 case HeapObjectRequest::kHeapNumber: {
249 Handle<HeapNumber> object =
250 isolate->factory()->NewHeapNumber<AllocationType::kOld>(
251 request.heap_number());
252 WriteUnalignedValue(pc, object);
253 break;
254 }
255 case HeapObjectRequest::kStringConstant: {
256 const StringConstantBase* str = request.string();
257 CHECK_NOT_NULL(str)do { if ((__builtin_expect(!!(!((str) != nullptr)), 0))) { V8_Fatal
("Check failed: %s.", "(str) != nullptr"); } } while (false)
;
258 Handle<String> allocated = str->AllocateStringConstant(isolate);
259 WriteUnalignedValue(pc, allocated);
260 break;
261 }
262 }
263 }
264}
265
266// Partial Constant Pool.
267bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
268 auto existing = entries_.find(data);
269 if (existing == entries_.end()) {
270 entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
271 return false;
272 }
273
274 // Make sure this is called with strictly ascending offsets.
275 DCHECK_GT(offset + kMoveImm64Offset, existing->second)((void) 0);
276
277 entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
278 return true;
279}
280
281bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
282 if (!FLAG_partial_constant_pool) return false;
283 DCHECK_WITH_MSG(void(0);
284 FLAG_text_is_readable,void(0);
285 "The partial constant pool requires a readable .text section")void(0);;
286 if (!RelocInfo::IsShareableRelocMode(mode)) return false;
287
288 // Currently, partial constant pool only handles the following kinds of
289 // RelocInfo.
290 if (mode != RelocInfo::NO_INFO && mode != RelocInfo::EXTERNAL_REFERENCE &&
291 mode != RelocInfo::OFF_HEAP_TARGET)
292 return false;
293
294 uint64_t raw_data = static_cast<uint64_t>(data);
295 int offset = assm_->pc_offset();
296 return AddSharedEntry(raw_data, offset);
297}
298
299bool ConstPool::IsMoveRipRelative(Address instr) {
300 return (ReadUnalignedValue<uint32_t>(instr) & kMoveRipRelativeMask) ==
301 kMoveRipRelativeInstr;
302}
303
304void ConstPool::Clear() { entries_.clear(); }
305
306void ConstPool::PatchEntries() {
307 for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
308 iter = entries_.upper_bound(iter->first)) {
309 std::pair<EntryMap::iterator, EntryMap::iterator> range =
310 entries_.equal_range(iter->first);
311 int constant_entry_offset = 0;
312 for (EntryMap::iterator it = range.first; it != range.second; it++) {
313 if (it == range.first) {
314 constant_entry_offset = it->second;
315 continue;
316 }
317
318 DCHECK_GT(constant_entry_offset, 0)((void) 0);
319 DCHECK_LT(constant_entry_offset, it->second)((void) 0);
320 int32_t disp32 =
321 constant_entry_offset - (it->second + kRipRelativeDispSize);
322 Address disp_addr = assm_->addr_at(it->second);
323
324 // Check if the instruction is actually a rip-relative move.
325 DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset))((void) 0);
326 // The displacement of the rip-relative move should be 0 before patching.
327 DCHECK(ReadUnalignedValue<uint32_t>(disp_addr) == 0)((void) 0);
328 WriteUnalignedValue(disp_addr, disp32);
329 }
330 }
331 Clear();
332}
333
334void Assembler::PatchConstPool() {
335 // There is nothing to do if there are no pending entries.
336 if (constpool_.IsEmpty()) {
337 return;
338 }
339 constpool_.PatchEntries();
340}
341
342bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
343 if (!FLAG_partial_constant_pool) return false;
344 return (rmode == RelocInfo::NO_INFO ||
345 rmode == RelocInfo::EXTERNAL_REFERENCE ||
346 rmode == RelocInfo::OFF_HEAP_TARGET);
347}
348
349// -----------------------------------------------------------------------------
350// Implementation of Assembler.
351
352Assembler::Assembler(const AssemblerOptions& options,
353 std::unique_ptr<AssemblerBuffer> buffer)
354 : AssemblerBase(options, std::move(buffer)), constpool_(this) {
355 reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
356 if (CpuFeatures::IsSupported(SSE4_2)) {
357 EnableCpuFeature(SSE4_1);
358 }
359 if (CpuFeatures::IsSupported(SSE4_1)) {
360 EnableCpuFeature(SSSE3);
361 }
362 if (CpuFeatures::IsSupported(SSSE3)) {
363 EnableCpuFeature(SSE3);
364 }
365
366#if defined(V8_OS_WIN_X64)
367 if (options.collect_win64_unwind_info) {
368 xdata_encoder_ = std::make_unique<win64_unwindinfo::XdataEncoder>(*this);
369 }
370#endif
371}
372
373void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
374 SafepointTableBuilder* safepoint_table_builder,
375 int handler_table_offset) {
376 // As a crutch to avoid having to add manual Align calls wherever we use a
377 // raw workflow to create Code objects (mostly in tests), add another Align
378 // call here. It does no harm - the end of the Code object is aligned to the
379 // (larger) kCodeAlignment anyways.
380 // TODO(jgruber): Consider moving responsibility for proper alignment to
381 // metadata table builders (safepoint, handler, constant pool, code
382 // comments).
383 DataAlign(Code::kMetadataAlignment);
384
385 PatchConstPool();
386 DCHECK(constpool_.IsEmpty())((void) 0);
387
388 const int code_comments_size = WriteCodeComments();
389
390 // At this point overflow() may be true, but the gap ensures
391 // that we are still not overlapping instructions and relocation info.
392 DCHECK(pc_ <= reloc_info_writer.pos())((void) 0); // No overlap.
393
394 AllocateAndInstallRequestedHeapObjects(isolate);
395
396 // Set up code descriptor.
397 // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
398 // this point to make CodeDesc initialization less fiddly.
399
400 static constexpr int kConstantPoolSize = 0;
401 const int instruction_size = pc_offset();
402 const int code_comments_offset = instruction_size - code_comments_size;
403 const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
404 const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
405 ? constant_pool_offset
406 : handler_table_offset;
407 const int safepoint_table_offset =
408 (safepoint_table_builder == kNoSafepointTable)
409 ? handler_table_offset2
410 : safepoint_table_builder->safepoint_table_offset();
411 const int reloc_info_offset =
412 static_cast<int>(reloc_info_writer.pos() - buffer_->start());
413 CodeDesc::Initialize(desc, this, safepoint_table_offset,
414 handler_table_offset2, constant_pool_offset,
415 code_comments_offset, reloc_info_offset);
416}
417
418void Assembler::FinalizeJumpOptimizationInfo() {
419 // Collection stage
420 auto jump_opt = jump_optimization_info();
421 if (jump_opt && jump_opt->is_collecting()) {
422 auto& bitmap = jump_opt->farjmp_bitmap();
423 int num = static_cast<int>(farjmp_positions_.size());
424 if (num && bitmap.empty()) {
425 bool can_opt = false;
426
427 bitmap.resize((num + 31) / 32, 0);
428 for (int i = 0; i < num; i++) {
429 int disp_pos = farjmp_positions_[i];
430 int disp = long_at(disp_pos);
431 if (is_int8(disp)) {
432 bitmap[i / 32] |= 1 << (i & 31);
433 can_opt = true;
434 }
435 }
436 if (can_opt) {
437 jump_opt->set_optimizable();
438 }
439 }
440 }
441}
442
443#if defined(V8_OS_WIN_X64)
444win64_unwindinfo::BuiltinUnwindInfo Assembler::GetUnwindInfo() const {
445 DCHECK(options().collect_win64_unwind_info)((void) 0);
446 DCHECK_NOT_NULL(xdata_encoder_)((void) 0);
447 return xdata_encoder_->unwinding_info();
448}
449#endif
450
451void Assembler::Align(int m) {
452 DCHECK(base::bits::IsPowerOfTwo(m))((void) 0);
453 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
454 Nop(delta);
455}
456
457void Assembler::CodeTargetAlign() {
458 Align(16); // Preferred alignment of jump targets on x64.
459}
460
461void Assembler::LoopHeaderAlign() {
462 Align(64); // Preferred alignment of loop header on x64.
463}
464
465bool Assembler::IsNop(Address addr) {
466 byte* a = reinterpret_cast<byte*>(addr);
467 while (*a == 0x66) a++;
468 if (*a == 0x90) return true;
469 if (a[0] == 0xF && a[1] == 0x1F) return true;
470 return false;
471}
472
473void Assembler::bind_to(Label* L, int pos) {
474 DCHECK(!L->is_bound())((void) 0); // Label may only be bound once.
475 DCHECK(0 <= pos && pos <= pc_offset())((void) 0); // Position must be valid.
476 if (L->is_linked()) {
477 int current = L->pos();
478 int next = long_at(current);
479 while (next != current) {
480 if (current >= 4 && long_at(current - 4) == 0) {
481 // Absolute address.
482 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
483 WriteUnalignedValue(addr_at(current - 4), imm64);
484 internal_reference_positions_.push_back(current - 4);
485 } else {
486 // Relative address, relative to point after address.
487 int imm32 = pos - (current + sizeof(int32_t));
488 long_at_put(current, imm32);
489 }
490 current = next;
491 next = long_at(next);
492 }
493 // Fix up last fixup on linked list.
494 if (current >= 4 && long_at(current - 4) == 0) {
495 // Absolute address.
496 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
497 WriteUnalignedValue(addr_at(current - 4), imm64);
498 internal_reference_positions_.push_back(current - 4);
499 } else {
500 // Relative address, relative to point after address.
501 int imm32 = pos - (current + sizeof(int32_t));
502 long_at_put(current, imm32);
503 }
504 }
505 while (L->is_near_linked()) {
506 int fixup_pos = L->near_link_pos();
507 int offset_to_next =
508 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
509 DCHECK_LE(offset_to_next, 0)((void) 0);
510 int disp = pos - (fixup_pos + sizeof(int8_t));
511 CHECK(is_int8(disp))do { if ((__builtin_expect(!!(!(is_int8(disp))), 0))) { V8_Fatal
("Check failed: %s.", "is_int8(disp)"); } } while (false)
;
512 set_byte_at(fixup_pos, disp);
513 if (offset_to_next < 0) {
514 L->link_to(fixup_pos + offset_to_next, Label::kNear);
515 } else {
516 L->UnuseNear();
517 }
518 }
519
520 // Optimization stage
521 auto jump_opt = jump_optimization_info();
522 if (jump_opt && jump_opt->is_optimizing()) {
523 auto it = label_farjmp_maps_.find(L);
524 if (it != label_farjmp_maps_.end()) {
525 auto& pos_vector = it->second;
526 for (auto fixup_pos : pos_vector) {
527 int disp = pos - (fixup_pos + sizeof(int8_t));
528 CHECK(is_int8(disp))do { if ((__builtin_expect(!!(!(is_int8(disp))), 0))) { V8_Fatal
("Check failed: %s.", "is_int8(disp)"); } } while (false)
;
529 set_byte_at(fixup_pos, disp);
530 }
531 label_farjmp_maps_.erase(it);
532 }
533 }
534 L->bind_to(pos);
535}
536
537void Assembler::bind(Label* L) { bind_to(L, pc_offset()); }
538
539void Assembler::record_farjmp_position(Label* L, int pos) {
540 auto& pos_vector = label_farjmp_maps_[L];
541 pos_vector.push_back(pos);
542}
543
544bool Assembler::is_optimizable_farjmp(int idx) {
545 if (predictable_code_size()) return false;
546
547 auto jump_opt = jump_optimization_info();
548 CHECK(jump_opt->is_optimizing())do { if ((__builtin_expect(!!(!(jump_opt->is_optimizing())
), 0))) { V8_Fatal("Check failed: %s.", "jump_opt->is_optimizing()"
); } } while (false)
;
549
550 auto& bitmap = jump_opt->farjmp_bitmap();
551 CHECK(idx < static_cast<int>(bitmap.size() * 32))do { if ((__builtin_expect(!!(!(idx < static_cast<int>
(bitmap.size() * 32))), 0))) { V8_Fatal("Check failed: %s.", "idx < static_cast<int>(bitmap.size() * 32)"
); } } while (false)
;
552 return !!(bitmap[idx / 32] & (1 << (idx & 31)));
553}
554
555void Assembler::GrowBuffer() {
556 DCHECK(buffer_overflow())((void) 0);
557
558 // Compute new buffer size.
559 DCHECK_EQ(buffer_start_, buffer_->start())((void) 0);
560 int old_size = buffer_->size();
561 int new_size = 2 * old_size;
562
563 // Some internal data structures overflow for very large buffers,
564 // they must ensure that kMaximalBufferSize is not too large.
565 if (new_size > kMaximalBufferSize) {
566 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
567 }
568
569 // Set up new buffer.
570 std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
571 DCHECK_EQ(new_size, new_buffer->size())((void) 0);
572 byte* new_start = new_buffer->start();
573
574 // Copy the data.
575 intptr_t pc_delta = new_start - buffer_start_;
576 intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
577 size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
578 MemMove(new_start, buffer_start_, pc_offset());
579 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
580 reloc_size);
581
582 // Switch buffers.
583 buffer_ = std::move(new_buffer);
584 buffer_start_ = new_start;
585 pc_ += pc_delta;
586 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
587 reloc_info_writer.last_pc() + pc_delta);
588
589 // Relocate internal references.
590 for (auto pos : internal_reference_positions_) {
591 Address p = reinterpret_cast<Address>(buffer_start_ + pos);
592 WriteUnalignedValue(p, ReadUnalignedValue<intptr_t>(p) + pc_delta);
593 }
594
595 DCHECK(!buffer_overflow())((void) 0);
596}
597
598void Assembler::emit_operand(int code, Operand adr) {
599 DCHECK(is_uint3(code))((void) 0);
600 const unsigned length = adr.data().len;
601 DCHECK_GT(length, 0)((void) 0);
602
603 // Emit updated ModR/M byte containing the given register.
604 DCHECK_EQ(adr.data().buf[0] & 0x38, 0)((void) 0);
605 *pc_++ = adr.data().buf[0] | code << 3;
606
607 // Recognize RIP relative addressing.
608 if (adr.data().buf[0] == 5) {
609 DCHECK_EQ(9u, length)((void) 0);
610 Label* label = ReadUnalignedValue<Label*>(
611 reinterpret_cast<Address>(&adr.data().buf[1]));
612 if (label->is_bound()) {
613 int offset =
614 label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
615 DCHECK_GE(0, offset)((void) 0);
616 emitl(offset);
617 } else if (label->is_linked()) {
618 emitl(label->pos());
619 label->link_to(pc_offset() - sizeof(int32_t));
620 } else {
621 DCHECK(label->is_unused())((void) 0);
622 int32_t current = pc_offset();
623 emitl(current);
624 label->link_to(current);
625 }
626 } else {
627 // Emit the rest of the encoded operand.
628 for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
629 }
630}
631
632// Assembler Instruction implementations.
633
634void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
635 EnsureSpace ensure_space(this);
636 emit_rex(reg, op, size);
637 emit(opcode);
638 emit_operand(reg, op);
639}
640
641void Assembler::arithmetic_op(byte opcode, Register reg, Register rm_reg,
642 int size) {
643 EnsureSpace ensure_space(this);
644 DCHECK_EQ(opcode & 0xC6, 2)((void) 0);
645 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
646 // Swap reg and rm_reg and change opcode operand order.
647 emit_rex(rm_reg, reg, size);
648 emit(opcode ^ 0x02);
649 emit_modrm(rm_reg, reg);
650 } else {
651 emit_rex(reg, rm_reg, size);
652 emit(opcode);
653 emit_modrm(reg, rm_reg);
654 }
655}
656
657void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
658 EnsureSpace ensure_space(this);
659 DCHECK_EQ(opcode & 0xC6, 2)((void) 0);
660 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
661 // Swap reg and rm_reg and change opcode operand order.
662 emit(0x66);
663 emit_optional_rex_32(rm_reg, reg);
664 emit(opcode ^ 0x02);
665 emit_modrm(rm_reg, reg);
666 } else {
667 emit(0x66);
668 emit_optional_rex_32(reg, rm_reg);
669 emit(opcode);
670 emit_modrm(reg, rm_reg);
671 }
672}
673
674void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
675 EnsureSpace ensure_space(this);
676 emit(0x66);
677 emit_optional_rex_32(reg, rm_reg);
678 emit(opcode);
679 emit_operand(reg, rm_reg);
680}
681
682void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
683 EnsureSpace ensure_space(this);
684 if (!reg.is_byte_register()) {
685 emit_rex_32(reg, op);
686 } else {
687 emit_optional_rex_32(reg, op);
688 }
689 emit(opcode);
690 emit_operand(reg, op);
691}
692
693void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
694 EnsureSpace ensure_space(this);
695 DCHECK_EQ(opcode & 0xC6, 2)((void) 0);
696 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
697 // Swap reg and rm_reg and change opcode operand order.
698 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
699 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
700 emit_rex_32(rm_reg, reg);
701 }
702 emit(opcode ^ 0x02);
703 emit_modrm(rm_reg, reg);
704 } else {
705 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
706 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
707 emit_rex_32(reg, rm_reg);
708 }
709 emit(opcode);
710 emit_modrm(reg, rm_reg);
711 }
712}
713
714void Assembler::immediate_arithmetic_op(byte subcode, Register dst,
715 Immediate src, int size) {
716 EnsureSpace ensure_space(this);
717 emit_rex(dst, size);
718 if (is_int8(src.value_) && RelocInfo::IsNoInfo(src.rmode_)) {
719 emit(0x83);
720 emit_modrm(subcode, dst);
721 emit(src.value_);
722 } else if (dst == rax) {
723 emit(0x05 | (subcode << 3));
724 emit(src);
725 } else {
726 emit(0x81);
727 emit_modrm(subcode, dst);
728 emit(src);
729 }
730}
731
732void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
733 Immediate src, int size) {
734 EnsureSpace ensure_space(this);
735 emit_rex(dst, size);
736 if (is_int8(src.value_) && RelocInfo::IsNoInfo(src.rmode_)) {
737 emit(0x83);
738 emit_operand(subcode, dst);
739 emit(src.value_);
740 } else {
741 emit(0x81);
742 emit_operand(subcode, dst);
743 emit(src);
744 }
745}
746
747void Assembler::immediate_arithmetic_op_16(byte subcode, Register dst,
748 Immediate src) {
749 EnsureSpace ensure_space(this);
750 emit(0x66); // Operand size override prefix.
751 emit_optional_rex_32(dst);
752 if (is_int8(src.value_)) {
753 emit(0x83);
754 emit_modrm(subcode, dst);
755 emit(src.value_);
756 } else if (dst == rax) {
757 emit(0x05 | (subcode << 3));
758 emitw(src.value_);
759 } else {
760 emit(0x81);
761 emit_modrm(subcode, dst);
762 emitw(src.value_);
763 }
764}
765
766void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
767 Immediate src) {
768 EnsureSpace ensure_space(this);
769 emit(0x66); // Operand size override prefix.
770 emit_optional_rex_32(dst);
771 if (is_int8(src.value_)) {
772 emit(0x83);
773 emit_operand(subcode, dst);
774 emit(src.value_);
775 } else {
776 emit(0x81);
777 emit_operand(subcode, dst);
778 emitw(src.value_);
779 }
780}
781
782void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
783 Immediate src) {
784 EnsureSpace ensure_space(this);
785 emit_optional_rex_32(dst);
786 DCHECK(is_int8(src.value_) || is_uint8(src.value_))((void) 0);
787 emit(0x80);
788 emit_operand(subcode, dst);
789 emit(src.value_);
790}
791
792void Assembler::immediate_arithmetic_op_8(byte subcode, Register dst,
793 Immediate src) {
794 EnsureSpace ensure_space(this);
795 if (!dst.is_byte_register()) {
796 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
797 emit_rex_32(dst);
798 }
799 DCHECK(is_int8(src.value_) || is_uint8(src.value_))((void) 0);
800 emit(0x80);
801 emit_modrm(subcode, dst);
802 emit(src.value_);
803}
804
805void Assembler::shift(Register dst, Immediate shift_amount, int subcode,
806 int size) {
807 EnsureSpace ensure_space(this);
808 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)((void) 0)
809 : is_uint5(shift_amount.value_))((void) 0);
810 if (shift_amount.value_ == 1) {
811 emit_rex(dst, size);
812 emit(0xD1);
813 emit_modrm(subcode, dst);
814 } else {
815 emit_rex(dst, size);
816 emit(0xC1);
817 emit_modrm(subcode, dst);
818 emit(shift_amount.value_);
819 }
820}
821
822void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
823 int size) {
824 EnsureSpace ensure_space(this);
825 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)((void) 0)
826 : is_uint5(shift_amount.value_))((void) 0);
827 if (shift_amount.value_ == 1) {
828 emit_rex(dst, size);
829 emit(0xD1);
830 emit_operand(subcode, dst);
831 } else {
832 emit_rex(dst, size);
833 emit(0xC1);
834 emit_operand(subcode, dst);
835 emit(shift_amount.value_);
836 }
837}
838
839void Assembler::shift(Register dst, int subcode, int size) {
840 EnsureSpace ensure_space(this);
841 emit_rex(dst, size);
842 emit(0xD3);
843 emit_modrm(subcode, dst);
844}
845
846void Assembler::shift(Operand dst, int subcode, int size) {
847 EnsureSpace ensure_space(this);
848 emit_rex(dst, size);
849 emit(0xD3);
850 emit_operand(subcode, dst);
851}
852
853void Assembler::bswapl(Register dst) {
854 EnsureSpace ensure_space(this);
855 emit_rex_32(dst);
856 emit(0x0F);
857 emit(0xC8 + dst.low_bits());
858}
859
860void Assembler::bswapq(Register dst) {
861 EnsureSpace ensure_space(this);
862 emit_rex_64(dst);
863 emit(0x0F);
864 emit(0xC8 + dst.low_bits());
865}
866
867void Assembler::btq(Operand dst, Register src) {
868 EnsureSpace ensure_space(this);
869 emit_rex_64(src, dst);
870 emit(0x0F);
871 emit(0xA3);
872 emit_operand(src, dst);
873}
874
875void Assembler::btsq(Operand dst, Register src) {
876 EnsureSpace ensure_space(this);
877 emit_rex_64(src, dst);
878 emit(0x0F);
879 emit(0xAB);
880 emit_operand(src, dst);
881}
882
883void Assembler::btsq(Register dst, Immediate imm8) {
884 EnsureSpace ensure_space(this);
885 emit_rex_64(dst);
886 emit(0x0F);
887 emit(0xBA);
888 emit_modrm(0x5, dst);
889 emit(imm8.value_);
890}
891
892void Assembler::btrq(Register dst, Immediate imm8) {
893 EnsureSpace ensure_space(this);
894 emit_rex_64(dst);
895 emit(0x0F);
896 emit(0xBA);
897 emit_modrm(0x6, dst);
898 emit(imm8.value_);
899}
900
901void Assembler::bsrl(Register dst, Register src) {
902 EnsureSpace ensure_space(this);
903 emit_optional_rex_32(dst, src);
904 emit(0x0F);
905 emit(0xBD);
906 emit_modrm(dst, src);
907}
908
909void Assembler::bsrl(Register dst, Operand src) {
910 EnsureSpace ensure_space(this);
911 emit_optional_rex_32(dst, src);
912 emit(0x0F);
913 emit(0xBD);
914 emit_operand(dst, src);
915}
916
917void Assembler::bsrq(Register dst, Register src) {
918 EnsureSpace ensure_space(this);
919 emit_rex_64(dst, src);
920 emit(0x0F);
921 emit(0xBD);
922 emit_modrm(dst, src);
923}
924
925void Assembler::bsrq(Register dst, Operand src) {
926 EnsureSpace ensure_space(this);
927 emit_rex_64(dst, src);
928 emit(0x0F);
929 emit(0xBD);
930 emit_operand(dst, src);
931}
932
933void Assembler::bsfl(Register dst, Register src) {
934 EnsureSpace ensure_space(this);
935 emit_optional_rex_32(dst, src);
936 emit(0x0F);
937 emit(0xBC);
938 emit_modrm(dst, src);
939}
940
941void Assembler::bsfl(Register dst, Operand src) {
942 EnsureSpace ensure_space(this);
943 emit_optional_rex_32(dst, src);
944 emit(0x0F);
945 emit(0xBC);
946 emit_operand(dst, src);
947}
948
949void Assembler::bsfq(Register dst, Register src) {
950 EnsureSpace ensure_space(this);
951 emit_rex_64(dst, src);
952 emit(0x0F);
953 emit(0xBC);
954 emit_modrm(dst, src);
955}
956
957void Assembler::bsfq(Register dst, Operand src) {
958 EnsureSpace ensure_space(this);
959 emit_rex_64(dst, src);
960 emit(0x0F);
961 emit(0xBC);
962 emit_operand(dst, src);
963}
964
965void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
966 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
967 emit(mask);
968}
969
970void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
971 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
972 emit(mask);
973}
974
975void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
976 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
977 emit(mask);
978}
979
980void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
981 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
982 emit(mask);
983}
984
985void Assembler::call(Label* L) {
986 EnsureSpace ensure_space(this);
987 // 1110 1000 #32-bit disp.
988 emit(0xE8);
989 if (L->is_bound()) {
990 int offset = L->pos() - pc_offset() - sizeof(int32_t);
991 DCHECK_LE(offset, 0)((void) 0);
992 emitl(offset);
993 } else if (L->is_linked()) {
994 emitl(L->pos());
995 L->link_to(pc_offset() - sizeof(int32_t));
996 } else {
997 DCHECK(L->is_unused())((void) 0);
998 int32_t current = pc_offset();
999 emitl(current);
1000 L->link_to(current);
1001 }
1002}
1003
1004void Assembler::call(Address entry, RelocInfo::Mode rmode) {
1005 DCHECK(RelocInfo::IsRuntimeEntry(rmode))((void) 0);
1006 EnsureSpace ensure_space(this);
1007 // 1110 1000 #32-bit disp.
1008 emit(0xE8);
1009 emit_runtime_entry(entry, rmode);
1010}
1011
1012void Assembler::call(Handle<CodeT> target, RelocInfo::Mode rmode) {
1013 DCHECK(RelocInfo::IsCodeTarget(rmode))((void) 0);
1014 DCHECK(FromCodeT(*target).IsExecutable())((void) 0);
1015 EnsureSpace ensure_space(this);
1016 // 1110 1000 #32-bit disp.
1017 emit(0xE8);
1018 RecordRelocInfo(rmode);
1019 int code_target_index = AddCodeTarget(target);
1020 emitl(code_target_index);
1021}
1022
1023void Assembler::near_call(intptr_t disp, RelocInfo::Mode rmode) {
1024 EnsureSpace ensure_space(this);
1025 emit(0xE8);
1026 DCHECK(is_int32(disp))((void) 0);
1027 RecordRelocInfo(rmode);
1028 emitl(static_cast<int32_t>(disp));
1029}
1030
1031void Assembler::near_jmp(intptr_t disp, RelocInfo::Mode rmode) {
1032 EnsureSpace ensure_space(this);
1033 emit(0xE9);
1034 DCHECK(is_int32(disp))((void) 0);
1035 if (!RelocInfo::IsNoInfo(rmode)) RecordRelocInfo(rmode);
1036 emitl(static_cast<int32_t>(disp));
1037}
1038
1039void Assembler::call(Register adr) {
1040 EnsureSpace ensure_space(this);
1041 // Opcode: FF /2 r64.
1042 emit_optional_rex_32(adr);
1043 emit(0xFF);
1044 emit_modrm(0x2, adr);
1045}
1046
1047void Assembler::call(Operand op) {
1048 EnsureSpace ensure_space(this);
1049 // Opcode: FF /2 m64.
1050 emit_optional_rex_32(op);
1051 emit(0xFF);
1052 emit_operand(0x2, op);
1053}
1054
1055void Assembler::clc() {
1056 EnsureSpace ensure_space(this);
1057 emit(0xF8);
1058}
1059
1060void Assembler::cld() {
1061 EnsureSpace ensure_space(this);
1062 emit(0xFC);
1063}
1064
1065void Assembler::cdq() {
1066 EnsureSpace ensure_space(this);
1067 emit(0x99);
1068}
1069
1070void Assembler::cmovq(Condition cc, Register dst, Register src) {
1071 if (cc == always) {
1072 movq(dst, src);
1073 } else if (cc == never) {
1074 return;
1075 }
1076 // No need to check CpuInfo for CMOV support, it's a required part of the
1077 // 64-bit architecture.
1078 DCHECK_GE(cc, 0)((void) 0); // Use mov for unconditional moves.
1079 EnsureSpace ensure_space(this);
1080 // Opcode: REX.W 0f 40 + cc /r.
1081 emit_rex_64(dst, src);
1082 emit(0x0F);
1083 emit(0x40 + cc);
1084 emit_modrm(dst, src);
1085}
1086
1087void Assembler::cmovq(Condition cc, Register dst, Operand src) {
1088 if (cc == always) {
1089 movq(dst, src);
1090 } else if (cc == never) {
1091 return;
1092 }
1093 DCHECK_GE(cc, 0)((void) 0);
1094 EnsureSpace ensure_space(this);
1095 // Opcode: REX.W 0f 40 + cc /r.
1096 emit_rex_64(dst, src);
1097 emit(0x0F);
1098 emit(0x40 + cc);
1099 emit_operand(dst, src);
1100}
1101
1102void Assembler::cmovl(Condition cc, Register dst, Register src) {
1103 if (cc == always) {
1104 movl(dst, src);
1105 } else if (cc == never) {
1106 return;
1107 }
1108 DCHECK_GE(cc, 0)((void) 0);
1109 EnsureSpace ensure_space(this);
1110 // Opcode: 0f 40 + cc /r.
1111 emit_optional_rex_32(dst, src);
1112 emit(0x0F);
1113 emit(0x40 + cc);
1114 emit_modrm(dst, src);
1115}
1116
1117void Assembler::cmovl(Condition cc, Register dst, Operand src) {
1118 if (cc == always) {
1119 movl(dst, src);
1120 } else if (cc == never) {
1121 return;
1122 }
1123 DCHECK_GE(cc, 0)((void) 0);
1124 EnsureSpace ensure_space(this);
1125 // Opcode: 0f 40 + cc /r.
1126 emit_optional_rex_32(dst, src);
1127 emit(0x0F);
1128 emit(0x40 + cc);
1129 emit_operand(dst, src);
1130}
1131
1132void Assembler::cmpb_al(Immediate imm8) {
1133 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_))((void) 0);
1134 EnsureSpace ensure_space(this);
1135 emit(0x3C);
1136 emit(imm8.value_);
1137}
1138
1139void Assembler::lock() {
1140 EnsureSpace ensure_space(this);
1141 emit(0xF0);
1142}
1143
1144void Assembler::xaddb(Operand dst, Register src) {
1145 EnsureSpace ensure_space(this);
1146 emit_optional_rex_8(src, dst);
1147 emit(0x0F);
1148 emit(0xC0);
1149 emit_operand(src, dst);
1150}
1151
1152void Assembler::xaddw(Operand dst, Register src) {
1153 EnsureSpace ensure_space(this);
1154 emit(0x66);
1155 emit_optional_rex_32(src, dst);
1156 emit(0x0F);
1157 emit(0xC1);
1158 emit_operand(src, dst);
1159}
1160
1161void Assembler::xaddl(Operand dst, Register src) {
1162 EnsureSpace ensure_space(this);
1163 emit_optional_rex_32(src, dst);
1164 emit(0x0F);
1165 emit(0xC1);
1166 emit_operand(src, dst);
1167}
1168
1169void Assembler::xaddq(Operand dst, Register src) {
1170 EnsureSpace ensure_space(this);
1171 emit_rex(src, dst, kInt64Size);
1172 emit(0x0F);
1173 emit(0xC1);
1174 emit_operand(src, dst);
1175}
1176
1177void Assembler::cmpxchgb(Operand dst, Register src) {
1178 EnsureSpace ensure_space(this);
1179 if (!src.is_byte_register()) {
1180 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1181 emit_rex_32(src, dst);
1182 } else {
1183 emit_optional_rex_32(src, dst);
1184 }
1185 emit(0x0F);
1186 emit(0xB0);
1187 emit_operand(src, dst);
1188}
1189
1190void Assembler::cmpxchgw(Operand dst, Register src) {
1191 EnsureSpace ensure_space(this);
1192 emit(0x66);
1193 emit_optional_rex_32(src, dst);
1194 emit(0x0F);
1195 emit(0xB1);
1196 emit_operand(src, dst);
1197}
1198
1199void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
1200 EnsureSpace ensure_space(this);
1201 emit_rex(src, dst, size);
1202 emit(0x0F);
1203 emit(0xB1);
1204 emit_operand(src, dst);
1205}
1206
1207void Assembler::mfence() {
1208 EnsureSpace ensure_space(this);
1209 emit(0x0F);
1210 emit(0xAE);
1211 emit(0xF0);
1212}
1213
1214void Assembler::lfence() {
1215 EnsureSpace ensure_space(this);
1216 emit(0x0F);
1217 emit(0xAE);
1218 emit(0xE8);
1219}
1220
1221void Assembler::cpuid() {
1222 EnsureSpace ensure_space(this);
1223 emit(0x0F);
1224 emit(0xA2);
1225}
1226
1227void Assembler::cqo() {
1228 EnsureSpace ensure_space(this);
1229 emit_rex_64();
1230 emit(0x99);
1231}
1232
1233void Assembler::emit_dec(Register dst, int size) {
1234 EnsureSpace ensure_space(this);
1235 emit_rex(dst, size);
1236 emit(0xFF);
1237 emit_modrm(0x1, dst);
1238}
1239
1240void Assembler::emit_dec(Operand dst, int size) {
1241 EnsureSpace ensure_space(this);
1242 emit_rex(dst, size);
1243 emit(0xFF);
1244 emit_operand(1, dst);
1245}
1246
1247void Assembler::decb(Register dst) {
1248 EnsureSpace ensure_space(this);
1249 if (!dst.is_byte_register()) {
1250 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1251 emit_rex_32(dst);
1252 }
1253 emit(0xFE);
1254 emit_modrm(0x1, dst);
1255}
1256
1257void Assembler::decb(Operand dst) {
1258 EnsureSpace ensure_space(this);
1259 emit_optional_rex_32(dst);
1260 emit(0xFE);
1261 emit_operand(1, dst);
1262}
1263
1264void Assembler::hlt() {
1265 EnsureSpace ensure_space(this);
1266 emit(0xF4);
1267}
1268
1269void Assembler::emit_idiv(Register src, int size) {
1270 EnsureSpace ensure_space(this);
1271 emit_rex(src, size);
1272 emit(0xF7);
1273 emit_modrm(0x7, src);
1274}
1275
1276void Assembler::emit_div(Register src, int size) {
1277 EnsureSpace ensure_space(this);
1278 emit_rex(src, size);
1279 emit(0xF7);
1280 emit_modrm(0x6, src);
1281}
1282
1283void Assembler::emit_imul(Register src, int size) {
1284 EnsureSpace ensure_space(this);
1285 emit_rex(src, size);
1286 emit(0xF7);
1287 emit_modrm(0x5, src);
1288}
1289
1290void Assembler::emit_imul(Operand src, int size) {
1291 EnsureSpace ensure_space(this);
1292 emit_rex(src, size);
1293 emit(0xF7);
1294 emit_operand(0x5, src);
1295}
1296
1297void Assembler::emit_imul(Register dst, Register src, int size) {
1298 EnsureSpace ensure_space(this);
1299 emit_rex(dst, src, size);
1300 emit(0x0F);
1301 emit(0xAF);
1302 emit_modrm(dst, src);
1303}
1304
1305void Assembler::emit_imul(Register dst, Operand src, int size) {
1306 EnsureSpace ensure_space(this);
1307 emit_rex(dst, src, size);
1308 emit(0x0F);
1309 emit(0xAF);
1310 emit_operand(dst, src);
1311}
1312
1313void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1314 EnsureSpace ensure_space(this);
1315 emit_rex(dst, src, size);
1316 if (is_int8(imm.value_)) {
1317 emit(0x6B);
1318 emit_modrm(dst, src);
1319 emit(imm.value_);
1320 } else {
1321 emit(0x69);
1322 emit_modrm(dst, src);
1323 emitl(imm.value_);
1324 }
1325}
1326
1327void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
1328 EnsureSpace ensure_space(this);
1329 emit_rex(dst, src, size);
1330 if (is_int8(imm.value_)) {
1331 emit(0x6B);
1332 emit_operand(dst, src);
1333 emit(imm.value_);
1334 } else {
1335 emit(0x69);
1336 emit_operand(dst, src);
1337 emitl(imm.value_);
1338 }
1339}
1340
1341void Assembler::emit_inc(Register dst, int size) {
1342 EnsureSpace ensure_space(this);
1343 emit_rex(dst, size);
1344 emit(0xFF);
1345 emit_modrm(0x0, dst);
1346}
1347
1348void Assembler::emit_inc(Operand dst, int size) {
1349 EnsureSpace ensure_space(this);
1350 emit_rex(dst, size);
1351 emit(0xFF);
1352 emit_operand(0, dst);
1353}
1354
1355void Assembler::int3() {
1356 EnsureSpace ensure_space(this);
1357 emit(0xCC);
1358}
1359
1360void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1361 if (cc == always) {
1362 jmp(L, distance);
1363 return;
1364 } else if (cc == never) {
1365 return;
1366 }
1367 EnsureSpace ensure_space(this);
1368 DCHECK(is_uint4(cc))((void) 0);
1369 if (L->is_bound()) {
1370 const int short_size = 2;
1371 const int long_size = 6;
1372 int offs = L->pos() - pc_offset();
1373 DCHECK_LE(offs, 0)((void) 0);
1374 // Determine whether we can use 1-byte offsets for backwards branches,
1375 // which have a max range of 128 bytes.
1376
1377 // We also need to check predictable_code_size() flag here, because on x64,
1378 // when the full code generator recompiles code for debugging, some places
1379 // need to be padded out to a certain size. The debugger is keeping track of
1380 // how often it did this so that it can adjust return addresses on the
1381 // stack, but if the size of jump instructions can also change, that's not
1382 // enough and the calculated offsets would be incorrect.
1383 if (is_int8(offs - short_size) && !predictable_code_size()) {
1384 // 0111 tttn #8-bit disp.
1385 emit(0x70 | cc);
1386 emit((offs - short_size) & 0xFF);
1387 } else {
1388 // 0000 1111 1000 tttn #32-bit disp.
1389 emit(0x0F);
1390 emit(0x80 | cc);
1391 emitl(offs - long_size);
1392 }
1393 } else if (distance == Label::kNear) {
1394 // 0111 tttn #8-bit disp
1395 emit(0x70 | cc);
1396 byte disp = 0x00;
1397 if (L->is_near_linked()) {
1398 int offset = L->near_link_pos() - pc_offset();
1399 DCHECK(is_int8(offset))((void) 0);
1400 disp = static_cast<byte>(offset & 0xFF);
1401 }
1402 L->link_to(pc_offset(), Label::kNear);
1403 emit(disp);
1404 } else {
1405 auto jump_opt = jump_optimization_info();
1406 if (V8_UNLIKELY(jump_opt)(__builtin_expect(!!(jump_opt), 0))) {
1407 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1408 // 0111 tttn #8-bit disp
1409 emit(0x70 | cc);
1410 record_farjmp_position(L, pc_offset());
1411 emit(0);
1412 return;
1413 }
1414 if (jump_opt->is_collecting()) {
1415 farjmp_positions_.push_back(pc_offset() + 2);
1416 }
1417 }
1418 if (L->is_linked()) {
1419 // 0000 1111 1000 tttn #32-bit disp.
1420 emit(0x0F);
1421 emit(0x80 | cc);
1422 emitl(L->pos());
1423 L->link_to(pc_offset() - sizeof(int32_t));
1424 } else {
1425 DCHECK(L->is_unused())((void) 0);
1426 emit(0x0F);
1427 emit(0x80 | cc);
1428 int32_t current = pc_offset();
1429 emitl(current);
1430 L->link_to(current);
1431 }
1432 }
1433}
1434
1435void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1436 DCHECK(RelocInfo::IsWasmStubCall(rmode))((void) 0);
1437 EnsureSpace ensure_space(this);
1438 DCHECK(is_uint4(cc))((void) 0);
1439 emit(0x0F);
1440 emit(0x80 | cc);
1441 RecordRelocInfo(rmode);
1442 emitl(static_cast<int32_t>(entry));
1443}
1444
1445void Assembler::j(Condition cc, Handle<CodeT> target, RelocInfo::Mode rmode) {
1446 if (cc == always) {
1447 jmp(target, rmode);
1448 return;
1449 } else if (cc == never) {
1450 return;
1451 }
1452 EnsureSpace ensure_space(this);
1453 DCHECK(is_uint4(cc))((void) 0);
1454 // 0000 1111 1000 tttn #32-bit disp.
1455 emit(0x0F);
1456 emit(0x80 | cc);
1457 DCHECK(RelocInfo::IsCodeTarget(rmode))((void) 0);
1458 RecordRelocInfo(rmode);
1459 int code_target_index = AddCodeTarget(target);
1460 emitl(code_target_index);
1461}
1462
1463void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1464 DCHECK(RelocInfo::IsRuntimeEntry(rmode))((void) 0);
1465 EnsureSpace ensure_space(this);
1466 // 1110 1001 #32-bit disp.
1467 emit(0xE9);
1468 emit_runtime_entry(entry, rmode);
1469}
1470
1471void Assembler::jmp_rel(int32_t offset) {
1472 EnsureSpace ensure_space(this);
1473 // The offset is encoded relative to the next instruction.
1474 constexpr int32_t kShortJmpDisplacement = 1 + sizeof(int8_t);
1475 constexpr int32_t kNearJmpDisplacement = 1 + sizeof(int32_t);
1476 DCHECK_LE(std::numeric_limits<int32_t>::min() + kNearJmpDisplacement, offset)((void) 0);
1477 if (is_int8(offset - kShortJmpDisplacement) && !predictable_code_size()) {
1478 // 0xEB #8-bit disp.
1479 emit(0xEB);
1480 emit(offset - kShortJmpDisplacement);
1481 } else {
1482 // 0xE9 #32-bit disp.
1483 emit(0xE9);
1484 emitl(offset - kNearJmpDisplacement);
1485 }
1486}
1487
1488void Assembler::jmp(Label* L, Label::Distance distance) {
1489 const int long_size = sizeof(int32_t);
1490
1491 if (L->is_bound()) {
1492 int offset = L->pos() - pc_offset();
1493 DCHECK_LE(offset, 0)((void) 0); // backward jump.
1494 jmp_rel(offset);
1495 return;
1496 }
1497
1498 EnsureSpace ensure_space(this);
1499 if (distance == Label::kNear) {
1500 emit(0xEB);
1501 byte disp = 0x00;
1502 if (L->is_near_linked()) {
1503 int offset = L->near_link_pos() - pc_offset();
1504 DCHECK(is_int8(offset))((void) 0);
1505 disp = static_cast<byte>(offset & 0xFF);
1506 }
1507 L->link_to(pc_offset(), Label::kNear);
1508 emit(disp);
1509 } else {
1510 auto jump_opt = jump_optimization_info();
1511 if (V8_UNLIKELY(jump_opt)(__builtin_expect(!!(jump_opt), 0))) {
1512 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1513 emit(0xEB);
1514 record_farjmp_position(L, pc_offset());
1515 emit(0);
1516 return;
1517 }
1518 if (jump_opt->is_collecting()) {
1519 farjmp_positions_.push_back(pc_offset() + 1);
1520 }
1521 }
1522 if (L->is_linked()) {
1523 // 1110 1001 #32-bit disp.
1524 emit(0xE9);
1525 emitl(L->pos());
1526 L->link_to(pc_offset() - long_size);
1527 } else {
1528 // 1110 1001 #32-bit disp.
1529 DCHECK(L->is_unused())((void) 0);
1530 emit(0xE9);
1531 int32_t current = pc_offset();
1532 emitl(current);
1533 L->link_to(current);
1534 }
1535 }
1536}
1537
1538void Assembler::jmp(Handle<CodeT> target, RelocInfo::Mode rmode) {
1539 DCHECK(RelocInfo::IsCodeTarget(rmode))((void) 0);
1540 EnsureSpace ensure_space(this);
1541 // 1110 1001 #32-bit disp.
1542 emit(0xE9);
1543 RecordRelocInfo(rmode);
1544 int code_target_index = AddCodeTarget(target);
1545 emitl(code_target_index);
1546}
1547
1548void Assembler::jmp(Register target) {
1549 EnsureSpace ensure_space(this);
1550 // Opcode FF/4 r64.
1551 emit_optional_rex_32(target);
1552 emit(0xFF);
1553 emit_modrm(0x4, target);
1554}
1555
1556void Assembler::jmp(Operand src) {
1557 EnsureSpace ensure_space(this);
1558 // Opcode FF/4 m64.
1559 emit_optional_rex_32(src);
1560 emit(0xFF);
1561 emit_operand(0x4, src);
1562}
1563
1564void Assembler::emit_lea(Register dst, Operand src, int size) {
1565 EnsureSpace ensure_space(this);
1566 emit_rex(dst, src, size);
1567 emit(0x8D);
1568 emit_operand(dst, src);
1569}
1570
1571void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
1572 EnsureSpace ensure_space(this);
1573 emit(0x48); // REX.W
1574 emit(0xA1);
1575 emit(Immediate64(value, mode));
1576}
1577
1578void Assembler::load_rax(ExternalReference ref) {
1579 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1580}
1581
1582void Assembler::leave() {
1583 EnsureSpace ensure_space(this);
1584 emit(0xC9);
1585}
1586
1587void Assembler::movb(Register dst, Operand src) {
1588 EnsureSpace ensure_space(this);
1589 if (!dst.is_byte_register()) {
1590 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1591 emit_rex_32(dst, src);
1592 } else {
1593 emit_optional_rex_32(dst, src);
1594 }
1595 emit(0x8A);
1596 emit_operand(dst, src);
1597}
1598
1599void Assembler::movb(Register dst, Immediate imm) {
1600 EnsureSpace ensure_space(this);
1601 if (!dst.is_byte_register()) {
1602 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1603 emit_rex_32(dst);
1604 }
1605 emit(0xB0 + dst.low_bits());
1606 emit(imm.value_);
1607}
1608
1609void Assembler::movb(Operand dst, Register src) {
1610 EnsureSpace ensure_space(this);
1611 if (!src.is_byte_register()) {
1612 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1613 emit_rex_32(src, dst);
1614 } else {
1615 emit_optional_rex_32(src, dst);
1616 }
1617 emit(0x88);
1618 emit_operand(src, dst);
1619}
1620
1621void Assembler::movb(Operand dst, Immediate imm) {
1622 EnsureSpace ensure_space(this);
1623 emit_optional_rex_32(dst);
1624 emit(0xC6);
1625 emit_operand(0x0, dst);
1626 emit(static_cast<byte>(imm.value_));
1627}
1628
1629void Assembler::movw(Register dst, Operand src) {
1630 EnsureSpace ensure_space(this);
1631 emit(0x66);
1632 emit_optional_rex_32(dst, src);
1633 emit(0x8B);
1634 emit_operand(dst, src);
1635}
1636
1637void Assembler::movw(Operand dst, Register src) {
1638 EnsureSpace ensure_space(this);
1639 emit(0x66);
1640 emit_optional_rex_32(src, dst);
1641 emit(0x89);
1642 emit_operand(src, dst);
1643}
1644
1645void Assembler::movw(Operand dst, Immediate imm) {
1646 EnsureSpace ensure_space(this);
1647 emit(0x66);
1648 emit_optional_rex_32(dst);
1649 emit(0xC7);
1650 emit_operand(0x0, dst);
1651 emit(static_cast<byte>(imm.value_ & 0xFF));
1652 emit(static_cast<byte>(imm.value_ >> 8));
1653}
1654
1655void Assembler::emit_mov(Register dst, Operand src, int size) {
1656 EnsureSpace ensure_space(this);
1657 emit_rex(dst, src, size);
1658 emit(0x8B);
1659 emit_operand(dst, src);
1660}
1661
1662void Assembler::emit_mov(Register dst, Register src, int size) {
1663 EnsureSpace ensure_space(this);
1664 if (src.low_bits() == 4) {
1665 emit_rex(src, dst, size);
1666 emit(0x89);
1667 emit_modrm(src, dst);
1668 } else {
1669 emit_rex(dst, src, size);
1670 emit(0x8B);
1671 emit_modrm(dst, src);
1672 }
1673
1674#if defined(V8_OS_WIN_X64)
1675 if (xdata_encoder_ && dst == rbp && src == rsp) {
1676 xdata_encoder_->onMovRbpRsp();
1677 }
1678#endif
1679}
1680
1681void Assembler::emit_mov(Operand dst, Register src, int size) {
1682 EnsureSpace ensure_space(this);
1683 emit_rex(src, dst, size);
1684 emit(0x89);
1685 emit_operand(src, dst);
1686}
1687
1688void Assembler::emit_mov(Register dst, Immediate value, int size) {
1689 EnsureSpace ensure_space(this);
1690 emit_rex(dst, size);
1691 if (size == kInt64Size) {
1692 emit(0xC7);
1693 emit_modrm(0x0, dst);
1694 } else {
1695 DCHECK_EQ(size, kInt32Size)((void) 0);
1696 emit(0xB8 + dst.low_bits());
1697 }
1698 emit(value);
1699}
1700
1701void Assembler::emit_mov(Operand dst, Immediate value, int size) {
1702 EnsureSpace ensure_space(this);
1703 emit_rex(dst, size);
1704 emit(0xC7);
1705 emit_operand(0x0, dst);
1706 emit(value);
1707}
1708
1709void Assembler::emit_mov(Register dst, Immediate64 value, int size) {
1710 DCHECK_EQ(size, kInt64Size)((void) 0);
1711 if (constpool_.TryRecordEntry(value.value_, value.rmode_)) {
1712 // Emit rip-relative move with offset = 0
1713 Label label;
1714 emit_mov(dst, Operand(&label, 0), size);
1715 bind(&label);
1716 } else {
1717 EnsureSpace ensure_space(this);
1718 emit_rex(dst, size);
1719 emit(0xB8 | dst.low_bits());
1720 emit(value);
1721 }
1722}
1723
1724void Assembler::movq_imm64(Register dst, int64_t value) {
1725 EnsureSpace ensure_space(this);
1726 emit_rex(dst, kInt64Size);
1727 emit(0xB8 | dst.low_bits());
1728 emitq(static_cast<uint64_t>(value));
1729}
1730
1731void Assembler::movq_heap_number(Register dst, double value) {
1732 EnsureSpace ensure_space(this);
1733 emit_rex(dst, kInt64Size);
1734 emit(0xB8 | dst.low_bits());
1735 RequestHeapObject(HeapObjectRequest(value));
1736 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1737}
1738
1739void Assembler::movq_string(Register dst, const StringConstantBase* str) {
1740 EnsureSpace ensure_space(this);
1741 emit_rex(dst, kInt64Size);
1742 emit(0xB8 | dst.low_bits());
1743 RequestHeapObject(HeapObjectRequest(str));
1744 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1745}
1746
1747// Loads the ip-relative location of the src label into the target location
1748// (as a 32-bit offset sign extended to 64-bit).
1749void Assembler::movl(Operand dst, Label* src) {
1750 EnsureSpace ensure_space(this);
1751 emit_optional_rex_32(dst);
1752 emit(0xC7);
1753 emit_operand(0, dst);
1754 if (src->is_bound()) {
1755 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1756 DCHECK_LE(offset, 0)((void) 0);
1757 emitl(offset);
1758 } else if (src->is_linked()) {
1759 emitl(src->pos());
1760 src->link_to(pc_offset() - sizeof(int32_t));
1761 } else {
1762 DCHECK(src->is_unused())((void) 0);
1763 int32_t current = pc_offset();
1764 emitl(current);
1765 src->link_to(current);
1766 }
1767}
1768
1769void Assembler::movsxbl(Register dst, Register src) {
1770 EnsureSpace ensure_space(this);
1771 if (!src.is_byte_register()) {
1772 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1773 emit_rex_32(dst, src);
1774 } else {
1775 emit_optional_rex_32(dst, src);
1776 }
1777 emit(0x0F);
1778 emit(0xBE);
1779 emit_modrm(dst, src);
1780}
1781
1782void Assembler::movsxbl(Register dst, Operand src) {
1783 EnsureSpace ensure_space(this);
1784 emit_optional_rex_32(dst, src);
1785 emit(0x0F);
1786 emit(0xBE);
1787 emit_operand(dst, src);
1788}
1789
1790void Assembler::movsxbq(Register dst, Operand src) {
1791 EnsureSpace ensure_space(this);
1792 emit_rex_64(dst, src);
1793 emit(0x0F);
1794 emit(0xBE);
1795 emit_operand(dst, src);
1796}
1797
1798void Assembler::movsxbq(Register dst, Register src) {
1799 EnsureSpace ensure_space(this);
1800 emit_rex_64(dst, src);
1801 emit(0x0F);
1802 emit(0xBE);
1803 emit_modrm(dst, src);
1804}
1805
1806void Assembler::movsxwl(Register dst, Register src) {
1807 EnsureSpace ensure_space(this);
1808 emit_optional_rex_32(dst, src);
1809 emit(0x0F);
1810 emit(0xBF);
1811 emit_modrm(dst, src);
1812}
1813
1814void Assembler::movsxwl(Register dst, Operand src) {
1815 EnsureSpace ensure_space(this);
1816 emit_optional_rex_32(dst, src);
1817 emit(0x0F);
1818 emit(0xBF);
1819 emit_operand(dst, src);
1820}
1821
1822void Assembler::movsxwq(Register dst, Operand src) {
1823 EnsureSpace ensure_space(this);
1824 emit_rex_64(dst, src);
1825 emit(0x0F);
1826 emit(0xBF);
1827 emit_operand(dst, src);
1828}
1829
1830void Assembler::movsxwq(Register dst, Register src) {
1831 EnsureSpace ensure_space(this);
1832 emit_rex_64(dst, src);
1833 emit(0x0F);
1834 emit(0xBF);
1835 emit_modrm(dst, src);
1836}
1837
1838void Assembler::movsxlq(Register dst, Register src) {
1839 EnsureSpace ensure_space(this);
1840 emit_rex_64(dst, src);
1841 emit(0x63);
1842 emit_modrm(dst, src);
1843}
1844
1845void Assembler::movsxlq(Register dst, Operand src) {
1846 EnsureSpace ensure_space(this);
1847 emit_rex_64(dst, src);
1848 emit(0x63);
1849 emit_operand(dst, src);
1850}
1851
1852void Assembler::emit_movzxb(Register dst, Operand src, int size) {
1853 EnsureSpace ensure_space(this);
1854 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1855 // there is no need to make this a 64 bit operation.
1856 emit_optional_rex_32(dst, src);
1857 emit(0x0F);
1858 emit(0xB6);
1859 emit_operand(dst, src);
1860}
1861
1862void Assembler::emit_movzxb(Register dst, Register src, int size) {
1863 EnsureSpace ensure_space(this);
1864 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1865 // there is no need to make this a 64 bit operation.
1866 if (!src.is_byte_register()) {
1867 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1868 emit_rex_32(dst, src);
1869 } else {
1870 emit_optional_rex_32(dst, src);
1871 }
1872 emit(0x0F);
1873 emit(0xB6);
1874 emit_modrm(dst, src);
1875}
1876
1877void Assembler::emit_movzxw(Register dst, Operand src, int size) {
1878 EnsureSpace ensure_space(this);
1879 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1880 // there is no need to make this a 64 bit operation.
1881 emit_optional_rex_32(dst, src);
1882 emit(0x0F);
1883 emit(0xB7);
1884 emit_operand(dst, src);
1885}
1886
1887void Assembler::emit_movzxw(Register dst, Register src, int size) {
1888 EnsureSpace ensure_space(this);
1889 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1890 // there is no need to make this a 64 bit operation.
1891 emit_optional_rex_32(dst, src);
1892 emit(0x0F);
1893 emit(0xB7);
1894 emit_modrm(dst, src);
1895}
1896
1897void Assembler::repmovsb() {
1898 EnsureSpace ensure_space(this);
1899 emit(0xF3);
1900 emit(0xA4);
1901}
1902
1903void Assembler::repmovsw() {
1904 EnsureSpace ensure_space(this);
1905 emit(0x66); // Operand size override.
1906 emit(0xF3);
1907 emit(0xA4);
1908}
1909
1910void Assembler::emit_repmovs(int size) {
1911 EnsureSpace ensure_space(this);
1912 emit(0xF3);
1913 emit_rex(size);
1914 emit(0xA5);
1915}
1916
1917void Assembler::repstosl() {
1918 EnsureSpace ensure_space(this);
1919 emit(0xF3);
1920 emit(0xAB);
1921}
1922
1923void Assembler::repstosq() {
1924 EnsureSpace ensure_space(this);
1925 emit(0xF3);
1926 emit_rex_64();
1927 emit(0xAB);
1928}
1929
1930void Assembler::mull(Register src) {
1931 EnsureSpace ensure_space(this);
1932 emit_optional_rex_32(src);
1933 emit(0xF7);
1934 emit_modrm(0x4, src);
1935}
1936
1937void Assembler::mull(Operand src) {
1938 EnsureSpace ensure_space(this);
1939 emit_optional_rex_32(src);
1940 emit(0xF7);
1941 emit_operand(0x4, src);
1942}
1943
1944void Assembler::mulq(Register src) {
1945 EnsureSpace ensure_space(this);
1946 emit_rex_64(src);
1947 emit(0xF7);
1948 emit_modrm(0x4, src);
1949}
1950
1951void Assembler::negb(Register reg) {
1952 EnsureSpace ensure_space(this);
1953 emit_optional_rex_8(reg);
1954 emit(0xF6);
1955 emit_modrm(0x3, reg);
1956}
1957
1958void Assembler::negw(Register reg) {
1959 EnsureSpace ensure_space(this);
1960 emit(0x66);
1961 emit_optional_rex_32(reg);
1962 emit(0xF7);
1963 emit_modrm(0x3, reg);
1964}
1965
1966void Assembler::negl(Register reg) {
1967 EnsureSpace ensure_space(this);
1968 emit_optional_rex_32(reg);
1969 emit(0xF7);
1970 emit_modrm(0x3, reg);
1971}
1972
1973void Assembler::negq(Register reg) {
1974 EnsureSpace ensure_space(this);
1975 emit_rex_64(reg);
1976 emit(0xF7);
1977 emit_modrm(0x3, reg);
1978}
1979
1980void Assembler::negb(Operand op) {
1981 EnsureSpace ensure_space(this);
1982 emit_optional_rex_32(op);
1983 emit(0xF6);
1984 emit_operand(0x3, op);
1985}
1986
1987void Assembler::negw(Operand op) {
1988 EnsureSpace ensure_space(this);
1989 emit(0x66);
1990 emit_optional_rex_32(op);
1991 emit(0xF7);
1992 emit_operand(0x3, op);
1993}
1994
1995void Assembler::negl(Operand op) {
1996 EnsureSpace ensure_space(this);
1997 emit_optional_rex_32(op);
1998 emit(0xF7);
1999 emit_operand(0x3, op);
2000}
2001
2002void Assembler::negq(Operand op) {
2003 EnsureSpace ensure_space(this);
2004 emit_rex_64(op);
2005 emit(0xF7);
2006 emit_operand(0x3, op);
2007}
2008
2009void Assembler::nop() {
2010 EnsureSpace ensure_space(this);
2011 emit(0x90);
2012}
2013
2014void Assembler::emit_not(Register dst, int size) {
2015 EnsureSpace ensure_space(this);
2016 emit_rex(dst, size);
2017 emit(0xF7);
2018 emit_modrm(0x2, dst);
2019}
2020
2021void Assembler::emit_not(Operand dst, int size) {
2022 EnsureSpace ensure_space(this);
2023 emit_rex(dst, size);
2024 emit(0xF7);
2025 emit_operand(2, dst);
2026}
2027
2028void Assembler::Nop(int n) {
2029 DCHECK_LE(0, n)((void) 0);
2030 // The recommended muti-byte sequences of NOP instructions from the Intel 64
2031 // and IA-32 Architectures Software Developer's Manual.
2032 //
2033 // Len Assembly Byte Sequence
2034 // 2 66 NOP 66 90H
2035 // 3 NOP DWORD ptr [EAX] 0F 1F 00H
2036 // 4 NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
2037 // 5 NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
2038 // 6 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
2039 // 7 NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
2040 // 8 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
2041 // 9 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 66 0F 1F 84 00 00 00 00 00H
2042
2043 constexpr const char* kNopSequences =
2044 "\x66\x90" // length 1 (@1) / 2 (@0)
2045 "\x0F\x1F\x00" // length 3 (@2)
2046 "\x0F\x1F\x40\x00" // length 4 (@5)
2047 "\x66\x0F\x1F\x44\x00\x00" // length 5 (@10) / 6 (@9)
2048 "\x0F\x1F\x80\x00\x00\x00\x00" // length 7 (@15)
2049 "\x66\x0F\x1F\x84\x00\x00\x00\x00\x00"; // length 8 (@23) / 9 (@22)
2050 constexpr int8_t kNopOffsets[10] = {0, 1, 0, 2, 5, 10, 9, 15, 23, 22};
2051
2052 do {
2053 EnsureSpace ensure_space(this);
2054 int nop_bytes = std::min(n, 9);
2055 const char* sequence = kNopSequences + kNopOffsets[nop_bytes];
2056 memcpy(pc_, sequence, nop_bytes);
2057 pc_ += nop_bytes;
2058 n -= nop_bytes;
2059 } while (n);
2060}
2061
2062void Assembler::popq(Register dst) {
2063 EnsureSpace ensure_space(this);
2064 emit_optional_rex_32(dst);
2065 emit(0x58 | dst.low_bits());
2066}
2067
2068void Assembler::popq(Operand dst) {
2069 EnsureSpace ensure_space(this);
2070 emit_optional_rex_32(dst);
2071 emit(0x8F);
2072 emit_operand(0, dst);
2073}
2074
2075void Assembler::popfq() {
2076 EnsureSpace ensure_space(this);
2077 emit(0x9D);
2078}
2079
2080void Assembler::pushq(Register src) {
2081 EnsureSpace ensure_space(this);
2082 emit_optional_rex_32(src);
2083 emit(0x50 | src.low_bits());
2084
2085#if defined(V8_OS_WIN_X64)
2086 if (xdata_encoder_ && src == rbp) {
2087 xdata_encoder_->onPushRbp();
2088 }
2089#endif
2090}
2091
2092void Assembler::pushq(Operand src) {
2093 EnsureSpace ensure_space(this);
2094 emit_optional_rex_32(src);
2095 emit(0xFF);
2096 emit_operand(6, src);
2097}
2098
2099void Assembler::pushq(Immediate value) {
2100 EnsureSpace ensure_space(this);
2101 if (is_int8(value.value_)) {
2102 emit(0x6A);
2103 emit(value.value_); // Emit low byte of value.
2104 } else {
2105 emit(0x68);
2106 emitl(value.value_);
2107 }
2108}
2109
2110void Assembler::pushq_imm32(int32_t imm32) {
2111 EnsureSpace ensure_space(this);
2112 emit(0x68);
2113 emitl(imm32);
2114}
2115
2116void Assembler::pushfq() {
2117 EnsureSpace ensure_space(this);
2118 emit(0x9C);
2119}
2120
2121void Assembler::incsspq(Register number_of_words) {
2122 EnsureSpace ensure_space(this);
2123 emit(0xF3);
2124 emit_rex_64(number_of_words);
2125 emit(0x0F);
2126 emit(0xAE);
2127 emit(0xE8 | number_of_words.low_bits());
2128}
2129
2130void Assembler::ret(int imm16) {
2131 EnsureSpace ensure_space(this);
2132 DCHECK(is_uint16(imm16))((void) 0);
2133 if (imm16 == 0) {
2134 emit(0xC3);
2135 } else {
2136 emit(0xC2);
2137 emit(imm16 & 0xFF);
2138 emit((imm16 >> 8) & 0xFF);
2139 }
2140}
2141
2142void Assembler::ud2() {
2143 EnsureSpace ensure_space(this);
2144 emit(0x0F);
2145 emit(0x0B);
2146}
2147
2148void Assembler::setcc(Condition cc, Register reg) {
2149 if (cc > last_condition) {
2150 movb(reg, Immediate(cc == always ? 1 : 0));
2151 return;
2152 }
2153 EnsureSpace ensure_space(this);
2154 DCHECK(is_uint4(cc))((void) 0);
2155 if (!reg.is_byte_register()) {
2156 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2157 emit_rex_32(reg);
2158 }
2159 emit(0x0F);
2160 emit(0x90 | cc);
2161 emit_modrm(0x0, reg);
2162}
2163
2164void Assembler::shld(Register dst, Register src) {
2165 EnsureSpace ensure_space(this);
2166 emit_rex_64(src, dst);
2167 emit(0x0F);
2168 emit(0xA5);
2169 emit_modrm(src, dst);
2170}
2171
2172void Assembler::shrd(Register dst, Register src) {
2173 EnsureSpace ensure_space(this);
2174 emit_rex_64(src, dst);
2175 emit(0x0F);
2176 emit(0xAD);
2177 emit_modrm(src, dst);
2178}
2179
2180void Assembler::xchgb(Register reg, Operand op) {
2181 EnsureSpace ensure_space(this);
2182 if (!reg.is_byte_register()) {
2183 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2184 emit_rex_32(reg, op);
2185 } else {
2186 emit_optional_rex_32(reg, op);
2187 }
2188 emit(0x86);
2189 emit_operand(reg, op);
2190}
2191
2192void Assembler::xchgw(Register reg, Operand op) {
2193 EnsureSpace ensure_space(this);
2194 emit(0x66);
2195 emit_optional_rex_32(reg, op);
2196 emit(0x87);
2197 emit_operand(reg, op);
2198}
2199
2200void Assembler::emit_xchg(Register dst, Register src, int size) {
2201 EnsureSpace ensure_space(this);
2202 if (src == rax || dst == rax) { // Single-byte encoding
2203 Register other = src == rax ? dst : src;
2204 emit_rex(other, size);
2205 emit(0x90 | other.low_bits());
2206 } else if (dst.low_bits() == 4) {
2207 emit_rex(dst, src, size);
2208 emit(0x87);
2209 emit_modrm(dst, src);
2210 } else {
2211 emit_rex(src, dst, size);
2212 emit(0x87);
2213 emit_modrm(src, dst);
2214 }
2215}
2216
2217void Assembler::emit_xchg(Register dst, Operand src, int size) {
2218 EnsureSpace ensure_space(this);
2219 emit_rex(dst, src, size);
2220 emit(0x87);
2221 emit_operand(dst, src);
2222}
2223
2224void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
2225 EnsureSpace ensure_space(this);
2226 emit(0x48); // REX.W
2227 emit(0xA3);
2228 emit(Immediate64(dst, mode));
2229}
2230
2231void Assembler::store_rax(ExternalReference ref) {
2232 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2233}
2234
2235void Assembler::sub_sp_32(uint32_t imm) {
2236 emit_rex_64();
2237 emit(0x81); // using a literal 32-bit immediate.
2238 emit_modrm(0x5, rsp);
2239 emitl(imm);
2240}
2241
2242void Assembler::testb(Register dst, Register src) {
2243 EnsureSpace ensure_space(this);
2244 emit_test(dst, src, sizeof(int8_t));
2245}
2246
2247void Assembler::testb(Register reg, Immediate mask) {
2248 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_))((void) 0);
2249 emit_test(reg, mask, sizeof(int8_t));
2250}
2251
2252void Assembler::testb(Operand op, Immediate mask) {
2253 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_))((void) 0);
2254 emit_test(op, mask, sizeof(int8_t));
2255}
2256
2257void Assembler::testb(Operand op, Register reg) {
2258 emit_test(op, reg, sizeof(int8_t));
2259}
2260
2261void Assembler::testw(Register dst, Register src) {
2262 emit_test(dst, src, sizeof(uint16_t));
2263}
2264
2265void Assembler::testw(Register reg, Immediate mask) {
2266 emit_test(reg, mask, sizeof(int16_t));
2267}
2268
2269void Assembler::testw(Operand op, Immediate mask) {
2270 emit_test(op, mask, sizeof(int16_t));
2271}
2272
2273void Assembler::testw(Operand op, Register reg) {
2274 emit_test(op, reg, sizeof(int16_t));
2275}
2276
2277void Assembler::emit_test(Register dst, Register src, int size) {
2278 EnsureSpace ensure_space(this);
2279 if (src.low_bits() == 4) std::swap(dst, src);
2280 if (size == sizeof(int16_t)) {
2281 emit(0x66);
2282 size = sizeof(int32_t);
2283 }
2284 bool byte_operand = size == sizeof(int8_t);
2285 if (byte_operand) {
2286 size = sizeof(int32_t);
2287 if (!src.is_byte_register() || !dst.is_byte_register()) {
2288 emit_rex_32(dst, src);
2289 }
2290 } else {
2291 emit_rex(dst, src, size);
2292 }
2293 emit(byte_operand ? 0x84 : 0x85);
2294 emit_modrm(dst, src);
2295}
2296
2297void Assembler::emit_test(Register reg, Immediate mask, int size) {
2298 if (is_uint8(mask.value_)) {
2299 size = sizeof(int8_t);
2300 } else if (is_uint16(mask.value_)) {
2301 size = sizeof(int16_t);
2302 }
2303 EnsureSpace ensure_space(this);
2304 bool half_word = size == sizeof(int16_t);
2305 if (half_word) {
2306 emit(0x66);
2307 size = sizeof(int32_t);
2308 }
2309 bool byte_operand = size == sizeof(int8_t);
2310 if (byte_operand) {
2311 size = sizeof(int32_t);
Value stored to 'size' is never read
2312 if (!reg.is_byte_register()) emit_rex_32(reg);
2313 } else {
2314 emit_rex(reg, size);
2315 }
2316 if (reg == rax) {
2317 emit(byte_operand ? 0xA8 : 0xA9);
2318 } else {
2319 emit(byte_operand ? 0xF6 : 0xF7);
2320 emit_modrm(0x0, reg);
2321 }
2322 if (byte_operand) {
2323 emit(mask.value_);
2324 } else if (half_word) {
2325 emitw(mask.value_);
2326 } else {
2327 emit(mask);
2328 }
2329}
2330
2331void Assembler::emit_test(Operand op, Immediate mask, int size) {
2332 if (is_uint8(mask.value_)) {
2333 size = sizeof(int8_t);
2334 } else if (is_uint16(mask.value_)) {
2335 size = sizeof(int16_t);
2336 }
2337 EnsureSpace ensure_space(this);
2338 bool half_word = size == sizeof(int16_t);
2339 if (half_word) {
2340 emit(0x66);
2341 size = sizeof(int32_t);
2342 }
2343 bool byte_operand = size == sizeof(int8_t);
2344 if (byte_operand) {
2345 size = sizeof(int32_t);
2346 }
2347 emit_rex(rax, op, size);
2348 emit(byte_operand ? 0xF6 : 0xF7);
2349 emit_operand(rax, op); // Operation code 0
2350 if (byte_operand) {
2351 emit(mask.value_);
2352 } else if (half_word) {
2353 emitw(mask.value_);
2354 } else {
2355 emit(mask);
2356 }
2357}
2358
2359void Assembler::emit_test(Operand op, Register reg, int size) {
2360 EnsureSpace ensure_space(this);
2361 if (size == sizeof(int16_t)) {
2362 emit(0x66);
2363 size = sizeof(int32_t);
2364 }
2365 bool byte_operand = size == sizeof(int8_t);
2366 if (byte_operand) {
2367 size = sizeof(int32_t);
2368 if (!reg.is_byte_register()) {
2369 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2370 emit_rex_32(reg, op);
2371 } else {
2372 emit_optional_rex_32(reg, op);
2373 }
2374 } else {
2375 emit_rex(reg, op, size);
2376 }
2377 emit(byte_operand ? 0x84 : 0x85);
2378 emit_operand(reg, op);
2379}
2380
2381// FPU instructions.
2382
2383void Assembler::fld(int i) {
2384 EnsureSpace ensure_space(this);
2385 emit_farith(0xD9, 0xC0, i);
2386}
2387
2388void Assembler::fld1() {
2389 EnsureSpace ensure_space(this);
2390 emit(0xD9);
2391 emit(0xE8);
2392}
2393
2394void Assembler::fldz() {
2395 EnsureSpace ensure_space(this);
2396 emit(0xD9);
2397 emit(0xEE);
2398}
2399
2400void Assembler::fldpi() {
2401 EnsureSpace ensure_space(this);
2402 emit(0xD9);
2403 emit(0xEB);
2404}
2405
2406void Assembler::fldln2() {
2407 EnsureSpace ensure_space(this);
2408 emit(0xD9);
2409 emit(0xED);
2410}
2411
2412void Assembler::fld_s(Operand adr) {
2413 EnsureSpace ensure_space(this);
2414 emit_optional_rex_32(adr);
2415 emit(0xD9);
2416 emit_operand(0, adr);
2417}
2418
2419void Assembler::fld_d(Operand adr) {
2420 EnsureSpace ensure_space(this);
2421 emit_optional_rex_32(adr);
2422 emit(0xDD);
2423 emit_operand(0, adr);
2424}
2425
2426void Assembler::fstp_s(Operand adr) {
2427 EnsureSpace ensure_space(this);
2428 emit_optional_rex_32(adr);
2429 emit(0xD9);
2430 emit_operand(3, adr);
2431}
2432
2433void Assembler::fstp_d(Operand adr) {
2434 EnsureSpace ensure_space(this);
2435 emit_optional_rex_32(adr);
2436 emit(0xDD);
2437 emit_operand(3, adr);
2438}
2439
2440void Assembler::fstp(int index) {
2441 DCHECK(is_uint3(index))((void) 0);
2442 EnsureSpace ensure_space(this);
2443 emit_farith(0xDD, 0xD8, index);
2444}
2445
2446void Assembler::fild_s(Operand adr) {
2447 EnsureSpace ensure_space(this);
2448 emit_optional_rex_32(adr);
2449 emit(0xDB);
2450 emit_operand(0, adr);
2451}
2452
2453void Assembler::fild_d(Operand adr) {
2454 EnsureSpace ensure_space(this);
2455 emit_optional_rex_32(adr);
2456 emit(0xDF);
2457 emit_operand(5, adr);
2458}
2459
2460void Assembler::fistp_s(Operand adr) {
2461 EnsureSpace ensure_space(this);
2462 emit_optional_rex_32(adr);
2463 emit(0xDB);
2464 emit_operand(3, adr);
2465}
2466
2467void Assembler::fisttp_s(Operand adr) {
2468 DCHECK(IsEnabled(SSE3))((void) 0);
2469 EnsureSpace ensure_space(this);
2470 emit_optional_rex_32(adr);
2471 emit(0xDB);
2472 emit_operand(1, adr);
2473}
2474
2475void Assembler::fisttp_d(Operand adr) {
2476 DCHECK(IsEnabled(SSE3))((void) 0);
2477 EnsureSpace ensure_space(this);
2478 emit_optional_rex_32(adr);
2479 emit(0xDD);
2480 emit_operand(1, adr);
2481}
2482
2483void Assembler::fist_s(Operand adr) {
2484 EnsureSpace ensure_space(this);
2485 emit_optional_rex_32(adr);
2486 emit(0xDB);
2487 emit_operand(2, adr);
2488}
2489
2490void Assembler::fistp_d(Operand adr) {
2491 EnsureSpace ensure_space(this);
2492 emit_optional_rex_32(adr);
2493 emit(0xDF);
2494 emit_operand(7, adr);
2495}
2496
2497void Assembler::fabs() {
2498 EnsureSpace ensure_space(this);
2499 emit(0xD9);
2500 emit(0xE1);
2501}
2502
2503void Assembler::fchs() {
2504 EnsureSpace ensure_space(this);
2505 emit(0xD9);
2506 emit(0xE0);
2507}
2508
2509void Assembler::fcos() {
2510 EnsureSpace ensure_space(this);
2511 emit(0xD9);
2512 emit(0xFF);
2513}
2514
2515void Assembler::fsin() {
2516 EnsureSpace ensure_space(this);
2517 emit(0xD9);
2518 emit(0xFE);
2519}
2520
2521void Assembler::fptan() {
2522 EnsureSpace ensure_space(this);
2523 emit(0xD9);
2524 emit(0xF2);
2525}
2526
2527void Assembler::fyl2x() {
2528 EnsureSpace ensure_space(this);
2529 emit(0xD9);
2530 emit(0xF1);
2531}
2532
2533void Assembler::f2xm1() {
2534 EnsureSpace ensure_space(this);
2535 emit(0xD9);
2536 emit(0xF0);
2537}
2538
2539void Assembler::fscale() {
2540 EnsureSpace ensure_space(this);
2541 emit(0xD9);
2542 emit(0xFD);
2543}
2544
2545void Assembler::fninit() {
2546 EnsureSpace ensure_space(this);
2547 emit(0xDB);
2548 emit(0xE3);
2549}
2550
2551void Assembler::fadd(int i) {
2552 EnsureSpace ensure_space(this);
2553 emit_farith(0xDC, 0xC0, i);
2554}
2555
2556void Assembler::fsub(int i) {
2557 EnsureSpace ensure_space(this);
2558 emit_farith(0xDC, 0xE8, i);
2559}
2560
2561void Assembler::fisub_s(Operand adr) {
2562 EnsureSpace ensure_space(this);
2563 emit_optional_rex_32(adr);
2564 emit(0xDA);
2565 emit_operand(4, adr);
2566}
2567
2568void Assembler::fmul(int i) {
2569 EnsureSpace ensure_space(this);
2570 emit_farith(0xDC, 0xC8, i);
2571}
2572
2573void Assembler::fdiv(int i) {
2574 EnsureSpace ensure_space(this);
2575 emit_farith(0xDC, 0xF8, i);
2576}
2577
2578void Assembler::faddp(int i) {
2579 EnsureSpace ensure_space(this);
2580 emit_farith(0xDE, 0xC0, i);
2581}
2582
2583void Assembler::fsubp(int i) {
2584 EnsureSpace ensure_space(this);
2585 emit_farith(0xDE, 0xE8, i);
2586}
2587
2588void Assembler::fsubrp(int i) {
2589 EnsureSpace ensure_space(this);
2590 emit_farith(0xDE, 0xE0, i);
2591}
2592
2593void Assembler::fmulp(int i) {
2594 EnsureSpace ensure_space(this);
2595 emit_farith(0xDE, 0xC8, i);
2596}
2597
2598void Assembler::fdivp(int i) {
2599 EnsureSpace ensure_space(this);
2600 emit_farith(0xDE, 0xF8, i);
2601}
2602
2603void Assembler::fprem() {
2604 EnsureSpace ensure_space(this);
2605 emit(0xD9);
2606 emit(0xF8);
2607}
2608
2609void Assembler::fprem1() {
2610 EnsureSpace ensure_space(this);
2611 emit(0xD9);
2612 emit(0xF5);
2613}
2614
2615void Assembler::fxch(int i) {
2616 EnsureSpace ensure_space(this);
2617 emit_farith(0xD9, 0xC8, i);
2618}
2619
2620void Assembler::fincstp() {
2621 EnsureSpace ensure_space(this);
2622 emit(0xD9);
2623 emit(0xF7);
2624}
2625
2626void Assembler::ffree(int i) {
2627 EnsureSpace ensure_space(this);
2628 emit_farith(0xDD, 0xC0, i);
2629}
2630
2631void Assembler::ftst() {
2632 EnsureSpace ensure_space(this);
2633 emit(0xD9);
2634 emit(0xE4);
2635}
2636
2637void Assembler::fucomp(int i) {
2638 EnsureSpace ensure_space(this);
2639 emit_farith(0xDD, 0xE8, i);
2640}
2641
2642void Assembler::fucompp() {
2643 EnsureSpace ensure_space(this);
2644 emit(0xDA);
2645 emit(0xE9);
2646}
2647
2648void Assembler::fucomi(int i) {
2649 EnsureSpace ensure_space(this);
2650 emit(0xDB);
2651 emit(0xE8 + i);
2652}
2653
2654void Assembler::fucomip() {
2655 EnsureSpace ensure_space(this);
2656 emit(0xDF);
2657 emit(0xE9);
2658}
2659
2660void Assembler::fcompp() {
2661 EnsureSpace ensure_space(this);
2662 emit(0xDE);
2663 emit(0xD9);
2664}
2665
2666void Assembler::fnstsw_ax() {
2667 EnsureSpace ensure_space(this);
2668 emit(0xDF);
2669 emit(0xE0);
2670}
2671
2672void Assembler::fwait() {
2673 EnsureSpace ensure_space(this);
2674 emit(0x9B);
2675}
2676
2677void Assembler::frndint() {
2678 EnsureSpace ensure_space(this);
2679 emit(0xD9);
2680 emit(0xFC);
2681}
2682
2683void Assembler::fnclex() {
2684 EnsureSpace ensure_space(this);
2685 emit(0xDB);
2686 emit(0xE2);
2687}
2688
2689void Assembler::sahf() {
2690 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2691 // in 64-bit mode. Test CpuID.
2692 DCHECK(IsEnabled(SAHF))((void) 0);
2693 EnsureSpace ensure_space(this);
2694 emit(0x9E);
2695}
2696
2697void Assembler::emit_farith(int b1, int b2, int i) {
2698 DCHECK(is_uint8(b1) && is_uint8(b2))((void) 0); // wrong opcode
2699 DCHECK(is_uint3(i))((void) 0); // illegal stack offset
2700 emit(b1);
2701 emit(b2 + i);
2702}
2703
2704// SSE 2 operations.
2705
2706void Assembler::movd(XMMRegister dst, Register src) {
2707 DCHECK(!IsEnabled(AVX))((void) 0);
2708 EnsureSpace ensure_space(this);
2709 emit(0x66);
2710 emit_optional_rex_32(dst, src);
2711 emit(0x0F);
2712 emit(0x6E);
2713 emit_sse_operand(dst, src);
2714}
2715
2716void Assembler::movd(XMMRegister dst, Operand src) {
2717 DCHECK(!IsEnabled(AVX))((void) 0);
2718 EnsureSpace ensure_space(this);
2719 emit(0x66);
2720 emit_optional_rex_32(dst, src);
2721 emit(0x0F);
2722 emit(0x6E);
2723 emit_sse_operand(dst, src);
2724}
2725
2726void Assembler::movd(Register dst, XMMRegister src) {
2727 DCHECK(!IsEnabled(AVX))((void) 0);
2728 EnsureSpace ensure_space(this);
2729 emit(0x66);
2730 emit_optional_rex_32(src, dst);
2731 emit(0x0F);
2732 emit(0x7E);
2733 emit_sse_operand(src, dst);
2734}
2735
2736void Assembler::movq(XMMRegister dst, Register src) {
2737 // Mixing AVX and non-AVX is expensive, catch those cases
2738 DCHECK(!IsEnabled(AVX))((void) 0);
2739 EnsureSpace ensure_space(this);
2740 emit(0x66);
2741 emit_rex_64(dst, src);
2742 emit(0x0F);
2743 emit(0x6E);
2744 emit_sse_operand(dst, src);
2745}
2746
2747void Assembler::movq(XMMRegister dst, Operand src) {
2748 // Mixing AVX and non-AVX is expensive, catch those cases
2749 DCHECK(!IsEnabled(AVX))((void) 0);
2750 EnsureSpace ensure_space(this);
2751 emit(0x66);
2752 emit_rex_64(dst, src);
2753 emit(0x0F);
2754 emit(0x6E);
2755 emit_sse_operand(dst, src);
2756}
2757
2758void Assembler::movq(Register dst, XMMRegister src) {
2759 // Mixing AVX and non-AVX is expensive, catch those cases
2760 DCHECK(!IsEnabled(AVX))((void) 0);
2761 EnsureSpace ensure_space(this);
2762 emit(0x66);
2763 emit_rex_64(src, dst);
2764 emit(0x0F);
2765 emit(0x7E);
2766 emit_sse_operand(src, dst);
2767}
2768
2769void Assembler::movq(XMMRegister dst, XMMRegister src) {
2770 // Mixing AVX and non-AVX is expensive, catch those cases
2771 DCHECK(!IsEnabled(AVX))((void) 0);
2772 EnsureSpace ensure_space(this);
2773 if (dst.low_bits() == 4) {
2774 // Avoid unnecessary SIB byte.
2775 emit(0xF3);
2776 emit_optional_rex_32(dst, src);
2777 emit(0x0F);
2778 emit(0x7E);
2779 emit_sse_operand(dst, src);
2780 } else {
2781 emit(0x66);
2782 emit_optional_rex_32(src, dst);
2783 emit(0x0F);
2784 emit(0xD6);
2785 emit_sse_operand(src, dst);
2786 }
2787}
2788
2789void Assembler::movdqa(Operand dst, XMMRegister src) {
2790 EnsureSpace ensure_space(this);
2791 emit(0x66);
2792 emit_rex_64(src, dst);
2793 emit(0x0F);
2794 emit(0x7F);
2795 emit_sse_operand(src, dst);
2796}
2797
2798void Assembler::movdqa(XMMRegister dst, Operand src) {
2799 EnsureSpace ensure_space(this);
2800 emit(0x66);
2801 emit_rex_64(dst, src);
2802 emit(0x0F);
2803 emit(0x6F);
2804 emit_sse_operand(dst, src);
2805}
2806
2807void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2808 EnsureSpace ensure_space(this);
2809 emit(0x66);
2810 emit_rex_64(src, dst);
2811 emit(0x0F);
2812 emit(0x7F);
2813 emit_sse_operand(src, dst);
2814}
2815
2816void Assembler::movdqu(Operand dst, XMMRegister src) {
2817 EnsureSpace ensure_space(this);
2818 emit(0xF3);
2819 emit_rex_64(src, dst);
2820 emit(0x0F);
2821 emit(0x7F);
2822 emit_sse_operand(src, dst);
2823}
2824
2825void Assembler::movdqu(XMMRegister dst, Operand src) {
2826 EnsureSpace ensure_space(this);
2827 emit(0xF3);
2828 emit_rex_64(dst, src);
2829 emit(0x0F);
2830 emit(0x6F);
2831 emit_sse_operand(dst, src);
2832}
2833
2834void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2835 EnsureSpace ensure_space(this);
2836 emit(0xF3);
2837 emit_rex_64(dst, src);
2838 emit(0x0F);
2839 emit(0x6F);
2840 emit_sse_operand(dst, src);
2841}
2842
2843void Assembler::pinsrw(XMMRegister dst, Register src, uint8_t imm8) {
2844 EnsureSpace ensure_space(this);
2845 emit(0x66);
2846 emit_optional_rex_32(dst, src);
2847 emit(0x0F);
2848 emit(0xC4);
2849 emit_sse_operand(dst, src);
2850 emit(imm8);
2851}
2852
2853void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t imm8) {
2854 EnsureSpace ensure_space(this);
2855 emit(0x66);
2856 emit_optional_rex_32(dst, src);
2857 emit(0x0F);
2858 emit(0xC4);
2859 emit_sse_operand(dst, src);
2860 emit(imm8);
2861}
2862
2863void Assembler::pextrq(Register dst, XMMRegister src, int8_t imm8) {
2864 DCHECK(IsEnabled(SSE4_1))((void) 0);
2865 EnsureSpace ensure_space(this);
2866 emit(0x66);
2867 emit_rex_64(src, dst);
2868 emit(0x0F);
2869 emit(0x3A);
2870 emit(0x16);
2871 emit_sse_operand(src, dst);
2872 emit(imm8);
2873}
2874
2875void Assembler::pinsrq(XMMRegister dst, Register src, uint8_t imm8) {
2876 DCHECK(IsEnabled(SSE4_1))((void) 0);
2877 EnsureSpace ensure_space(this);
2878 emit(0x66);
2879 emit_rex_64(dst, src);
2880 emit(0x0F);
2881 emit(0x3A);
2882 emit(0x22);
2883 emit_sse_operand(dst, src);
2884 emit(imm8);
2885}
2886
2887void Assembler::pinsrq(XMMRegister dst, Operand src, uint8_t imm8) {
2888 DCHECK(IsEnabled(SSE4_1))((void) 0);
2889 EnsureSpace ensure_space(this);
2890 emit(0x66);
2891 emit_rex_64(dst, src);
2892 emit(0x0F);
2893 emit(0x3A);
2894 emit(0x22);
2895 emit_sse_operand(dst, src);
2896 emit(imm8);
2897}
2898
2899void Assembler::pinsrd(XMMRegister dst, Register src, uint8_t imm8) {
2900 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22, imm8);
2901}
2902
2903void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
2904 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22);
2905 emit(imm8);
2906}
2907
2908void Assembler::pinsrb(XMMRegister dst, Register src, uint8_t imm8) {
2909 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20, imm8);
2910}
2911
2912void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t imm8) {
2913 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20);
2914 emit(imm8);
2915}
2916
2917void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2918 DCHECK(is_uint8(imm8))((void) 0);
2919 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2920 emit(imm8);
2921}
2922
2923void Assembler::insertps(XMMRegister dst, Operand src, byte imm8) {
2924 DCHECK(is_uint8(imm8))((void) 0);
2925 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2926 emit(imm8);
2927}
2928
2929void Assembler::movsd(Operand dst, XMMRegister src) {
2930 DCHECK(!IsEnabled(AVX))((void) 0);
2931 EnsureSpace ensure_space(this);
2932 emit(0xF2); // double
2933 emit_optional_rex_32(src, dst);
2934 emit(0x0F);
2935 emit(0x11); // store
2936 emit_sse_operand(src, dst);
2937}
2938
2939void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2940 DCHECK(!IsEnabled(AVX))((void) 0);
2941 EnsureSpace ensure_space(this);
2942 emit(0xF2); // double
2943 emit_optional_rex_32(dst, src);
2944 emit(0x0F);
2945 emit(0x10); // load
2946 emit_sse_operand(dst, src);
2947}
2948
2949void Assembler::movsd(XMMRegister dst, Operand src) {
2950 DCHECK(!IsEnabled(AVX))((void) 0);
2951 EnsureSpace ensure_space(this);
2952 emit(0xF2); // double
2953 emit_optional_rex_32(dst, src);
2954 emit(0x0F);
2955 emit(0x10); // load
2956 emit_sse_operand(dst, src);
2957}
2958
2959void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2960 DCHECK(!IsEnabled(AVX))((void) 0);
2961 EnsureSpace ensure_space(this);
2962 if (src.low_bits() == 4) {
2963 // Try to avoid an unnecessary SIB byte.
2964 emit_optional_rex_32(src, dst);
2965 emit(0x0F);
2966 emit(0x29);
2967 emit_sse_operand(src, dst);
2968 } else {
2969 emit_optional_rex_32(dst, src);
2970 emit(0x0F);
2971 emit(0x28);
2972 emit_sse_operand(dst, src);
2973 }
2974}
2975
2976void Assembler::movaps(XMMRegister dst, Operand src) {
2977 DCHECK(!IsEnabled(AVX))((void) 0);
2978 EnsureSpace ensure_space(this);
2979 emit_optional_rex_32(dst, src);
2980 emit(0x0F);
2981 emit(0x28);
2982 emit_sse_operand(dst, src);
2983}
2984
2985void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2986 DCHECK(is_uint8(imm8))((void) 0);
2987 EnsureSpace ensure_space(this);
2988 emit_optional_rex_32(dst, src);
2989 emit(0x0F);
2990 emit(0xC6);
2991 emit_sse_operand(dst, src);
2992 emit(imm8);
2993}
2994
2995void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2996 DCHECK(!IsEnabled(AVX))((void) 0);
2997 EnsureSpace ensure_space(this);
2998 if (src.low_bits() == 4) {
2999 // Try to avoid an unnecessary SIB byte.
3000 emit(0x66);
3001 emit_optional_rex_32(src, dst);
3002 emit(0x0F);
3003 emit(0x29);
3004 emit_sse_operand(src, dst);
3005 } else {
3006 emit(0x66);
3007 emit_optional_rex_32(dst, src);
3008 emit(0x0F);
3009 emit(0x28);
3010 emit_sse_operand(dst, src);
3011 }
3012}
3013
3014void Assembler::movupd(XMMRegister dst, Operand src) {
3015 EnsureSpace ensure_space(this);
3016 emit(0x66);
3017 emit_optional_rex_32(dst, src);
3018 emit(0x0F);
3019 emit(0x10);
3020 emit_sse_operand(dst, src);
3021}
3022
3023void Assembler::movupd(Operand dst, XMMRegister src) {
3024 EnsureSpace ensure_space(this);
3025 emit(0x66);
3026 emit_optional_rex_32(src, dst);
3027 emit(0x0F);
3028 emit(0x11);
3029 emit_sse_operand(src, dst);
3030}
3031
3032void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3033 DCHECK(!IsEnabled(AVX))((void) 0);
3034 EnsureSpace ensure_space(this);
3035 emit_optional_rex_32(dst, src);
3036 emit(0x0F);
3037 emit(0x2E);
3038 emit_sse_operand(dst, src);
3039}
3040
3041void Assembler::ucomiss(XMMRegister dst, Operand src) {
3042 DCHECK(!IsEnabled(AVX))((void) 0);
3043 EnsureSpace ensure_space(this);
3044 emit_optional_rex_32(dst, src);
3045 emit(0x0F);
3046 emit(0x2E);
3047 emit_sse_operand(dst, src);
3048}
3049
3050void Assembler::movss(XMMRegister dst, XMMRegister src) {
3051 DCHECK(!IsEnabled(AVX))((void) 0);
3052 EnsureSpace ensure_space(this);
3053 emit(0xF3); // single
3054 emit_optional_rex_32(dst, src);
3055 emit(0x0F);
3056 emit(0x10); // load
3057 emit_sse_operand(dst, src);
3058}
3059
3060void Assembler::movss(XMMRegister dst, Operand src) {
3061 DCHECK(!IsEnabled(AVX))((void) 0);
3062 EnsureSpace ensure_space(this);
3063 emit(0xF3); // single
3064 emit_optional_rex_32(dst, src);
3065 emit(0x0F);
3066 emit(0x10); // load
3067 emit_sse_operand(dst, src);
3068}
3069
3070void Assembler::movss(Operand src, XMMRegister dst) {
3071 DCHECK(!IsEnabled(AVX))((void) 0);
3072 EnsureSpace ensure_space(this);
3073 emit(0xF3); // single
3074 emit_optional_rex_32(dst, src);
3075 emit(0x0F);
3076 emit(0x11); // store
3077 emit_sse_operand(dst, src);
3078}
3079
3080void Assembler::movlps(XMMRegister dst, Operand src) {
3081 DCHECK(!IsEnabled(AVX))((void) 0);
3082 EnsureSpace ensure_space(this);
3083 emit_optional_rex_32(dst, src);
3084 emit(0x0F);
3085 emit(0x12);
3086 emit_sse_operand(dst, src);
3087}
3088
3089void Assembler::movlps(Operand src, XMMRegister dst) {
3090 DCHECK(!IsEnabled(AVX))((void) 0);
3091 EnsureSpace ensure_space(this);
3092 emit_optional_rex_32(dst, src);
3093 emit(0x0F);
3094 emit(0x13);
3095 emit_sse_operand(dst, src);
3096}
3097
3098void Assembler::movhps(XMMRegister dst, Operand src) {
3099 DCHECK(!IsEnabled(AVX))((void) 0);
3100 EnsureSpace ensure_space(this);
3101 emit_optional_rex_32(dst, src);
3102 emit(0x0F);
3103 emit(0x16);
3104 emit_sse_operand(dst, src);
3105}
3106
3107void Assembler::movhps(Operand src, XMMRegister dst) {
3108 DCHECK(!IsEnabled(AVX))((void) 0);
3109 EnsureSpace ensure_space(this);
3110 emit_optional_rex_32(dst, src);
3111 emit(0x0F);
3112 emit(0x17);
3113 emit_sse_operand(dst, src);
3114}
3115
3116void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3117 EnsureSpace ensure_space(this);
3118 emit_optional_rex_32(dst, src);
3119 emit(0x0F);
3120 emit(0xC2);
3121 emit_sse_operand(dst, src);
3122 emit(cmp);
3123}
3124
3125void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
3126 EnsureSpace ensure_space(this);
3127 emit_optional_rex_32(dst, src);
3128 emit(0x0F);
3129 emit(0xC2);
3130 emit_sse_operand(dst, src);
3131 emit(cmp);
3132}
3133
3134void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3135 EnsureSpace ensure_space(this);
3136 emit(0x66);
3137 emit_optional_rex_32(dst, src);
3138 emit(0x0F);
3139 emit(0xC2);
3140 emit_sse_operand(dst, src);
3141 emit(cmp);
3142}
3143
3144void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
3145 EnsureSpace ensure_space(this);
3146 emit(0x66);
3147 emit_optional_rex_32(dst, src);
3148 emit(0x0F);
3149 emit(0xC2);
3150 emit_sse_operand(dst, src);
3151 emit(cmp);
3152}
3153
3154void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
3155 sse2_instr(dst, src, 0xF3, 0x0F, 0xE6);
3156}
3157
3158void Assembler::cvttss2si(Register dst, Operand src) {
3159 DCHECK(!IsEnabled(AVX))((void) 0);
3160 EnsureSpace ensure_space(this);
3161 emit(0xF3);
3162 emit_optional_rex_32(dst, src);
3163 emit(0x0F);
3164 emit(0x2C);
3165 emit_operand(dst, src);
3166}
3167
3168void Assembler::cvttss2si(Register dst, XMMRegister src) {
3169 DCHECK(!IsEnabled(AVX))((void) 0);
3170 EnsureSpace ensure_space(this);
3171 emit(0xF3);
3172 emit_optional_rex_32(dst, src);
3173 emit(0x0F);
3174 emit(0x2C);
3175 emit_sse_operand(dst, src);
3176}
3177
3178void Assembler::cvttsd2si(Register dst, Operand src) {
3179 DCHECK(!IsEnabled(AVX))((void) 0);
3180 EnsureSpace ensure_space(this);
3181 emit(0xF2);
3182 emit_optional_rex_32(dst, src);
3183 emit(0x0F);
3184 emit(0x2C);
3185 emit_operand(dst, src);
3186}
3187
3188void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3189 DCHECK(!IsEnabled(AVX))((void) 0);
3190 EnsureSpace ensure_space(this);
3191 emit(0xF2);
3192 emit_optional_rex_32(dst, src);
3193 emit(0x0F);
3194 emit(0x2C);
3195 emit_sse_operand(dst, src);
3196}
3197
3198void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3199 DCHECK(!IsEnabled(AVX))((void) 0);
3200 EnsureSpace ensure_space(this);
3201 emit(0xF3);
3202 emit_rex_64(dst, src);
3203 emit(0x0F);
3204 emit(0x2C);
3205 emit_sse_operand(dst, src);
3206}
3207
3208void Assembler::cvttss2siq(Register dst, Operand src) {
3209 DCHECK(!IsEnabled(AVX))((void) 0);
3210 EnsureSpace ensure_space(this);
3211 emit(0xF3);
3212 emit_rex_64(dst, src);
3213 emit(0x0F);
3214 emit(0x2C);
3215 emit_sse_operand(dst, src);
3216}
3217
3218void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3219 DCHECK(!IsEnabled(AVX))((void) 0);
3220 EnsureSpace ensure_space(this);
3221 emit(0xF2);
3222 emit_rex_64(dst, src);
3223 emit(0x0F);
3224 emit(0x2C);
3225 emit_sse_operand(dst, src);
3226}
3227
3228void Assembler::cvttsd2siq(Register dst, Operand src) {
3229 DCHECK(!IsEnabled(AVX))((void) 0);
3230 EnsureSpace ensure_space(this);
3231 emit(0xF2);
3232 emit_rex_64(dst, src);
3233 emit(0x0F);
3234 emit(0x2C);
3235 emit_sse_operand(dst, src);
3236}
3237
3238void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
3239 EnsureSpace ensure_space(this);
3240 emit(0xF3);
3241 emit_rex_64(dst, src);
3242 emit(0x0F);
3243 emit(0x5B);
3244 emit_sse_operand(dst, src);
3245}
3246
3247void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
3248 EnsureSpace ensure_space(this);
3249 emit(0xF3);
3250 emit_rex_64(dst, src);
3251 emit(0x0F);
3252 emit(0x5B);
3253 emit_sse_operand(dst, src);
3254}
3255
3256void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
3257 DCHECK(!IsEnabled(AVX))((void) 0);
3258 EnsureSpace ensure_space(this);
3259 emit(0xF2);
3260 emit_optional_rex_32(dst, src);
3261 emit(0x0F);
3262 emit(0x2A);
3263 emit_sse_operand(dst, src);
3264}
3265
3266void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3267 DCHECK(!IsEnabled(AVX))((void) 0);
3268 EnsureSpace ensure_space(this);
3269 emit(0xF2);
3270 emit_optional_rex_32(dst, src);
3271 emit(0x0F);
3272 emit(0x2A);
3273 emit_sse_operand(dst, src);
3274}
3275
3276void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
3277 DCHECK(!IsEnabled(AVX))((void) 0);
3278 EnsureSpace ensure_space(this);
3279 emit(0xF3);
3280 emit_optional_rex_32(dst, src);
3281 emit(0x0F);
3282 emit(0x2A);
3283 emit_sse_operand(dst, src);
3284}
3285
3286void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3287 EnsureSpace ensure_space(this);
3288 emit(0xF3);
3289 emit_optional_rex_32(dst, src);
3290 emit(0x0F);
3291 emit(0x2A);
3292 emit_sse_operand(dst, src);
3293}
3294
3295void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
3296 DCHECK(!IsEnabled(AVX))((void) 0);
3297 EnsureSpace ensure_space(this);
3298 emit(0xF3);
3299 emit_rex_64(dst, src);
3300 emit(0x0F);
3301 emit(0x2A);
3302 emit_sse_operand(dst, src);
3303}
3304
3305void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3306 DCHECK(!IsEnabled(AVX))((void) 0);
3307 EnsureSpace ensure_space(this);
3308 emit(0xF3);
3309 emit_rex_64(dst, src);
3310 emit(0x0F);
3311 emit(0x2A);
3312 emit_sse_operand(dst, src);
3313}
3314
3315void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
3316 DCHECK(!IsEnabled(AVX))((void) 0);
3317 EnsureSpace ensure_space(this);
3318 emit(0xF2);
3319 emit_rex_64(dst, src);
3320 emit(0x0F);
3321 emit(0x2A);
3322 emit_sse_operand(dst, src);
3323}
3324
3325void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3326 DCHECK(!IsEnabled(AVX))((void) 0);
3327 EnsureSpace ensure_space(this);
3328 emit(0xF2);
3329 emit_rex_64(dst, src);
3330 emit(0x0F);
3331 emit(0x2A);
3332 emit_sse_operand(dst, src);
3333}
3334
3335void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3336 DCHECK(!IsEnabled(AVX))((void) 0);
3337 EnsureSpace ensure_space(this);
3338 emit(0xF2);
3339 emit_optional_rex_32(dst, src);
3340 emit(0x0F);
3341 emit(0x2D);
3342 emit_sse_operand(dst, src);
3343}
3344
3345void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3346 DCHECK(!IsEnabled(AVX))((void) 0);
3347 EnsureSpace ensure_space(this);
3348 emit(0xF2);
3349 emit_rex_64(dst, src);
3350 emit(0x0F);
3351 emit(0x2D);
3352 emit_sse_operand(dst, src);
3353}
3354
3355void Assembler::haddps(XMMRegister dst, XMMRegister src) {
3356 DCHECK(IsEnabled(SSE3))((void) 0);
3357 EnsureSpace ensure_space(this);
3358 emit(0xF2);
3359 emit_optional_rex_32(dst, src);
3360 emit(0x0F);
3361 emit(0x7C);
3362 emit_sse_operand(dst, src);
3363}
3364
3365void Assembler::haddps(XMMRegister dst, Operand src) {
3366 DCHECK(IsEnabled(SSE3))((void) 0);
3367 EnsureSpace ensure_space(this);
3368 emit(0xF2);
3369 emit_optional_rex_32(dst, src);
3370 emit(0x0F);
3371 emit(0x7C);
3372 emit_sse_operand(dst, src);
3373}
3374
3375void Assembler::cmpeqss(XMMRegister dst, XMMRegister src) {
3376 DCHECK(!IsEnabled(AVX))((void) 0);
3377 EnsureSpace ensure_space(this);
3378 emit(0xF3);
3379 emit_optional_rex_32(dst, src);
3380 emit(0x0F);
3381 emit(0xC2);
3382 emit_sse_operand(dst, src);
3383 emit(0x00); // EQ == 0
3384}
3385
3386void Assembler::cmpeqsd(XMMRegister dst, XMMRegister src) {
3387 DCHECK(!IsEnabled(AVX))((void) 0);
3388 EnsureSpace ensure_space(this);
3389 emit(0xF2);
3390 emit_optional_rex_32(dst, src);
3391 emit(0x0F);
3392 emit(0xC2);
3393 emit_sse_operand(dst, src);
3394 emit(0x00); // EQ == 0
3395}
3396
3397void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3398 EnsureSpace ensure_space(this);
3399 emit(0xF2);
3400 emit_optional_rex_32(dst, src);
3401 emit(0x0F);
3402 emit(0xC2);
3403 emit_sse_operand(dst, src);
3404 emit(0x01); // LT == 1
3405}
3406
3407void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3408 DCHECK(!IsEnabled(AVX))((void) 0);
3409 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
3410 // Mask precision exception.
3411 emit(static_cast<byte>(mode) | 0x8);
3412}
3413
3414void Assembler::roundss(XMMRegister dst, Operand src, RoundingMode mode) {
3415 DCHECK(!IsEnabled(AVX))((void) 0);
3416 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
3417 // Mask precision exception.
3418 emit(static_cast<byte>(mode) | 0x8);
3419}
3420
3421void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3422 DCHECK(!IsEnabled(AVX))((void) 0);
3423 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
3424 // Mask precision exception.
3425 emit(static_cast<byte>(mode) | 0x8);
3426}
3427
3428void Assembler::roundsd(XMMRegister dst, Operand src, RoundingMode mode) {
3429 DCHECK(!IsEnabled(AVX))((void) 0);
3430 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
3431 // Mask precision exception.
3432 emit(static_cast<byte>(mode) | 0x8);
3433}
3434
3435void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3436 DCHECK(!IsEnabled(AVX))((void) 0);
3437 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x08);
3438 // Mask precision exception.
3439 emit(static_cast<byte>(mode) | 0x8);
3440}
3441
3442void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3443 DCHECK(!IsEnabled(AVX))((void) 0);
3444 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x09);
3445 // Mask precision exception.
3446 emit(static_cast<byte>(mode) | 0x8);
3447}
3448
3449void Assembler::movmskpd(Register dst, XMMRegister src) {
3450 EnsureSpace ensure_space(this);
3451 emit(0x66);
3452 emit_optional_rex_32(dst, src);
3453 emit(0x0F);
3454 emit(0x50);
3455 emit_sse_operand(dst, src);
3456}
3457
3458void Assembler::movmskps(Register dst, XMMRegister src) {
3459 EnsureSpace ensure_space(this);
3460 emit_optional_rex_32(dst, src);
3461 emit(0x0F);
3462 emit(0x50);
3463 emit_sse_operand(dst, src);
3464}
3465
3466void Assembler::pmovmskb(Register dst, XMMRegister src) {
3467 EnsureSpace ensure_space(this);
3468 emit(0x66);
3469 emit_optional_rex_32(dst, src);
3470 emit(0x0F);
3471 emit(0xD7);
3472 emit_sse_operand(dst, src);
3473}
3474
3475// AVX instructions
3476#define VMOV_DUP(SIMDRegister, length) \
3477 void Assembler::vmovddup(SIMDRegister dst, SIMDRegister src) { \
3478 DCHECK(IsEnabled(AVX))((void) 0); \
3479 EnsureSpace ensure_space(this); \
3480 emit_vex_prefix(dst, xmm0, src, k##length, kF2, k0F, kWIG); \
3481 emit(0x12); \
3482 emit_sse_operand(dst, src); \
3483 } \
3484 \
3485 void Assembler::vmovddup(SIMDRegister dst, Operand src) { \
3486 DCHECK(IsEnabled(AVX))((void) 0); \
3487 EnsureSpace ensure_space(this); \
3488 emit_vex_prefix(dst, xmm0, src, k##length, kF2, k0F, kWIG); \
3489 emit(0x12); \
3490 emit_sse_operand(dst, src); \
3491 } \
3492 \
3493 void Assembler::vmovshdup(SIMDRegister dst, SIMDRegister src) { \
3494 DCHECK(IsEnabled(AVX))((void) 0); \
3495 EnsureSpace ensure_space(this); \
3496 emit_vex_prefix(dst, xmm0, src, k##length, kF3, k0F, kWIG); \
3497 emit(0x16); \
3498 emit_sse_operand(dst, src); \
3499 }
3500VMOV_DUP(XMMRegister, L128)
3501VMOV_DUP(YMMRegister, L256)
3502#undef VMOV_DUP
3503
3504#define BROADCASTSS(SIMDRegister, length) \
3505 void Assembler::vbroadcastss(SIMDRegister dst, Operand src) { \
3506 DCHECK(IsEnabled(AVX))((void) 0); \
3507 EnsureSpace ensure_space(this); \
3508 emit_vex_prefix(dst, xmm0, src, k##length, k66, k0F38, kW0); \
3509 emit(0x18); \
3510 emit_sse_operand(dst, src); \
3511 } \
3512 void Assembler::vbroadcastss(SIMDRegister dst, XMMRegister src) { \
3513 DCHECK(IsEnabled(AVX2))((void) 0); \
3514 EnsureSpace ensure_space(this); \
3515 emit_vex_prefix(dst, xmm0, src, k##length, k66, k0F38, kW0); \
3516 emit(0x18); \
3517 emit_sse_operand(dst, src); \
3518 }
3519BROADCASTSS(XMMRegister, L128)
3520BROADCASTSS(YMMRegister, L256)
3521#undef BROADCASTSS
3522
3523void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3524 XMMRegister src2, VectorLength l, SIMDPrefix pp,
3525 LeadingOpcode m, VexW w) {
3526 DCHECK(IsEnabled(FMA3))((void) 0);
3527 EnsureSpace ensure_space(this);
3528 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3529 emit(op);
3530 emit_sse_operand(dst, src2);
3531}
3532
3533void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3534 Operand src2, VectorLength l, SIMDPrefix pp,
3535 LeadingOpcode m, VexW w) {
3536 DCHECK(IsEnabled(FMA3))((void) 0);
3537 EnsureSpace ensure_space(this);
3538 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3539 emit(op);
3540 emit_sse_operand(dst, src2);
3541}
3542
3543void Assembler::vmovd(XMMRegister dst, Register src) {
3544 DCHECK(IsEnabled(AVX))((void) 0);
3545 EnsureSpace ensure_space(this);
3546 XMMRegister isrc = XMMRegister::from_code(src.code());
3547 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
3548 emit(0x6E);
3549 emit_sse_operand(dst, src);
3550}
3551
3552void Assembler::vmovd(XMMRegister dst, Operand src) {
3553 DCHECK(IsEnabled(AVX))((void) 0);
3554 EnsureSpace ensure_space(this);
3555 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
3556 emit(0x6E);
3557 emit_sse_operand(dst, src);
3558}
3559
3560void Assembler::vmovd(Register dst, XMMRegister src) {
3561 DCHECK(IsEnabled(AVX))((void) 0);
3562 EnsureSpace ensure_space(this);
3563 XMMRegister idst = XMMRegister::from_code(dst.code());
3564 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
3565 emit(0x7E);
3566 emit_sse_operand(src, dst);
3567}
3568
3569void Assembler::vmovq(XMMRegister dst, Register src) {
3570 DCHECK(IsEnabled(AVX))((void) 0);
3571 EnsureSpace ensure_space(this);
3572 XMMRegister isrc = XMMRegister::from_code(src.code());
3573 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
3574 emit(0x6E);
3575 emit_sse_operand(dst, src);
3576}
3577
3578void Assembler::vmovq(XMMRegister dst, Operand src) {
3579 DCHECK(IsEnabled(AVX))((void) 0);
3580 EnsureSpace ensure_space(this);
3581 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
3582 emit(0x6E);
3583 emit_sse_operand(dst, src);
3584}
3585
3586void Assembler::vmovq(Register dst, XMMRegister src) {
3587 DCHECK(IsEnabled(AVX))((void) 0);
3588 EnsureSpace ensure_space(this);
3589 XMMRegister idst = XMMRegister::from_code(dst.code());
3590 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
3591 emit(0x7E);
3592 emit_sse_operand(src, dst);
3593}
3594
3595void Assembler::vmovdqa(XMMRegister dst, Operand src) {
3596 DCHECK(IsEnabled(AVX))((void) 0);
3597 EnsureSpace ensure_space(this);
3598 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kWIG);
3599 emit(0x6F);
3600 emit_sse_operand(dst, src);
3601}
3602
3603void Assembler::vmovdqa(XMMRegister dst, XMMRegister src) {
3604 DCHECK(IsEnabled(AVX))((void) 0);
3605 EnsureSpace ensure_space(this);
3606 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kWIG);
3607 emit(0x6F);
3608 emit_sse_operand(dst, src);
3609}
3610
3611void Assembler::vmovdqa(YMMRegister dst, Operand src) {
3612 DCHECK(IsEnabled(AVX))((void) 0);
3613 EnsureSpace ensure_space(this);
3614 emit_vex_prefix(dst, ymm0, src, kL256, k66, k0F, kWIG);
3615 emit(0x6F);
3616 emit_sse_operand(dst, src);
3617}
3618
3619void Assembler::vmovdqa(YMMRegister dst, YMMRegister src) {
3620 DCHECK(IsEnabled(AVX))((void) 0);
3621 EnsureSpace ensure_space(this);
3622 emit_vex_prefix(dst, ymm0, src, kL256, k66, k0F, kWIG);
3623 emit(0x6F);
3624 emit_sse_operand(dst, src);
3625}
3626
3627void Assembler::vmovdqu(XMMRegister dst, Operand src) {
3628 DCHECK(IsEnabled(AVX))((void) 0);
3629 EnsureSpace ensure_space(this);
3630 emit_vex_prefix(dst, xmm0, src, kL128, kF3, k0F, kWIG);
3631 emit(0x6F);
3632 emit_sse_operand(dst, src);
3633}
3634
3635void Assembler::vmovdqu(Operand dst, XMMRegister src) {
3636 DCHECK(IsEnabled(AVX))((void) 0);
3637 EnsureSpace ensure_space(this);
3638 emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
3639 emit(0x7F);
3640 emit_sse_operand(src, dst);
3641}
3642
3643void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3644 DCHECK(IsEnabled(AVX))((void) 0);
3645 EnsureSpace ensure_space(this);
3646 emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
3647 emit(0x7F);
3648 emit_sse_operand(src, dst);
3649}
3650
3651void Assembler::vmovdqu(YMMRegister dst, Operand src) {
3652 DCHECK(IsEnabled(AVX))((void) 0);
3653 EnsureSpace ensure_space(this);
3654 emit_vex_prefix(dst, ymm0, src, kL256, kF3, k0F, kWIG);
3655 emit(0x6F);
3656 emit_sse_operand(dst, src);
3657}
3658
3659void Assembler::vmovdqu(Operand dst, YMMRegister src) {
3660 DCHECK(IsEnabled(AVX))((void) 0);
3661 EnsureSpace ensure_space(this);
3662 emit_vex_prefix(src, ymm0, dst, kL256, kF3, k0F, kWIG);
3663 emit(0x7F);
3664 emit_sse_operand(src, dst);
3665}
3666
3667void Assembler::vmovdqu(YMMRegister dst, YMMRegister src) {
3668 DCHECK(IsEnabled(AVX))((void) 0);
3669 EnsureSpace ensure_space(this);
3670 emit_vex_prefix(src, ymm0, dst, kL256, kF3, k0F, kWIG);
3671 emit(0x7F);
3672 emit_sse_operand(src, dst);
3673}
3674
3675void Assembler::vmovlps(XMMRegister dst, XMMRegister src1, Operand src2) {
3676 DCHECK(IsEnabled(AVX))((void) 0);
3677 EnsureSpace ensure_space(this);
3678 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3679 emit(0x12);
3680 emit_sse_operand(dst, src2);
3681}
3682
3683void Assembler::vmovlps(Operand dst, XMMRegister src) {
3684 DCHECK(IsEnabled(AVX))((void) 0);
3685 EnsureSpace ensure_space(this);
3686 emit_vex_prefix(src, xmm0, dst, kL128, kNoPrefix, k0F, kWIG);
3687 emit(0x13);
3688 emit_sse_operand(src, dst);
3689}
3690
3691void Assembler::vmovhps(XMMRegister dst, XMMRegister src1, Operand src2) {
3692 DCHECK(IsEnabled(AVX))((void) 0);
3693 EnsureSpace ensure_space(this);
3694 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3695 emit(0x16);
3696 emit_sse_operand(dst, src2);
3697}
3698
3699void Assembler::vmovhps(Operand dst, XMMRegister src) {
3700 DCHECK(IsEnabled(AVX))((void) 0);
3701 EnsureSpace ensure_space(this);
3702 emit_vex_prefix(src, xmm0, dst, kL128, kNoPrefix, k0F, kWIG);
3703 emit(0x17);
3704 emit_sse_operand(src, dst);
3705}
3706
3707void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
3708 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, VexW w,
3709 CpuFeature feature) {
3710 DCHECK(IsEnabled(feature))((void) 0);
3711 DCHECK(feature == AVX || feature == AVX2)((void) 0);
3712 EnsureSpace ensure_space(this);
3713 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3714 emit(op);
3715 emit_sse_operand(dst, src2);
3716}
3717
3718void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
3719 SIMDPrefix pp, LeadingOpcode m, VexW w,
3720 CpuFeature feature) {
3721 DCHECK(IsEnabled(feature))((void) 0);
3722 DCHECK(feature == AVX || feature == AVX2)((void) 0);
3723 EnsureSpace ensure_space(this);
3724 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3725 emit(op);
3726 emit_sse_operand(dst, src2);
3727}
3728
3729template <typename Reg1, typename Reg2, typename Op>
3730void Assembler::vinstr(byte op, Reg1 dst, Reg2 src1, Op src2, SIMDPrefix pp,
3731 LeadingOpcode m, VexW w, CpuFeature feature) {
3732 DCHECK(IsEnabled(feature))((void) 0);
3733 DCHECK(feature == AVX || feature == AVX2)((void) 0);
3734 EnsureSpace ensure_space(this);
3735 emit_vex_prefix(dst, src1, src2, kL256, pp, m, w);
3736 emit(op);
3737 emit_sse_operand(dst, src2);
3738}
3739
3740template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3741 byte op, YMMRegister dst, YMMRegister src1, YMMRegister src2, SIMDPrefix pp,
3742 LeadingOpcode m, VexW w, CpuFeature feature);
3743template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3744 byte op, YMMRegister dst, XMMRegister src1, XMMRegister src2, SIMDPrefix pp,
3745 LeadingOpcode m, VexW w, CpuFeature feature);
3746template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3747 byte op, YMMRegister dst, YMMRegister src1, Operand src2, SIMDPrefix pp,
3748 LeadingOpcode m, VexW w, CpuFeature feature);
3749template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3750 byte op, YMMRegister dst, YMMRegister src1, XMMRegister src2, SIMDPrefix pp,
3751 LeadingOpcode m, VexW w, CpuFeature feature);
3752template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3753 byte op, YMMRegister dst, XMMRegister src1, Operand src2, SIMDPrefix pp,
3754 LeadingOpcode m, VexW w, CpuFeature feature);
3755
3756void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3757 XMMRegister src2) {
3758 DCHECK(IsEnabled(AVX))((void) 0);
3759 EnsureSpace ensure_space(this);
3760 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3761 emit(op);
3762 emit_sse_operand(dst, src2);
3763}
3764
3765void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1,
3766 YMMRegister src2) {
3767 DCHECK(IsEnabled(AVX))((void) 0);
3768 EnsureSpace ensure_space(this);
3769 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3770 emit(op);
3771 emit_sse_operand(dst, src2);
3772}
3773
3774void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3775 DCHECK(IsEnabled(AVX))((void) 0);
3776 EnsureSpace ensure_space(this);
3777 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3778 emit(op);
3779 emit_sse_operand(dst, src2);
3780}
3781
3782void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1, Operand src2) {
3783 DCHECK(IsEnabled(AVX))((void) 0);
3784 EnsureSpace ensure_space(this);
3785 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3786 emit(op);
3787 emit_sse_operand(dst, src2);
3788}
3789
3790void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3791 XMMRegister src2, byte imm8) {
3792 DCHECK(IsEnabled(AVX))((void) 0);
3793 EnsureSpace ensure_space(this);
3794 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3795 emit(op);
3796 emit_sse_operand(dst, src2);
3797 emit(imm8);
3798}
3799
3800void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1,
3801 YMMRegister src2, byte imm8) {
3802 DCHECK(IsEnabled(AVX))((void) 0);
3803 EnsureSpace ensure_space(this);
3804 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3805 emit(op);
3806 emit_sse_operand(dst, src2);
3807 emit(imm8);
3808}
3809
3810#define VPD(SIMDRegister, length) \
3811 void Assembler::vpd(byte op, SIMDRegister dst, SIMDRegister src1, \
3812 SIMDRegister src2) { \
3813 DCHECK(IsEnabled(AVX))((void) 0); \
3814 EnsureSpace ensure_space(this); \
3815 emit_vex_prefix(dst, src1, src2, k##length, k66, k0F, kWIG); \
3816 emit(op); \
3817 emit_sse_operand(dst, src2); \
3818 } \
3819 \
3820 void Assembler::vpd(byte op, SIMDRegister dst, SIMDRegister src1, \
3821 Operand src2) { \
3822 DCHECK(IsEnabled(AVX))((void) 0); \
3823 EnsureSpace ensure_space(this); \
3824 emit_vex_prefix(dst, src1, src2, k##length, k66, k0F, kWIG); \
3825 emit(op); \
3826 emit_sse_operand(dst, src2); \
3827 }
3828VPD(XMMRegister, L128)
3829VPD(YMMRegister, L256)
3830#undef VPD
3831
3832void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
3833 DCHECK(IsEnabled(AVX))((void) 0);
3834 EnsureSpace ensure_space(this);
3835 emit_vex_prefix(dst, xmm0, src, kLIG, kNoPrefix, k0F, kWIG);
3836 emit(0x2E);
3837 emit_sse_operand(dst, src);
3838}
3839
3840void Assembler::vucomiss(XMMRegister dst, Operand src) {
3841 DCHECK(IsEnabled(AVX))((void) 0);
3842 EnsureSpace ensure_space(this);
3843 emit_vex_prefix(dst, xmm0, src, kLIG, kNoPrefix, k0F, kWIG);
3844 emit(0x2E);
3845 emit_sse_operand(dst, src);
3846}
3847
3848void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3849 XMMRegister idst = XMMRegister::from_code(dst.code());
3850 DCHECK(IsEnabled(AVX))((void) 0);
3851 EnsureSpace ensure_space(this);
3852 emit_vex_prefix(idst, xmm0, src, kL128, k66, k0F, kWIG);
3853 emit(0xD7);
3854 emit_sse_operand(idst, src);
3855}
3856
3857void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
3858 XMMRegister src2) {
3859 DCHECK(IsEnabled(AVX))((void) 0);
3860 EnsureSpace ensure_space(this);
3861 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3862 emit(op);
3863 emit_sse_operand(dst, src2);
3864}
3865
3866void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3867 DCHECK(IsEnabled(AVX))((void) 0);
3868 EnsureSpace ensure_space(this);
3869 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3870 emit(op);
3871 emit_sse_operand(dst, src2);
3872}
3873
3874void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
3875 DCHECK(IsEnabled(BMI1))((void) 0);
3876 EnsureSpace ensure_space(this);
3877 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW1);
3878 emit(op);
3879 emit_modrm(reg, rm);
3880}
3881
3882void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
3883 DCHECK(IsEnabled(BMI1))((void) 0);
3884 EnsureSpace ensure_space(this);
3885 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW1);
3886 emit(op);
3887 emit_operand(reg, rm);
3888}
3889
3890void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
3891 DCHECK(IsEnabled(BMI1))((void) 0);
3892 EnsureSpace ensure_space(this);
3893 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW0);
3894 emit(op);
3895 emit_modrm(reg, rm);
3896}
3897
3898void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
3899 DCHECK(IsEnabled(BMI1))((void) 0);
3900 EnsureSpace ensure_space(this);
3901 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW0);
3902 emit(op);
3903 emit_operand(reg, rm);
3904}
3905
3906void Assembler::tzcntq(Register dst, Register src) {
3907 DCHECK(IsEnabled(BMI1))((void) 0);
3908 EnsureSpace ensure_space(this);
3909 emit(0xF3);
3910 emit_rex_64(dst, src);
3911 emit(0x0F);
3912 emit(0xBC);
3913 emit_modrm(dst, src);
3914}
3915
3916void Assembler::tzcntq(Register dst, Operand src) {
3917 DCHECK(IsEnabled(BMI1))((void) 0);
3918 EnsureSpace ensure_space(this);
3919 emit(0xF3);
3920 emit_rex_64(dst, src);
3921 emit(0x0F);
3922 emit(0xBC);
3923 emit_operand(dst, src);
3924}
3925
3926void Assembler::tzcntl(Register dst, Register src) {
3927 DCHECK(IsEnabled(BMI1))((void) 0);
3928 EnsureSpace ensure_space(this);
3929 emit(0xF3);
3930 emit_optional_rex_32(dst, src);
3931 emit(0x0F);
3932 emit(0xBC);
3933 emit_modrm(dst, src);
3934}
3935
3936void Assembler::tzcntl(Register dst, Operand src) {
3937 DCHECK(IsEnabled(BMI1))((void) 0);
3938 EnsureSpace ensure_space(this);
3939 emit(0xF3);
3940 emit_optional_rex_32(dst, src);
3941 emit(0x0F);
3942 emit(0xBC);
3943 emit_operand(dst, src);
3944}
3945
3946void Assembler::lzcntq(Register dst, Register src) {
3947 DCHECK(IsEnabled(LZCNT))((void) 0);
3948 EnsureSpace ensure_space(this);
3949 emit(0xF3);
3950 emit_rex_64(dst, src);
3951 emit(0x0F);
3952 emit(0xBD);
3953 emit_modrm(dst, src);
3954}
3955
3956void Assembler::lzcntq(Register dst, Operand src) {
3957 DCHECK(IsEnabled(LZCNT))((void) 0);
3958 EnsureSpace ensure_space(this);
3959 emit(0xF3);
3960 emit_rex_64(dst, src);
3961 emit(0x0F);
3962 emit(0xBD);
3963 emit_operand(dst, src);
3964}
3965
3966void Assembler::lzcntl(Register dst, Register src) {
3967 DCHECK(IsEnabled(LZCNT))((void) 0);
3968 EnsureSpace ensure_space(this);
3969 emit(0xF3);
3970 emit_optional_rex_32(dst, src);
3971 emit(0x0F);
3972 emit(0xBD);
3973 emit_modrm(dst, src);
3974}
3975
3976void Assembler::lzcntl(Register dst, Operand src) {
3977 DCHECK(IsEnabled(LZCNT))((void) 0);
3978 EnsureSpace ensure_space(this);
3979 emit(0xF3);
3980 emit_optional_rex_32(dst, src);
3981 emit(0x0F);
3982 emit(0xBD);
3983 emit_operand(dst, src);
3984}
3985
3986void Assembler::popcntq(Register dst, Register src) {
3987 DCHECK(IsEnabled(POPCNT))((void) 0);
3988 EnsureSpace ensure_space(this);
3989 emit(0xF3);
3990 emit_rex_64(dst, src);
3991 emit(0x0F);
3992 emit(0xB8);
3993 emit_modrm(dst, src);
3994}
3995
3996void Assembler::popcntq(Register dst, Operand src) {
3997 DCHECK(IsEnabled(POPCNT))((void) 0);
3998 EnsureSpace ensure_space(this);
3999 emit(0xF3);
4000 emit_rex_64(dst, src);
4001 emit(0x0F);
4002 emit(0xB8);
4003 emit_operand(dst, src);
4004}
4005
4006void Assembler::popcntl(Register dst, Register src) {
4007 DCHECK(IsEnabled(POPCNT))((void) 0);
4008 EnsureSpace ensure_space(this);
4009 emit(0xF3);
4010 emit_optional_rex_32(dst, src);
4011 emit(0x0F);
4012 emit(0xB8);
4013 emit_modrm(dst, src);
4014}
4015
4016void Assembler::popcntl(Register dst, Operand src) {
4017 DCHECK(IsEnabled(POPCNT))((void) 0);
4018 EnsureSpace ensure_space(this);
4019 emit(0xF3);
4020 emit_optional_rex_32(dst, src);
4021 emit(0x0F);
4022 emit(0xB8);
4023 emit_operand(dst, src);
4024}
4025
4026void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4027 Register rm) {
4028 DCHECK(IsEnabled(BMI2))((void) 0);
4029 EnsureSpace ensure_space(this);
4030 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4031 emit(op);
4032 emit_modrm(reg, rm);
4033}
4034
4035void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4036 Operand rm) {
4037 DCHECK(IsEnabled(BMI2))((void) 0);
4038 EnsureSpace ensure_space(this);
4039 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4040 emit(op);
4041 emit_operand(reg, rm);
4042}
4043
4044void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4045 Register rm) {
4046 DCHECK(IsEnabled(BMI2))((void) 0);
4047 EnsureSpace ensure_space(this);
4048 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4049 emit(op);
4050 emit_modrm(reg, rm);
4051}
4052
4053void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4054 Operand rm) {
4055 DCHECK(IsEnabled(BMI2))((void) 0);
4056 EnsureSpace ensure_space(this);
4057 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4058 emit(op);
4059 emit_operand(reg, rm);
4060}
4061
4062void Assembler::rorxq(Register dst, Register src, byte imm8) {
4063 DCHECK(IsEnabled(BMI2))((void) 0);
4064 DCHECK(is_uint8(imm8))((void) 0);
4065 Register vreg = Register::from_code(0); // VEX.vvvv unused
4066 EnsureSpace ensure_space(this);
4067 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4068 emit(0xF0);
4069 emit_modrm(dst, src);
4070 emit(imm8);
4071}
4072
4073void Assembler::rorxq(Register dst, Operand src, byte imm8) {
4074 DCHECK(IsEnabled(BMI2))((void) 0);
4075 DCHECK(is_uint8(imm8))((void) 0);
4076 Register vreg = Register::from_code(0); // VEX.vvvv unused
4077 EnsureSpace ensure_space(this);
4078 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4079 emit(0xF0);
4080 emit_operand(dst, src);
4081 emit(imm8);
4082}
4083
4084void Assembler::rorxl(Register dst, Register src, byte imm8) {
4085 DCHECK(IsEnabled(BMI2))((void) 0);
4086 DCHECK(is_uint8(imm8))((void) 0);
4087 Register vreg = Register::from_code(0); // VEX.vvvv unused
4088 EnsureSpace ensure_space(this);
4089 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4090 emit(0xF0);
4091 emit_modrm(dst, src);
4092 emit(imm8);
4093}
4094
4095void Assembler::rorxl(Register dst, Operand src, byte imm8) {
4096 DCHECK(IsEnabled(BMI2))((void) 0);
4097 DCHECK(is_uint8(imm8))((void) 0);
4098 Register vreg = Register::from_code(0); // VEX.vvvv unused
4099 EnsureSpace ensure_space(this);
4100 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4101 emit(0xF0);
4102 emit_operand(dst, src);
4103 emit(imm8);
4104}
4105
4106void Assembler::pause() {
4107 emit(0xF3);
4108 emit(0x90);
4109}
4110
4111void Assembler::movups(XMMRegister dst, XMMRegister src) {
4112 EnsureSpace ensure_space(this);
4113 if (src.low_bits() == 4) {
4114 // Try to avoid an unnecessary SIB byte.
4115 emit_optional_rex_32(src, dst);
4116 emit(0x0F);
4117 emit(0x11);
4118 emit_sse_operand(src, dst);
4119 } else {
4120 emit_optional_rex_32(dst, src);
4121 emit(0x0F);
4122 emit(0x10);
4123 emit_sse_operand(dst, src);
4124 }
4125}
4126
4127void Assembler::movups(XMMRegister dst, Operand src) {
4128 EnsureSpace ensure_space(this);
4129 emit_optional_rex_32(dst, src);
4130 emit(0x0F);
4131 emit(0x10);
4132 emit_sse_operand(dst, src);
4133}
4134
4135void Assembler::movups(Operand dst, XMMRegister src) {
4136 EnsureSpace ensure_space(this);
4137 emit_optional_rex_32(src, dst);
4138 emit(0x0F);
4139 emit(0x11);
4140 emit_sse_operand(src, dst);
4141}
4142
4143void Assembler::sse_instr(XMMRegister dst, XMMRegister src, byte escape,
4144 byte opcode) {
4145 EnsureSpace ensure_space(this);
4146 emit_optional_rex_32(dst, src);
4147 emit(escape);
4148 emit(opcode);
4149 emit_sse_operand(dst, src);
4150}
4151
4152void Assembler::sse_instr(XMMRegister dst, Operand src, byte escape,
4153 byte opcode) {
4154 EnsureSpace ensure_space(this);
4155 emit_optional_rex_32(dst, src);
4156 emit(escape);
4157 emit(opcode);
4158 emit_sse_operand(dst, src);
4159}
4160
4161void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4162 byte escape, byte opcode) {
4163 EnsureSpace ensure_space(this);
4164 emit(prefix);
4165 emit_optional_rex_32(dst, src);
4166 emit(escape);
4167 emit(opcode);
4168 emit_sse_operand(dst, src);
4169}
4170
4171void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
4172 byte escape, byte opcode) {
4173 EnsureSpace ensure_space(this);
4174 emit(prefix);
4175 emit_optional_rex_32(dst, src);
4176 emit(escape);
4177 emit(opcode);
4178 emit_sse_operand(dst, src);
4179}
4180
4181void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4182 byte escape1, byte escape2, byte opcode) {
4183 DCHECK(IsEnabled(SSSE3))((void) 0);
4184 EnsureSpace ensure_space(this);
4185 emit(prefix);
4186 emit_optional_rex_32(dst, src);
4187 emit(escape1);
4188 emit(escape2);
4189 emit(opcode);
4190 emit_sse_operand(dst, src);
4191}
4192
4193void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
4194 byte escape1, byte escape2, byte opcode) {
4195 DCHECK(IsEnabled(SSSE3))((void) 0);
4196 EnsureSpace ensure_space(this);
4197 emit(prefix);
4198 emit_optional_rex_32(dst, src);
4199 emit(escape1);
4200 emit(escape2);
4201 emit(opcode);
4202 emit_sse_operand(dst, src);
4203}
4204
4205void Assembler::sse4_instr(XMMRegister dst, Register src, byte prefix,
4206 byte escape1, byte escape2, byte opcode,
4207 int8_t imm8) {
4208 DCHECK(is_uint8(imm8))((void) 0);
4209 DCHECK(IsEnabled(SSE4_1))((void) 0);
4210 EnsureSpace ensure_space(this);
4211 emit(prefix);
4212 emit_optional_rex_32(dst, src);
4213 emit(escape1);
4214 emit(escape2);
4215 emit(opcode);
4216 emit_sse_operand(dst, src);
4217 emit(imm8);
4218}
4219
4220void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4221 byte escape1, byte escape2, byte opcode) {
4222 DCHECK(IsEnabled(SSE4_1))((void) 0);
4223 EnsureSpace ensure_space(this);
4224 emit(prefix);
4225 emit_optional_rex_32(dst, src);
4226 emit(escape1);
4227 emit(escape2);
4228 emit(opcode);
4229 emit_sse_operand(dst, src);
4230}
4231
4232void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
4233 byte escape1, byte escape2, byte opcode) {
4234 DCHECK(IsEnabled(SSE4_1))((void) 0);
4235 EnsureSpace ensure_space(this);
4236 emit(prefix);
4237 emit_optional_rex_32(dst, src);
4238 emit(escape1);
4239 emit(escape2);
4240 emit(opcode);
4241 emit_sse_operand(dst, src);
4242}
4243
4244void Assembler::sse4_instr(Register dst, XMMRegister src, byte prefix,
4245 byte escape1, byte escape2, byte opcode,
4246 int8_t imm8) {
4247 DCHECK(is_uint8(imm8))((void) 0);
4248 DCHECK(IsEnabled(SSE4_1))((void) 0);
4249 EnsureSpace ensure_space(this);
4250 emit(prefix);
4251 emit_optional_rex_32(src, dst);
4252 emit(escape1);
4253 emit(escape2);
4254 emit(opcode);
4255 emit_sse_operand(src, dst);
4256 emit(imm8);
4257}
4258
4259void Assembler::sse4_instr(Operand dst, XMMRegister src, byte prefix,
4260 byte escape1, byte escape2, byte opcode,
4261 int8_t imm8) {
4262 DCHECK(is_uint8(imm8))((void) 0);
4263 DCHECK(IsEnabled(SSE4_1))((void) 0);
4264 EnsureSpace ensure_space(this);
4265 emit(prefix);
4266 emit_optional_rex_32(src, dst);
4267 emit(escape1);
4268 emit(escape2);
4269 emit(opcode);
4270 emit_sse_operand(src, dst);
4271 emit(imm8);
4272}
4273
4274void Assembler::sse4_2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4275 byte escape1, byte escape2, byte opcode) {
4276 DCHECK(IsEnabled(SSE4_2))((void) 0);
4277 EnsureSpace ensure_space(this);
4278 emit(prefix);
4279 emit_optional_rex_32(dst, src);
4280 emit(escape1);
4281 emit(escape2);
4282 emit(opcode);
4283 emit_sse_operand(dst, src);
4284}
4285
4286void Assembler::sse4_2_instr(XMMRegister dst, Operand src, byte prefix,
4287 byte escape1, byte escape2, byte opcode) {
4288 DCHECK(IsEnabled(SSE4_2))((void) 0);
4289 EnsureSpace ensure_space(this);
4290 emit(prefix);
4291 emit_optional_rex_32(dst, src);
4292 emit(escape1);
4293 emit(escape2);
4294 emit(opcode);
4295 emit_sse_operand(dst, src);
4296}
4297
4298void Assembler::lddqu(XMMRegister dst, Operand src) {
4299 DCHECK(IsEnabled(SSE3))((void) 0);
4300 EnsureSpace ensure_space(this);
4301 emit(0xF2);
4302 emit_optional_rex_32(dst, src);
4303 emit(0x0F);
4304 emit(0xF0);
4305 emit_sse_operand(dst, src);
4306}
4307
4308void Assembler::movddup(XMMRegister dst, XMMRegister src) {
4309 DCHECK(IsEnabled(SSE3))((void) 0);
4310 EnsureSpace ensure_space(this);
4311 emit(0xF2);
4312 emit_optional_rex_32(dst, src);
4313 emit(0x0F);
4314 emit(0x12);
4315 emit_sse_operand(dst, src);
4316}
4317
4318void Assembler::movddup(XMMRegister dst, Operand src) {
4319 DCHECK(IsEnabled(SSE3))((void) 0);
4320 EnsureSpace ensure_space(this);
4321 emit(0xF2);
4322 emit_optional_rex_32(dst, src);
4323 emit(0x0F);
4324 emit(0x12);
4325 emit_sse_operand(dst, src);
4326}
4327
4328void Assembler::movshdup(XMMRegister dst, XMMRegister src) {
4329 DCHECK(IsEnabled(SSE3))((void) 0);
4330 EnsureSpace ensure_space(this);
4331 emit(0xF3);
4332 emit_optional_rex_32(dst, src);
4333 emit(0x0F);
4334 emit(0x16);
4335 emit_sse_operand(dst, src);
4336}
4337
4338void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4339 EnsureSpace ensure_space(this);
4340 emit(0x66);
4341 emit_optional_rex_32(dst);
4342 emit(0x0F);
4343 emit(0x73);
4344 emit_sse_operand(dst);
4345 emit(shift);
4346}
4347
4348void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4349 EnsureSpace ensure_space(this);
4350 emit(0xF3);
4351 emit_optional_rex_32(dst, src);
4352 emit(0x0F);
4353 emit(0x70);
4354 emit_sse_operand(dst, src);
4355 emit(shuffle);
4356}
4357
4358void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
4359 EnsureSpace ensure_space(this);
4360 emit(0xF3);
4361 emit_optional_rex_32(dst, src);
4362 emit(0x0F);
4363 emit(0x70);
4364 emit_sse_operand(dst, src);
4365 emit(shuffle);
4366}
4367
4368void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4369 EnsureSpace ensure_space(this);
4370 emit(0xF2);
4371 emit_optional_rex_32(dst, src);
4372 emit(0x0F);
4373 emit(0x70);
4374 emit_sse_operand(dst, src);
4375 emit(shuffle);
4376}
4377
4378void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
4379 EnsureSpace ensure_space(this);
4380 emit(0xF2);
4381 emit_optional_rex_32(dst, src);
4382 emit(0x0F);
4383 emit(0x70);
4384 emit_sse_operand(dst, src);
4385 emit(shuffle);
4386}
4387
4388void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4389 EnsureSpace ensure_space(this);
4390 emit(0x66);
4391 emit_optional_rex_32(dst, src);
4392 emit(0x0F);
4393 emit(0x70);
4394 emit_sse_operand(dst, src);
4395 emit(shuffle);
4396}
4397
4398void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
4399 EnsureSpace ensure_space(this);
4400 emit(0x66);
4401 emit_optional_rex_32(dst, src);
4402 emit(0x0F);
4403 emit(0x70);
4404 emit_sse_operand(dst, src);
4405 emit(shuffle);
4406}
4407
4408void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
4409 Register ireg = Register::from_code(reg.code());
4410 emit_operand(ireg, adr);
4411}
4412
4413void Assembler::emit_sse_operand(Register reg, Operand adr) {
4414 emit_operand(reg, adr);
4415}
4416
4417void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4418 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4419}
4420
4421void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4422 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4423}
4424
4425void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4426 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4427}
4428
4429void Assembler::emit_sse_operand(XMMRegister dst) {
4430 emit(0xD8 | dst.low_bits());
4431}
4432
4433void Assembler::db(uint8_t data) {
4434 EnsureSpace ensure_space(this);
4435 emit(data);
4436}
4437
4438void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) {
4439 EnsureSpace ensure_space(this);
4440 if (!RelocInfo::IsNoInfo(rmode)) {
4441 DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) ||((void) 0)
4442 RelocInfo::IsLiteralConstant(rmode))((void) 0);
4443 RecordRelocInfo(rmode);
4444 }
4445 emitl(data);
4446}
4447
4448void Assembler::dq(uint64_t data, RelocInfo::Mode rmode) {
4449 EnsureSpace ensure_space(this);
4450 if (!RelocInfo::IsNoInfo(rmode)) {
4451 DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) ||((void) 0)
4452 RelocInfo::IsLiteralConstant(rmode))((void) 0);
4453 RecordRelocInfo(rmode);
4454 }
4455 emitq(data);
4456}
4457
4458void Assembler::dq(Label* label) {
4459 EnsureSpace ensure_space(this);
4460 if (label->is_bound()) {
4461 internal_reference_positions_.push_back(pc_offset());
4462 emit(Immediate64(reinterpret_cast<Address>(buffer_start_) + label->pos(),
4463 RelocInfo::INTERNAL_REFERENCE));
4464 } else {
4465 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4466 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4467 if (label->is_linked()) {
4468 emitl(label->pos());
4469 label->link_to(pc_offset() - sizeof(int32_t));
4470 } else {
4471 DCHECK(label->is_unused())((void) 0);
4472 int32_t current = pc_offset();
4473 emitl(current);
4474 label->link_to(current);
4475 }
4476 }
4477}
4478
4479// Relocation information implementations.
4480
4481void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4482 if (!ShouldRecordRelocInfo(rmode)) return;
4483 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
4484 reloc_info_writer.Write(&rinfo);
4485}
4486
4487const int RelocInfo::kApplyMask =
4488 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
4489 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
4490 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
4491 RelocInfo::ModeMask(RelocInfo::WASM_CALL);
4492
4493bool RelocInfo::IsCodedSpecially() {
4494 // The deserializer needs to know whether a pointer is specially coded. Being
4495 // specially coded on x64 means that it is a relative 32 bit address, as used
4496 // by branch instructions.
4497 return (1 << rmode_) & kApplyMask;
4498}
4499
4500bool RelocInfo::IsInConstantPool() { return false; }
4501
4502} // namespace internal
4503} // namespace v8
4504
4505#endif // V8_TARGET_ARCH_X64