Bug Summary

File:out/../deps/v8/src/compiler/wasm-compiler.cc
Warning:line 1729, column 18
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name wasm-compiler.cc -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=all -relaxed-aliasing -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/home/maurizio/node-v18.6.0/out -resource-dir /usr/local/lib/clang/16.0.0 -D _GLIBCXX_USE_CXX11_ABI=1 -D NODE_OPENSSL_CONF_NAME=nodejs_conf -D NODE_OPENSSL_HAS_QUIC -D V8_GYP_BUILD -D V8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=64 -D __STDC_FORMAT_MACROS -D OPENSSL_NO_PINSHARED -D OPENSSL_THREADS -D V8_TARGET_ARCH_X64 -D V8_HAVE_TARGET_OS -D V8_TARGET_OS_LINUX -D V8_EMBEDDER_STRING="-node.8" -D ENABLE_DISASSEMBLER -D V8_PROMISE_INTERNAL_FIELD_COUNT=1 -D V8_SHORT_BUILTIN_CALLS -D OBJECT_PRINT -D V8_INTL_SUPPORT -D V8_ATOMIC_OBJECT_FIELD_WRITES -D V8_ENABLE_LAZY_SOURCE_POSITIONS -D V8_USE_SIPHASH -D V8_SHARED_RO_HEAP -D V8_WIN64_UNWINDING_INFO -D V8_ENABLE_REGEXP_INTERPRETER_THREADED_DISPATCH -D V8_SNAPSHOT_COMPRESSION -D V8_ENABLE_WEBASSEMBLY -D V8_ENABLE_JAVASCRIPT_PROMISE_HOOKS -D V8_ALLOCATION_FOLDING -D V8_ALLOCATION_SITE_TRACKING -D V8_SCRIPTORMODULE_LEGACY_LIFETIME -D V8_ADVANCED_BIGINT_ALGORITHMS -D UCONFIG_NO_SERVICE=1 -D U_ENABLE_DYLOAD=0 -D U_STATIC_IMPLEMENTATION=1 -D U_HAVE_STD_STRING=1 -D UCONFIG_NO_BREAK_ITERATION=0 -I ../deps/v8 -I ../deps/v8/include -I /home/maurizio/node-v18.6.0/out/Release/obj/gen/generate-bytecode-output-root -I /home/maurizio/node-v18.6.0/out/Release/obj/gen -I ../deps/icu-small/source/i18n -I ../deps/icu-small/source/common -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/x86_64-redhat-linux -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/backward -internal-isystem /usr/local/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-redhat-linux/8/../../../../x86_64-redhat-linux/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-unused-parameter -Wno-return-type -std=gnu++17 -fdeprecated-macro -fdebug-compilation-dir=/home/maurizio/node-v18.6.0/out -ferror-limit 19 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-08-22-142216-507842-1 -x c++ ../deps/v8/src/compiler/wasm-compiler.cc
1// Copyright 2015 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/compiler/wasm-compiler.h"
6
7#include <memory>
8
9#include "src/api/api-inl.h"
10#include "src/base/optional.h"
11#include "src/base/platform/elapsed-timer.h"
12#include "src/base/platform/platform.h"
13#include "src/base/platform/wrappers.h"
14#include "src/base/small-vector.h"
15#include "src/base/v8-fallthrough.h"
16#include "src/base/vector.h"
17#include "src/codegen/assembler-inl.h"
18#include "src/codegen/assembler.h"
19#include "src/codegen/code-factory.h"
20#include "src/codegen/compiler.h"
21#include "src/codegen/interface-descriptors-inl.h"
22#include "src/codegen/machine-type.h"
23#include "src/codegen/optimized-compilation-info.h"
24#include "src/compiler/backend/code-generator.h"
25#include "src/compiler/backend/instruction-selector.h"
26#include "src/compiler/common-operator.h"
27#include "src/compiler/compiler-source-position-table.h"
28#include "src/compiler/diamond.h"
29#include "src/compiler/fast-api-calls.h"
30#include "src/compiler/graph-assembler.h"
31#include "src/compiler/graph-visualizer.h"
32#include "src/compiler/graph.h"
33#include "src/compiler/int64-lowering.h"
34#include "src/compiler/linkage.h"
35#include "src/compiler/machine-operator.h"
36#include "src/compiler/node-matchers.h"
37#include "src/compiler/node-origin-table.h"
38#include "src/compiler/node-properties.h"
39#include "src/compiler/pipeline.h"
40#include "src/compiler/zone-stats.h"
41#include "src/execution/isolate-inl.h"
42#include "src/execution/simulator.h"
43#include "src/heap/factory.h"
44#include "src/logging/counters.h"
45#include "src/logging/log.h"
46#include "src/objects/heap-number.h"
47#include "src/objects/instance-type.h"
48#include "src/roots/roots.h"
49#include "src/tracing/trace-event.h"
50#include "src/trap-handler/trap-handler.h"
51#include "src/wasm/code-space-access.h"
52#include "src/wasm/function-body-decoder-impl.h"
53#include "src/wasm/function-compiler.h"
54#include "src/wasm/graph-builder-interface.h"
55#include "src/wasm/jump-table-assembler.h"
56#include "src/wasm/memory-tracing.h"
57#include "src/wasm/object-access.h"
58#include "src/wasm/wasm-code-manager.h"
59#include "src/wasm/wasm-constants.h"
60#include "src/wasm/wasm-engine.h"
61#include "src/wasm/wasm-limits.h"
62#include "src/wasm/wasm-linkage.h"
63#include "src/wasm/wasm-module.h"
64#include "src/wasm/wasm-objects-inl.h"
65#include "src/wasm/wasm-opcodes-inl.h"
66
67namespace v8 {
68namespace internal {
69namespace compiler {
70
71namespace {
72
73#define FATAL_UNSUPPORTED_OPCODE(opcode) \
74 FATAL("Unsupported opcode 0x%x:%s", (opcode), \V8_Fatal("Unsupported opcode 0x%x:%s", (opcode), wasm::WasmOpcodes
::OpcodeName(opcode))
75 wasm::WasmOpcodes::OpcodeName(opcode))V8_Fatal("Unsupported opcode 0x%x:%s", (opcode), wasm::WasmOpcodes
::OpcodeName(opcode))
;
76
77MachineType assert_size(int expected_size, MachineType type) {
78 DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()))((void) 0);
79 return type;
80}
81
82#define WASM_INSTANCE_OBJECT_SIZE(name) \
83 (WasmInstanceObject::k##name##OffsetEnd - \
84 WasmInstanceObject::k##name##Offset + 1) // NOLINT(whitespace/indent)
85
86#define LOAD_MUTABLE_INSTANCE_FIELD(name, type) \
87 gasm_->LoadFromObject( \
88 assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type), GetInstance(), \
89 wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset))
90
91#define LOAD_INSTANCE_FIELD(name, type) \
92 gasm_->LoadImmutable( \
93 assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type), GetInstance(), \
94 wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset))
95
96#define LOAD_INSTANCE_FIELD_NO_ELIMINATION(name, type)gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type
), GetInstance(), wasm::ObjectAccess::ToTagged(WasmInstanceObject
::knameOffset))
\
97 gasm_->Load( \
98 assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type), GetInstance(), \
99 wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset))
100
101// Use MachineType::Pointer() over Tagged() to load root pointers because they
102// do not get compressed.
103#define LOAD_ROOT(root_name, factory_name) \
104 (parameter_mode_ == kNoSpecialParameterMode \
105 ? graph()->NewNode(mcgraph()->common()->HeapConstant( \
106 isolate_->factory()->factory_name())) \
107 : gasm_->LoadImmutable( \
108 MachineType::Pointer(), BuildLoadIsolateRoot(), \
109 IsolateData::root_slot_offset(RootIndex::k##root_name)))
110
111bool ContainsSimd(const wasm::FunctionSig* sig) {
112 for (auto type : sig->all()) {
113 if (type == wasm::kWasmS128) return true;
114 }
115 return false;
116}
117
118bool ContainsInt64(const wasm::FunctionSig* sig) {
119 for (auto type : sig->all()) {
120 if (type == wasm::kWasmI64) return true;
121 }
122 return false;
123}
124
125constexpr Builtin WasmRuntimeStubIdToBuiltinName(
126 wasm::WasmCode::RuntimeStubId runtime_stub_id) {
127 switch (runtime_stub_id) {
128#define DEF_CASE(name) \
129 case wasm::WasmCode::k##name: \
130 return Builtin::k##name;
131#define DEF_TRAP_CASE(name) DEF_CASE(ThrowWasm##name)
132 WASM_RUNTIME_STUB_LIST(DEF_CASE, DEF_TRAP_CASE)DEF_TRAP_CASE(TrapUnreachable) DEF_TRAP_CASE(TrapMemOutOfBounds
) DEF_TRAP_CASE(TrapUnalignedAccess) DEF_TRAP_CASE(TrapDivByZero
) DEF_TRAP_CASE(TrapDivUnrepresentable) DEF_TRAP_CASE(TrapRemByZero
) DEF_TRAP_CASE(TrapFloatUnrepresentable) DEF_TRAP_CASE(TrapFuncSigMismatch
) DEF_TRAP_CASE(TrapDataSegmentOutOfBounds) DEF_TRAP_CASE(TrapElemSegmentDropped
) DEF_TRAP_CASE(TrapTableOutOfBounds) DEF_TRAP_CASE(TrapRethrowNull
) DEF_TRAP_CASE(TrapNullDereference) DEF_TRAP_CASE(TrapIllegalCast
) DEF_TRAP_CASE(TrapArrayOutOfBounds) DEF_TRAP_CASE(TrapArrayTooLarge
) DEF_CASE(WasmCompileLazy) DEF_CASE(WasmTriggerTierUp) DEF_CASE
(WasmDebugBreak) DEF_CASE(WasmInt32ToHeapNumber) DEF_CASE(WasmTaggedNonSmiToInt32
) DEF_CASE(WasmFloat32ToNumber) DEF_CASE(WasmFloat64ToNumber)
DEF_CASE(WasmTaggedToFloat64) DEF_CASE(WasmAllocateJSArray) DEF_CASE
(WasmAtomicNotify) DEF_CASE(WasmI32AtomicWait32) DEF_CASE(WasmI32AtomicWait64
) DEF_CASE(WasmI64AtomicWait32) DEF_CASE(WasmI64AtomicWait64)
DEF_CASE(WasmGetOwnProperty) DEF_CASE(WasmRefFunc) DEF_CASE(
WasmMemoryGrow) DEF_CASE(WasmTableInit) DEF_CASE(WasmTableCopy
) DEF_CASE(WasmTableFill) DEF_CASE(WasmTableGrow) DEF_CASE(WasmTableGet
) DEF_CASE(WasmTableSet) DEF_CASE(WasmStackGuard) DEF_CASE(WasmStackOverflow
) DEF_CASE(WasmAllocateFixedArray) DEF_CASE(WasmThrow) DEF_CASE
(WasmRethrow) DEF_CASE(WasmRethrowExplicitContext) DEF_CASE(WasmTraceEnter
) DEF_CASE(WasmTraceExit) DEF_CASE(WasmTraceMemory) DEF_CASE(
BigIntToI32Pair) DEF_CASE(BigIntToI64) DEF_CASE(CallRefIC) DEF_CASE
(DoubleToI) DEF_CASE(I32PairToBigInt) DEF_CASE(I64ToBigInt) DEF_CASE
(RecordWriteEmitRememberedSetSaveFP) DEF_CASE(RecordWriteOmitRememberedSetSaveFP
) DEF_CASE(RecordWriteEmitRememberedSetIgnoreFP) DEF_CASE(RecordWriteOmitRememberedSetIgnoreFP
) DEF_CASE(ToNumber) DEF_CASE(WasmAllocateArray_Uninitialized
) DEF_CASE(WasmAllocateArray_InitNull) DEF_CASE(WasmAllocateArray_InitZero
) DEF_CASE(WasmArrayCopy) DEF_CASE(WasmArrayCopyWithChecks) DEF_CASE
(WasmArrayInitFromData) DEF_CASE(WasmAllocateStructWithRtt) DEF_CASE
(WasmSubtypeCheck) DEF_CASE(WasmOnStackReplace) DEF_CASE(WasmSuspend
)
133#undef DEF_CASE
134#undef DEF_TRAP_CASE
135 default:
136 UNREACHABLE()V8_Fatal("unreachable code");
137 }
138}
139
140CallDescriptor* GetBuiltinCallDescriptor(
141 Builtin name, Zone* zone, StubCallMode stub_mode,
142 bool needs_frame_state = false,
143 Operator::Properties properties = Operator::kNoProperties) {
144 CallInterfaceDescriptor interface_descriptor =
145 Builtins::CallInterfaceDescriptorFor(name);
146 return Linkage::GetStubCallDescriptor(
147 zone, // zone
148 interface_descriptor, // descriptor
149 interface_descriptor.GetStackParameterCount(), // stack parameter count
150 needs_frame_state ? CallDescriptor::kNeedsFrameState
151 : CallDescriptor::kNoFlags, // flags
152 properties, // properties
153 stub_mode); // stub call mode
154}
155
156ObjectAccess ObjectAccessForGCStores(wasm::ValueType type) {
157 return ObjectAccess(
158 MachineType::TypeForRepresentation(type.machine_representation(),
159 !type.is_packed()),
160 type.is_reference() ? kFullWriteBarrier : kNoWriteBarrier);
161}
162} // namespace
163
164JSWasmCallData::JSWasmCallData(const wasm::FunctionSig* wasm_signature)
165 : result_needs_conversion_(wasm_signature->return_count() == 1 &&
166 wasm_signature->GetReturn().kind() ==
167 wasm::kI64) {
168 arg_needs_conversion_.resize(wasm_signature->parameter_count());
169 for (size_t i = 0; i < wasm_signature->parameter_count(); i++) {
170 wasm::ValueType type = wasm_signature->GetParam(i);
171 arg_needs_conversion_[i] = type.kind() == wasm::kI64;
172 }
173}
174
175class WasmGraphAssembler : public GraphAssembler {
176 public:
177 WasmGraphAssembler(MachineGraph* mcgraph, Zone* zone)
178 : GraphAssembler(mcgraph, zone), simplified_(zone) {}
179
180 template <typename... Args>
181 Node* CallRuntimeStub(wasm::WasmCode::RuntimeStubId stub_id,
182 Operator::Properties properties, Args*... args) {
183 auto* call_descriptor = GetBuiltinCallDescriptor(
184 WasmRuntimeStubIdToBuiltinName(stub_id), temp_zone(),
185 StubCallMode::kCallWasmRuntimeStub, false, properties);
186 // A direct call to a wasm runtime stub defined in this module.
187 // Just encode the stub index. This will be patched at relocation.
188 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
189 stub_id, RelocInfo::WASM_STUB_CALL);
190 return Call(call_descriptor, call_target, args...);
191 }
192
193 template <typename... Args>
194 Node* CallBuiltin(Builtin name, Operator::Properties properties,
195 Args*... args) {
196 auto* call_descriptor = GetBuiltinCallDescriptor(
197 name, temp_zone(), StubCallMode::kCallBuiltinPointer, false,
198 properties);
199 Node* call_target = GetBuiltinPointerTarget(name);
200 return Call(call_descriptor, call_target, args...);
201 }
202
203 void MergeControlToEnd(Node* node) {
204 NodeProperties::MergeControlToEnd(graph(), mcgraph()->common(), node);
205 }
206
207 void AssertFalse(Node* condition) {
208#if DEBUG
209 if (FLAG_debug_code) {
210 auto ok = MakeLabel();
211 GotoIfNot(condition, &ok);
212 Unreachable();
213 Bind(&ok);
214 }
215#endif
216 }
217
218 Node* GetBuiltinPointerTarget(Builtin builtin) {
219 static_assert(std::is_same<Smi, BuiltinPtr>(), "BuiltinPtr must be Smi");
220 return NumberConstant(static_cast<int>(builtin));
221 }
222
223 // Sets {true_node} and {false_node} to their corresponding Branch outputs.
224 // Returns the Branch node. Does not change control().
225 Node* Branch(Node* cond, Node** true_node, Node** false_node,
226 BranchHint hint) {
227 DCHECK_NOT_NULL(cond)((void) 0);
228 Node* branch =
229 graph()->NewNode(mcgraph()->common()->Branch(hint), cond, control());
230 *true_node = graph()->NewNode(mcgraph()->common()->IfTrue(), branch);
231 *false_node = graph()->NewNode(mcgraph()->common()->IfFalse(), branch);
232 return branch;
233 }
234
235 Node* NumberConstant(volatile double value) {
236 return graph()->NewNode(mcgraph()->common()->NumberConstant(value));
237 }
238
239 // Helper functions for dealing with HeapObjects.
240 // Rule of thumb: if access to a given field in an object is required in
241 // at least two places, put a helper function here.
242
243 Node* Allocate(int size) {
244 AllowLargeObjects allow_large = size < kMaxRegularHeapObjectSize
245 ? AllowLargeObjects::kFalse
246 : AllowLargeObjects::kTrue;
247 return Allocate(Int32Constant(size), allow_large);
248 }
249
250 Node* Allocate(Node* size,
251 AllowLargeObjects allow_large = AllowLargeObjects::kTrue) {
252 return AddNode(
253 graph()->NewNode(simplified_.AllocateRaw(
254 Type::Any(), AllocationType::kYoung, allow_large),
255 size, effect(), control()));
256 }
257
258 Node* LoadFromObject(MachineType type, Node* base, Node* offset) {
259 return AddNode(graph()->NewNode(
260 simplified_.LoadFromObject(ObjectAccess(type, kNoWriteBarrier)), base,
261 offset, effect(), control()));
262 }
263
264 Node* LoadFromObject(MachineType type, Node* base, int offset) {
265 return LoadFromObject(type, base, IntPtrConstant(offset));
266 }
267
268 Node* LoadImmutableFromObject(MachineType type, Node* base, Node* offset) {
269 return AddNode(graph()->NewNode(simplified_.LoadImmutableFromObject(
270 ObjectAccess(type, kNoWriteBarrier)),
271 base, offset, effect(), control()));
272 }
273
274 Node* LoadImmutableFromObject(MachineType type, Node* base, int offset) {
275 return LoadImmutableFromObject(type, base, IntPtrConstant(offset));
276 }
277
278 Node* LoadImmutable(LoadRepresentation rep, Node* base, Node* offset) {
279 return AddNode(graph()->NewNode(mcgraph()->machine()->LoadImmutable(rep),
280 base, offset));
281 }
282
283 Node* LoadImmutable(LoadRepresentation rep, Node* base, int offset) {
284 return LoadImmutable(rep, base, IntPtrConstant(offset));
285 }
286
287 Node* StoreToObject(ObjectAccess access, Node* base, Node* offset,
288 Node* value) {
289 return AddNode(graph()->NewNode(simplified_.StoreToObject(access), base,
290 offset, value, effect(), control()));
291 }
292
293 Node* StoreToObject(ObjectAccess access, Node* base, int offset,
294 Node* value) {
295 return StoreToObject(access, base, IntPtrConstant(offset), value);
296 }
297
298 Node* InitializeImmutableInObject(ObjectAccess access, Node* base,
299 Node* offset, Node* value) {
300 return AddNode(
301 graph()->NewNode(simplified_.InitializeImmutableInObject(access), base,
302 offset, value, effect(), control()));
303 }
304
305 Node* InitializeImmutableInObject(ObjectAccess access, Node* base, int offset,
306 Node* value) {
307 return InitializeImmutableInObject(access, base, IntPtrConstant(offset),
308 value);
309 }
310
311 Node* IsI31(Node* object) {
312 if (COMPRESS_POINTERS_BOOLfalse) {
313 return Word32Equal(Word32And(object, Int32Constant(kSmiTagMask)),
314 Int32Constant(kSmiTag));
315 } else {
316 return WordEqual(WordAnd(object, IntPtrConstant(kSmiTagMask)),
317 IntPtrConstant(kSmiTag));
318 }
319 }
320
321 // Maps and their contents.
322 Node* LoadMap(Node* object) {
323 Node* map_word =
324 LoadImmutableFromObject(MachineType::TaggedPointer(), object,
325 HeapObject::kMapOffset - kHeapObjectTag);
326#ifdef V8_MAP_PACKING
327 return UnpackMapWord(map_word);
328#else
329 return map_word;
330#endif
331 }
332
333 void StoreMap(Node* heap_object, Node* map) {
334 ObjectAccess access(MachineType::TaggedPointer(), kMapWriteBarrier);
335#ifdef V8_MAP_PACKING
336 map = PackMapWord(TNode<Map>::UncheckedCast(map));
337#endif
338 InitializeImmutableInObject(access, heap_object,
339 HeapObject::kMapOffset - kHeapObjectTag, map);
340 }
341
342 Node* LoadInstanceType(Node* map) {
343 return LoadImmutableFromObject(
344 MachineType::Uint16(), map,
345 wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset));
346 }
347 Node* LoadWasmTypeInfo(Node* map) {
348 int offset = Map::kConstructorOrBackPointerOrNativeContextOffset;
349 return LoadImmutableFromObject(MachineType::TaggedPointer(), map,
350 wasm::ObjectAccess::ToTagged(offset));
351 }
352
353 Node* LoadSupertypes(Node* wasm_type_info) {
354 return LoadImmutableFromObject(
355 MachineType::TaggedPointer(), wasm_type_info,
356 wasm::ObjectAccess::ToTagged(WasmTypeInfo::kSupertypesOffset));
357 }
358
359 // FixedArrays.
360
361 Node* LoadFixedArrayLengthAsSmi(Node* fixed_array) {
362 return LoadImmutableFromObject(
363 MachineType::TaggedSigned(), fixed_array,
364 wasm::ObjectAccess::ToTagged(FixedArray::kLengthOffset));
365 }
366
367 Node* LoadFixedArrayElement(Node* fixed_array, Node* index_intptr,
368 MachineType type = MachineType::AnyTagged()) {
369 Node* offset = IntAdd(
370 IntMul(index_intptr, IntPtrConstant(kTaggedSize)),
371 IntPtrConstant(wasm::ObjectAccess::ToTagged(FixedArray::kHeaderSize)));
372 return LoadFromObject(type, fixed_array, offset);
373 }
374
375 Node* LoadImmutableFixedArrayElement(
376 Node* fixed_array, Node* index_intptr,
377 MachineType type = MachineType::AnyTagged()) {
378 Node* offset = IntAdd(
379 IntMul(index_intptr, IntPtrConstant(kTaggedSize)),
380 IntPtrConstant(wasm::ObjectAccess::ToTagged(FixedArray::kHeaderSize)));
381 return LoadImmutableFromObject(type, fixed_array, offset);
382 }
383
384 Node* LoadFixedArrayElement(Node* array, int index, MachineType type) {
385 return LoadFromObject(
386 type, array,
387 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index));
388 }
389
390 Node* LoadFixedArrayElementSmi(Node* array, int index) {
391 return LoadFixedArrayElement(array, index, MachineType::TaggedSigned());
392 }
393
394 Node* LoadFixedArrayElementPtr(Node* array, int index) {
395 return LoadFixedArrayElement(array, index, MachineType::TaggedPointer());
396 }
397
398 Node* LoadFixedArrayElementAny(Node* array, int index) {
399 return LoadFixedArrayElement(array, index, MachineType::AnyTagged());
400 }
401
402 Node* StoreFixedArrayElement(Node* array, int index, Node* value,
403 ObjectAccess access) {
404 return StoreToObject(
405 access, array,
406 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), value);
407 }
408
409 Node* StoreFixedArrayElementSmi(Node* array, int index, Node* value) {
410 return StoreFixedArrayElement(
411 array, index, value,
412 ObjectAccess(MachineType::TaggedSigned(), kNoWriteBarrier));
413 }
414
415 Node* StoreFixedArrayElementAny(Node* array, int index, Node* value) {
416 return StoreFixedArrayElement(
417 array, index, value,
418 ObjectAccess(MachineType::AnyTagged(), kFullWriteBarrier));
419 }
420
421 // Functions, SharedFunctionInfos, FunctionData.
422
423 Node* LoadSharedFunctionInfo(Node* js_function) {
424 return LoadFromObject(
425 MachineType::TaggedPointer(), js_function,
426 wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction());
427 }
428 Node* LoadContextFromJSFunction(Node* js_function) {
429 return LoadFromObject(
430 MachineType::TaggedPointer(), js_function,
431 wasm::ObjectAccess::ContextOffsetInTaggedJSFunction());
432 }
433
434 Node* LoadFunctionDataFromJSFunction(Node* js_function) {
435 Node* shared = LoadSharedFunctionInfo(js_function);
436 return LoadFromObject(
437 MachineType::TaggedPointer(), shared,
438 wasm::ObjectAccess::ToTagged(SharedFunctionInfo::kFunctionDataOffset));
439 }
440
441 Node* LoadExportedFunctionIndexAsSmi(Node* exported_function_data) {
442 return LoadImmutableFromObject(
443 MachineType::TaggedSigned(), exported_function_data,
444 wasm::ObjectAccess::ToTagged(
445 WasmExportedFunctionData::kFunctionIndexOffset));
446 }
447 Node* LoadExportedFunctionInstance(Node* exported_function_data) {
448 return LoadImmutableFromObject(
449 MachineType::TaggedPointer(), exported_function_data,
450 wasm::ObjectAccess::ToTagged(
451 WasmExportedFunctionData::kInstanceOffset));
452 }
453
454 // JavaScript objects.
455
456 Node* LoadJSArrayElements(Node* js_array) {
457 return LoadFromObject(
458 MachineType::AnyTagged(), js_array,
459 wasm::ObjectAccess::ToTagged(JSObject::kElementsOffset));
460 }
461
462 // WasmGC objects.
463
464 Node* FieldOffset(const wasm::StructType* type, uint32_t field_index) {
465 return IntPtrConstant(wasm::ObjectAccess::ToTagged(
466 WasmStruct::kHeaderSize + type->field_offset(field_index)));
467 }
468
469 Node* StoreStructField(Node* struct_object, const wasm::StructType* type,
470 uint32_t field_index, Node* value) {
471 ObjectAccess access = ObjectAccessForGCStores(type->field(field_index));
472 return type->mutability(field_index)
473 ? StoreToObject(access, struct_object,
474 FieldOffset(type, field_index), value)
475 : InitializeImmutableInObject(access, struct_object,
476 FieldOffset(type, field_index),
477 value);
478 }
479
480 Node* WasmArrayElementOffset(Node* index, wasm::ValueType element_type) {
481 Node* index_intptr =
482 mcgraph()->machine()->Is64() ? ChangeUint32ToUint64(index) : index;
483 return IntAdd(
484 IntPtrConstant(wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize)),
485 IntMul(index_intptr, IntPtrConstant(element_type.value_kind_size())));
486 }
487
488 Node* LoadWasmArrayLength(Node* array) {
489 return LoadImmutableFromObject(
490 MachineType::Uint32(), array,
491 wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset));
492 }
493
494 Node* IsDataRefMap(Node* map) {
495 Node* instance_type = LoadInstanceType(map);
496 // We're going to test a range of WasmObject instance types with a single
497 // unsigned comparison.
498 Node* comparison_value =
499 Int32Sub(instance_type, Int32Constant(FIRST_WASM_OBJECT_TYPE));
500 return Uint32LessThanOrEqual(
501 comparison_value,
502 Int32Constant(LAST_WASM_OBJECT_TYPE - FIRST_WASM_OBJECT_TYPE));
503 }
504
505 // Generic HeapObject helpers.
506
507 Node* HasInstanceType(Node* heap_object, InstanceType type) {
508 Node* map = LoadMap(heap_object);
509 Node* instance_type = LoadInstanceType(map);
510 return Word32Equal(instance_type, Int32Constant(type));
511 }
512
513 SimplifiedOperatorBuilder* simplified() { return &simplified_; }
514
515 private:
516 SimplifiedOperatorBuilder simplified_;
517};
518
519WasmGraphBuilder::WasmGraphBuilder(
520 wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
521 const wasm::FunctionSig* sig,
522 compiler::SourcePositionTable* source_position_table,
523 Parameter0Mode parameter_mode, Isolate* isolate)
524 : gasm_(std::make_unique<WasmGraphAssembler>(mcgraph, zone)),
525 zone_(zone),
526 mcgraph_(mcgraph),
527 env_(env),
528 has_simd_(ContainsSimd(sig)),
529 sig_(sig),
530 source_position_table_(source_position_table),
531 parameter_mode_(parameter_mode),
532 isolate_(isolate) {
533 DCHECK_EQ(isolate == nullptr, parameter_mode_ != kNoSpecialParameterMode)((void) 0);
534 DCHECK_IMPLIES(env && env->bounds_checks == wasm::kTrapHandler,((void) 0)
535 trap_handler::IsTrapHandlerEnabled())((void) 0);
536 DCHECK_NOT_NULL(mcgraph_)((void) 0);
537}
538
539// Destructor define here where the definition of {WasmGraphAssembler} is
540// available.
541WasmGraphBuilder::~WasmGraphBuilder() = default;
542
543void WasmGraphBuilder::Start(unsigned params) {
544 Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
545 graph()->SetStart(start);
546 SetEffectControl(start);
547 // Initialize parameter nodes.
548 parameters_ = zone_->NewArray<Node*>(params);
549 for (unsigned i = 0; i < params; i++) {
550 parameters_[i] = nullptr;
551 }
552 // Initialize instance node.
553 switch (parameter_mode_) {
554 case kInstanceMode:
555 instance_node_ = Param(wasm::kWasmInstanceParameterIndex);
556 break;
557 case kNoSpecialParameterMode:
558 instance_node_ = gasm_->LoadExportedFunctionInstance(
559 gasm_->LoadFunctionDataFromJSFunction(
560 Param(Linkage::kJSCallClosureParamIndex, "%closure")));
561 break;
562 case kWasmApiFunctionRefMode:
563 // We need an instance node anyway, because FromJS() needs to pass it to
564 // the WasmIsValidRefValue runtime function.
565 instance_node_ = UndefinedValue();
566 break;
567 }
568 graph()->SetEnd(graph()->NewNode(mcgraph()->common()->End(0)));
569}
570
571Node* WasmGraphBuilder::Param(int index, const char* debug_name) {
572 DCHECK_NOT_NULL(graph()->start())((void) 0);
573 // Turbofan allows negative parameter indices.
574 static constexpr int kMinParameterIndex = -1;
575 DCHECK_GE(index, kMinParameterIndex)((void) 0);
576 int array_index = index - kMinParameterIndex;
577 if (parameters_[array_index] == nullptr) {
578 parameters_[array_index] = graph()->NewNode(
579 mcgraph()->common()->Parameter(index, debug_name), graph()->start());
580 }
581 return parameters_[array_index];
582}
583
584Node* WasmGraphBuilder::Loop(Node* entry) {
585 return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
586}
587
588void WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
589 Node* terminate =
590 graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
591 gasm_->MergeControlToEnd(terminate);
592}
593
594Node* WasmGraphBuilder::LoopExit(Node* loop_node) {
595 DCHECK(loop_node->opcode() == IrOpcode::kLoop)((void) 0);
596 Node* loop_exit =
597 graph()->NewNode(mcgraph()->common()->LoopExit(), control(), loop_node);
598 Node* loop_exit_effect = graph()->NewNode(
599 mcgraph()->common()->LoopExitEffect(), effect(), loop_exit);
600 SetEffectControl(loop_exit_effect, loop_exit);
601 return loop_exit;
602}
603
604Node* WasmGraphBuilder::LoopExitValue(Node* value,
605 MachineRepresentation representation) {
606 DCHECK(control()->opcode() == IrOpcode::kLoopExit)((void) 0);
607 return graph()->NewNode(mcgraph()->common()->LoopExitValue(representation),
608 value, control());
609}
610
611void WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
612 Node* terminate =
613 graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
614 gasm_->MergeControlToEnd(terminate);
615}
616
617bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
618 return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
619 NodeProperties::GetControlInput(phi) == merge;
620}
621
622bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
623 Node** if_exception) {
624 if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
625 return false;
626 }
627
628 *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
629 *if_exception =
630 graph()->NewNode(mcgraph()->common()->IfException(), node, node);
631
632 return true;
633}
634
635void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
636 DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()))((void) 0);
637 merge->AppendInput(mcgraph()->zone(), from);
638 int new_size = merge->InputCount();
639 NodeProperties::ChangeOp(
640 merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
641}
642
643void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
644 DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()))((void) 0);
645 int new_size = phi->InputCount();
646 phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
647 NodeProperties::ChangeOp(
648 phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
649}
650
651template <typename... Nodes>
652Node* WasmGraphBuilder::Merge(Node* fst, Nodes*... args) {
653 return graph()->NewNode(this->mcgraph()->common()->Merge(1 + sizeof...(args)),
654 fst, args...);
655}
656
657Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
658 return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
659}
660
661Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count,
662 Node** vals_and_control) {
663 DCHECK(IrOpcode::IsMergeOpcode(vals_and_control[count]->opcode()))((void) 0);
664 DCHECK_EQ(vals_and_control[count]->op()->ControlInputCount(), count)((void) 0);
665 return graph()->NewNode(
666 mcgraph()->common()->Phi(type.machine_representation(), count), count + 1,
667 vals_and_control);
668}
669
670Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects_and_control) {
671 DCHECK(IrOpcode::IsMergeOpcode(effects_and_control[count]->opcode()))((void) 0);
672 return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
673 effects_and_control);
674}
675
676Node* WasmGraphBuilder::RefNull() { return LOAD_ROOT(NullValue, null_value); }
677
678Node* WasmGraphBuilder::RefFunc(uint32_t function_index) {
679 return gasm_->CallRuntimeStub(wasm::WasmCode::kWasmRefFunc,
680 Operator::kNoThrow,
681 gasm_->Uint32Constant(function_index));
682}
683
684Node* WasmGraphBuilder::RefAsNonNull(Node* arg,
685 wasm::WasmCodePosition position) {
686 if (!FLAG_experimental_wasm_skip_null_checks) {
687 TrapIfTrue(wasm::kTrapIllegalCast, IsNull(arg), position);
688 }
689 return arg;
690}
691
692Node* WasmGraphBuilder::NoContextConstant() {
693 return mcgraph()->IntPtrConstant(0);
694}
695
696Node* WasmGraphBuilder::GetInstance() { return instance_node_.get(); }
697
698Node* WasmGraphBuilder::BuildLoadIsolateRoot() {
699 switch (parameter_mode_) {
700 case kInstanceMode:
701 // For wasm functions, the IsolateRoot is loaded from the instance node so
702 // that the generated code is Isolate independent.
703 return LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
704 case kWasmApiFunctionRefMode:
705 // Note: Even if V8_SANDBOXED_EXTERNAL_POINTERS, the pointer to the
706 // isolate root is not encoded, much like the case above. TODO(manoskouk):
707 // Decode the pointer here if that changes.
708 return gasm_->Load(
709 MachineType::Pointer(), Param(0),
710 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kIsolateRootOffset));
711 case kNoSpecialParameterMode:
712 return mcgraph()->IntPtrConstant(isolate_->isolate_root());
713 }
714}
715
716Node* WasmGraphBuilder::Int32Constant(int32_t value) {
717 return mcgraph()->Int32Constant(value);
718}
719
720Node* WasmGraphBuilder::Int64Constant(int64_t value) {
721 return mcgraph()->Int64Constant(value);
722}
723
724Node* WasmGraphBuilder::UndefinedValue() {
725 return LOAD_ROOT(UndefinedValue, undefined_value);
726}
727
728void WasmGraphBuilder::StackCheck(
729 WasmInstanceCacheNodes* shared_memory_instance_cache,
730 wasm::WasmCodePosition position) {
731 DCHECK_NOT_NULL(env_)((void) 0); // Wrappers don't get stack checks.
732 if (!FLAG_wasm_stack_checks || !env_->runtime_exception_support) {
733 return;
734 }
735
736 Node* limit_address =
737 LOAD_INSTANCE_FIELD(StackLimitAddress, MachineType::Pointer());
738 Node* limit = gasm_->LoadFromObject(MachineType::Pointer(), limit_address, 0);
739
740 Node* check = SetEffect(graph()->NewNode(
741 mcgraph()->machine()->StackPointerGreaterThan(StackCheckKind::kWasm),
742 limit, effect()));
743
744 Node* if_true;
745 Node* if_false;
746 BranchExpectTrue(check, &if_true, &if_false);
747
748 if (stack_check_call_operator_ == nullptr) {
749 // Build and cache the stack check call operator and the constant
750 // representing the stack check code.
751
752 // A direct call to a wasm runtime stub defined in this module.
753 // Just encode the stub index. This will be patched at relocation.
754 stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
755 wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
756
757 constexpr Operator::Properties properties =
758 Operator::kNoThrow | Operator::kNoWrite;
759 // If we ever want to mark this call as kNoDeopt, we'll have to make it
760 // non-eliminatable some other way.
761 STATIC_ASSERT((properties & Operator::kEliminatable) !=static_assert((properties & Operator::kEliminatable) != Operator
::kEliminatable, "(properties & Operator::kEliminatable) != Operator::kEliminatable"
)
762 Operator::kEliminatable)static_assert((properties & Operator::kEliminatable) != Operator
::kEliminatable, "(properties & Operator::kEliminatable) != Operator::kEliminatable"
)
;
763 auto call_descriptor = Linkage::GetStubCallDescriptor(
764 mcgraph()->zone(), // zone
765 NoContextDescriptor{}, // descriptor
766 0, // stack parameter count
767 CallDescriptor::kNoFlags, // flags
768 properties, // properties
769 StubCallMode::kCallWasmRuntimeStub); // stub call mode
770 stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
771 }
772
773 Node* call =
774 graph()->NewNode(stack_check_call_operator_.get(),
775 stack_check_code_node_.get(), effect(), if_false);
776 SetSourcePosition(call, position);
777
778 DCHECK_GT(call->op()->EffectOutputCount(), 0)((void) 0);
779 DCHECK_EQ(call->op()->ControlOutputCount(), 0)((void) 0);
780
781 SetEffectControl(call, if_false);
782
783 Node* merge = Merge(if_true, control());
784 Node* ephi_inputs[] = {check, effect(), merge};
785 Node* ephi = EffectPhi(2, ephi_inputs);
786
787 // We only need to refresh the size of a shared memory, as its start can never
788 // change.
789 if (shared_memory_instance_cache != nullptr) {
790 // We handle caching of the instance cache nodes manually, and we may reload
791 // them in contexts where load elimination would eliminate the reload.
792 // Therefore, we use plain Load nodes which are not subject to load
793 // elimination.
794 Node* new_memory_size =
795 LOAD_INSTANCE_FIELD_NO_ELIMINATION(MemorySize, MachineType::UintPtr())gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(MemorySize
), MachineType::UintPtr()), GetInstance(), wasm::ObjectAccess
::ToTagged(WasmInstanceObject::kMemorySizeOffset))
;
796 shared_memory_instance_cache->mem_size = CreateOrMergeIntoPhi(
797 MachineType::PointerRepresentation(), merge,
798 shared_memory_instance_cache->mem_size, new_memory_size);
799 }
800
801 SetEffectControl(ephi, merge);
802}
803
804void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
805 if (!needs_stack_check_) return;
806
807 Node* start = graph()->start();
808 // Place a stack check which uses a dummy node as control and effect.
809 Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
810 SetEffectControl(dummy);
811 // The function-prologue stack check is associated with position 0, which
812 // is never a position of any instruction in the function.
813 // We pass the null instance cache, as we are at the beginning of the function
814 // and do not need to update it.
815 StackCheck(nullptr, 0);
816
817 // In testing, no stack checks were emitted. Nothing to rewire then.
818 if (effect() == dummy) return;
819
820 // Now patch all control uses of {start} to use {control} and all effect uses
821 // to use {effect} instead. We exclude Projection nodes: Projections pointing
822 // to start are floating control, and we want it to point directly to start
823 // because of restrictions later in the pipeline (specifically, loop
824 // unrolling).
825 // Then rewire the dummy node to use start instead.
826 NodeProperties::ReplaceUses(start, start, effect(), control());
827 {
828 // We need an intermediate vector because we are not allowed to modify a use
829 // while traversing uses().
830 std::vector<Node*> projections;
831 for (Node* use : control()->uses()) {
832 if (use->opcode() == IrOpcode::kProjection) projections.emplace_back(use);
833 }
834 for (Node* use : projections) {
835 use->ReplaceInput(NodeProperties::FirstControlIndex(use), start);
836 }
837 }
838 NodeProperties::ReplaceUses(dummy, nullptr, start, start);
839}
840
841Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
842 wasm::WasmCodePosition position) {
843 const Operator* op;
844 MachineOperatorBuilder* m = mcgraph()->machine();
845 switch (opcode) {
846 case wasm::kExprI32Add:
847 op = m->Int32Add();
848 break;
849 case wasm::kExprI32Sub:
850 op = m->Int32Sub();
851 break;
852 case wasm::kExprI32Mul:
853 op = m->Int32Mul();
854 break;
855 case wasm::kExprI32DivS:
856 return BuildI32DivS(left, right, position);
857 case wasm::kExprI32DivU:
858 return BuildI32DivU(left, right, position);
859 case wasm::kExprI32RemS:
860 return BuildI32RemS(left, right, position);
861 case wasm::kExprI32RemU:
862 return BuildI32RemU(left, right, position);
863 case wasm::kExprI32And:
864 op = m->Word32And();
865 break;
866 case wasm::kExprI32Ior:
867 op = m->Word32Or();
868 break;
869 case wasm::kExprI32Xor:
870 op = m->Word32Xor();
871 break;
872 case wasm::kExprI32Shl:
873 op = m->Word32Shl();
874 right = MaskShiftCount32(right);
875 break;
876 case wasm::kExprI32ShrU:
877 op = m->Word32Shr();
878 right = MaskShiftCount32(right);
879 break;
880 case wasm::kExprI32ShrS:
881 op = m->Word32Sar();
882 right = MaskShiftCount32(right);
883 break;
884 case wasm::kExprI32Ror:
885 op = m->Word32Ror();
886 right = MaskShiftCount32(right);
887 break;
888 case wasm::kExprI32Rol:
889 if (m->Word32Rol().IsSupported()) {
890 op = m->Word32Rol().op();
891 right = MaskShiftCount32(right);
892 break;
893 }
894 return BuildI32Rol(left, right);
895 case wasm::kExprI32Eq:
896 op = m->Word32Equal();
897 break;
898 case wasm::kExprI32Ne:
899 return Invert(Binop(wasm::kExprI32Eq, left, right));
900 case wasm::kExprI32LtS:
901 op = m->Int32LessThan();
902 break;
903 case wasm::kExprI32LeS:
904 op = m->Int32LessThanOrEqual();
905 break;
906 case wasm::kExprI32LtU:
907 op = m->Uint32LessThan();
908 break;
909 case wasm::kExprI32LeU:
910 op = m->Uint32LessThanOrEqual();
911 break;
912 case wasm::kExprI32GtS:
913 op = m->Int32LessThan();
914 std::swap(left, right);
915 break;
916 case wasm::kExprI32GeS:
917 op = m->Int32LessThanOrEqual();
918 std::swap(left, right);
919 break;
920 case wasm::kExprI32GtU:
921 op = m->Uint32LessThan();
922 std::swap(left, right);
923 break;
924 case wasm::kExprI32GeU:
925 op = m->Uint32LessThanOrEqual();
926 std::swap(left, right);
927 break;
928 case wasm::kExprI64And:
929 op = m->Word64And();
930 break;
931 case wasm::kExprI64Add:
932 op = m->Int64Add();
933 break;
934 case wasm::kExprI64Sub:
935 op = m->Int64Sub();
936 break;
937 case wasm::kExprI64Mul:
938 op = m->Int64Mul();
939 break;
940 case wasm::kExprI64DivS:
941 return BuildI64DivS(left, right, position);
942 case wasm::kExprI64DivU:
943 return BuildI64DivU(left, right, position);
944 case wasm::kExprI64RemS:
945 return BuildI64RemS(left, right, position);
946 case wasm::kExprI64RemU:
947 return BuildI64RemU(left, right, position);
948 case wasm::kExprI64Ior:
949 op = m->Word64Or();
950 break;
951 case wasm::kExprI64Xor:
952 op = m->Word64Xor();
953 break;
954 case wasm::kExprI64Shl:
955 op = m->Word64Shl();
956 right = MaskShiftCount64(right);
957 break;
958 case wasm::kExprI64ShrU:
959 op = m->Word64Shr();
960 right = MaskShiftCount64(right);
961 break;
962 case wasm::kExprI64ShrS:
963 op = m->Word64Sar();
964 right = MaskShiftCount64(right);
965 break;
966 case wasm::kExprI64Eq:
967 op = m->Word64Equal();
968 break;
969 case wasm::kExprI64Ne:
970 return Invert(Binop(wasm::kExprI64Eq, left, right));
971 case wasm::kExprI64LtS:
972 op = m->Int64LessThan();
973 break;
974 case wasm::kExprI64LeS:
975 op = m->Int64LessThanOrEqual();
976 break;
977 case wasm::kExprI64LtU:
978 op = m->Uint64LessThan();
979 break;
980 case wasm::kExprI64LeU:
981 op = m->Uint64LessThanOrEqual();
982 break;
983 case wasm::kExprI64GtS:
984 op = m->Int64LessThan();
985 std::swap(left, right);
986 break;
987 case wasm::kExprI64GeS:
988 op = m->Int64LessThanOrEqual();
989 std::swap(left, right);
990 break;
991 case wasm::kExprI64GtU:
992 op = m->Uint64LessThan();
993 std::swap(left, right);
994 break;
995 case wasm::kExprI64GeU:
996 op = m->Uint64LessThanOrEqual();
997 std::swap(left, right);
998 break;
999 case wasm::kExprI64Ror:
1000 right = MaskShiftCount64(right);
1001 return m->Is64() ? graph()->NewNode(m->Word64Ror(), left, right)
1002 : graph()->NewNode(m->Word64RorLowerable(), left, right,
1003 control());
1004 case wasm::kExprI64Rol:
1005 if (m->Word64Rol().IsSupported()) {
1006 return m->Is64() ? graph()->NewNode(m->Word64Rol().op(), left,
1007 MaskShiftCount64(right))
1008 : graph()->NewNode(m->Word64RolLowerable().op(), left,
1009 MaskShiftCount64(right), control());
1010 } else if (m->Word32Rol().IsSupported()) {
1011 return graph()->NewNode(m->Word64RolLowerable().placeholder(), left,
1012 right, control());
1013 }
1014 return BuildI64Rol(left, right);
1015 case wasm::kExprF32CopySign:
1016 return BuildF32CopySign(left, right);
1017 case wasm::kExprF64CopySign:
1018 return BuildF64CopySign(left, right);
1019 case wasm::kExprF32Add:
1020 op = m->Float32Add();
1021 break;
1022 case wasm::kExprF32Sub:
1023 op = m->Float32Sub();
1024 break;
1025 case wasm::kExprF32Mul:
1026 op = m->Float32Mul();
1027 break;
1028 case wasm::kExprF32Div:
1029 op = m->Float32Div();
1030 break;
1031 case wasm::kExprF32Eq:
1032 op = m->Float32Equal();
1033 break;
1034 case wasm::kExprF32Ne:
1035 return Invert(Binop(wasm::kExprF32Eq, left, right));
1036 case wasm::kExprF32Lt:
1037 op = m->Float32LessThan();
1038 break;
1039 case wasm::kExprF32Ge:
1040 op = m->Float32LessThanOrEqual();
1041 std::swap(left, right);
1042 break;
1043 case wasm::kExprF32Gt:
1044 op = m->Float32LessThan();
1045 std::swap(left, right);
1046 break;
1047 case wasm::kExprF32Le:
1048 op = m->Float32LessThanOrEqual();
1049 break;
1050 case wasm::kExprF64Add:
1051 op = m->Float64Add();
1052 break;
1053 case wasm::kExprF64Sub:
1054 op = m->Float64Sub();
1055 break;
1056 case wasm::kExprF64Mul:
1057 op = m->Float64Mul();
1058 break;
1059 case wasm::kExprF64Div:
1060 op = m->Float64Div();
1061 break;
1062 case wasm::kExprF64Eq:
1063 op = m->Float64Equal();
1064 break;
1065 case wasm::kExprF64Ne:
1066 return Invert(Binop(wasm::kExprF64Eq, left, right));
1067 case wasm::kExprF64Lt:
1068 op = m->Float64LessThan();
1069 break;
1070 case wasm::kExprF64Le:
1071 op = m->Float64LessThanOrEqual();
1072 break;
1073 case wasm::kExprF64Gt:
1074 op = m->Float64LessThan();
1075 std::swap(left, right);
1076 break;
1077 case wasm::kExprF64Ge:
1078 op = m->Float64LessThanOrEqual();
1079 std::swap(left, right);
1080 break;
1081 case wasm::kExprF32Min:
1082 op = m->Float32Min();
1083 break;
1084 case wasm::kExprF64Min:
1085 op = m->Float64Min();
1086 break;
1087 case wasm::kExprF32Max:
1088 op = m->Float32Max();
1089 break;
1090 case wasm::kExprF64Max:
1091 op = m->Float64Max();
1092 break;
1093 case wasm::kExprF64Pow:
1094 return BuildF64Pow(left, right);
1095 case wasm::kExprF64Atan2:
1096 op = m->Float64Atan2();
1097 break;
1098 case wasm::kExprF64Mod:
1099 return BuildF64Mod(left, right);
1100 case wasm::kExprRefEq:
1101 return gasm_->TaggedEqual(left, right);
1102 case wasm::kExprI32AsmjsDivS:
1103 return BuildI32AsmjsDivS(left, right);
1104 case wasm::kExprI32AsmjsDivU:
1105 return BuildI32AsmjsDivU(left, right);
1106 case wasm::kExprI32AsmjsRemS:
1107 return BuildI32AsmjsRemS(left, right);
1108 case wasm::kExprI32AsmjsRemU:
1109 return BuildI32AsmjsRemU(left, right);
1110 case wasm::kExprI32AsmjsStoreMem8:
1111 return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
1112 case wasm::kExprI32AsmjsStoreMem16:
1113 return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
1114 case wasm::kExprI32AsmjsStoreMem:
1115 return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
1116 case wasm::kExprF32AsmjsStoreMem:
1117 return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
1118 case wasm::kExprF64AsmjsStoreMem:
1119 return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
1120 default:
1121 FATAL_UNSUPPORTED_OPCODE(opcode);
1122 }
1123 return graph()->NewNode(op, left, right);
1124}
1125
1126Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
1127 wasm::WasmCodePosition position) {
1128 const Operator* op;
1129 MachineOperatorBuilder* m = mcgraph()->machine();
1130 switch (opcode) {
1131 case wasm::kExprI32Eqz:
1132 return gasm_->Word32Equal(input, Int32Constant(0));
1133 case wasm::kExprF32Abs:
1134 op = m->Float32Abs();
1135 break;
1136 case wasm::kExprF32Neg: {
1137 op = m->Float32Neg();
1138 break;
1139 }
1140 case wasm::kExprF32Sqrt:
1141 op = m->Float32Sqrt();
1142 break;
1143 case wasm::kExprF64Abs:
1144 op = m->Float64Abs();
1145 break;
1146 case wasm::kExprF64Neg: {
1147 op = m->Float64Neg();
1148 break;
1149 }
1150 case wasm::kExprF64Sqrt:
1151 op = m->Float64Sqrt();
1152 break;
1153 case wasm::kExprI32SConvertF32:
1154 case wasm::kExprI32UConvertF32:
1155 case wasm::kExprI32SConvertF64:
1156 case wasm::kExprI32UConvertF64:
1157 case wasm::kExprI32SConvertSatF64:
1158 case wasm::kExprI32UConvertSatF64:
1159 case wasm::kExprI32SConvertSatF32:
1160 case wasm::kExprI32UConvertSatF32:
1161 return BuildIntConvertFloat(input, position, opcode);
1162 case wasm::kExprI32AsmjsSConvertF64:
1163 return BuildI32AsmjsSConvertF64(input);
1164 case wasm::kExprI32AsmjsUConvertF64:
1165 return BuildI32AsmjsUConvertF64(input);
1166 case wasm::kExprF32ConvertF64:
1167 op = m->TruncateFloat64ToFloat32();
1168 break;
1169 case wasm::kExprF64SConvertI32:
1170 op = m->ChangeInt32ToFloat64();
1171 break;
1172 case wasm::kExprF64UConvertI32:
1173 op = m->ChangeUint32ToFloat64();
1174 break;
1175 case wasm::kExprF32SConvertI32:
1176 op = m->RoundInt32ToFloat32();
1177 break;
1178 case wasm::kExprF32UConvertI32:
1179 op = m->RoundUint32ToFloat32();
1180 break;
1181 case wasm::kExprI32AsmjsSConvertF32:
1182 return BuildI32AsmjsSConvertF32(input);
1183 case wasm::kExprI32AsmjsUConvertF32:
1184 return BuildI32AsmjsUConvertF32(input);
1185 case wasm::kExprF64ConvertF32:
1186 op = m->ChangeFloat32ToFloat64();
1187 break;
1188 case wasm::kExprF32ReinterpretI32:
1189 op = m->BitcastInt32ToFloat32();
1190 break;
1191 case wasm::kExprI32ReinterpretF32:
1192 op = m->BitcastFloat32ToInt32();
1193 break;
1194 case wasm::kExprI32Clz:
1195 op = m->Word32Clz();
1196 break;
1197 case wasm::kExprI32Ctz: {
1198 if (m->Word32Ctz().IsSupported()) {
1199 op = m->Word32Ctz().op();
1200 break;
1201 } else if (m->Word32ReverseBits().IsSupported()) {
1202 Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
1203 Node* result = graph()->NewNode(m->Word32Clz(), reversed);
1204 return result;
1205 } else {
1206 return BuildI32Ctz(input);
1207 }
1208 }
1209 case wasm::kExprI32Popcnt: {
1210 if (m->Word32Popcnt().IsSupported()) {
1211 op = m->Word32Popcnt().op();
1212 break;
1213 } else {
1214 return BuildI32Popcnt(input);
1215 }
1216 }
1217 case wasm::kExprF32Floor: {
1218 if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
1219 op = m->Float32RoundDown().op();
1220 break;
1221 }
1222 case wasm::kExprF32Ceil: {
1223 if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
1224 op = m->Float32RoundUp().op();
1225 break;
1226 }
1227 case wasm::kExprF32Trunc: {
1228 if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
1229 op = m->Float32RoundTruncate().op();
1230 break;
1231 }
1232 case wasm::kExprF32NearestInt: {
1233 if (!m->Float32RoundTiesEven().IsSupported())
1234 return BuildF32NearestInt(input);
1235 op = m->Float32RoundTiesEven().op();
1236 break;
1237 }
1238 case wasm::kExprF64Floor: {
1239 if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
1240 op = m->Float64RoundDown().op();
1241 break;
1242 }
1243 case wasm::kExprF64Ceil: {
1244 if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
1245 op = m->Float64RoundUp().op();
1246 break;
1247 }
1248 case wasm::kExprF64Trunc: {
1249 if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
1250 op = m->Float64RoundTruncate().op();
1251 break;
1252 }
1253 case wasm::kExprF64NearestInt: {
1254 if (!m->Float64RoundTiesEven().IsSupported())
1255 return BuildF64NearestInt(input);
1256 op = m->Float64RoundTiesEven().op();
1257 break;
1258 }
1259 case wasm::kExprF64Acos: {
1260 return BuildF64Acos(input);
1261 }
1262 case wasm::kExprF64Asin: {
1263 return BuildF64Asin(input);
1264 }
1265 case wasm::kExprF64Atan:
1266 op = m->Float64Atan();
1267 break;
1268 case wasm::kExprF64Cos: {
1269 op = m->Float64Cos();
1270 break;
1271 }
1272 case wasm::kExprF64Sin: {
1273 op = m->Float64Sin();
1274 break;
1275 }
1276 case wasm::kExprF64Tan: {
1277 op = m->Float64Tan();
1278 break;
1279 }
1280 case wasm::kExprF64Exp: {
1281 op = m->Float64Exp();
1282 break;
1283 }
1284 case wasm::kExprF64Log:
1285 op = m->Float64Log();
1286 break;
1287 case wasm::kExprI32ConvertI64:
1288 op = m->TruncateInt64ToInt32();
1289 break;
1290 case wasm::kExprI64SConvertI32:
1291 op = m->ChangeInt32ToInt64();
1292 break;
1293 case wasm::kExprI64UConvertI32:
1294 op = m->ChangeUint32ToUint64();
1295 break;
1296 case wasm::kExprF64ReinterpretI64:
1297 op = m->BitcastInt64ToFloat64();
1298 break;
1299 case wasm::kExprI64ReinterpretF64:
1300 op = m->BitcastFloat64ToInt64();
1301 break;
1302 case wasm::kExprI64Clz:
1303 return m->Is64()
1304 ? graph()->NewNode(m->Word64Clz(), input)
1305 : graph()->NewNode(m->Word64ClzLowerable(), input, control());
1306 case wasm::kExprI64Ctz: {
1307 if (m->Word64Ctz().IsSupported()) {
1308 return m->Is64() ? graph()->NewNode(m->Word64Ctz().op(), input)
1309 : graph()->NewNode(m->Word64CtzLowerable().op(), input,
1310 control());
1311 } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
1312 return graph()->NewNode(m->Word64CtzLowerable().placeholder(), input,
1313 control());
1314 } else if (m->Word64ReverseBits().IsSupported()) {
1315 Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
1316 Node* result = m->Is64() ? graph()->NewNode(m->Word64Clz(), reversed)
1317 : graph()->NewNode(m->Word64ClzLowerable(),
1318 reversed, control());
1319 return result;
1320 } else {
1321 return BuildI64Ctz(input);
1322 }
1323 }
1324 case wasm::kExprI64Popcnt: {
1325 OptionalOperator popcnt64 = m->Word64Popcnt();
1326 if (popcnt64.IsSupported()) {
1327 op = popcnt64.op();
1328 } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
1329 op = popcnt64.placeholder();
1330 } else {
1331 return BuildI64Popcnt(input);
1332 }
1333 break;
1334 }
1335 case wasm::kExprI64Eqz:
1336 return gasm_->Word64Equal(input, Int64Constant(0));
1337 case wasm::kExprF32SConvertI64:
1338 if (m->Is32()) {
1339 return BuildF32SConvertI64(input);
1340 }
1341 op = m->RoundInt64ToFloat32();
1342 break;
1343 case wasm::kExprF32UConvertI64:
1344 if (m->Is32()) {
1345 return BuildF32UConvertI64(input);
1346 }
1347 op = m->RoundUint64ToFloat32();
1348 break;
1349 case wasm::kExprF64SConvertI64:
1350 if (m->Is32()) {
1351 return BuildF64SConvertI64(input);
1352 }
1353 op = m->RoundInt64ToFloat64();
1354 break;
1355 case wasm::kExprF64UConvertI64:
1356 if (m->Is32()) {
1357 return BuildF64UConvertI64(input);
1358 }
1359 op = m->RoundUint64ToFloat64();
1360 break;
1361 case wasm::kExprI32SExtendI8:
1362 op = m->SignExtendWord8ToInt32();
1363 break;
1364 case wasm::kExprI32SExtendI16:
1365 op = m->SignExtendWord16ToInt32();
1366 break;
1367 case wasm::kExprI64SExtendI8:
1368 op = m->SignExtendWord8ToInt64();
1369 break;
1370 case wasm::kExprI64SExtendI16:
1371 op = m->SignExtendWord16ToInt64();
1372 break;
1373 case wasm::kExprI64SExtendI32:
1374 op = m->SignExtendWord32ToInt64();
1375 break;
1376 case wasm::kExprI64SConvertF32:
1377 case wasm::kExprI64UConvertF32:
1378 case wasm::kExprI64SConvertF64:
1379 case wasm::kExprI64UConvertF64:
1380 case wasm::kExprI64SConvertSatF32:
1381 case wasm::kExprI64UConvertSatF32:
1382 case wasm::kExprI64SConvertSatF64:
1383 case wasm::kExprI64UConvertSatF64:
1384 return mcgraph()->machine()->Is32()
1385 ? BuildCcallConvertFloat(input, position, opcode)
1386 : BuildIntConvertFloat(input, position, opcode);
1387 case wasm::kExprRefIsNull:
1388 return IsNull(input);
1389 // We abuse ref.as_non_null, which isn't otherwise used in this switch, as
1390 // a sentinel for the negation of ref.is_null.
1391 case wasm::kExprRefAsNonNull:
1392 return gasm_->Int32Sub(gasm_->Int32Constant(1), IsNull(input));
1393 case wasm::kExprI32AsmjsLoadMem8S:
1394 return BuildAsmjsLoadMem(MachineType::Int8(), input);
1395 case wasm::kExprI32AsmjsLoadMem8U:
1396 return BuildAsmjsLoadMem(MachineType::Uint8(), input);
1397 case wasm::kExprI32AsmjsLoadMem16S:
1398 return BuildAsmjsLoadMem(MachineType::Int16(), input);
1399 case wasm::kExprI32AsmjsLoadMem16U:
1400 return BuildAsmjsLoadMem(MachineType::Uint16(), input);
1401 case wasm::kExprI32AsmjsLoadMem:
1402 return BuildAsmjsLoadMem(MachineType::Int32(), input);
1403 case wasm::kExprF32AsmjsLoadMem:
1404 return BuildAsmjsLoadMem(MachineType::Float32(), input);
1405 case wasm::kExprF64AsmjsLoadMem:
1406 return BuildAsmjsLoadMem(MachineType::Float64(), input);
1407 default:
1408 FATAL_UNSUPPORTED_OPCODE(opcode);
1409 }
1410 return graph()->NewNode(op, input);
1411}
1412
1413Node* WasmGraphBuilder::Float32Constant(float value) {
1414 return mcgraph()->Float32Constant(value);
1415}
1416
1417Node* WasmGraphBuilder::Float64Constant(double value) {
1418 return mcgraph()->Float64Constant(value);
1419}
1420
1421Node* WasmGraphBuilder::Simd128Constant(const uint8_t value[16]) {
1422 has_simd_ = true;
1423 return graph()->NewNode(mcgraph()->machine()->S128Const(value));
1424}
1425
1426Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
1427 Node** false_node) {
1428 return gasm_->Branch(cond, true_node, false_node, BranchHint::kNone);
1429}
1430
1431Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
1432 Node** false_node) {
1433 return gasm_->Branch(cond, true_node, false_node, BranchHint::kFalse);
1434}
1435
1436Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
1437 Node** false_node) {
1438 return gasm_->Branch(cond, true_node, false_node, BranchHint::kTrue);
1439}
1440
1441Node* WasmGraphBuilder::Select(Node *cond, Node* true_node,
1442 Node* false_node, wasm::ValueType type) {
1443 MachineOperatorBuilder* m = mcgraph()->machine();
1444 wasm::ValueKind kind = type.kind();
1445 // Lower to select if supported.
1446 if (kind == wasm::kF32 && m->Float32Select().IsSupported()) {
1447 return mcgraph()->graph()->NewNode(m->Float32Select().op(), cond,
1448 true_node, false_node);
1449 }
1450 if (kind == wasm::kF64 && m->Float64Select().IsSupported()) {
1451 return mcgraph()->graph()->NewNode(m->Float64Select().op(), cond,
1452 true_node, false_node);
1453 }
1454 if (kind == wasm::kI32 && m->Word32Select().IsSupported()) {
1455 return mcgraph()->graph()->NewNode(m->Word32Select().op(), cond, true_node,
1456 false_node);
1457 }
1458 if (kind == wasm::kI64 && m->Word64Select().IsSupported()) {
1459 return mcgraph()->graph()->NewNode(m->Word64Select().op(), cond, true_node,
1460 false_node);
1461 }
1462 // Default to control-flow.
1463 Node* controls[2];
1464 BranchNoHint(cond, &controls[0], &controls[1]);
1465 Node* merge = Merge(2, controls);
1466 SetControl(merge);
1467 Node* inputs[] = {true_node, false_node, merge};
1468 return Phi(type, 2, inputs);
1469}
1470
1471TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
1472 // TODO(wasm): "!env_" should not happen when compiling an actual wasm
1473 // function.
1474 if (!env_ || !env_->runtime_exception_support) {
1475 // We use TrapId::kInvalid as a marker to tell the code generator
1476 // to generate a call to a testing c-function instead of a runtime
1477 // stub. This code should only be called from a cctest.
1478 return TrapId::kInvalid;
1479 }
1480
1481 switch (reason) {
1482#define TRAPREASON_TO_TRAPID(name) \
1483 case wasm::k##name: \
1484 static_assert( \
1485 static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
1486 "trap id mismatch"); \
1487 return TrapId::k##name;
1488 FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)TRAPREASON_TO_TRAPID(TrapUnreachable) TRAPREASON_TO_TRAPID(TrapMemOutOfBounds
) TRAPREASON_TO_TRAPID(TrapUnalignedAccess) TRAPREASON_TO_TRAPID
(TrapDivByZero) TRAPREASON_TO_TRAPID(TrapDivUnrepresentable) TRAPREASON_TO_TRAPID
(TrapRemByZero) TRAPREASON_TO_TRAPID(TrapFloatUnrepresentable
) TRAPREASON_TO_TRAPID(TrapFuncSigMismatch) TRAPREASON_TO_TRAPID
(TrapDataSegmentOutOfBounds) TRAPREASON_TO_TRAPID(TrapElemSegmentDropped
) TRAPREASON_TO_TRAPID(TrapTableOutOfBounds) TRAPREASON_TO_TRAPID
(TrapRethrowNull) TRAPREASON_TO_TRAPID(TrapNullDereference) TRAPREASON_TO_TRAPID
(TrapIllegalCast) TRAPREASON_TO_TRAPID(TrapArrayOutOfBounds) TRAPREASON_TO_TRAPID
(TrapArrayTooLarge)
1489#undef TRAPREASON_TO_TRAPID
1490 default:
1491 UNREACHABLE()V8_Fatal("unreachable code");
1492 }
1493}
1494
1495void WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
1496 wasm::WasmCodePosition position) {
1497 TrapId trap_id = GetTrapIdForTrap(reason);
1498 Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
1499 cond, effect(), control()));
1500 SetSourcePosition(node, position);
1501}
1502
1503void WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
1504 wasm::WasmCodePosition position) {
1505 TrapId trap_id = GetTrapIdForTrap(reason);
1506 Node* node = SetControl(graph()->NewNode(
1507 mcgraph()->common()->TrapUnless(trap_id), cond, effect(), control()));
1508 SetSourcePosition(node, position);
1509}
1510
1511// Add a check that traps if {node} is equal to {val}.
1512void WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
1513 int32_t val,
1514 wasm::WasmCodePosition position) {
1515 Int32Matcher m(node);
1516 if (m.HasResolvedValue() && !m.Is(val)) return;
1517 if (val == 0) {
1518 TrapIfFalse(reason, node, position);
1519 } else {
1520 TrapIfTrue(reason, gasm_->Word32Equal(node, Int32Constant(val)), position);
1521 }
1522}
1523
1524// Add a check that traps if {node} is zero.
1525void WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
1526 wasm::WasmCodePosition position) {
1527 TrapIfEq32(reason, node, 0, position);
1528}
1529
1530// Add a check that traps if {node} is equal to {val}.
1531void WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
1532 int64_t val,
1533 wasm::WasmCodePosition position) {
1534 Int64Matcher m(node);
1535 if (m.HasResolvedValue() && !m.Is(val)) return;
1536 TrapIfTrue(reason, gasm_->Word64Equal(node, Int64Constant(val)), position);
1537}
1538
1539// Add a check that traps if {node} is zero.
1540void WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
1541 wasm::WasmCodePosition position) {
1542 TrapIfEq64(reason, node, 0, position);
1543}
1544
1545Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
1546 // The instruction selector will use {kArchTableSwitch} for large switches,
1547 // which has limited input count, see {InstructionSelector::EmitTableSwitch}.
1548 DCHECK_LE(count, Instruction::kMaxInputCount - 2)((void) 0); // value_range + 2
1549 DCHECK_LE(count, wasm::kV8MaxWasmFunctionBrTableSize + 1)((void) 0); // plus IfDefault
1550 return graph()->NewNode(mcgraph()->common()->Switch(count), key, control());
1551}
1552
1553Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1554 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode())((void) 0);
1555 return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1556}
1557
1558Node* WasmGraphBuilder::IfDefault(Node* sw) {
1559 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode())((void) 0);
1560 return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1561}
1562
1563Node* WasmGraphBuilder::Return(base::Vector<Node*> vals) {
1564 unsigned count = static_cast<unsigned>(vals.size());
1565 base::SmallVector<Node*, 8> buf(count + 3);
1566
1567 buf[0] = Int32Constant(0);
1568 if (count > 0) {
1569 memcpy(buf.data() + 1, vals.begin(), sizeof(void*) * count);
1570 }
1571 buf[count + 1] = effect();
1572 buf[count + 2] = control();
1573 Node* ret = graph()->NewNode(mcgraph()->common()->Return(count), count + 3,
1574 buf.data());
1575
1576 gasm_->MergeControlToEnd(ret);
1577 return ret;
1578}
1579
1580void WasmGraphBuilder::Trap(wasm::TrapReason reason,
1581 wasm::WasmCodePosition position) {
1582 TrapIfFalse(reason, Int32Constant(0), position);
1583 // Connect control to end via a Throw() node.
1584 TerminateThrow(effect(), control());
1585}
1586
1587Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1588 static const int32_t kMask32 = 0x1F;
1589 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1590 // Shifts by constants are so common we pattern-match them here.
1591 Int32Matcher match(node);
1592 if (match.HasResolvedValue()) {
1593 int32_t masked = (match.ResolvedValue() & kMask32);
1594 if (match.ResolvedValue() != masked) node = Int32Constant(masked);
1595 } else {
1596 node = gasm_->Word32And(node, Int32Constant(kMask32));
1597 }
1598 }
1599 return node;
1600}
1601
1602Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1603 static const int64_t kMask64 = 0x3F;
1604 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1605 // Shifts by constants are so common we pattern-match them here.
1606 Int64Matcher match(node);
1607 if (match.HasResolvedValue()) {
1608 int64_t masked = (match.ResolvedValue() & kMask64);
1609 if (match.ResolvedValue() != masked) node = Int64Constant(masked);
1610 } else {
1611 node = gasm_->Word64And(node, Int64Constant(kMask64));
1612 }
1613 }
1614 return node;
1615}
1616
1617namespace {
1618
1619bool ReverseBytesSupported(MachineOperatorBuilder* m, size_t size_in_bytes) {
1620 switch (size_in_bytes) {
1621 case 4:
1622 case 16:
1623 return true;
1624 case 8:
1625 return m->Is64();
1626 default:
1627 break;
1628 }
1629 return false;
1630}
1631
1632} // namespace
1633
1634Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1635 Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1636 Node* result;
1
'result' declared without an initial value
1637 Node* value = node;
1638 MachineOperatorBuilder* m = mcgraph()->machine();
1639 int valueSizeInBytes = wasmtype.value_kind_size();
1640 int valueSizeInBits = 8 * valueSizeInBytes;
1641 bool isFloat = false;
1642
1643 switch (wasmtype.kind()) {
2
Control jumps to 'case kS128:' at line 1658
1644 case wasm::kF64:
1645 value = gasm_->BitcastFloat64ToInt64(node);
1646 isFloat = true;
1647 V8_FALLTHROUGH[[clang::fallthrough]];
1648 case wasm::kI64:
1649 result = Int64Constant(0);
1650 break;
1651 case wasm::kF32:
1652 value = gasm_->BitcastFloat32ToInt32(node);
1653 isFloat = true;
1654 V8_FALLTHROUGH[[clang::fallthrough]];
1655 case wasm::kI32:
1656 result = Int32Constant(0);
1657 break;
1658 case wasm::kS128:
1659 DCHECK(ReverseBytesSupported(m, valueSizeInBytes))((void) 0);
1660 break;
1661 default:
1662 UNREACHABLE()V8_Fatal("unreachable code");
1663 }
1664
1665 if (mem_rep == MachineRepresentation::kWord8) {
3
Execution continues on line 1665
4
Assuming 'mem_rep' is not equal to kWord8
1666 // No need to change endianness for byte size, return original node
1667 return node;
1668 }
1669 if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1670 // In case we store lower part of WasmI64 expression, we can truncate
1671 // upper 32bits
1672 value = gasm_->TruncateInt64ToInt32(value);
1673 valueSizeInBytes = wasm::kWasmI32.value_kind_size();
1674 valueSizeInBits = 8 * valueSizeInBytes;
1675 if (mem_rep == MachineRepresentation::kWord16) {
1676 value = gasm_->Word32Shl(value, Int32Constant(16));
1677 }
1678 } else if (wasmtype == wasm::kWasmI32 &&
1679 mem_rep == MachineRepresentation::kWord16) {
1680 value = gasm_->Word32Shl(value, Int32Constant(16));
1681 }
1682
1683 int i;
1684 uint32_t shiftCount;
1685
1686 if (ReverseBytesSupported(m, valueSizeInBytes)) {
5
Taking false branch
1687 switch (valueSizeInBytes) {
1688 case 4:
1689 result = gasm_->Word32ReverseBytes(value);
1690 break;
1691 case 8:
1692 result = gasm_->Word64ReverseBytes(value);
1693 break;
1694 case 16:
1695 result = graph()->NewNode(m->Simd128ReverseBytes(), value);
1696 break;
1697 default:
1698 UNREACHABLE()V8_Fatal("unreachable code");
1699 }
1700 } else {
1701 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
6
Assuming the condition is true
7
Loop condition is true. Entering loop body
1702 i += 8, shiftCount -= 16) {
1703 Node* shiftLower;
1704 Node* shiftHigher;
1705 Node* lowerByte;
1706 Node* higherByte;
1707
1708 DCHECK_LT(0, shiftCount)((void) 0);
1709 DCHECK_EQ(0, (shiftCount + 8) % 16)((void) 0);
1710
1711 if (valueSizeInBits > 32) {
8
Assuming 'valueSizeInBits' is <= 32
9
Taking false branch
1712 shiftLower = gasm_->Word64Shl(value, Int64Constant(shiftCount));
1713 shiftHigher = gasm_->Word64Shr(value, Int64Constant(shiftCount));
1714 lowerByte = gasm_->Word64And(
1715 shiftLower, Int64Constant(static_cast<uint64_t>(0xFF)
1716 << (valueSizeInBits - 8 - i)));
1717 higherByte = gasm_->Word64And(
1718 shiftHigher, Int64Constant(static_cast<uint64_t>(0xFF) << i));
1719 result = gasm_->Word64Or(result, lowerByte);
1720 result = gasm_->Word64Or(result, higherByte);
1721 } else {
1722 shiftLower = gasm_->Word32Shl(value, Int32Constant(shiftCount));
1723 shiftHigher = gasm_->Word32Shr(value, Int32Constant(shiftCount));
1724 lowerByte = gasm_->Word32And(
1725 shiftLower, Int32Constant(static_cast<uint32_t>(0xFF)
1726 << (valueSizeInBits - 8 - i)));
1727 higherByte = gasm_->Word32And(
1728 shiftHigher, Int32Constant(static_cast<uint32_t>(0xFF) << i));
1729 result = gasm_->Word32Or(result, lowerByte);
10
1st function call argument is an uninitialized value
1730 result = gasm_->Word32Or(result, higherByte);
1731 }
1732 }
1733 }
1734
1735 if (isFloat) {
1736 switch (wasmtype.kind()) {
1737 case wasm::kF64:
1738 result = gasm_->BitcastInt64ToFloat64(result);
1739 break;
1740 case wasm::kF32:
1741 result = gasm_->BitcastInt32ToFloat32(result);
1742 break;
1743 default:
1744 UNREACHABLE()V8_Fatal("unreachable code");
1745 }
1746 }
1747
1748 return result;
1749}
1750
1751Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1752 MachineType memtype,
1753 wasm::ValueType wasmtype) {
1754 Node* result;
1755 Node* value = node;
1756 MachineOperatorBuilder* m = mcgraph()->machine();
1757 int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1758 int valueSizeInBits = 8 * valueSizeInBytes;
1759 bool isFloat = false;
1760
1761 switch (memtype.representation()) {
1762 case MachineRepresentation::kFloat64:
1763 value = gasm_->BitcastFloat64ToInt64(node);
1764 isFloat = true;
1765 V8_FALLTHROUGH[[clang::fallthrough]];
1766 case MachineRepresentation::kWord64:
1767 result = Int64Constant(0);
1768 break;
1769 case MachineRepresentation::kFloat32:
1770 value = gasm_->BitcastFloat32ToInt32(node);
1771 isFloat = true;
1772 V8_FALLTHROUGH[[clang::fallthrough]];
1773 case MachineRepresentation::kWord32:
1774 case MachineRepresentation::kWord16:
1775 result = Int32Constant(0);
1776 break;
1777 case MachineRepresentation::kWord8:
1778 // No need to change endianness for byte size, return original node
1779 return node;
1780 case MachineRepresentation::kSimd128:
1781 DCHECK(ReverseBytesSupported(m, valueSizeInBytes))((void) 0);
1782 break;
1783 default:
1784 UNREACHABLE()V8_Fatal("unreachable code");
1785 }
1786
1787 int i;
1788 uint32_t shiftCount;
1789
1790 if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1791 switch (valueSizeInBytes) {
1792 case 2:
1793 result = gasm_->Word32ReverseBytes(
1794 gasm_->Word32Shl(value, Int32Constant(16)));
1795 break;
1796 case 4:
1797 result = gasm_->Word32ReverseBytes(value);
1798 break;
1799 case 8:
1800 result = gasm_->Word64ReverseBytes(value);
1801 break;
1802 case 16:
1803 result = graph()->NewNode(m->Simd128ReverseBytes(), value);
1804 break;
1805 default:
1806 UNREACHABLE()V8_Fatal("unreachable code");
1807 }
1808 } else {
1809 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1810 i += 8, shiftCount -= 16) {
1811 Node* shiftLower;
1812 Node* shiftHigher;
1813 Node* lowerByte;
1814 Node* higherByte;
1815
1816 DCHECK_LT(0, shiftCount)((void) 0);
1817 DCHECK_EQ(0, (shiftCount + 8) % 16)((void) 0);
1818
1819 if (valueSizeInBits > 32) {
1820 shiftLower = gasm_->Word64Shl(value, Int64Constant(shiftCount));
1821 shiftHigher = gasm_->Word64Shr(value, Int64Constant(shiftCount));
1822 lowerByte = gasm_->Word64And(
1823 shiftLower, Int64Constant(static_cast<uint64_t>(0xFF)
1824 << (valueSizeInBits - 8 - i)));
1825 higherByte = gasm_->Word64And(
1826 shiftHigher, Int64Constant(static_cast<uint64_t>(0xFF) << i));
1827 result = gasm_->Word64Or(result, lowerByte);
1828 result = gasm_->Word64Or(result, higherByte);
1829 } else {
1830 shiftLower = gasm_->Word32Shl(value, Int32Constant(shiftCount));
1831 shiftHigher = gasm_->Word32Shr(value, Int32Constant(shiftCount));
1832 lowerByte = gasm_->Word32And(
1833 shiftLower, Int32Constant(static_cast<uint32_t>(0xFF)
1834 << (valueSizeInBits - 8 - i)));
1835 higherByte = gasm_->Word32And(
1836 shiftHigher, Int32Constant(static_cast<uint32_t>(0xFF) << i));
1837 result = gasm_->Word32Or(result, lowerByte);
1838 result = gasm_->Word32Or(result, higherByte);
1839 }
1840 }
1841 }
1842
1843 if (isFloat) {
1844 switch (memtype.representation()) {
1845 case MachineRepresentation::kFloat64:
1846 result = gasm_->BitcastInt64ToFloat64(result);
1847 break;
1848 case MachineRepresentation::kFloat32:
1849 result = gasm_->BitcastInt32ToFloat32(result);
1850 break;
1851 default:
1852 UNREACHABLE()V8_Fatal("unreachable code");
1853 }
1854 }
1855
1856 // We need to sign or zero extend the value
1857 if (memtype.IsSigned()) {
1858 DCHECK(!isFloat)((void) 0);
1859 if (valueSizeInBits < 32) {
1860 Node* shiftBitCount;
1861 // Perform sign extension using following trick
1862 // result = (x << machine_width - type_width) >> (machine_width -
1863 // type_width)
1864 if (wasmtype == wasm::kWasmI64) {
1865 shiftBitCount = Int32Constant(64 - valueSizeInBits);
1866 result = gasm_->Word64Sar(
1867 gasm_->Word64Shl(gasm_->ChangeInt32ToInt64(result), shiftBitCount),
1868 shiftBitCount);
1869 } else if (wasmtype == wasm::kWasmI32) {
1870 shiftBitCount = Int32Constant(32 - valueSizeInBits);
1871 result = gasm_->Word32Sar(gasm_->Word32Shl(result, shiftBitCount),
1872 shiftBitCount);
1873 }
1874 }
1875 } else if (wasmtype == wasm::kWasmI64 && valueSizeInBits < 64) {
1876 result = gasm_->ChangeUint32ToUint64(result);
1877 }
1878
1879 return result;
1880}
1881
1882Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1883 Node* result = Unop(
1884 wasm::kExprF32ReinterpretI32,
1885 Binop(wasm::kExprI32Ior,
1886 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1887 Int32Constant(0x7FFFFFFF)),
1888 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1889 Int32Constant(0x80000000))));
1890
1891 return result;
1892}
1893
1894Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1895 if (mcgraph()->machine()->Is64()) {
1896 return gasm_->BitcastInt64ToFloat64(
1897 gasm_->Word64Or(gasm_->Word64And(gasm_->BitcastFloat64ToInt64(left),
1898 Int64Constant(0x7FFFFFFFFFFFFFFF)),
1899 gasm_->Word64And(gasm_->BitcastFloat64ToInt64(right),
1900 Int64Constant(0x8000000000000000))));
1901 }
1902
1903 DCHECK(mcgraph()->machine()->Is32())((void) 0);
1904
1905 Node* high_word_left = gasm_->Float64ExtractHighWord32(left);
1906 Node* high_word_right = gasm_->Float64ExtractHighWord32(right);
1907
1908 Node* new_high_word = gasm_->Word32Or(
1909 gasm_->Word32And(high_word_left, Int32Constant(0x7FFFFFFF)),
1910 gasm_->Word32And(high_word_right, Int32Constant(0x80000000)));
1911
1912 return gasm_->Float64InsertHighWord32(left, new_high_word);
1913}
1914
1915namespace {
1916
1917MachineType IntConvertType(wasm::WasmOpcode opcode) {
1918 switch (opcode) {
1919 case wasm::kExprI32SConvertF32:
1920 case wasm::kExprI32SConvertF64:
1921 case wasm::kExprI32SConvertSatF32:
1922 case wasm::kExprI32SConvertSatF64:
1923 return MachineType::Int32();
1924 case wasm::kExprI32UConvertF32:
1925 case wasm::kExprI32UConvertF64:
1926 case wasm::kExprI32UConvertSatF32:
1927 case wasm::kExprI32UConvertSatF64:
1928 return MachineType::Uint32();
1929 case wasm::kExprI64SConvertF32:
1930 case wasm::kExprI64SConvertF64:
1931 case wasm::kExprI64SConvertSatF32:
1932 case wasm::kExprI64SConvertSatF64:
1933 return MachineType::Int64();
1934 case wasm::kExprI64UConvertF32:
1935 case wasm::kExprI64UConvertF64:
1936 case wasm::kExprI64UConvertSatF32:
1937 case wasm::kExprI64UConvertSatF64:
1938 return MachineType::Uint64();
1939 default:
1940 UNREACHABLE()V8_Fatal("unreachable code");
1941 }
1942}
1943
1944MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1945 switch (opcode) {
1946 case wasm::kExprI32SConvertF32:
1947 case wasm::kExprI32UConvertF32:
1948 case wasm::kExprI32SConvertSatF32:
1949 case wasm::kExprI64SConvertF32:
1950 case wasm::kExprI64UConvertF32:
1951 case wasm::kExprI32UConvertSatF32:
1952 case wasm::kExprI64SConvertSatF32:
1953 case wasm::kExprI64UConvertSatF32:
1954 return MachineType::Float32();
1955 case wasm::kExprI32SConvertF64:
1956 case wasm::kExprI32UConvertF64:
1957 case wasm::kExprI64SConvertF64:
1958 case wasm::kExprI64UConvertF64:
1959 case wasm::kExprI32SConvertSatF64:
1960 case wasm::kExprI32UConvertSatF64:
1961 case wasm::kExprI64SConvertSatF64:
1962 case wasm::kExprI64UConvertSatF64:
1963 return MachineType::Float64();
1964 default:
1965 UNREACHABLE()V8_Fatal("unreachable code");
1966 }
1967}
1968
1969const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1970 switch (opcode) {
1971 case wasm::kExprI32SConvertF32:
1972 return builder->mcgraph()->machine()->TruncateFloat32ToInt32(
1973 TruncateKind::kSetOverflowToMin);
1974 case wasm::kExprI32SConvertSatF32:
1975 return builder->mcgraph()->machine()->TruncateFloat32ToInt32(
1976 TruncateKind::kArchitectureDefault);
1977 case wasm::kExprI32UConvertF32:
1978 return builder->mcgraph()->machine()->TruncateFloat32ToUint32(
1979 TruncateKind::kSetOverflowToMin);
1980 case wasm::kExprI32UConvertSatF32:
1981 return builder->mcgraph()->machine()->TruncateFloat32ToUint32(
1982 TruncateKind::kArchitectureDefault);
1983 case wasm::kExprI32SConvertF64:
1984 case wasm::kExprI32SConvertSatF64:
1985 return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1986 case wasm::kExprI32UConvertF64:
1987 case wasm::kExprI32UConvertSatF64:
1988 return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1989 case wasm::kExprI64SConvertF32:
1990 case wasm::kExprI64SConvertSatF32:
1991 return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1992 case wasm::kExprI64UConvertF32:
1993 case wasm::kExprI64UConvertSatF32:
1994 return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1995 case wasm::kExprI64SConvertF64:
1996 case wasm::kExprI64SConvertSatF64:
1997 return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1998 case wasm::kExprI64UConvertF64:
1999 case wasm::kExprI64UConvertSatF64:
2000 return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
2001 default:
2002 UNREACHABLE()V8_Fatal("unreachable code");
2003 }
2004}
2005
2006wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
2007 switch (opcode) {
2008 case wasm::kExprI32SConvertF32:
2009 case wasm::kExprI32SConvertSatF32:
2010 return wasm::kExprF32SConvertI32;
2011 case wasm::kExprI32UConvertF32:
2012 case wasm::kExprI32UConvertSatF32:
2013 return wasm::kExprF32UConvertI32;
2014 case wasm::kExprI32SConvertF64:
2015 case wasm::kExprI32SConvertSatF64:
2016 return wasm::kExprF64SConvertI32;
2017 case wasm::kExprI32UConvertF64:
2018 case wasm::kExprI32UConvertSatF64:
2019 return wasm::kExprF64UConvertI32;
2020 default:
2021 UNREACHABLE()V8_Fatal("unreachable code");
2022 }
2023}
2024
2025bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
2026 switch (opcode) {
2027 case wasm::kExprI32SConvertF32:
2028 case wasm::kExprI32UConvertF32:
2029 case wasm::kExprI32SConvertF64:
2030 case wasm::kExprI32UConvertF64:
2031 case wasm::kExprI64SConvertF32:
2032 case wasm::kExprI64UConvertF32:
2033 case wasm::kExprI64SConvertF64:
2034 case wasm::kExprI64UConvertF64:
2035 return true;
2036 case wasm::kExprI32SConvertSatF64:
2037 case wasm::kExprI32UConvertSatF64:
2038 case wasm::kExprI32SConvertSatF32:
2039 case wasm::kExprI32UConvertSatF32:
2040 case wasm::kExprI64SConvertSatF32:
2041 case wasm::kExprI64UConvertSatF32:
2042 case wasm::kExprI64SConvertSatF64:
2043 case wasm::kExprI64UConvertSatF64:
2044 return false;
2045 default:
2046 UNREACHABLE()V8_Fatal("unreachable code");
2047 }
2048}
2049
2050Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
2051 switch (ty.representation()) {
2052 case MachineRepresentation::kWord32:
2053 return builder->Int32Constant(0);
2054 case MachineRepresentation::kWord64:
2055 return builder->Int64Constant(0);
2056 case MachineRepresentation::kFloat32:
2057 return builder->Float32Constant(0.0);
2058 case MachineRepresentation::kFloat64:
2059 return builder->Float64Constant(0.0);
2060 default:
2061 UNREACHABLE()V8_Fatal("unreachable code");
2062 }
2063}
2064
2065Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
2066 switch (ty.semantic()) {
2067 case MachineSemantic::kInt32:
2068 return builder->Int32Constant(std::numeric_limits<int32_t>::min());
2069 case MachineSemantic::kUint32:
2070 return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
2071 case MachineSemantic::kInt64:
2072 return builder->Int64Constant(std::numeric_limits<int64_t>::min());
2073 case MachineSemantic::kUint64:
2074 return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
2075 default:
2076 UNREACHABLE()V8_Fatal("unreachable code");
2077 }
2078}
2079
2080Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
2081 switch (ty.semantic()) {
2082 case MachineSemantic::kInt32:
2083 return builder->Int32Constant(std::numeric_limits<int32_t>::max());
2084 case MachineSemantic::kUint32:
2085 return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
2086 case MachineSemantic::kInt64:
2087 return builder->Int64Constant(std::numeric_limits<int64_t>::max());
2088 case MachineSemantic::kUint64:
2089 return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
2090 default:
2091 UNREACHABLE()V8_Fatal("unreachable code");
2092 }
2093}
2094
2095wasm::WasmOpcode TruncOp(const MachineType& ty) {
2096 switch (ty.representation()) {
2097 case MachineRepresentation::kFloat32:
2098 return wasm::kExprF32Trunc;
2099 case MachineRepresentation::kFloat64:
2100 return wasm::kExprF64Trunc;
2101 default:
2102 UNREACHABLE()V8_Fatal("unreachable code");
2103 }
2104}
2105
2106wasm::WasmOpcode NeOp(const MachineType& ty) {
2107 switch (ty.representation()) {
2108 case MachineRepresentation::kFloat32:
2109 return wasm::kExprF32Ne;
2110 case MachineRepresentation::kFloat64:
2111 return wasm::kExprF64Ne;
2112 default:
2113 UNREACHABLE()V8_Fatal("unreachable code");
2114 }
2115}
2116
2117wasm::WasmOpcode LtOp(const MachineType& ty) {
2118 switch (ty.representation()) {
2119 case MachineRepresentation::kFloat32:
2120 return wasm::kExprF32Lt;
2121 case MachineRepresentation::kFloat64:
2122 return wasm::kExprF64Lt;
2123 default:
2124 UNREACHABLE()V8_Fatal("unreachable code");
2125 }
2126}
2127
2128Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
2129 const MachineType& int_ty, const MachineType& float_ty,
2130 Node* trunc, Node* converted_value) {
2131 if (int_ty.representation() == MachineRepresentation::kWord32) {
2132 Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
2133 return builder->Binop(NeOp(float_ty), trunc, check);
2134 }
2135 return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
2136 trunc, builder->graph()->start());
2137}
2138
2139Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
2140 const MachineType& int_ty,
2141 const MachineType& float_ty, Node* trunc,
2142 Node* converted_value) {
2143 Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
2144 converted_value);
2145 if (int_ty.representation() == MachineRepresentation::kWord64) {
2146 test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
2147 }
2148 return test;
2149}
2150
2151} // namespace
2152
2153Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
2154 wasm::WasmCodePosition position,
2155 wasm::WasmOpcode opcode) {
2156 const MachineType int_ty = IntConvertType(opcode);
2157 const MachineType float_ty = FloatConvertType(opcode);
2158 const Operator* conv_op = ConvertOp(this, opcode);
2159 Node* trunc = nullptr;
2160 Node* converted_value = nullptr;
2161 const bool is_int32 =
2162 int_ty.representation() == MachineRepresentation::kWord32;
2163 if (is_int32) {
2164 trunc = Unop(TruncOp(float_ty), input);
2165 converted_value = graph()->NewNode(conv_op, trunc);
2166 } else {
2167 trunc = graph()->NewNode(conv_op, input);
2168 converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
2169 trunc, graph()->start());
2170 }
2171 if (IsTrappingConvertOp(opcode)) {
2172 Node* test =
2173 ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
2174 if (is_int32) {
2175 TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
2176 } else {
2177 ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
2178 }
2179 return converted_value;
2180 }
2181 if (mcgraph()->machine()->SatConversionIsSafe()) {
2182 return converted_value;
2183 }
2184 Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
2185 converted_value);
2186 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2187 tl_d.Chain(control());
2188 Node* nan_test = Binop(NeOp(float_ty), input, input);
2189 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2190 nan_d.Nest(tl_d, true);
2191 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2192 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2193 sat_d.Nest(nan_d, false);
2194 Node* sat_val =
2195 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2196 Node* nan_val =
2197 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2198 return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
2199}
2200
2201Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
2202 // asm.js must use the wacky JS semantics.
2203 return gasm_->TruncateFloat64ToWord32(gasm_->ChangeFloat32ToFloat64(input));
2204}
2205
2206Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
2207 // asm.js must use the wacky JS semantics.
2208 return gasm_->TruncateFloat64ToWord32(input);
2209}
2210
2211Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
2212 // asm.js must use the wacky JS semantics.
2213 return gasm_->TruncateFloat64ToWord32(gasm_->ChangeFloat32ToFloat64(input));
2214}
2215
2216Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
2217 // asm.js must use the wacky JS semantics.
2218 return gasm_->TruncateFloat64ToWord32(input);
2219}
2220
2221Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
2222 MachineRepresentation input_type) {
2223 Node* stack_slot_param = StoreArgsInStackSlot({{input_type, input}});
2224
2225 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2226 MachineSignature sig(1, 1, sig_types);
2227
2228 Node* function = gasm_->ExternalConstant(ref);
2229
2230 return BuildCCall(&sig, function, stack_slot_param);
2231}
2232
2233Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
2234 return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
2235 MachineRepresentation::kWord32);
2236}
2237
2238Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
2239 return Unop(wasm::kExprI64UConvertI32,
2240 BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
2241 MachineRepresentation::kWord64));
2242}
2243
2244Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
2245 return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
2246 MachineRepresentation::kWord32);
2247}
2248
2249Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
2250 return Unop(
2251 wasm::kExprI64UConvertI32,
2252 BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
2253 MachineRepresentation::kWord64));
2254}
2255
2256Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
2257 MachineType type = MachineType::Float32();
2258 ExternalReference ref = ExternalReference::wasm_f32_trunc();
2259
2260 return BuildCFuncInstruction(ref, type, input);
2261}
2262
2263Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
2264 MachineType type = MachineType::Float32();
2265 ExternalReference ref = ExternalReference::wasm_f32_floor();
2266 return BuildCFuncInstruction(ref, type, input);
2267}
2268
2269Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
2270 MachineType type = MachineType::Float32();
2271 ExternalReference ref = ExternalReference::wasm_f32_ceil();
2272 return BuildCFuncInstruction(ref, type, input);
2273}
2274
2275Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
2276 MachineType type = MachineType::Float32();
2277 ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
2278 return BuildCFuncInstruction(ref, type, input);
2279}
2280
2281Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
2282 MachineType type = MachineType::Float64();
2283 ExternalReference ref = ExternalReference::wasm_f64_trunc();
2284 return BuildCFuncInstruction(ref, type, input);
2285}
2286
2287Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
2288 MachineType type = MachineType::Float64();
2289 ExternalReference ref = ExternalReference::wasm_f64_floor();
2290 return BuildCFuncInstruction(ref, type, input);
2291}
2292
2293Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
2294 MachineType type = MachineType::Float64();
2295 ExternalReference ref = ExternalReference::wasm_f64_ceil();
2296 return BuildCFuncInstruction(ref, type, input);
2297}
2298
2299Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
2300 MachineType type = MachineType::Float64();
2301 ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
2302 return BuildCFuncInstruction(ref, type, input);
2303}
2304
2305Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
2306 MachineType type = MachineType::Float64();
2307 ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
2308 return BuildCFuncInstruction(ref, type, input);
2309}
2310
2311Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
2312 MachineType type = MachineType::Float64();
2313 ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
2314 return BuildCFuncInstruction(ref, type, input);
2315}
2316
2317Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
2318 MachineType type = MachineType::Float64();
2319 ExternalReference ref = ExternalReference::wasm_float64_pow();
2320 return BuildCFuncInstruction(ref, type, left, right);
2321}
2322
2323Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
2324 MachineType type = MachineType::Float64();
2325 ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
2326 return BuildCFuncInstruction(ref, type, left, right);
2327}
2328
2329Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
2330 MachineType type, Node* input0,
2331 Node* input1) {
2332 // We do truncation by calling a C function which calculates the result.
2333 // The input is passed to the C function as a byte buffer holding the two
2334 // input doubles. We reserve this byte buffer as a stack slot, store the
2335 // parameters in this buffer slots, pass a pointer to the buffer to the C
2336 // function, and after calling the C function we collect the return value from
2337 // the buffer.
2338 Node* stack_slot;
2339 if (input1) {
2340 stack_slot = StoreArgsInStackSlot(
2341 {{type.representation(), input0}, {type.representation(), input1}});
2342 } else {
2343 stack_slot = StoreArgsInStackSlot({{type.representation(), input0}});
2344 }
2345
2346 MachineType sig_types[] = {MachineType::Pointer()};
2347 MachineSignature sig(0, 1, sig_types);
2348 Node* function = gasm_->ExternalConstant(ref);
2349 BuildCCall(&sig, function, stack_slot);
2350
2351 return gasm_->LoadFromObject(type, stack_slot, 0);
2352}
2353
2354Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
2355 // TODO(titzer/bradnelson): Check handlng of asm.js case.
2356 return BuildIntToFloatConversionInstruction(
2357 input, ExternalReference::wasm_int64_to_float32(),
2358 MachineRepresentation::kWord64, MachineType::Float32());
2359}
2360Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
2361 // TODO(titzer/bradnelson): Check handlng of asm.js case.
2362 return BuildIntToFloatConversionInstruction(
2363 input, ExternalReference::wasm_uint64_to_float32(),
2364 MachineRepresentation::kWord64, MachineType::Float32());
2365}
2366Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
2367 return BuildIntToFloatConversionInstruction(
2368 input, ExternalReference::wasm_int64_to_float64(),
2369 MachineRepresentation::kWord64, MachineType::Float64());
2370}
2371Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
2372 return BuildIntToFloatConversionInstruction(
2373 input, ExternalReference::wasm_uint64_to_float64(),
2374 MachineRepresentation::kWord64, MachineType::Float64());
2375}
2376
2377Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
2378 Node* input, ExternalReference ref,
2379 MachineRepresentation parameter_representation,
2380 const MachineType result_type) {
2381 int stack_slot_size =
2382 std::max(ElementSizeInBytes(parameter_representation),
2383 ElementSizeInBytes(result_type.representation()));
2384 Node* stack_slot =
2385 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2386 auto store_rep =
2387 StoreRepresentation(parameter_representation, kNoWriteBarrier);
2388 gasm_->Store(store_rep, stack_slot, 0, input);
2389 MachineType sig_types[] = {MachineType::Pointer()};
2390 MachineSignature sig(0, 1, sig_types);
2391 Node* function = gasm_->ExternalConstant(ref);
2392 BuildCCall(&sig, function, stack_slot);
2393 return gasm_->LoadFromObject(result_type, stack_slot, 0);
2394}
2395
2396namespace {
2397
2398ExternalReference convert_ccall_ref(wasm::WasmOpcode opcode) {
2399 switch (opcode) {
2400 case wasm::kExprI64SConvertF32:
2401 case wasm::kExprI64SConvertSatF32:
2402 return ExternalReference::wasm_float32_to_int64();
2403 case wasm::kExprI64UConvertF32:
2404 case wasm::kExprI64UConvertSatF32:
2405 return ExternalReference::wasm_float32_to_uint64();
2406 case wasm::kExprI64SConvertF64:
2407 case wasm::kExprI64SConvertSatF64:
2408 return ExternalReference::wasm_float64_to_int64();
2409 case wasm::kExprI64UConvertF64:
2410 case wasm::kExprI64UConvertSatF64:
2411 return ExternalReference::wasm_float64_to_uint64();
2412 default:
2413 UNREACHABLE()V8_Fatal("unreachable code");
2414 }
2415}
2416
2417} // namespace
2418
2419Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
2420 wasm::WasmCodePosition position,
2421 wasm::WasmOpcode opcode) {
2422 const MachineType int_ty = IntConvertType(opcode);
2423 const MachineType float_ty = FloatConvertType(opcode);
2424 ExternalReference call_ref = convert_ccall_ref(opcode);
2425 int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
2426 ElementSizeInBytes(float_ty.representation()));
2427 Node* stack_slot =
2428 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2429 auto store_rep =
2430 StoreRepresentation(float_ty.representation(), kNoWriteBarrier);
2431 gasm_->Store(store_rep, stack_slot, 0, input);
2432 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2433 MachineSignature sig(1, 1, sig_types);
2434 Node* function = gasm_->ExternalConstant(call_ref);
2435 Node* overflow = BuildCCall(&sig, function, stack_slot);
2436 if (IsTrappingConvertOp(opcode)) {
2437 ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
2438 return gasm_->LoadFromObject(int_ty, stack_slot, 0);
2439 }
2440 Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
2441 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2442 tl_d.Chain(control());
2443 Node* nan_test = Binop(NeOp(float_ty), input, input);
2444 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2445 nan_d.Nest(tl_d, true);
2446 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2447 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2448 sat_d.Nest(nan_d, false);
2449 Node* sat_val =
2450 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2451 Node* load = gasm_->LoadFromObject(int_ty, stack_slot, 0);
2452 Node* nan_val =
2453 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2454 return tl_d.Phi(int_ty.representation(), nan_val, load);
2455}
2456
2457Node* WasmGraphBuilder::MemoryGrow(Node* input) {
2458 needs_stack_check_ = true;
2459 if (!env_->module->is_memory64) {
2460 // For 32-bit memories, just call the builtin.
2461 return gasm_->CallRuntimeStub(wasm::WasmCode::kWasmMemoryGrow,
2462 Operator::kNoThrow, input);
2463 }
2464
2465 // If the input is not a positive int32, growing will always fail
2466 // (growing negative or requesting >= 256 TB).
2467 Node* old_effect = effect();
2468 Diamond is_32_bit(graph(), mcgraph()->common(),
2469 gasm_->Uint64LessThanOrEqual(input, Int64Constant(kMaxInt)),
2470 BranchHint::kTrue);
2471 is_32_bit.Chain(control());
2472
2473 SetControl(is_32_bit.if_true);
2474
2475 Node* grow_result = gasm_->ChangeInt32ToInt64(gasm_->CallRuntimeStub(
2476 wasm::WasmCode::kWasmMemoryGrow, Operator::kNoThrow,
2477 gasm_->TruncateInt64ToInt32(input)));
2478
2479 Node* diamond_result = is_32_bit.Phi(MachineRepresentation::kWord64,
2480 grow_result, gasm_->Int64Constant(-1));
2481 SetEffectControl(is_32_bit.EffectPhi(effect(), old_effect), is_32_bit.merge);
2482 return diamond_result;
2483}
2484
2485Node* WasmGraphBuilder::Throw(uint32_t tag_index, const wasm::WasmTag* tag,
2486 const base::Vector<Node*> values,
2487 wasm::WasmCodePosition position) {
2488 needs_stack_check_ = true;
2489 uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(tag);
2490
2491 Node* values_array = gasm_->CallRuntimeStub(
2492 wasm::WasmCode::kWasmAllocateFixedArray, Operator::kNoThrow,
2493 gasm_->IntPtrConstant(encoded_size));
2494 SetSourcePosition(values_array, position);
2495
2496 uint32_t index = 0;
2497 const wasm::WasmTagSig* sig = tag->sig;
2498 MachineOperatorBuilder* m = mcgraph()->machine();
2499 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2500 Node* value = values[i];
2501 switch (sig->GetParam(i).kind()) {
2502 case wasm::kF32:
2503 value = gasm_->BitcastFloat32ToInt32(value);
2504 V8_FALLTHROUGH[[clang::fallthrough]];
2505 case wasm::kI32:
2506 BuildEncodeException32BitValue(values_array, &index, value);
2507 break;
2508 case wasm::kF64:
2509 value = gasm_->BitcastFloat64ToInt64(value);
2510 V8_FALLTHROUGH[[clang::fallthrough]];
2511 case wasm::kI64: {
2512 Node* upper32 = gasm_->TruncateInt64ToInt32(
2513 Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2514 BuildEncodeException32BitValue(values_array, &index, upper32);
2515 Node* lower32 = gasm_->TruncateInt64ToInt32(value);
2516 BuildEncodeException32BitValue(values_array, &index, lower32);
2517 break;
2518 }
2519 case wasm::kS128:
2520 BuildEncodeException32BitValue(
2521 values_array, &index,
2522 graph()->NewNode(m->I32x4ExtractLane(0), value));
2523 BuildEncodeException32BitValue(
2524 values_array, &index,
2525 graph()->NewNode(m->I32x4ExtractLane(1), value));
2526 BuildEncodeException32BitValue(
2527 values_array, &index,
2528 graph()->NewNode(m->I32x4ExtractLane(2), value));
2529 BuildEncodeException32BitValue(
2530 values_array, &index,
2531 graph()->NewNode(m->I32x4ExtractLane(3), value));
2532 break;
2533 case wasm::kRef:
2534 case wasm::kOptRef:
2535 case wasm::kRtt:
2536 gasm_->StoreFixedArrayElementAny(values_array, index, value);
2537 ++index;
2538 break;
2539 case wasm::kI8:
2540 case wasm::kI16:
2541 case wasm::kVoid:
2542 case wasm::kBottom:
2543 UNREACHABLE()V8_Fatal("unreachable code");
2544 }
2545 }
2546 DCHECK_EQ(encoded_size, index)((void) 0);
2547
2548 Node* exception_tag = LoadTagFromTable(tag_index);
2549
2550 Node* throw_call = gasm_->CallRuntimeStub(wasm::WasmCode::kWasmThrow,
2551 Operator::kNoProperties,
2552 exception_tag, values_array);
2553 SetSourcePosition(throw_call, position);
2554 return throw_call;
2555}
2556
2557void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
2558 uint32_t* index,
2559 Node* value) {
2560 Node* upper_halfword_as_smi =
2561 BuildChangeUint31ToSmi(gasm_->Word32Shr(value, Int32Constant(16)));
2562 gasm_->StoreFixedArrayElementSmi(values_array, *index, upper_halfword_as_smi);
2563 ++(*index);
2564 Node* lower_halfword_as_smi =
2565 BuildChangeUint31ToSmi(gasm_->Word32And(value, Int32Constant(0xFFFFu)));
2566 gasm_->StoreFixedArrayElementSmi(values_array, *index, lower_halfword_as_smi);
2567 ++(*index);
2568}
2569
2570Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
2571 uint32_t* index) {
2572 Node* upper = BuildChangeSmiToInt32(
2573 gasm_->LoadFixedArrayElementSmi(values_array, *index));
2574 (*index)++;
2575 upper = gasm_->Word32Shl(upper, Int32Constant(16));
2576 Node* lower = BuildChangeSmiToInt32(
2577 gasm_->LoadFixedArrayElementSmi(values_array, *index));
2578 (*index)++;
2579 Node* value = gasm_->Word32Or(upper, lower);
2580 return value;
2581}
2582
2583Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
2584 uint32_t* index) {
2585 Node* upper = Binop(wasm::kExprI64Shl,
2586 Unop(wasm::kExprI64UConvertI32,
2587 BuildDecodeException32BitValue(values_array, index)),
2588 Int64Constant(32));
2589 Node* lower = Unop(wasm::kExprI64UConvertI32,
2590 BuildDecodeException32BitValue(values_array, index));
2591 return Binop(wasm::kExprI64Ior, upper, lower);
2592}
2593
2594Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
2595 // TODO(v8:8091): Currently the message of the original exception is not being
2596 // preserved when rethrown to the console. The pending message will need to be
2597 // saved when caught and restored here while being rethrown.
2598 return gasm_->CallRuntimeStub(wasm::WasmCode::kWasmRethrow,
2599 Operator::kNoProperties, except_obj);
2600}
2601
2602Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
2603 Node* expected_tag) {
2604 return gasm_->WordEqual(caught_tag, expected_tag);
2605}
2606
2607Node* WasmGraphBuilder::LoadTagFromTable(uint32_t tag_index) {
2608 Node* tags_table =
2609 LOAD_INSTANCE_FIELD(TagsTable, MachineType::TaggedPointer());
2610 Node* tag = gasm_->LoadFixedArrayElementPtr(tags_table, tag_index);
2611 return tag;
2612}
2613
2614Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
2615 return gasm_->CallBuiltin(
2616 Builtin::kWasmGetOwnProperty, Operator::kEliminatable, except_obj,
2617 LOAD_ROOT(wasm_exception_tag_symbol, wasm_exception_tag_symbol),
2618 LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer()));
2619}
2620
2621Node* WasmGraphBuilder::GetExceptionValues(Node* except_obj,
2622 const wasm::WasmTag* tag,
2623 base::Vector<Node*> values) {
2624 Node* values_array = gasm_->CallBuiltin(
2625 Builtin::kWasmGetOwnProperty, Operator::kEliminatable, except_obj,
2626 LOAD_ROOT(wasm_exception_values_symbol, wasm_exception_values_symbol),
2627 LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer()));
2628 uint32_t index = 0;
2629 const wasm::WasmTagSig* sig = tag->sig;
2630 DCHECK_EQ(sig->parameter_count(), values.size())((void) 0);
2631 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2632 Node* value;
2633 switch (sig->GetParam(i).kind()) {
2634 case wasm::kI32:
2635 value = BuildDecodeException32BitValue(values_array, &index);
2636 break;
2637 case wasm::kI64:
2638 value = BuildDecodeException64BitValue(values_array, &index);
2639 break;
2640 case wasm::kF32: {
2641 value = Unop(wasm::kExprF32ReinterpretI32,
2642 BuildDecodeException32BitValue(values_array, &index));
2643 break;
2644 }
2645 case wasm::kF64: {
2646 value = Unop(wasm::kExprF64ReinterpretI64,
2647 BuildDecodeException64BitValue(values_array, &index));
2648 break;
2649 }
2650 case wasm::kS128:
2651 value = graph()->NewNode(
2652 mcgraph()->machine()->I32x4Splat(),
2653 BuildDecodeException32BitValue(values_array, &index));
2654 value = graph()->NewNode(
2655 mcgraph()->machine()->I32x4ReplaceLane(1), value,
2656 BuildDecodeException32BitValue(values_array, &index));
2657 value = graph()->NewNode(
2658 mcgraph()->machine()->I32x4ReplaceLane(2), value,
2659 BuildDecodeException32BitValue(values_array, &index));
2660 value = graph()->NewNode(
2661 mcgraph()->machine()->I32x4ReplaceLane(3), value,
2662 BuildDecodeException32BitValue(values_array, &index));
2663 break;
2664 case wasm::kRef:
2665 case wasm::kOptRef:
2666 case wasm::kRtt:
2667 value = gasm_->LoadFixedArrayElementAny(values_array, index);
2668 ++index;
2669 break;
2670 case wasm::kI8:
2671 case wasm::kI16:
2672 case wasm::kVoid:
2673 case wasm::kBottom:
2674 UNREACHABLE()V8_Fatal("unreachable code");
2675 }
2676 values[i] = value;
2677 }
2678 DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(tag))((void) 0);
2679 return values_array;
2680}
2681
2682Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2683 wasm::WasmCodePosition position) {
2684 ZeroCheck32(wasm::kTrapDivByZero, right, position);
2685 Node* before = control();
2686 Node* denom_is_m1;
2687 Node* denom_is_not_m1;
2688 BranchExpectFalse(gasm_->Word32Equal(right, Int32Constant(-1)), &denom_is_m1,
2689 &denom_is_not_m1);
2690 SetControl(denom_is_m1);
2691 TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2692 if (control() != denom_is_m1) {
2693 SetControl(Merge(denom_is_not_m1, control()));
2694 } else {
2695 SetControl(before);
2696 }
2697 return gasm_->Int32Div(left, right);
2698}
2699
2700Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2701 wasm::WasmCodePosition position) {
2702 MachineOperatorBuilder* m = mcgraph()->machine();
2703
2704 ZeroCheck32(wasm::kTrapRemByZero, right, position);
2705
2706 Diamond d(graph(), mcgraph()->common(),
2707 gasm_->Word32Equal(right, Int32Constant(-1)), BranchHint::kFalse);
2708 d.Chain(control());
2709
2710 return d.Phi(MachineRepresentation::kWord32, Int32Constant(0),
2711 graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2712}
2713
2714Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2715 wasm::WasmCodePosition position) {
2716 ZeroCheck32(wasm::kTrapDivByZero, right, position);
2717 return gasm_->Uint32Div(left, right);
2718}
2719
2720Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2721 wasm::WasmCodePosition position) {
2722 ZeroCheck32(wasm::kTrapRemByZero, right, position);
2723 return gasm_->Uint32Mod(left, right);
2724}
2725
2726Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2727 MachineOperatorBuilder* m = mcgraph()->machine();
2728
2729 Int32Matcher mr(right);
2730 if (mr.HasResolvedValue()) {
2731 if (mr.ResolvedValue() == 0) {
2732 return Int32Constant(0);
2733 } else if (mr.ResolvedValue() == -1) {
2734 // The result is the negation of the left input.
2735 return gasm_->Int32Sub(Int32Constant(0), left);
2736 }
2737 return gasm_->Int32Div(left, right);
2738 }
2739
2740 // asm.js semantics return 0 on divide or mod by zero.
2741 if (m->Int32DivIsSafe()) {
2742 // The hardware instruction does the right thing (e.g. arm).
2743 return gasm_->Int32Div(left, right);
2744 }
2745
2746 // Check denominator for zero.
2747 Diamond z(graph(), mcgraph()->common(),
2748 gasm_->Word32Equal(right, Int32Constant(0)), BranchHint::kFalse);
2749 z.Chain(control());
2750
2751 // Check denominator for -1. (avoid minint / -1 case).
2752 Diamond n(graph(), mcgraph()->common(),
2753 gasm_->Word32Equal(right, Int32Constant(-1)), BranchHint::kFalse);
2754 n.Chain(z.if_false);
2755
2756 Node* div = graph()->NewNode(m->Int32Div(), left, right, n.if_false);
2757
2758 Node* neg = gasm_->Int32Sub(Int32Constant(0), left);
2759
2760 return z.Phi(MachineRepresentation::kWord32, Int32Constant(0),
2761 n.Phi(MachineRepresentation::kWord32, neg, div));
2762}
2763
2764Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2765 CommonOperatorBuilder* c = mcgraph()->common();
2766 MachineOperatorBuilder* m = mcgraph()->machine();
2767 Node* const zero = Int32Constant(0);
2768
2769 Int32Matcher mr(right);
2770 if (mr.HasResolvedValue()) {
2771 if (mr.ResolvedValue() == 0 || mr.ResolvedValue() == -1) {
2772 return zero;
2773 }
2774 return gasm_->Int32Mod(left, right);
2775 }
2776
2777 // General case for signed integer modulus, with optimization for (unknown)
2778 // power of 2 right hand side.
2779 //
2780 // if 0 < right then
2781 // msk = right - 1
2782 // if right & msk != 0 then
2783 // left % right
2784 // else
2785 // if left < 0 then
2786 // -(-left & msk)
2787 // else
2788 // left & msk
2789 // else
2790 // if right < -1 then
2791 // left % right
2792 // else
2793 // zero
2794 //
2795 // Note: We do not use the Diamond helper class here, because it really hurts
2796 // readability with nested diamonds.
2797 Node* const minus_one = Int32Constant(-1);
2798
2799 const Operator* const merge_op = c->Merge(2);
2800 const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2801
2802 Node* check0 = gasm_->Int32LessThan(zero, right);
2803 Node* branch0 =
2804 graph()->NewNode(c->Branch(BranchHint::kTrue), check0, control());
2805
2806 Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2807 Node* true0;
2808 {
2809 Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2810
2811 Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2812 Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2813
2814 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2815 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2816
2817 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2818 Node* false1;
2819 {
2820 Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2821 Node* branch2 =
2822 graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2823
2824 Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2825 Node* true2 = graph()->NewNode(
2826 m->Int32Sub(), zero,
2827 graph()->NewNode(m->Word32And(),
2828 graph()->NewNode(m->Int32Sub(), zero, left), msk));
2829
2830 Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2831 Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2832
2833 if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2834 false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2835 }
2836
2837 if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2838 true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2839 }
2840
2841 Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2842 Node* false0;
2843 {
2844 Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2845 Node* branch1 =
2846 graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2847
2848 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2849 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2850
2851 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2852 Node* false1 = zero;
2853
2854 if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2855 false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2856 }
2857
2858 Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2859 return graph()->NewNode(phi_op, true0, false0, merge0);
2860}
2861
2862Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2863 MachineOperatorBuilder* m = mcgraph()->machine();
2864 // asm.js semantics return 0 on divide or mod by zero.
2865 if (m->Uint32DivIsSafe()) {
2866 // The hardware instruction does the right thing (e.g. arm).
2867 return gasm_->Uint32Div(left, right);
2868 }
2869
2870 // Explicit check for x % 0.
2871 Diamond z(graph(), mcgraph()->common(),
2872 gasm_->Word32Equal(right, Int32Constant(0)), BranchHint::kFalse);
2873 z.Chain(control());
2874
2875 return z.Phi(MachineRepresentation::kWord32, Int32Constant(0),
2876 graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2877 z.if_false));
2878}
2879
2880Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2881 // asm.js semantics return 0 on divide or mod by zero.
2882 // Explicit check for x % 0.
2883 Diamond z(graph(), mcgraph()->common(),
2884 gasm_->Word32Equal(right, Int32Constant(0)), BranchHint::kFalse);
2885 z.Chain(control());
2886
2887 Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2888 z.if_false);
2889 return z.Phi(MachineRepresentation::kWord32, Int32Constant(0), rem);
2890}
2891
2892Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2893 wasm::WasmCodePosition position) {
2894 if (mcgraph()->machine()->Is32()) {
2895 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2896 MachineType::Int64(), wasm::kTrapDivByZero, position);
2897 }
2898 ZeroCheck64(wasm::kTrapDivByZero, right, position);
2899 Node* before = control();
2900 Node* denom_is_m1;
2901 Node* denom_is_not_m1;
2902 BranchExpectFalse(gasm_->Word64Equal(right, Int64Constant(-1)), &denom_is_m1,
2903 &denom_is_not_m1);
2904 SetControl(denom_is_m1);
2905 TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2906 std::numeric_limits<int64_t>::min(), position);
2907 if (control() != denom_is_m1) {
2908 SetControl(Merge(denom_is_not_m1, control()));
2909 } else {
2910 SetControl(before);
2911 }
2912 return gasm_->Int64Div(left, right);
2913}
2914
2915Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2916 wasm::WasmCodePosition position) {
2917 if (mcgraph()->machine()->Is32()) {
2918 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2919 MachineType::Int64(), wasm::kTrapRemByZero, position);
2920 }
2921 ZeroCheck64(wasm::kTrapRemByZero, right, position);
2922 Diamond d(mcgraph()->graph(), mcgraph()->common(),
2923 gasm_->Word64Equal(right, Int64Constant(-1)));
2924
2925 d.Chain(control());
2926
2927 Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2928 d.if_false);
2929
2930 return d.Phi(MachineRepresentation::kWord64, Int64Constant(0), rem);
2931}
2932
2933Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2934 wasm::WasmCodePosition position) {
2935 if (mcgraph()->machine()->Is32()) {
2936 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2937 MachineType::Int64(), wasm::kTrapDivByZero, position);
2938 }
2939 ZeroCheck64(wasm::kTrapDivByZero, right, position);
2940 return gasm_->Uint64Div(left, right);
2941}
2942Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2943 wasm::WasmCodePosition position) {
2944 if (mcgraph()->machine()->Is32()) {
2945 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2946 MachineType::Int64(), wasm::kTrapRemByZero, position);
2947 }
2948 ZeroCheck64(wasm::kTrapRemByZero, right, position);
2949 return gasm_->Uint64Mod(left, right);
2950}
2951
2952Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2953 ExternalReference ref,
2954 MachineType result_type,
2955 wasm::TrapReason trap_zero,
2956 wasm::WasmCodePosition position) {
2957 Node* stack_slot =
2958 StoreArgsInStackSlot({{MachineRepresentation::kWord64, left},
2959 {MachineRepresentation::kWord64, right}});
2960
2961 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2962 MachineSignature sig(1, 1, sig_types);
2963
2964 Node* function = gasm_->ExternalConstant(ref);
2965 Node* call = BuildCCall(&sig, function, stack_slot);
2966
2967 ZeroCheck32(trap_zero, call, position);
2968 TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2969 return gasm_->LoadFromObject(result_type, stack_slot, 0);
2970}
2971
2972Node* WasmGraphBuilder::IsNull(Node* object) {
2973 return gasm_->TaggedEqual(object, RefNull());
2974}
2975
2976template <typename... Args>
2977Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2978 Args... args) {
2979 DCHECK_LE(sig->return_count(), 1)((void) 0);
2980 DCHECK_EQ(sizeof...(args), sig->parameter_count())((void) 0);
2981 Node* call_args[] = {function, args..., effect(), control()};
2982
2983 auto call_descriptor =
2984 Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2985
2986 return gasm_->Call(call_descriptor, arraysize(call_args)(sizeof(ArraySizeHelper(call_args))), call_args);
2987}
2988
2989Node* WasmGraphBuilder::BuildCallNode(const wasm::FunctionSig* sig,
2990 base::Vector<Node*> args,
2991 wasm::WasmCodePosition position,
2992 Node* instance_node, const Operator* op,
2993 Node* frame_state) {
2994 if (instance_node == nullptr) {
2995 instance_node = GetInstance();
2996 }
2997 needs_stack_check_ = true;
2998 const size_t params = sig->parameter_count();
2999 const size_t has_frame_state = frame_state != nullptr ? 1 : 0;
3000 const size_t extra = 3; // instance_node, effect, and control.
3001 const size_t count = 1 + params + extra + has_frame_state;
3002
3003 // Reallocate the buffer to make space for extra inputs.
3004 base::SmallVector<Node*, 16 + extra> inputs(count);
3005 DCHECK_EQ(1 + params, args.size())((void) 0);
3006
3007 // Make room for the instance_node parameter at index 1, just after code.
3008 inputs[0] = args[0]; // code
3009 inputs[1] = instance_node;
3010 if (params > 0) memcpy(&inputs[2], &args[1], params * sizeof(Node*));
3011
3012 // Add effect and control inputs.
3013 if (has_frame_state != 0) inputs[params + 2] = frame_state;
3014 inputs[params + has_frame_state + 2] = effect();
3015 inputs[params + has_frame_state + 3] = control();
3016
3017 Node* call = graph()->NewNode(op, static_cast<int>(count), inputs.begin());
3018 // Return calls have no effect output. Other calls are the new effect node.
3019 if (op->EffectOutputCount() > 0) SetEffect(call);
3020 DCHECK(position == wasm::kNoCodePosition || position > 0)((void) 0);
3021 if (position > 0) SetSourcePosition(call, position);
3022
3023 return call;
3024}
3025
3026Node* WasmGraphBuilder::BuildWasmCall(const wasm::FunctionSig* sig,
3027 base::Vector<Node*> args,
3028 base::Vector<Node*> rets,
3029 wasm::WasmCodePosition position,
3030 Node* instance_node, Node* frame_state) {
3031 CallDescriptor* call_descriptor = GetWasmCallDescriptor(
3032 mcgraph()->zone(), sig, kWasmFunction, frame_state != nullptr);
3033 const Operator* op = mcgraph()->common()->Call(call_descriptor);
3034 Node* call =
3035 BuildCallNode(sig, args, position, instance_node, op, frame_state);
3036 // TODO(manoskouk): If we have kNoThrow calls, do not set them as control.
3037 DCHECK_GT(call->op()->ControlOutputCount(), 0)((void) 0);
3038 SetControl(call);
3039
3040 size_t ret_count = sig->return_count();
3041 if (ret_count == 0) return call; // No return value.
3042
3043 DCHECK_EQ(ret_count, rets.size())((void) 0);
3044 if (ret_count == 1) {
3045 // Only a single return value.
3046 rets[0] = call;
3047 } else {
3048 // Create projections for all return values.
3049 for (size_t i = 0; i < ret_count; i++) {
3050 rets[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
3051 graph()->start());
3052 }
3053 }
3054 return call;
3055}
3056
3057Node* WasmGraphBuilder::BuildWasmReturnCall(const wasm::FunctionSig* sig,
3058 base::Vector<Node*> args,
3059 wasm::WasmCodePosition position,
3060 Node* instance_node) {
3061 CallDescriptor* call_descriptor =
3062 GetWasmCallDescriptor(mcgraph()->zone(), sig);
3063 const Operator* op = mcgraph()->common()->TailCall(call_descriptor);
3064 Node* call = BuildCallNode(sig, args, position, instance_node, op);
3065
3066 // TODO(manoskouk): If we have kNoThrow calls, do not merge them to end.
3067 DCHECK_GT(call->op()->ControlOutputCount(), 0)((void) 0);
3068 gasm_->MergeControlToEnd(call);
3069
3070 return call;
3071}
3072
3073Node* WasmGraphBuilder::BuildImportCall(const wasm::FunctionSig* sig,
3074 base::Vector<Node*> args,
3075 base::Vector<Node*> rets,
3076 wasm::WasmCodePosition position,
3077 int func_index,
3078 IsReturnCall continuation) {
3079 return BuildImportCall(sig, args, rets, position,
3080 gasm_->Uint32Constant(func_index), continuation);
3081}
3082
3083Node* WasmGraphBuilder::BuildImportCall(const wasm::FunctionSig* sig,
3084 base::Vector<Node*> args,
3085 base::Vector<Node*> rets,
3086 wasm::WasmCodePosition position,
3087 Node* func_index,
3088 IsReturnCall continuation) {
3089 // Load the imported function refs array from the instance.
3090 Node* imported_function_refs =
3091 LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
3092 // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
3093 Node* func_index_intptr = BuildChangeUint32ToUintPtr(func_index);
3094 Node* ref_node = gasm_->LoadFixedArrayElement(
3095 imported_function_refs, func_index_intptr, MachineType::TaggedPointer());
3096
3097 // Load the target from the imported_targets array at the offset of
3098 // {func_index}.
3099 Node* func_index_times_pointersize = gasm_->IntMul(
3100 func_index_intptr, gasm_->IntPtrConstant(kSystemPointerSize));
3101 Node* imported_targets =
3102 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
3103 Node* target_node = gasm_->LoadImmutableFromObject(
3104 MachineType::Pointer(), imported_targets, func_index_times_pointersize);
3105 args[0] = target_node;
3106
3107 switch (continuation) {
3108 case kCallContinues:
3109 return BuildWasmCall(sig, args, rets, position, ref_node);
3110 case kReturnCall:
3111 DCHECK(rets.empty())((void) 0);
3112 return BuildWasmReturnCall(sig, args, position, ref_node);
3113 }
3114}
3115
3116Node* WasmGraphBuilder::CallDirect(uint32_t index, wasm::FunctionSig* real_sig,
3117 base::Vector<Node*> args,
3118 base::Vector<Node*> rets,
3119 wasm::WasmCodePosition position) {
3120 DCHECK_NULL(args[0])((void) 0);
3121
3122 if (env_ && index < env_->module->num_imported_functions) {
3123 // Call to an imported function.
3124 return BuildImportCall(real_sig, args, rets, position, index,
3125 kCallContinues);
3126 }
3127
3128 // A direct call to a wasm function defined in this module.
3129 // Just encode the function index. This will be patched at instantiation.
3130 Address code = static_cast<Address>(index);
3131 args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
3132
3133 return BuildWasmCall(real_sig, args, rets, position, nullptr);
3134}
3135
3136Node* WasmGraphBuilder::CallIndirect(uint32_t table_index, uint32_t sig_index,
3137 wasm::FunctionSig* sig,
3138 base::Vector<Node*> args,
3139 base::Vector<Node*> rets,
3140 wasm::WasmCodePosition position) {
3141 return BuildIndirectCall(table_index, sig_index, sig, args, rets, position,
3142 kCallContinues);
3143}
3144
3145void WasmGraphBuilder::LoadIndirectFunctionTable(uint32_t table_index,
3146 Node** ift_size,
3147 Node** ift_sig_ids,
3148 Node** ift_targets,
3149 Node** ift_instances) {
3150 bool needs_dynamic_size = true;
3151 const wasm::WasmTable& table = env_->module->tables[table_index];
3152 if (table.has_maximum_size && table.maximum_size == table.initial_size) {
3153 *ift_size = Int32Constant(table.initial_size);
3154 needs_dynamic_size = false;
3155 }
3156
3157 if (table_index == 0) {
3158 if (needs_dynamic_size) {
3159 *ift_size = LOAD_MUTABLE_INSTANCE_FIELD(IndirectFunctionTableSize,
3160 MachineType::Uint32());
3161 }
3162 *ift_sig_ids = LOAD_MUTABLE_INSTANCE_FIELD(IndirectFunctionTableSigIds,
3163 MachineType::Pointer());
3164 *ift_targets = LOAD_MUTABLE_INSTANCE_FIELD(IndirectFunctionTableTargets,
3165 MachineType::Pointer());
3166 *ift_instances = LOAD_MUTABLE_INSTANCE_FIELD(IndirectFunctionTableRefs,
3167 MachineType::TaggedPointer());
3168 return;
3169 }
3170
3171 Node* ift_tables = LOAD_MUTABLE_INSTANCE_FIELD(IndirectFunctionTables,
3172 MachineType::TaggedPointer());
3173 Node* ift_table = gasm_->LoadFixedArrayElementAny(ift_tables, table_index);
3174
3175 if (needs_dynamic_size) {
3176 *ift_size = gasm_->LoadFromObject(
3177 MachineType::Int32(), ift_table,
3178 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kSizeOffset));
3179 }
3180
3181 *ift_sig_ids = gasm_->LoadFromObject(
3182 MachineType::Pointer(), ift_table,
3183 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kSigIdsOffset));
3184
3185 *ift_targets = gasm_->LoadFromObject(
3186 MachineType::Pointer(), ift_table,
3187 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kTargetsOffset));
3188
3189 *ift_instances = gasm_->LoadFromObject(
3190 MachineType::TaggedPointer(), ift_table,
3191 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kRefsOffset));
3192}
3193
3194Node* WasmGraphBuilder::BuildIndirectCall(
3195 uint32_t table_index, uint32_t sig_index, wasm::FunctionSig* real_sig,
3196 base::Vector<Node*> args, base::Vector<Node*> rets,
3197 wasm::WasmCodePosition position, IsReturnCall continuation) {
3198 DCHECK_NOT_NULL(args[0])((void) 0);
3199 DCHECK_NOT_NULL(env_)((void) 0);
3200
3201 // First we have to load the table.
3202 Node* ift_size;
3203 Node* ift_sig_ids;
3204 Node* ift_targets;
3205 Node* ift_instances;
3206 LoadIndirectFunctionTable(table_index, &ift_size, &ift_sig_ids, &ift_targets,
3207 &ift_instances);
3208
3209 Node* key = args[0];
3210
3211 // Bounds check against the table size.
3212 Node* in_bounds = gasm_->Uint32LessThan(key, ift_size);
3213 TrapIfFalse(wasm::kTrapTableOutOfBounds, in_bounds, position);
3214
3215 const wasm::ValueType table_type = env_->module->tables[table_index].type;
3216 // Check that the table entry is not null and that the type of the function is
3217 // **identical with** the function type declared at the call site (no
3218 // subtyping of functions is allowed).
3219 // Note: Since null entries are identified by having ift_sig_id (-1), we only
3220 // need one comparison.
3221 // TODO(9495): Change this if we should do full function subtyping instead.
3222 const bool needs_signature_check =
3223 FLAG_experimental_wasm_gc ||
3224 table_type.is_reference_to(wasm::HeapType::kFunc) ||
3225 table_type.is_nullable();
3226 if (needs_signature_check) {
3227 Node* int32_scaled_key =
3228 BuildChangeUint32ToUintPtr(gasm_->Word32Shl(key, Int32Constant(2)));
3229
3230 Node* loaded_sig = gasm_->LoadFromObject(MachineType::Int32(), ift_sig_ids,
3231 int32_scaled_key);
3232 int32_t expected_sig_id = env_->module->canonicalized_type_ids[sig_index];
3233 Node* sig_match =
3234 gasm_->Word32Equal(loaded_sig, Int32Constant(expected_sig_id));
3235 TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
3236 }
3237
3238 Node* key_intptr = BuildChangeUint32ToUintPtr(key);
3239
3240 Node* target_instance = gasm_->LoadFixedArrayElement(
3241 ift_instances, key_intptr, MachineType::TaggedPointer());
3242
3243 Node* intptr_scaled_key =
3244 gasm_->IntMul(key_intptr, gasm_->IntPtrConstant(kSystemPointerSize));
3245
3246 Node* target = gasm_->LoadFromObject(MachineType::Pointer(), ift_targets,
3247 intptr_scaled_key);
3248
3249 args[0] = target;
3250
3251 switch (continuation) {
3252 case kCallContinues:
3253 return BuildWasmCall(real_sig, args, rets, position, target_instance);
3254 case kReturnCall:
3255 return BuildWasmReturnCall(real_sig, args, position, target_instance);
3256 }
3257}
3258
3259Node* WasmGraphBuilder::BuildLoadExternalPointerFromObject(
3260 Node* object, int offset, ExternalPointerTag tag) {
3261#ifdef V8_SANDBOXED_EXTERNAL_POINTERS
3262 Node* external_pointer = gasm_->LoadFromObject(
3263 MachineType::Uint32(), object, wasm::ObjectAccess::ToTagged(offset));
3264 STATIC_ASSERT(kExternalPointerIndexShift > kSystemPointerSizeLog2)static_assert(kExternalPointerIndexShift > kSystemPointerSizeLog2
, "kExternalPointerIndexShift > kSystemPointerSizeLog2")
;
3265 Node* shift_amount =
3266 gasm_->Int32Constant(kExternalPointerIndexShift - kSystemPointerSizeLog2);
3267 Node* scaled_index = gasm_->Word32Shr(external_pointer, shift_amount);
3268 Node* isolate_root = BuildLoadIsolateRoot();
3269 Node* table =
3270 gasm_->LoadFromObject(MachineType::Pointer(), isolate_root,
3271 IsolateData::external_pointer_table_offset() +
3272 Internals::kExternalPointerTableBufferOffset);
3273 Node* decoded_ptr = gasm_->Load(MachineType::Pointer(), table, scaled_index);
3274 return gasm_->WordAnd(decoded_ptr, gasm_->IntPtrConstant(~tag));
3275#else
3276 return gasm_->LoadFromObject(MachineType::Pointer(), object,
3277 wasm::ObjectAccess::ToTagged(offset));
3278#endif
3279}
3280
3281Node* WasmGraphBuilder::BuildLoadCallTargetFromExportedFunctionData(
3282 Node* function) {
3283 Node* internal = gasm_->LoadFromObject(
3284 MachineType::TaggedPointer(), function,
3285 wasm::ObjectAccess::ToTagged(WasmExportedFunctionData::kInternalOffset));
3286 return BuildLoadExternalPointerFromObject(
3287 internal, WasmInternalFunction::kForeignAddressOffset);
3288}
3289
3290// TODO(9495): Support CAPI function refs.
3291Node* WasmGraphBuilder::BuildCallRef(const wasm::FunctionSig* real_sig,
3292 base::Vector<Node*> args,
3293 base::Vector<Node*> rets,
3294 CheckForNull null_check,
3295 IsReturnCall continuation,
3296 wasm::WasmCodePosition position) {
3297 if (null_check == kWithNullCheck) {
3298 TrapIfTrue(wasm::kTrapNullDereference, IsNull(args[0]), position);
3299 }
3300
3301 Node* function = args[0];
3302
3303 auto load_target = gasm_->MakeLabel();
3304 auto end_label = gasm_->MakeLabel(MachineType::PointerRepresentation());
3305
3306 Node* ref_node = gasm_->LoadImmutableFromObject(
3307 MachineType::TaggedPointer(), function,
3308 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset));
3309
3310 Node* target = BuildLoadExternalPointerFromObject(
3311 function, WasmInternalFunction::kForeignAddressOffset);
3312 Node* is_null_target = gasm_->WordEqual(target, gasm_->IntPtrConstant(0));
3313 gasm_->GotoIfNot(is_null_target, &end_label, target);
3314 {
3315 // Compute the call target from the (on-heap) wrapper code. The cached
3316 // target can only be null for WasmJSFunctions.
3317 Node* wrapper_code = gasm_->LoadImmutableFromObject(
3318 MachineType::TaggedPointer(), function,
3319 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kCodeOffset));
3320 Node* call_target;
3321 if (V8_EXTERNAL_CODE_SPACE_BOOLfalse) {
3322 call_target = BuildLoadExternalPointerFromObject(
3323 wrapper_code, CodeDataContainer::kCodeEntryPointOffset,
3324 kCodeEntryPointTag);
3325
3326 } else {
3327 call_target = gasm_->IntAdd(
3328 wrapper_code, gasm_->IntPtrConstant(
3329 wasm::ObjectAccess::ToTagged(Code::kHeaderSize)));
3330 }
3331 gasm_->Goto(&end_label, call_target);
3332 }
3333
3334 gasm_->Bind(&end_label);
3335
3336 args[0] = end_label.PhiAt(0);
3337
3338 Node* call = continuation == kCallContinues
3339 ? BuildWasmCall(real_sig, args, rets, position, ref_node)
3340 : BuildWasmReturnCall(real_sig, args, position, ref_node);
3341 return call;
3342}
3343
3344void WasmGraphBuilder::CompareToInternalFunctionAtIndex(
3345 Node* func_ref, uint32_t function_index, Node** success_control,
3346 Node** failure_control) {
3347 // Since we are comparing to a function reference, it is guaranteed that
3348 // instance->wasm_internal_functions() has been initialized.
3349 Node* internal_functions = gasm_->LoadImmutable(
3350 MachineType::TaggedPointer(), GetInstance(),
3351 wasm::ObjectAccess::ToTagged(
3352 WasmInstanceObject::kWasmInternalFunctionsOffset));
3353 Node* function_ref_at_index = gasm_->LoadFixedArrayElement(
3354 internal_functions, gasm_->IntPtrConstant(function_index),
3355 MachineType::AnyTagged());
3356 gasm_->Branch(gasm_->TaggedEqual(function_ref_at_index, func_ref),
3357 success_control, failure_control, BranchHint::kTrue);
3358}
3359
3360Node* WasmGraphBuilder::CallRef(const wasm::FunctionSig* real_sig,
3361 base::Vector<Node*> args,
3362 base::Vector<Node*> rets,
3363 WasmGraphBuilder::CheckForNull null_check,
3364 wasm::WasmCodePosition position) {
3365 return BuildCallRef(real_sig, args, rets, null_check,
3366 IsReturnCall::kCallContinues, position);
3367}
3368
3369Node* WasmGraphBuilder::ReturnCallRef(const wasm::FunctionSig* real_sig,
3370 base::Vector<Node*> args,
3371 WasmGraphBuilder::CheckForNull null_check,
3372 wasm::WasmCodePosition position) {
3373 return BuildCallRef(real_sig, args, {}, null_check, IsReturnCall::kReturnCall,
3374 position);
3375}
3376
3377Node* WasmGraphBuilder::ReturnCall(uint32_t index,
3378 const wasm::FunctionSig* real_sig,
3379 base::Vector<Node*> args,
3380 wasm::WasmCodePosition position) {
3381 DCHECK_NULL(args[0])((void) 0);
3382
3383 if (env_ && index < env_->module->num_imported_functions) {
3384 // Return Call to an imported function.
3385 return BuildImportCall(real_sig, args, {}, position, index, kReturnCall);
3386 }
3387
3388 // A direct tail call to a wasm function defined in this module.
3389 // Just encode the function index. This will be patched during code
3390 // generation.
3391 Address code = static_cast<Address>(index);
3392 args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
3393
3394 return BuildWasmReturnCall(real_sig, args, position, nullptr);
3395}
3396
3397Node* WasmGraphBuilder::ReturnCallIndirect(uint32_t table_index,
3398 uint32_t sig_index,
3399 wasm::FunctionSig* real_sig,
3400 base::Vector<Node*> args,
3401 wasm::WasmCodePosition position) {
3402 return BuildIndirectCall(table_index, sig_index, real_sig, args, {}, position,
3403 kReturnCall);
3404}
3405
3406void WasmGraphBuilder::BrOnNull(Node* ref_object, Node** null_node,
3407 Node** non_null_node) {
3408 BranchExpectFalse(IsNull(ref_object), null_node, non_null_node);
3409}
3410
3411Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
3412 // Implement Rol by Ror since TurboFan does not have Rol opcode.
3413 // TODO(weiliang): support Word32Rol opcode in TurboFan.
3414 Int32Matcher m(right);
3415 if (m.HasResolvedValue()) {
3416 return Binop(wasm::kExprI32Ror, left,
3417 Int32Constant(32 - (m.ResolvedValue() & 0x1F)));
3418 } else {
3419 return Binop(wasm::kExprI32Ror, left,
3420 Binop(wasm::kExprI32Sub, Int32Constant(32), right));
3421 }
3422}
3423
3424Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
3425 // Implement Rol by Ror since TurboFan does not have Rol opcode.
3426 // TODO(weiliang): support Word64Rol opcode in TurboFan.
3427 Int64Matcher m(right);
3428 Node* inv_right = m.HasResolvedValue()
3429 ? Int64Constant(64 - (m.ResolvedValue() & 0x3F))
3430 : Binop(wasm::kExprI64Sub, Int64Constant(64), right);
3431 return Binop(wasm::kExprI64Ror, left, inv_right);
3432}
3433
3434Node* WasmGraphBuilder::Invert(Node* node) {
3435 return Unop(wasm::kExprI32Eqz, node);
3436}
3437
3438Node* WasmGraphBuilder::BuildTruncateIntPtrToInt32(Node* value) {
3439 return mcgraph()->machine()->Is64() ? gasm_->TruncateInt64ToInt32(value)
3440 : value;
3441}
3442
3443Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
3444 return mcgraph()->machine()->Is64() ? gasm_->ChangeInt32ToInt64(value)
3445 : value;
3446}
3447
3448Node* WasmGraphBuilder::BuildChangeIntPtrToInt64(Node* value) {
3449 return mcgraph()->machine()->Is32() ? gasm_->ChangeInt32ToInt64(value)
3450 : value;
3451}
3452
3453Node* WasmGraphBuilder::BuildChangeUint32ToUintPtr(Node* node) {
3454 if (mcgraph()->machine()->Is32()) return node;
3455 // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3456 Uint32Matcher matcher(node);
3457 if (matcher.HasResolvedValue()) {
3458 uintptr_t value = matcher.ResolvedValue();
3459 return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3460 }
3461 return gasm_->ChangeUint32ToUint64(node);
3462}
3463
3464Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
3465 return gasm_->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
3466}
3467
3468Node* WasmGraphBuilder::BuildSmiShiftBitsConstant32() {
3469 return Int32Constant(kSmiShiftSize + kSmiTagSize);
3470}
3471
3472Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
3473 // With pointer compression, only the lower 32 bits are used.
3474 return COMPRESS_POINTERS_BOOLfalse
3475 ? gasm_->Word32Shl(value, BuildSmiShiftBitsConstant32())
3476 : gasm_->WordShl(BuildChangeInt32ToIntPtr(value),
3477 BuildSmiShiftBitsConstant());
3478}
3479
3480Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
3481 return COMPRESS_POINTERS_BOOLfalse
3482 ? gasm_->Word32Shl(value, BuildSmiShiftBitsConstant32())
3483 : gasm_->WordShl(BuildChangeUint32ToUintPtr(value),
3484 BuildSmiShiftBitsConstant());
3485}
3486
3487Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
3488 return COMPRESS_POINTERS_BOOLfalse
3489 ? gasm_->Word32Sar(value, BuildSmiShiftBitsConstant32())
3490 : BuildTruncateIntPtrToInt32(
3491 gasm_->WordSar(value, BuildSmiShiftBitsConstant()));
3492}
3493
3494Node* WasmGraphBuilder::BuildChangeSmiToIntPtr(Node* value) {
3495 return COMPRESS_POINTERS_BOOLfalse
3496 ? BuildChangeInt32ToIntPtr(
3497 gasm_->Word32Sar(value, BuildSmiShiftBitsConstant32()))
3498 : gasm_->WordSar(value, BuildSmiShiftBitsConstant());
3499}
3500
3501Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
3502 uint32_t maxval) {
3503 DCHECK(Smi::IsValid(maxval))((void) 0);
3504 Node* max = mcgraph()->Uint32Constant(maxval);
3505 Node* check = gasm_->Uint32LessThanOrEqual(value, max);
3506 Node* valsmi = BuildChangeUint31ToSmi(value);
3507 Node* maxsmi = gasm_->NumberConstant(maxval);
3508 Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
3509 d.Chain(control());
3510 return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
3511}
3512
3513void WasmGraphBuilder::InitInstanceCache(
3514 WasmInstanceCacheNodes* instance_cache) {
3515 // We handle caching of the instance cache nodes manually, and we may reload
3516 // them in contexts where load elimination would eliminate the reload.
3517 // Therefore, we use plain Load nodes which are not subject to load
3518 // elimination.
3519
3520 // Load the memory start.
3521#ifdef V8_SANDBOXED_POINTERS
3522 instance_cache->mem_start = LOAD_INSTANCE_FIELD_NO_ELIMINATION(gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(MemoryStart
), MachineType::SandboxedPointer()), GetInstance(), wasm::ObjectAccess
::ToTagged(WasmInstanceObject::kMemoryStartOffset))
3523 MemoryStart, MachineType::SandboxedPointer())gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(MemoryStart
), MachineType::SandboxedPointer()), GetInstance(), wasm::ObjectAccess
::ToTagged(WasmInstanceObject::kMemoryStartOffset))
;
3524#else
3525 instance_cache->mem_start =
3526 LOAD_INSTANCE_FIELD_NO_ELIMINATION(MemoryStart, MachineType::UintPtr())gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(MemoryStart
), MachineType::UintPtr()), GetInstance(), wasm::ObjectAccess
::ToTagged(WasmInstanceObject::kMemoryStartOffset))
;
3527#endif
3528
3529 // Load the memory size.
3530 instance_cache->mem_size =
3531 LOAD_INSTANCE_FIELD_NO_ELIMINATION(MemorySize, MachineType::UintPtr())gasm_->Load( assert_size(WASM_INSTANCE_OBJECT_SIZE(MemorySize
), MachineType::UintPtr()), GetInstance(), wasm::ObjectAccess
::ToTagged(WasmInstanceObject::kMemorySizeOffset))
;
3532}
3533
3534void WasmGraphBuilder::PrepareInstanceCacheForLoop(
3535 WasmInstanceCacheNodes* instance_cache, Node* control) {
3536#define INTRODUCE_PHI(field, rep) \
3537 instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
3538 instance_cache->field, control);
3539
3540 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
3541 INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
3542#undef INTRODUCE_PHI
3543}
3544
3545void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
3546 WasmInstanceCacheNodes* from,
3547 Node* merge) {
3548#define INTRODUCE_PHI(field, rep) \
3549 if (to->field != from->field) { \
3550 Node* vals[] = {to->field, from->field, merge}; \
3551 to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
3552 }
3553
3554 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
3555 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
3556#undef INTRODUCE_PHI
3557}
3558
3559void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
3560 WasmInstanceCacheNodes* from,
3561 Node* merge) {
3562 to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3563 merge, to->mem_size, from->mem_size);
3564 to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3565 merge, to->mem_start, from->mem_start);
3566}
3567
3568Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
3569 Node* merge, Node* tnode,
3570 Node* fnode) {
3571 if (IsPhiWithMerge(tnode, merge)) {
3572 AppendToPhi(tnode, fnode);
3573 } else if (tnode != fnode) {
3574 // Note that it is not safe to use {Buffer} here since this method is used
3575 // via {CheckForException} while the {Buffer} is in use by another method.
3576 uint32_t count = merge->InputCount();
3577 // + 1 for the merge node.
3578 base::SmallVector<Node*, 9> inputs(count + 1);
3579 for (uint32_t j = 0; j < count - 1; j++) inputs[j] = tnode;
3580 inputs[count - 1] = fnode;
3581 inputs[count] = merge;
3582 tnode = graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
3583 inputs.begin());
3584 }
3585 return tnode;
3586}
3587
3588Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
3589 Node* fnode) {
3590 if (IsPhiWithMerge(tnode, merge)) {
3591 AppendToPhi(tnode, fnode);
3592 } else if (tnode != fnode) {
3593 // Note that it is not safe to use {Buffer} here since this method is used
3594 // via {CheckForException} while the {Buffer} is in use by another method.
3595 uint32_t count = merge->InputCount();
3596 // + 1 for the merge node.
3597 base::SmallVector<Node*, 9> inputs(count + 1);
3598 for (uint32_t j = 0; j < count - 1; j++) {
3599 inputs[j] = tnode;
3600 }
3601 inputs[count - 1] = fnode;
3602 inputs[count] = merge;
3603 tnode = graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
3604 inputs.begin());
3605 }
3606 return tnode;
3607}
3608
3609Node* WasmGraphBuilder::effect() { return gasm_->effect(); }
3610
3611Node* WasmGraphBuilder::control() { return gasm_->control(); }
3612
3613Node* WasmGraphBuilder::SetEffect(Node* node) {
3614 SetEffectControl(node, control());
3615 return node;
3616}
3617
3618Node* WasmGraphBuilder::SetControl(Node* node) {
3619 SetEffectControl(effect(), node);
3620 return node;
3621}
3622
3623void WasmGraphBuilder::SetEffectControl(Node* effect, Node* control) {
3624 gasm_->InitializeEffectControl(effect, control);
3625}
3626
3627Node* WasmGraphBuilder::MemBuffer(uintptr_t offset) {
3628 DCHECK_NOT_NULL(instance_cache_)((void) 0);
3629 Node* mem_start = instance_cache_->mem_start;
3630 DCHECK_NOT_NULL(mem_start)((void) 0);
3631 if (offset == 0) return mem_start;
3632 return gasm_->IntAdd(mem_start, gasm_->UintPtrConstant(offset));
3633}
3634
3635Node* WasmGraphBuilder::CurrentMemoryPages() {
3636 // CurrentMemoryPages can not be called from asm.js.
3637 DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin)((void) 0);
3638 DCHECK_NOT_NULL(instance_cache_)((void) 0);
3639 Node* mem_size = instance_cache_->mem_size;
3640 DCHECK_NOT_NULL(mem_size)((void) 0);
3641 Node* result =
3642 gasm_->WordShr(mem_size, Int32Constant(wasm::kWasmPageSizeLog2));
3643 result = env_->module->is_memory64 ? BuildChangeIntPtrToInt64(result)
3644 : BuildTruncateIntPtrToInt32(result);
3645 return result;
3646}
3647
3648// Only call this function for code which is not reused across instantiations,
3649// as we do not patch the embedded js_context.
3650Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
3651 Node* js_context,
3652 Node** parameters,
3653 int parameter_count) {
3654 const Runtime::Function* fun = Runtime::FunctionForId(f);
3655 auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3656 mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
3657 CallDescriptor::kNoFlags);
3658 // The CEntryStub is loaded from the IsolateRoot so that generated code is
3659 // Isolate independent. At the moment this is only done for CEntryStub(1).
3660 Node* isolate_root = BuildLoadIsolateRoot();
3661 DCHECK_EQ(1, fun->result_size)((void) 0);
3662 auto centry_id =
3663 Builtin::kCEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit;
3664 int builtin_slot_offset = IsolateData::BuiltinSlotOffset(centry_id);
3665 Node* centry_stub = gasm_->LoadFromObject(MachineType::Pointer(),
3666 isolate_root, builtin_slot_offset);
3667 // TODO(titzer): allow arbitrary number of runtime arguments
3668 // At the moment we only allow 5 parameters. If more parameters are needed,
3669 // increase this constant accordingly.
3670 static const int kMaxParams = 5;
3671 DCHECK_GE(kMaxParams, parameter_count)((void) 0);
3672 Node* inputs[kMaxParams + 6];
3673 int count = 0;
3674 inputs[count++] = centry_stub;
3675 for (int i = 0; i < parameter_count; i++) {
3676 inputs[count++] = parameters[i];
3677 }
3678 inputs[count++] =
3679 mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
3680 inputs[count++] = Int32Constant(fun->nargs); // arity
3681 inputs[count++] = js_context; // js_context
3682 inputs[count++] = effect();
3683 inputs[count++] = control();
3684
3685 return gasm_->Call(call_descriptor, count, inputs);
3686}
3687
3688Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
3689 Node** parameters,
3690 int parameter_count) {
3691 return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3692 parameter_count);
3693}
3694
3695void WasmGraphBuilder::GetGlobalBaseAndOffset(const wasm::WasmGlobal& global,
3696 Node** base, Node** offset) {
3697 if (global.mutability && global.imported) {
3698 Node* base_or_index = gasm_->LoadFromObject(
3699 MachineType::UintPtr(),
3700 LOAD_INSTANCE_FIELD(ImportedMutableGlobals, MachineType::UintPtr()),
3701 Int32Constant(global.index * kSystemPointerSize));
3702 if (global.type.is_reference()) {
3703 // Load the base from the ImportedMutableGlobalsBuffer of the instance.
3704 Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
3705 MachineType::TaggedPointer());
3706 *base = gasm_->LoadFixedArrayElementAny(buffers, global.index);
3707
3708 // For this case, {base_or_index} gives the index of the global in the
3709 // buffer. From the index, calculate the actual offset in the FixedArray.
3710 // This is kHeaderSize + (index * kTaggedSize).
3711 *offset = gasm_->IntAdd(
3712 gasm_->IntMul(base_or_index, gasm_->IntPtrConstant(kTaggedSize)),
3713 gasm_->IntPtrConstant(
3714 wasm::ObjectAccess::ToTagged(FixedArray::kObjectsOffset)));
3715 } else {
3716 *base = base_or_index;
3717 *offset = gasm_->IntPtrConstant(0);
3718 }
3719 } else if (global.type.is_reference()) {
3720 *base =
3721 LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3722 *offset = gasm_->IntPtrConstant(
3723 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(global.offset));
3724 } else {
3725 *base = LOAD_INSTANCE_FIELD(GlobalsStart, MachineType::UintPtr());
3726 *offset = gasm_->IntPtrConstant(global.offset);
3727 }
3728}
3729
3730Node* WasmGraphBuilder::GlobalGet(uint32_t index) {
3731 const wasm::WasmGlobal& global = env_->module->globals[index];
3732 if (global.type == wasm::kWasmS128) has_simd_ = true;
3733 Node* base = nullptr;
3734 Node* offset = nullptr;
3735 GetGlobalBaseAndOffset(global, &base, &offset);
3736 MachineType mem_type = global.type.machine_type();
3737 return global.mutability ? gasm_->LoadFromObject(mem_type, base, offset)
3738 : gasm_->LoadImmutable(mem_type, base, offset);
3739}
3740
3741void WasmGraphBuilder::GlobalSet(uint32_t index, Node* val) {
3742 const wasm::WasmGlobal& global = env_->module->globals[index];
3743 if (global.type == wasm::kWasmS128) has_simd_ = true;
3744 Node* base = nullptr;
3745 Node* offset = nullptr;
3746 GetGlobalBaseAndOffset(global, &base, &offset);
3747 ObjectAccess access(global.type.machine_type(), global.type.is_reference()
3748 ? kFullWriteBarrier
3749 : kNoWriteBarrier);
3750 gasm_->StoreToObject(access, base, offset, val);
3751}
3752
3753Node* WasmGraphBuilder::TableGet(uint32_t table_index, Node* index,
3754 wasm::WasmCodePosition position) {
3755 return gasm_->CallRuntimeStub(wasm::WasmCode::kWasmTableGet,
3756 Operator::kNoThrow,
3757 gasm_->IntPtrConstant(table_index), index);
3758}
3759
3760void WasmGraphBuilder::TableSet(uint32_t table_index, Node* index, Node* val,
3761 wasm::WasmCodePosition position) {
3762 gasm_->CallRuntimeStub(wasm::WasmCode::kWasmTableSet, Operator::kNoThrow,
3763 gasm_->IntPtrConstant(table_index), index, val);
3764}
3765
3766Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3767 int8_t access_size, Node* index, uint64_t offset,
3768 wasm::WasmCodePosition position) {
3769 // Atomic operations need bounds checks until the backend can emit protected
3770 // loads.
3771 index =
3772 BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck)
3773 .first;
3774
3775 const uintptr_t align_mask = access_size - 1;
3776
3777 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
3778 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
3779 // Don't emit an alignment check if the index is a constant.
3780 // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
3781 UintPtrMatcher match(index);
3782 if (match.HasResolvedValue()) {
3783 uintptr_t effective_offset = match.ResolvedValue() + capped_offset;
3784 if ((effective_offset & align_mask) != 0) {
3785 // statically known to be unaligned; trap.
3786 TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
3787 }
3788 return index;
3789 }
3790
3791 // Unlike regular memory accesses, atomic memory accesses should trap if
3792 // the effective offset is misaligned.
3793 // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
3794 Node* effective_offset = gasm_->IntAdd(MemBuffer(capped_offset), index);
3795
3796 Node* cond =
3797 gasm_->WordAnd(effective_offset, gasm_->IntPtrConstant(align_mask));
3798 TrapIfFalse(wasm::kTrapUnalignedAccess,
3799 gasm_->Word32Equal(cond, Int32Constant(0)), position);
3800 return index;
3801}
3802
3803// Insert code to bounds check a memory access if necessary. Return the
3804// bounds-checked index, which is guaranteed to have (the equivalent of)
3805// {uintptr_t} representation.
3806std::pair<Node*, WasmGraphBuilder::BoundsCheckResult>
3807WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3808 uint64_t offset,
3809 wasm::WasmCodePosition position,
3810 EnforceBoundsCheck enforce_check) {
3811 DCHECK_LE(1, access_size)((void) 0);
3812
3813 // If the offset does not fit in a uintptr_t, this can never succeed on this
3814 // machine.
3815 if (offset > std::numeric_limits<uintptr_t>::max() ||
3816 !base::IsInBounds<uintptr_t>(offset, access_size,
3817 env_->max_memory_size)) {
3818 // The access will be out of bounds, even for the largest memory.
3819 TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3820 return {gasm_->UintPtrConstant(0), kOutOfBounds};
3821 }
3822
3823 // Convert the index to uintptr.
3824 if (!env_->module->is_memory64) {
3825 index = BuildChangeUint32ToUintPtr(index);
3826 } else if (kSystemPointerSize == kInt32Size) {
3827 // In memory64 mode on 32-bit systems, the upper 32 bits need to be zero to
3828 // succeed the bounds check.
3829 DCHECK_NE(wasm::kTrapHandler, env_->bounds_checks)((void) 0);
3830 if (env_->bounds_checks == wasm::kExplicitBoundsChecks) {
3831 Node* high_word = gasm_->TruncateInt64ToInt32(
3832 gasm_->Word64Shr(index, Int32Constant(32)));
3833 TrapIfTrue(wasm::kTrapMemOutOfBounds, high_word, position);
3834 }
3835 // Only use the low word for the following bounds check.
3836 index = gasm_->TruncateInt64ToInt32(index);
3837 }
3838
3839 // If no bounds checks should be performed (for testing), just return the
3840 // converted index and assume it to be in-bounds.
3841 if (env_->bounds_checks == wasm::kNoBoundsChecks) return {index, kInBounds};
3842
3843 // The accessed memory is [index + offset, index + end_offset].
3844 // Check that the last read byte (at {index + end_offset}) is in bounds.
3845 // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3846 // compute {effective_size} as {mem_size - end_offset)}.
3847 // {effective_size} is >= 1 if condition 1) holds.
3848 // 2) Check that {index + end_offset < mem_size} by
3849 // - computing {effective_size} as {mem_size - end_offset} and
3850 // - checking that {index < effective_size}.
3851
3852 uintptr_t end_offset = offset + access_size - 1u;
3853
3854 UintPtrMatcher match(index);
3855 if (match.HasResolvedValue() && end_offset <= env_->min_memory_size &&
3856 match.ResolvedValue() < env_->min_memory_size - end_offset) {
3857 // The input index is a constant and everything is statically within
3858 // bounds of the smallest possible memory.
3859 return {index, kInBounds};
3860 }
3861
3862 if (env_->bounds_checks == wasm::kTrapHandler &&
3863 enforce_check == kCanOmitBoundsCheck) {
3864 return {index, kTrapHandler};
3865 }
3866
3867 Node* mem_size = instance_cache_->mem_size;
3868 Node* end_offset_node = mcgraph_->UintPtrConstant(end_offset);
3869 if (end_offset > env_->min_memory_size) {
3870 // The end offset is larger than the smallest memory.
3871 // Dynamically check the end offset against the dynamic memory size.
3872 Node* cond = gasm_->UintLessThan(end_offset_node, mem_size);
3873 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3874 }
3875
3876 // This produces a positive number since {end_offset <= min_size <= mem_size}.
3877 Node* effective_size = gasm_->IntSub(mem_size, end_offset_node);
3878
3879 // Introduce the actual bounds check.
3880 Node* cond = gasm_->UintLessThan(index, effective_size);
3881 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3882 return {index, kDynamicallyChecked};
3883}
3884
3885const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3886 wasm::ValueType type) {
3887 int alignment = offset % type.value_kind_size();
3888 MachineType mach_type = type.machine_type();
3889 if (COMPRESS_POINTERS_BOOLfalse && mach_type.IsTagged()) {
3890 // We are loading tagged value from off-heap location, so we need to load
3891 // it as a full word otherwise we will not be able to decompress it.
3892 mach_type = MachineType::Pointer();
3893 }
3894 if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3895 type.machine_representation())) {
3896 return mcgraph()->machine()->Load(mach_type);
3897 }
3898 return mcgraph()->machine()->UnalignedLoad(mach_type);
3899}
3900
3901const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3902 wasm::ValueType type) {
3903 int alignment = offset % type.value_kind_size();
3904 MachineRepresentation rep = type.machine_representation();
3905 if (COMPRESS_POINTERS_BOOLfalse && IsAnyTagged(rep)) {
3906 // We are storing tagged value to off-heap location, so we need to store
3907 // it as a full word otherwise we will not be able to decompress it.
3908 rep = MachineType::PointerRepresentation();
3909 }
3910 if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3911 StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3912 return mcgraph()->machine()->Store(store_rep);
3913 }
3914 UnalignedStoreRepresentation store_rep(rep);
3915 return mcgraph()->machine()->UnalignedStore(store_rep);
3916}
3917
3918void WasmGraphBuilder::TraceFunctionEntry(wasm::WasmCodePosition position) {
3919 Node* call = BuildCallToRuntime(Runtime::kWasmTraceEnter, nullptr, 0);
3920 SetSourcePosition(call, position);
3921}
3922
3923void WasmGraphBuilder::TraceFunctionExit(base::Vector<Node*> vals,
3924 wasm::WasmCodePosition position) {
3925 Node* info = gasm_->IntPtrConstant(0);
3926 size_t num_returns = vals.size();
3927 if (num_returns == 1) {
3928 wasm::ValueType return_type = sig_->GetReturn(0);
3929 MachineRepresentation rep = return_type.machine_representation();
3930 int size = ElementSizeInBytes(rep);
3931 info = gasm_->StackSlot(size, size);
3932
3933 gasm_->Store(StoreRepresentation(rep, kNoWriteBarrier), info,
3934 Int32Constant(0), vals[0]);
3935 }
3936
3937 Node* call = BuildCallToRuntime(Runtime::kWasmTraceExit, &info, 1);
3938 SetSourcePosition(call, position);
3939}
3940
3941void WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3942 MachineRepresentation rep,
3943 Node* index, uintptr_t offset,
3944 wasm::WasmCodePosition position) {
3945 int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3946 TNode<RawPtrT> info =
3947 gasm_->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign);
3948
3949 Node* effective_offset = gasm_->IntAdd(gasm_->UintPtrConstant(offset), index);
3950 auto store = [&](int field_offset, MachineRepresentation rep, Node* data) {
3951 gasm_->Store(StoreRepresentation(rep, kNoWriteBarrier), info,
3952 Int32Constant(field_offset), data);
3953 };
3954 // Store effective_offset, is_store, and mem_rep.
3955 store(offsetof(wasm::MemoryTracingInfo, offset)__builtin_offsetof(wasm::MemoryTracingInfo, offset),
3956 MachineType::PointerRepresentation(), effective_offset);
3957 store(offsetof(wasm::MemoryTracingInfo, is_store)__builtin_offsetof(wasm::MemoryTracingInfo, is_store),
3958 MachineRepresentation::kWord8, Int32Constant(is_store ? 1 : 0));
3959 store(offsetof(wasm::MemoryTracingInfo, mem_rep)__builtin_offsetof(wasm::MemoryTracingInfo, mem_rep),
3960 MachineRepresentation::kWord8, Int32Constant(static_cast<int>(rep)));
3961
3962 Node* args[] = {info};
3963 Node* call =
3964 BuildCallToRuntime(Runtime::kWasmTraceMemory, args, arraysize(args)(sizeof(ArraySizeHelper(args))));
3965 SetSourcePosition(call, position);
3966}
3967
3968namespace {
3969LoadTransformation GetLoadTransformation(
3970 MachineType memtype, wasm::LoadTransformationKind transform) {
3971 switch (transform) {
3972 case wasm::LoadTransformationKind::kSplat: {
3973 if (memtype == MachineType::Int8()) {
3974 return LoadTransformation::kS128Load8Splat;
3975 } else if (memtype == MachineType::Int16()) {
3976 return LoadTransformation::kS128Load16Splat;
3977 } else if (memtype == MachineType::Int32()) {
3978 return LoadTransformation::kS128Load32Splat;
3979 } else if (memtype == MachineType::Int64()) {
3980 return LoadTransformation::kS128Load64Splat;
3981 }
3982 break;
3983 }
3984 case wasm::LoadTransformationKind::kExtend: {
3985 if (memtype == MachineType::Int8()) {
3986 return LoadTransformation::kS128Load8x8S;
3987 } else if (memtype == MachineType::Uint8()) {
3988 return LoadTransformation::kS128Load8x8U;
3989 } else if (memtype == MachineType::Int16()) {
3990 return LoadTransformation::kS128Load16x4S;
3991 } else if (memtype == MachineType::Uint16()) {
3992 return LoadTransformation::kS128Load16x4U;
3993 } else if (memtype == MachineType::Int32()) {
3994 return LoadTransformation::kS128Load32x2S;
3995 } else if (memtype == MachineType::Uint32()) {
3996 return LoadTransformation::kS128Load32x2U;
3997 }
3998 break;
3999 }
4000 case wasm::LoadTransformationKind::kZeroExtend: {
4001 if (memtype == MachineType::Int32()) {
4002 return LoadTransformation::kS128Load32Zero;
4003 } else if (memtype == MachineType::Int64()) {
4004 return LoadTransformation::kS128Load64Zero;
4005 }
4006 break;
4007 }
4008 }
4009 UNREACHABLE()V8_Fatal("unreachable code");
4010}
4011
4012MemoryAccessKind GetMemoryAccessKind(
4013 MachineGraph* mcgraph, MachineRepresentation memrep,
4014 WasmGraphBuilder::BoundsCheckResult bounds_check_result) {
4015 if (bounds_check_result == WasmGraphBuilder::kTrapHandler) {
4016 // Protected instructions do not come in an 'unaligned' flavor, so the trap
4017 // handler can currently only be used on systems where all memory accesses
4018 // are allowed to be unaligned.
4019 DCHECK(memrep == MachineRepresentation::kWord8 ||((void) 0)
4020 mcgraph->machine()->UnalignedLoadSupported(memrep))((void) 0);
4021 return MemoryAccessKind::kProtected;
4022 }
4023 if (memrep != MachineRepresentation::kWord8 &&
4024 !mcgraph->machine()->UnalignedLoadSupported(memrep)) {
4025 return MemoryAccessKind::kUnaligned;
4026 }
4027 return MemoryAccessKind::kNormal;
4028}
4029} // namespace
4030
4031Node* WasmGraphBuilder::LoadLane(wasm::ValueType type, MachineType memtype,
4032 Node* value, Node* index, uint64_t offset,
4033 uint32_t alignment, uint8_t laneidx,
4034 wasm::WasmCodePosition position) {
4035 has_simd_ = true;
4036 Node* load;
4037 uint8_t access_size = memtype.MemSize();
4038 BoundsCheckResult bounds_check_result;
4039 std::tie(index, bounds_check_result) =
4040 BoundsCheckMem(access_size, index, offset, position, kCanOmitBoundsCheck);
4041
4042 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
4043 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
4044 MemoryAccessKind load_kind = GetMemoryAccessKind(
4045 mcgraph_, memtype.representation(), bounds_check_result);
4046
4047 load = SetEffect(graph()->NewNode(
4048 mcgraph()->machine()->LoadLane(load_kind, memtype, laneidx),
4049 MemBuffer(capped_offset), index, value, effect(), control()));
4050
4051 if (load_kind == MemoryAccessKind::kProtected) {
4052 SetSourcePosition(load, position);
4053 }
4054 if (FLAG_trace_wasm_memory) {
4055 TraceMemoryOperation(false, memtype.representation(), index, capped_offset,
4056 position);
4057 }
4058
4059 return load;
4060}
4061
4062Node* WasmGraphBuilder::LoadTransform(wasm::ValueType type, MachineType memtype,
4063 wasm::LoadTransformationKind transform,
4064 Node* index, uint64_t offset,
4065 uint32_t alignment,
4066 wasm::WasmCodePosition position) {
4067 has_simd_ = true;
4068
4069 Node* load;
4070 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
4071 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
4072
4073 // Wasm semantics throw on OOB. Introduce explicit bounds check and
4074 // conditioning when not using the trap handler.
4075
4076 // Load extends always load 8 bytes.
4077 uint8_t access_size = transform == wasm::LoadTransformationKind::kExtend
4078 ? 8
4079 : memtype.MemSize();
4080 BoundsCheckResult bounds_check_result;
4081 std::tie(index, bounds_check_result) =
4082 BoundsCheckMem(access_size, index, offset, position, kCanOmitBoundsCheck);
4083
4084 LoadTransformation transformation = GetLoadTransformation(memtype, transform);
4085 MemoryAccessKind load_kind = GetMemoryAccessKind(
4086 mcgraph_, memtype.representation(), bounds_check_result);
4087
4088 load = SetEffect(graph()->NewNode(
4089 mcgraph()->machine()->LoadTransform(load_kind, transformation),
4090 MemBuffer(capped_offset), index, effect(), control()));
4091
4092 if (load_kind == MemoryAccessKind::kProtected) {
4093 SetSourcePosition(load, position);
4094 }
4095
4096 if (FLAG_trace_wasm_memory) {
4097 TraceMemoryOperation(false, memtype.representation(), index, capped_offset,
4098 position);
4099 }
4100 return load;
4101}
4102
4103Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
4104 Node* index, uint64_t offset,
4105 uint32_t alignment,
4106 wasm::WasmCodePosition position) {
4107 Node* load;
4108
4109 if (memtype.representation() == MachineRepresentation::kSimd128) {
4110 has_simd_ = true;
4111 }
4112
4113 // Wasm semantics throw on OOB. Introduce explicit bounds check and
4114 // conditioning when not using the trap handler.
4115 BoundsCheckResult bounds_check_result;
4116 std::tie(index, bounds_check_result) = BoundsCheckMem(
4117 memtype.MemSize(), index, offset, position, kCanOmitBoundsCheck);
4118
4119 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
4120 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
4121
4122 switch (GetMemoryAccessKind(mcgraph_, memtype.representation(),
4123 bounds_check_result)) {
4124 case MemoryAccessKind::kUnaligned:
4125 load = gasm_->LoadUnaligned(memtype, MemBuffer(capped_offset), index);
4126 break;
4127 case MemoryAccessKind::kProtected:
4128 load = gasm_->ProtectedLoad(memtype, MemBuffer(capped_offset), index);
4129 SetSourcePosition(load, position);
4130 break;
4131 case MemoryAccessKind::kNormal:
4132 load = gasm_->Load(memtype, MemBuffer(capped_offset), index);
4133 break;
4134 }
4135
4136#if defined(V8_TARGET_BIG_ENDIAN)
4137 load = BuildChangeEndiannessLoad(load, memtype, type);
4138#endif
4139
4140 if (type == wasm::kWasmI64 &&
4141 ElementSizeInBytes(memtype.representation()) < 8) {
4142 // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
4143 load = memtype.IsSigned()
4144 ? gasm_->ChangeInt32ToInt64(load) // sign extend
4145 : gasm_->ChangeUint32ToUint64(load); // zero extend
4146 }
4147
4148 if (FLAG_trace_wasm_memory) {
4149 TraceMemoryOperation(false, memtype.representation(), index, capped_offset,
4150 position);
4151 }
4152
4153 return load;
4154}
4155
4156void WasmGraphBuilder::StoreLane(MachineRepresentation mem_rep, Node* index,
4157 uint64_t offset, uint32_t alignment, Node* val,
4158 uint8_t laneidx,
4159 wasm::WasmCodePosition position,
4160 wasm::ValueType type) {
4161 has_simd_ = true;
4162 BoundsCheckResult bounds_check_result;
4163 std::tie(index, bounds_check_result) =
4164 BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset, position,
4165 kCanOmitBoundsCheck);
4166
4167 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
4168 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
4169 MemoryAccessKind load_kind =
4170 GetMemoryAccessKind(mcgraph_, mem_rep, bounds_check_result);
4171
4172 Node* store = SetEffect(graph()->NewNode(
4173 mcgraph()->machine()->StoreLane(load_kind, mem_rep, laneidx),
4174 MemBuffer(capped_offset), index, val, effect(), control()));
4175
4176 if (load_kind == MemoryAccessKind::kProtected) {
4177 SetSourcePosition(store, position);
4178 }
4179 if (FLAG_trace_wasm_memory) {
4180 TraceMemoryOperation(true, mem_rep, index, capped_offset, position);
4181 }
4182}
4183
4184void WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
4185 uint64_t offset, uint32_t alignment, Node* val,
4186 wasm::WasmCodePosition position,
4187 wasm::ValueType type) {
4188 if (mem_rep == MachineRepresentation::kSimd128) {
4189 has_simd_ = true;
4190 }
4191
4192 BoundsCheckResult bounds_check_result;
4193 std::tie(index, bounds_check_result) =
4194 BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset, position,
4195 kCanOmitBoundsCheck);
4196
4197#if defined(V8_TARGET_BIG_ENDIAN)
4198 val = BuildChangeEndiannessStore(val, mem_rep, type);
4199#endif
4200
4201 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
4202 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
4203
4204 switch (GetMemoryAccessKind(mcgraph_, mem_rep, bounds_check_result)) {
4205 case MemoryAccessKind::kUnaligned:
4206 gasm_->StoreUnaligned(UnalignedStoreRepresentation{mem_rep},
4207 MemBuffer(capped_offset), index, val);
4208 break;
4209 case MemoryAccessKind::kProtected:
4210 SetSourcePosition(
4211 gasm_->ProtectedStore(mem_rep, MemBuffer(capped_offset), index, val),
4212 position);
4213 break;
4214 case MemoryAccessKind::kNormal:
4215 gasm_->Store(StoreRepresentation{mem_rep, kNoWriteBarrier},
4216 MemBuffer(capped_offset), index, val);
4217 break;
4218 }
4219
4220 if (FLAG_trace_wasm_memory) {
4221 TraceMemoryOperation(true, mem_rep, index, capped_offset, position);
4222 }
4223}
4224
4225Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
4226 DCHECK_NOT_NULL(instance_cache_)((void) 0);
4227 Node* mem_start = instance_cache_->mem_start;
4228 Node* mem_size = instance_cache_->mem_size;
4229 DCHECK_NOT_NULL(mem_start)((void) 0);
4230 DCHECK_NOT_NULL(mem_size)((void) 0);
4231
4232 // Asm.js semantics are defined in terms of typed arrays, hence OOB
4233 // reads return {undefined} coerced to the result type (0 for integers, NaN
4234 // for float and double).
4235 // Note that we check against the memory size ignoring the size of the
4236 // stored value, which is conservative if misaligned. Technically, asm.js
4237 // should never have misaligned accesses.
4238 index = BuildChangeUint32ToUintPtr(index);
4239 Diamond bounds_check(graph(), mcgraph()->common(),
4240 gasm_->UintLessThan(index, mem_size), BranchHint::kTrue);
4241 bounds_check.Chain(control());
4242
4243 Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
4244 index, effect(), bounds_check.if_true);
4245 SetEffectControl(bounds_check.EffectPhi(load, effect()), bounds_check.merge);
4246
4247 Node* oob_value;
4248 switch (type.representation()) {
4249 case MachineRepresentation::kWord8:
4250 case MachineRepresentation::kWord16:
4251 case MachineRepresentation::kWord32:
4252 oob_value = Int32Constant(0);
4253 break;
4254 case MachineRepresentation::kWord64:
4255 oob_value = Int64Constant(0);
4256 break;
4257 case MachineRepresentation::kFloat32:
4258 oob_value = Float32Constant(std::numeric_limits<float>::quiet_NaN());
4259 break;
4260 case MachineRepresentation::kFloat64:
4261 oob_value = Float64Constant(std::numeric_limits<double>::quiet_NaN());
4262 break;
4263 default:
4264 UNREACHABLE()V8_Fatal("unreachable code");
4265 }
4266
4267 return bounds_check.Phi(type.representation(), load, oob_value);
4268}
4269
4270Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
4271 Node* val) {
4272 DCHECK_NOT_NULL(instance_cache_)((void) 0);
4273 Node* mem_start = instance_cache_->mem_start;
4274 Node* mem_size = instance_cache_->mem_size;
4275 DCHECK_NOT_NULL(mem_start)((void) 0);
4276 DCHECK_NOT_NULL(mem_size)((void) 0);
4277
4278 // Asm.js semantics are to ignore OOB writes.
4279 // Note that we check against the memory size ignoring the size of the
4280 // stored value, which is conservative if misaligned. Technically, asm.js
4281 // should never have misaligned accesses.
4282 Diamond bounds_check(graph(), mcgraph()->common(),
4283 gasm_->Uint32LessThan(index, mem_size),
4284 BranchHint::kTrue);
4285 bounds_check.Chain(control());
4286
4287 index = BuildChangeUint32ToUintPtr(index);
4288 const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
4289 type.representation(), WriteBarrierKind::kNoWriteBarrier));
4290 Node* store = graph()->NewNode(store_op, mem_start, index, val, effect(),
4291 bounds_check.if_true);
4292 SetEffectControl(bounds_check.EffectPhi(store, effect()), bounds_check.merge);
4293 return val;
4294}
4295
4296Node* WasmGraphBuilder::BuildF64x2Ceil(Node* input) {
4297 MachineType type = MachineType::Simd128();
4298 ExternalReference ref = ExternalReference::wasm_f64x2_ceil();
4299 return BuildCFuncInstruction(ref, type, input);
4300}
4301
4302Node* WasmGraphBuilder::BuildF64x2Floor(Node* input) {
4303 MachineType type = MachineType::Simd128();
4304 ExternalReference ref = ExternalReference::wasm_f64x2_floor();
4305 return BuildCFuncInstruction(ref, type, input);
4306}
4307
4308Node* WasmGraphBuilder::BuildF64x2Trunc(Node* input) {
4309 MachineType type = MachineType::Simd128();
4310 ExternalReference ref = ExternalReference::wasm_f64x2_trunc();
4311 return BuildCFuncInstruction(ref, type, input);
4312}
4313
4314Node* WasmGraphBuilder::BuildF64x2NearestInt(Node* input) {
4315 MachineType type = MachineType::Simd128();
4316 ExternalReference ref = ExternalReference::wasm_f64x2_nearest_int();
4317 return BuildCFuncInstruction(ref, type, input);
4318}
4319
4320Node* WasmGraphBuilder::BuildF32x4Ceil(Node* input) {
4321 MachineType type = MachineType::Simd128();
4322 ExternalReference ref = ExternalReference::wasm_f32x4_ceil();
4323 return BuildCFuncInstruction(ref, type, input);
4324}
4325
4326Node* WasmGraphBuilder::BuildF32x4Floor(Node* input) {
4327 MachineType type = MachineType::Simd128();
4328 ExternalReference ref = ExternalReference::wasm_f32x4_floor();
4329 return BuildCFuncInstruction(ref, type, input);
4330}
4331
4332Node* WasmGraphBuilder::BuildF32x4Trunc(Node* input) {
4333 MachineType type = MachineType::Simd128();
4334 ExternalReference ref = ExternalReference::wasm_f32x4_trunc();
4335 return BuildCFuncInstruction(ref, type, input);
4336}
4337
4338Node* WasmGraphBuilder::BuildF32x4NearestInt(Node* input) {
4339 MachineType type = MachineType::Simd128();
4340 ExternalReference ref = ExternalReference::wasm_f32x4_nearest_int();
4341 return BuildCFuncInstruction(ref, type, input);
4342}
4343
4344void WasmGraphBuilder::PrintDebugName(Node* node) {
4345 PrintF("#%d:%s", node->id(), node->op()->mnemonic());
4346}
4347
4348Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
4349
4350Zone* WasmGraphBuilder::graph_zone() { return graph()->zone(); }
4351
4352namespace {
4353Signature<MachineRepresentation>* CreateMachineSignature(
4354 Zone* zone, const wasm::FunctionSig* sig,
4355 WasmGraphBuilder::CallOrigin origin) {
4356 Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
4357 sig->parameter_count());
4358 for (auto ret : sig->returns()) {
4359 if (origin == WasmGraphBuilder::kCalledFromJS) {
4360 builder.AddReturn(MachineRepresentation::kTagged);
4361 } else {
4362 builder.AddReturn(ret.machine_representation());
4363 }
4364 }
4365
4366 for (auto param : sig->parameters()) {
4367 if (origin == WasmGraphBuilder::kCalledFromJS) {
4368 // Parameters coming from JavaScript are always tagged values. Especially
4369 // when the signature says that it's an I64 value, then a BigInt object is
4370 // provided by JavaScript, and not two 32-bit parameters.
4371 builder.AddParam(MachineRepresentation::kTagged);
4372 } else {
4373 builder.AddParam(param.machine_representation());
4374 }
4375 }
4376 return builder.Build();
4377}
4378
4379} // namespace
4380
4381void WasmGraphBuilder::AddInt64LoweringReplacement(
4382 CallDescriptor* original, CallDescriptor* replacement) {
4383 if (!lowering_special_case_) {
4384 lowering_special_case_ = std::make_unique<Int64LoweringSpecialCase>();
4385 }
4386 lowering_special_case_->replacements.insert({original, replacement});
4387}
4388
4389CallDescriptor* WasmGraphBuilder::GetI32AtomicWaitCallDescriptor() {
4390 if (i32_atomic_wait_descriptor_) return i32_atomic_wait_descriptor_;
4391
4392 i32_atomic_wait_descriptor_ = GetBuiltinCallDescriptor(
4393 Builtin::kWasmI32AtomicWait64, zone_, StubCallMode::kCallWasmRuntimeStub);
4394
4395 AddInt64LoweringReplacement(
4396 i32_atomic_wait_descriptor_,
4397 GetBuiltinCallDescriptor(Builtin::kWasmI32AtomicWait32, zone_,
4398 StubCallMode::kCallWasmRuntimeStub));
4399
4400 return i32_atomic_wait_descriptor_;
4401}
4402
4403CallDescriptor* WasmGraphBuilder::GetI64AtomicWaitCallDescriptor() {
4404 if (i64_atomic_wait_descriptor_) return i64_atomic_wait_descriptor_;
4405
4406 i64_atomic_wait_descriptor_ = GetBuiltinCallDescriptor(
4407 Builtin::kWasmI64AtomicWait64, zone_, StubCallMode::kCallWasmRuntimeStub);
4408
4409 AddInt64LoweringReplacement(
4410 i64_atomic_wait_descriptor_,
4411 GetBuiltinCallDescriptor(Builtin::kWasmI64AtomicWait32, zone_,
4412 StubCallMode::kCallWasmRuntimeStub));
4413
4414 return i64_atomic_wait_descriptor_;
4415}
4416
4417void WasmGraphBuilder::LowerInt64(Signature<MachineRepresentation>* sig) {
4418 if (mcgraph()->machine()->Is64()) return;
4419 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
4420 gasm_->simplified(), mcgraph()->zone(), sig,
4421 std::move(lowering_special_case_));
4422 r.LowerGraph();
4423}
4424
4425void WasmGraphBuilder::LowerInt64(CallOrigin origin) {
4426 LowerInt64(CreateMachineSignature(mcgraph()->zone(), sig_, origin));
4427}
4428
4429void WasmGraphBuilder::SetSourcePosition(Node* node,
4430 wasm::WasmCodePosition position) {
4431 DCHECK_NE(position, wasm::kNoCodePosition)((void) 0);
4432 if (source_position_table_) {
4433 source_position_table_->SetSourcePosition(node, SourcePosition(position));
4434 }
4435}
4436
4437Node* WasmGraphBuilder::S128Zero() {
4438 has_simd_ = true;
4439 return graph()->NewNode(mcgraph()->machine()->S128Zero());
4440}
4441
4442Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
4443 has_simd_ = true;
4444 switch (opcode) {
4445 case wasm::kExprF64x2Splat:
4446 return graph()->NewNode(mcgraph()->machine()->F64x2Splat(), inputs[0]);
4447 case wasm::kExprF64x2Abs:
4448 return graph()->NewNode(mcgraph()->machine()->F64x2Abs(), inputs[0]);
4449 case wasm::kExprF64x2Neg:
4450 return graph()->NewNode(mcgraph()->machine()->F64x2Neg(), inputs[0]);
4451 case wasm::kExprF64x2Sqrt:
4452 return graph()->NewNode(mcgraph()->machine()->F64x2Sqrt(), inputs[0]);
4453 case wasm::kExprF64x2Add:
4454 return graph()->NewNode(mcgraph()->machine()->F64x2Add(), inputs[0],
4455 inputs[1]);
4456 case wasm::kExprF64x2Sub:
4457 return graph()->NewNode(mcgraph()->machine()->F64x2Sub(), inputs[0],
4458 inputs[1]);
4459 case wasm::kExprF64x2Mul:
4460 return graph()->NewNode(mcgraph()->machine()->F64x2Mul(), inputs[0],
4461 inputs[1]);
4462 case wasm::kExprF64x2Div:
4463 return graph()->NewNode(mcgraph()->machine()->F64x2Div(), inputs[0],
4464 inputs[1]);
4465 case wasm::kExprF64x2Min:
4466 return graph()->NewNode(mcgraph()->machine()->F64x2Min(), inputs[0],
4467 inputs[1]);
4468 case wasm::kExprF64x2Max:
4469 return graph()->NewNode(mcgraph()->machine()->F64x2Max(), inputs[0],
4470 inputs[1]);
4471 case wasm::kExprF64x2Eq:
4472 return graph()->NewNode(mcgraph()->machine()->F64x2Eq(), inputs[0],
4473 inputs[1]);
4474 case wasm::kExprF64x2Ne:
4475 return graph()->NewNode(mcgraph()->machine()->F64x2Ne(), inputs[0],
4476 inputs[1]);
4477 case wasm::kExprF64x2Lt:
4478 return graph()->NewNode(mcgraph()->machine()->F64x2Lt(), inputs[0],
4479 inputs[1]);
4480 case wasm::kExprF64x2Le:
4481 return graph()->NewNode(mcgraph()->machine()->F64x2Le(), inputs[0],
4482 inputs[1]);
4483 case wasm::kExprF64x2Gt:
4484 return graph()->NewNode(mcgraph()->machine()->F64x2Lt(), inputs[1],
4485 inputs[0]);
4486 case wasm::kExprF64x2Ge:
4487 return graph()->NewNode(mcgraph()->machine()->F64x2Le(), inputs[1],
4488 inputs[0]);
4489 case wasm::kExprF64x2Qfma:
4490 return graph()->NewNode(mcgraph()->machine()->F64x2Qfma(), inputs[0],
4491 inputs[1], inputs[2]);
4492 case wasm::kExprF64x2Qfms:
4493 return graph()->NewNode(mcgraph()->machine()->F64x2Qfms(), inputs[0],
4494 inputs[1], inputs[2]);
4495 case wasm::kExprF64x2Pmin:
4496 return graph()->NewNode(mcgraph()->machine()->F64x2Pmin(), inputs[0],
4497 inputs[1]);
4498 case wasm::kExprF64x2Pmax:
4499 return graph()->NewNode(mcgraph()->machine()->F64x2Pmax(), inputs[0],
4500 inputs[1]);
4501 case wasm::kExprF64x2Ceil:
4502 // Architecture support for F64x2Ceil and Float64RoundUp is the same.
4503 if (!mcgraph()->machine()->Float64RoundUp().IsSupported())
4504 return BuildF64x2Ceil(inputs[0]);
4505 return graph()->NewNode(mcgraph()->machine()->F64x2Ceil(), inputs[0]);
4506 case wasm::kExprF64x2Floor:
4507 // Architecture support for F64x2Floor and Float64RoundDown is the same.
4508 if (!mcgraph()->machine()->Float64RoundDown().IsSupported())
4509 return BuildF64x2Floor(inputs[0]);
4510 return graph()->NewNode(mcgraph()->machine()->F64x2Floor(), inputs[0]);
4511 case wasm::kExprF64x2Trunc:
4512 // Architecture support for F64x2Trunc and Float64RoundTruncate is the
4513 // same.
4514 if (!mcgraph()->machine()->Float64RoundTruncate().IsSupported())
4515 return BuildF64x2Trunc(inputs[0]);
4516 return graph()->NewNode(mcgraph()->machine()->F64x2Trunc(), inputs[0]);
4517 case wasm::kExprF64x2NearestInt:
4518 // Architecture support for F64x2NearestInt and Float64RoundTiesEven is
4519 // the same.
4520 if (!mcgraph()->machine()->Float64RoundTiesEven().IsSupported())
4521 return BuildF64x2NearestInt(inputs[0]);
4522 return graph()->NewNode(mcgraph()->machine()->F64x2NearestInt(),
4523 inputs[0]);
4524 case wasm::kExprF64x2ConvertLowI32x4S:
4525 return graph()->NewNode(mcgraph()->machine()->F64x2ConvertLowI32x4S(),
4526 inputs[0]);
4527 case wasm::kExprF64x2ConvertLowI32x4U:
4528 return graph()->NewNode(mcgraph()->machine()->F64x2ConvertLowI32x4U(),
4529 inputs[0]);
4530 case wasm::kExprF64x2PromoteLowF32x4:
4531 return graph()->NewNode(mcgraph()->machine()->F64x2PromoteLowF32x4(),
4532 inputs[0]);
4533 case wasm::kExprF32x4Splat:
4534 return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
4535 case wasm::kExprF32x4SConvertI32x4:
4536 return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
4537 inputs[0]);
4538 case wasm::kExprF32x4UConvertI32x4:
4539 return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
4540 inputs[0]);
4541 case wasm::kExprF32x4Abs:
4542 return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
4543 case wasm::kExprF32x4Neg:
4544 return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
4545 case wasm::kExprF32x4Sqrt:
4546 return graph()->NewNode(mcgraph()->machine()->F32x4Sqrt(), inputs[0]);
4547 case wasm::kExprF32x4RecipApprox:
4548 return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
4549 inputs[0]);
4550 case wasm::kExprF32x4RecipSqrtApprox:
4551 return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
4552 inputs[0]);
4553 case wasm::kExprF32x4Add:
4554 return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
4555 inputs[1]);
4556 case wasm::kExprF32x4Sub:
4557 return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
4558 inputs[1]);
4559 case wasm::kExprF32x4Mul:
4560 return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
4561 inputs[1]);
4562 case wasm::kExprF32x4Div:
4563 return graph()->NewNode(mcgraph()->machine()->F32x4Div(), inputs[0],
4564 inputs[1]);
4565 case wasm::kExprF32x4Min:
4566 return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
4567 inputs[1]);
4568 case wasm::kExprF32x4Max:
4569 return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
4570 inputs[1]);
4571 case wasm::kExprF32x4Eq:
4572 return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
4573 inputs[1]);
4574 case wasm::kExprF32x4Ne:
4575 return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
4576 inputs[1]);
4577 case wasm::kExprF32x4Lt:
4578 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
4579 inputs[1]);
4580 case wasm::kExprF32x4Le:
4581 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
4582 inputs[1]);
4583 case wasm::kExprF32x4Gt:
4584 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
4585 inputs[0]);
4586 case wasm::kExprF32x4Ge:
4587 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
4588 inputs[0]);
4589 case wasm::kExprF32x4Qfma:
4590 return graph()->NewNode(mcgraph()->machine()->F32x4Qfma(), inputs[0],
4591 inputs[1], inputs[2]);
4592 case wasm::kExprF32x4Qfms:
4593 return graph()->NewNode(mcgraph()->machine()->F32x4Qfms(), inputs[0],
4594 inputs[1], inputs[2]);
4595 case wasm::kExprF32x4Pmin:
4596 return graph()->NewNode(mcgraph()->machine()->F32x4Pmin(), inputs[0],
4597 inputs[1]);
4598 case wasm::kExprF32x4Pmax:
4599 return graph()->NewNode(mcgraph()->machine()->F32x4Pmax(), inputs[0],
4600 inputs[1]);
4601 case wasm::kExprF32x4Ceil:
4602 // Architecture support for F32x4Ceil and Float32RoundUp is the same.
4603 if (!mcgraph()->machine()->Float32RoundUp().IsSupported())
4604 return BuildF32x4Ceil(inputs[0]);
4605 return graph()->NewNode(mcgraph()->machine()->F32x4Ceil(), inputs[0]);
4606 case wasm::kExprF32x4Floor:
4607 // Architecture support for F32x4Floor and Float32RoundDown is the same.
4608 if (!mcgraph()->machine()->Float32RoundDown().IsSupported())
4609 return BuildF32x4Floor(inputs[0]);
4610 return graph()->NewNode(mcgraph()->machine()->F32x4Floor(), inputs[0]);
4611 case wasm::kExprF32x4Trunc:
4612 // Architecture support for F32x4Trunc and Float32RoundTruncate is the
4613 // same.
4614 if (!mcgraph()->machine()->Float32RoundTruncate().IsSupported())
4615 return BuildF32x4Trunc(inputs[0]);
4616 return graph()->NewNode(mcgraph()->machine()->F32x4Trunc(), inputs[0]);
4617 case wasm::kExprF32x4NearestInt:
4618 // Architecture support for F32x4NearestInt and Float32RoundTiesEven is
4619 // the same.
4620 if (!mcgraph()->machine()->Float32RoundTiesEven().IsSupported())
4621 return BuildF32x4NearestInt(inputs[0]);
4622 return graph()->NewNode(mcgraph()->machine()->F32x4NearestInt(),
4623 inputs[0]);
4624 case wasm::kExprF32x4DemoteF64x2Zero:
4625 return graph()->NewNode(mcgraph()->machine()->F32x4DemoteF64x2Zero(),
4626 inputs[0]);
4627 case wasm::kExprI64x2Splat:
4628 return graph()->NewNode(mcgraph()->machine()->I64x2Splat(), inputs[0]);
4629 case wasm::kExprI64x2Abs:
4630 return graph()->NewNode(mcgraph()->machine()->I64x2Abs(), inputs[0]);
4631 case wasm::kExprI64x2Neg:
4632 return graph()->NewNode(mcgraph()->machine()->I64x2Neg(), inputs[0]);
4633 case wasm::kExprI64x2SConvertI32x4Low:
4634 return graph()->NewNode(mcgraph()->machine()->I64x2SConvertI32x4Low(),
4635 inputs[0]);
4636 case wasm::kExprI64x2SConvertI32x4High:
4637 return graph()->NewNode(mcgraph()->machine()->I64x2SConvertI32x4High(),
4638 inputs[0]);
4639 case wasm::kExprI64x2UConvertI32x4Low:
4640 return graph()->NewNode(mcgraph()->machine()->I64x2UConvertI32x4Low(),
4641 inputs[0]);
4642 case wasm::kExprI64x2UConvertI32x4High:
4643 return graph()->NewNode(mcgraph()->machine()->I64x2UConvertI32x4High(),
4644 inputs[0]);
4645 case wasm::kExprI64x2BitMask:
4646 return graph()->NewNode(mcgraph()->machine()->I64x2BitMask(), inputs[0]);
4647 case wasm::kExprI64x2Shl:
4648 return graph()->NewNode(mcgraph()->machine()->I64x2Shl(), inputs[0],
4649 inputs[1]);
4650 case wasm::kExprI64x2ShrS:
4651 return graph()->NewNode(mcgraph()->machine()->I64x2ShrS(), inputs[0],
4652 inputs[1]);
4653 case wasm::kExprI64x2Add:
4654 return graph()->NewNode(mcgraph()->machine()->I64x2Add(), inputs[0],
4655 inputs[1]);
4656 case wasm::kExprI64x2Sub:
4657 return graph()->NewNode(mcgraph()->machine()->I64x2Sub(), inputs[0],
4658 inputs[1]);
4659 case wasm::kExprI64x2Mul:
4660 return graph()->NewNode(mcgraph()->machine()->I64x2Mul(), inputs[0],
4661 inputs[1]);
4662 case wasm::kExprI64x2Eq:
4663 return graph()->NewNode(mcgraph()->machine()->I64x2Eq(), inputs[0],
4664 inputs[1]);
4665 case wasm::kExprI64x2Ne:
4666 return graph()->NewNode(mcgraph()->machine()->I64x2Ne(), inputs[0],
4667 inputs[1]);
4668 case wasm::kExprI64x2LtS:
4669 return graph()->NewNode(mcgraph()->machine()->I64x2GtS(), inputs[1],
4670 inputs[0]);
4671 case wasm::kExprI64x2LeS:
4672 return graph()->NewNode(mcgraph()->machine()->I64x2GeS(), inputs[1],
4673 inputs[0]);
4674 case wasm::kExprI64x2GtS:
4675 return graph()->NewNode(mcgraph()->machine()->I64x2GtS(), inputs[0],
4676 inputs[1]);
4677 case wasm::kExprI64x2GeS:
4678 return graph()->NewNode(mcgraph()->machine()->I64x2GeS(), inputs[0],
4679 inputs[1]);
4680 case wasm::kExprI64x2ShrU:
4681 return graph()->NewNode(mcgraph()->machine()->I64x2ShrU(), inputs[0],
4682 inputs[1]);
4683 case wasm::kExprI64x2ExtMulLowI32x4S:
4684 return graph()->NewNode(mcgraph()->machine()->I64x2ExtMulLowI32x4S(),
4685 inputs[0], inputs[1]);
4686 case wasm::kExprI64x2ExtMulHighI32x4S:
4687 return graph()->NewNode(mcgraph()->machine()->I64x2ExtMulHighI32x4S(),
4688 inputs[0], inputs[1]);
4689 case wasm::kExprI64x2ExtMulLowI32x4U:
4690 return graph()->NewNode(mcgraph()->machine()->I64x2ExtMulLowI32x4U(),
4691 inputs[0], inputs[1]);
4692 case wasm::kExprI64x2ExtMulHighI32x4U:
4693 return graph()->NewNode(mcgraph()->machine()->I64x2ExtMulHighI32x4U(),
4694 inputs[0], inputs[1]);
4695 case wasm::kExprI32x4Splat:
4696 return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
4697 case wasm::kExprI32x4SConvertF32x4:
4698 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
4699 inputs[0]);
4700 case wasm::kExprI32x4UConvertF32x4:
4701 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
4702 inputs[0]);
4703 case wasm::kExprI32x4SConvertI16x8Low:
4704 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
4705 inputs[0]);
4706 case wasm::kExprI32x4SConvertI16x8High:
4707 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
4708 inputs[0]);
4709 case wasm::kExprI32x4Neg:
4710 return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
4711 case wasm::kExprI32x4Shl:
4712 return graph()->NewNode(mcgraph()->machine()->I32x4Shl(), inputs[0],
4713 inputs[1]);
4714 case wasm::kExprI32x4ShrS:
4715 return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(), inputs[0],
4716 inputs[1]);
4717 case wasm::kExprI32x4Add:
4718 return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
4719 inputs[1]);
4720 case wasm::kExprI32x4Sub:
4721 return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
4722 inputs[1]);
4723 case wasm::kExprI32x4Mul:
4724 return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
4725 inputs[1]);
4726 case wasm::kExprI32x4MinS:
4727 return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
4728 inputs[1]);
4729 case wasm::kExprI32x4MaxS:
4730 return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
4731 inputs[1]);
4732 case wasm::kExprI32x4Eq:
4733 return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
4734 inputs[1]);
4735 case wasm::kExprI32x4Ne:
4736 return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
4737 inputs[1]);
4738 case wasm::kExprI32x4LtS:
4739 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
4740 inputs[0]);
4741 case wasm::kExprI32x4LeS:
4742 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
4743 inputs[0]);
4744 case wasm::kExprI32x4GtS:
4745 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
4746 inputs[1]);
4747 case wasm::kExprI32x4GeS:
4748 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
4749 inputs[1]);
4750 case wasm::kExprI32x4UConvertI16x8Low:
4751 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
4752 inputs[0]);
4753 case wasm::kExprI32x4UConvertI16x8High:
4754 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
4755 inputs[0]);
4756 case wasm::kExprI32x4ShrU:
4757 return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(), inputs[0],
4758 inputs[1]);
4759 case wasm::kExprI32x4MinU:
4760 return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
4761 inputs[1]);
4762 case wasm::kExprI32x4MaxU:
4763 return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
4764 inputs[1]);
4765 case wasm::kExprI32x4LtU:
4766 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
4767 inputs[0]);
4768 case wasm::kExprI32x4LeU:
4769 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
4770 inputs[0]);
4771 case wasm::kExprI32x4GtU:
4772 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
4773 inputs[1]);
4774 case wasm::kExprI32x4GeU:
4775 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
4776 inputs[1]);
4777 case wasm::kExprI32x4Abs:
4778 return graph()->NewNode(mcgraph()->machine()->I32x4Abs(), inputs[0]);
4779 case wasm::kExprI32x4BitMask:
4780 return graph()->NewNode(mcgraph()->machine()->I32x4BitMask(), inputs[0]);
4781 case wasm::kExprI32x4DotI16x8S:
4782 return graph()->NewNode(mcgraph()->machine()->I32x4DotI16x8S(), inputs[0],
4783 inputs[1]);
4784 case wasm::kExprI32x4ExtMulLowI16x8S:
4785 return graph()->NewNode(mcgraph()->machine()->I32x4ExtMulLowI16x8S(),
4786 inputs[0], inputs[1]);
4787 case wasm::kExprI32x4ExtMulHighI16x8S:
4788 return graph()->NewNode(mcgraph()->machine()->I32x4ExtMulHighI16x8S(),
4789 inputs[0], inputs[1]);
4790 case wasm::kExprI32x4ExtMulLowI16x8U:
4791 return graph()->NewNode(mcgraph()->machine()->I32x4ExtMulLowI16x8U(),
4792 inputs[0], inputs[1]);
4793 case wasm::kExprI32x4ExtMulHighI16x8U:
4794 return graph()->NewNode(mcgraph()->machine()->I32x4ExtMulHighI16x8U(),
4795 inputs[0], inputs[1]);
4796 case wasm::kExprI32x4ExtAddPairwiseI16x8S:
4797 return graph()->NewNode(mcgraph()->machine()->I32x4ExtAddPairwiseI16x8S(),
4798 inputs[0]);
4799 case wasm::kExprI32x4ExtAddPairwiseI16x8U:
4800 return graph()->NewNode(mcgraph()->machine()->I32x4ExtAddPairwiseI16x8U(),
4801 inputs[0]);
4802 case wasm::kExprI32x4TruncSatF64x2SZero:
4803 return graph()->NewNode(mcgraph()->machine()->I32x4TruncSatF64x2SZero(),
4804 inputs[0]);
4805 case wasm::kExprI32x4TruncSatF64x2UZero:
4806 return graph()->NewNode(mcgraph()->machine()->I32x4TruncSatF64x2UZero(),
4807 inputs[0]);
4808 case wasm::kExprI16x8Splat:
4809 return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
4810 case wasm::kExprI16x8SConvertI8x16Low:
4811 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
4812 inputs[0]);
4813 case wasm::kExprI16x8SConvertI8x16High:
4814 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
4815 inputs[0]);
4816 case wasm::kExprI16x8Shl:
4817 return graph()->NewNode(mcgraph()->machine()->I16x8Shl(), inputs[0],
4818 inputs[1]);
4819 case wasm::kExprI16x8ShrS:
4820 return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(), inputs[0],
4821 inputs[1]);
4822 case wasm::kExprI16x8Neg:
4823 return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
4824 case wasm::kExprI16x8SConvertI32x4:
4825 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
4826 inputs[0], inputs[1]);
4827 case wasm::kExprI16x8Add:
4828 return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
4829 inputs[1]);
4830 case wasm::kExprI16x8AddSatS:
4831 return graph()->NewNode(mcgraph()->machine()->I16x8AddSatS(), inputs[0],
4832 inputs[1]);
4833 case wasm::kExprI16x8Sub:
4834 return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
4835 inputs[1]);
4836 case wasm::kExprI16x8SubSatS:
4837 return graph()->NewNode(mcgraph()->machine()->I16x8SubSatS(), inputs[0],
4838 inputs[1]);
4839 case wasm::kExprI16x8Mul:
4840 return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
4841 inputs[1]);
4842 case wasm::kExprI16x8MinS:
4843 return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
4844 inputs[1]);
4845 case wasm::kExprI16x8MaxS:
4846 return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
4847 inputs[1]);
4848 case wasm::kExprI16x8Eq:
4849 return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
4850 inputs[1]);
4851 case wasm::kExprI16x8Ne:
4852 return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
4853 inputs[1]);
4854 case wasm::kExprI16x8LtS:
4855 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
4856 inputs[0]);
4857 case wasm::kExprI16x8LeS:
4858 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
4859 inputs[0]);
4860 case wasm::kExprI16x8GtS:
4861 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
4862 inputs[1]);
4863 case wasm::kExprI16x8GeS:
4864 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
4865 inputs[1]);
4866 case wasm::kExprI16x8UConvertI8x16Low:
4867 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
4868 inputs[0]);
4869 case wasm::kExprI16x8UConvertI8x16High:
4870 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
4871 inputs[0]);
4872 case wasm::kExprI16x8UConvertI32x4:
4873 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
4874 inputs[0], inputs[1]);
4875 case wasm::kExprI16x8ShrU:
4876 return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(), inputs[0],
4877 inputs[1]);
4878 case wasm::kExprI16x8AddSatU:
4879 return graph()->NewNode(mcgraph()->machine()->I16x8AddSatU(), inputs[0],
4880 inputs[1]);
4881 case wasm::kExprI16x8SubSatU:
4882 return graph()->NewNode(mcgraph()->machine()->I16x8SubSatU(), inputs[0],
4883 inputs[1]);
4884 case wasm::kExprI16x8MinU:
4885 return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
4886 inputs[1]);
4887 case wasm::kExprI16x8MaxU:
4888 return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
4889 inputs[1]);
4890 case wasm::kExprI16x8LtU:
4891 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
4892 inputs[0]);
4893 case wasm::kExprI16x8LeU:
4894 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
4895 inputs[0]);
4896 case wasm::kExprI16x8GtU:
4897 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
4898 inputs[1]);
4899 case wasm::kExprI16x8GeU:
4900 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
4901 inputs[1]);
4902 case wasm::kExprI16x8RoundingAverageU:
4903 return graph()->NewNode(mcgraph()->machine()->I16x8RoundingAverageU(),
4904 inputs[0], inputs[1]);
4905 case wasm::kExprI16x8Q15MulRSatS:
4906 return graph()->NewNode(mcgraph()->machine()->I16x8Q15MulRSatS(),
4907 inputs[0], inputs[1]);
4908 case wasm::kExprI16x8Abs:
4909 return graph()->NewNode(mcgraph()->machine()->I16x8Abs(), inputs[0]);
4910 case wasm::kExprI16x8BitMask:
4911 return graph()->NewNode(mcgraph()->machine()->I16x8BitMask(), inputs[0]);
4912 case wasm::kExprI16x8ExtMulLowI8x16S:
4913 return graph()->NewNode(mcgraph()->machine()->I16x8ExtMulLowI8x16S(),
4914 inputs[0], inputs[1]);
4915 case wasm::kExprI16x8ExtMulHighI8x16S:
4916 return graph()->NewNode(mcgraph()->machine()->I16x8ExtMulHighI8x16S(),
4917 inputs[0], inputs[1]);
4918 case wasm::kExprI16x8ExtMulLowI8x16U:
4919 return graph()->NewNode(mcgraph()->machine()->I16x8ExtMulLowI8x16U(),
4920 inputs[0], inputs[1]);
4921 case wasm::kExprI16x8ExtMulHighI8x16U:
4922 return graph()->NewNode(mcgraph()->machine()->I16x8ExtMulHighI8x16U(),
4923 inputs[0], inputs[1]);
4924 case wasm::kExprI16x8ExtAddPairwiseI8x16S:
4925 return graph()->NewNode(mcgraph()->machine()->I16x8ExtAddPairwiseI8x16S(),
4926 inputs[0]);
4927 case wasm::kExprI16x8ExtAddPairwiseI8x16U:
4928 return graph()->NewNode(mcgraph()->machine()->I16x8ExtAddPairwiseI8x16U(),
4929 inputs[0]);
4930 case wasm::kExprI8x16Splat:
4931 return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
4932 case wasm::kExprI8x16Neg:
4933 return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
4934 case wasm::kExprI8x16Shl:
4935 return graph()->NewNode(mcgraph()->machine()->I8x16Shl(), inputs[0],
4936 inputs[1]);
4937 case wasm::kExprI8x16ShrS:
4938 return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(), inputs[0],
4939 inputs[1]);
4940 case wasm::kExprI8x16SConvertI16x8:
4941 return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
4942 inputs[0], inputs[1]);
4943 case wasm::kExprI8x16Add:
4944 return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
4945 inputs[1]);
4946 case wasm::kExprI8x16AddSatS:
4947 return graph()->NewNode(mcgraph()->machine()->I8x16AddSatS(), inputs[0],
4948 inputs[1]);
4949 case wasm::kExprI8x16Sub:
4950 return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
4951 inputs[1]);
4952 case wasm::kExprI8x16SubSatS:
4953 return graph()->NewNode(mcgraph()->machine()->I8x16SubSatS(), inputs[0],
4954 inputs[1]);
4955 case wasm::kExprI8x16MinS:
4956 return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
4957 inputs[1]);
4958 case wasm::kExprI8x16MaxS:
4959 return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
4960 inputs[1]);
4961 case wasm::kExprI8x16Eq:
4962 return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
4963 inputs[1]);
4964 case wasm::kExprI8x16Ne:
4965 return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
4966 inputs[1]);
4967 case wasm::kExprI8x16LtS:
4968 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
4969 inputs[0]);
4970 case wasm::kExprI8x16LeS:
4971 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
4972 inputs[0]);
4973 case wasm::kExprI8x16GtS:
4974 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
4975 inputs[1]);
4976 case wasm::kExprI8x16GeS:
4977 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
4978 inputs[1]);
4979 case wasm::kExprI8x16ShrU:
4980 return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(), inputs[0],
4981 inputs[1]);
4982 case wasm::kExprI8x16UConvertI16x8:
4983 return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
4984 inputs[0], inputs[1]);
4985 case wasm::kExprI8x16AddSatU:
4986 return graph()->NewNode(mcgraph()->machine()->I8x16AddSatU(), inputs[0],
4987 inputs[1]);
4988 case wasm::kExprI8x16SubSatU:
4989 return graph()->NewNode(mcgraph()->machine()->I8x16SubSatU(), inputs[0],
4990 inputs[1]);
4991 case wasm::kExprI8x16MinU:
4992 return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
4993 inputs[1]);
4994 case wasm::kExprI8x16MaxU:
4995 return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
4996 inputs[1]);
4997 case wasm::kExprI8x16LtU:
4998 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
4999 inputs[0]);
5000 case wasm::kExprI8x16LeU:
5001 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
5002 inputs[0]);
5003 case wasm::kExprI8x16GtU:
5004 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
5005 inputs[1]);
5006 case wasm::kExprI8x16GeU:
5007 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
5008 inputs[1]);
5009 case wasm::kExprI8x16RoundingAverageU:
5010 return graph()->NewNode(mcgraph()->machine()->I8x16RoundingAverageU(),
5011 inputs[0], inputs[1]);
5012 case wasm::kExprI8x16Popcnt:
5013 return graph()->NewNode(mcgraph()->machine()->I8x16Popcnt(), inputs[0]);
5014 case wasm::kExprI8x16Abs:
5015 return graph()->NewNode(mcgraph()->machine()->I8x16Abs(), inputs[0]);
5016 case wasm::kExprI8x16BitMask:
5017 return graph()->NewNode(mcgraph()->machine()->I8x16BitMask(), inputs[0]);
5018 case wasm::kExprS128And:
5019 return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
5020 inputs[1]);
5021 case wasm::kExprS128Or:
5022 return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
5023 inputs[1]);
5024 case wasm::kExprS128Xor:
5025 return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
5026 inputs[1]);
5027 case wasm::kExprS128Not:
5028 return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
5029 case wasm::kExprS128Select:
5030 return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
5031 inputs[0], inputs[1]);
5032 case wasm::kExprS128AndNot:
5033 return graph()->NewNode(mcgraph()->machine()->S128AndNot(), inputs[0],
5034 inputs[1]);
5035 case wasm::kExprI64x2AllTrue:
5036 return graph()->NewNode(mcgraph()->machine()->I64x2AllTrue(), inputs[0]);
5037 case wasm::kExprI32x4AllTrue:
5038 return graph()->NewNode(mcgraph()->machine()->I32x4AllTrue(), inputs[0]);
5039 case wasm::kExprI16x8AllTrue:
5040 return graph()->NewNode(mcgraph()->machine()->I16x8AllTrue(), inputs[0]);
5041 case wasm::kExprV128AnyTrue:
5042 return graph()->NewNode(mcgraph()->machine()->V128AnyTrue(), inputs[0]);
5043 case wasm::kExprI8x16AllTrue:
5044 return graph()->NewNode(mcgraph()->machine()->I8x16AllTrue(), inputs[0]);
5045 case wasm::kExprI8x16Swizzle:
5046 return graph()->NewNode(mcgraph()->machine()->I8x16Swizzle(false),
5047 inputs[0], inputs[1]);
5048 case wasm::kExprI8x16RelaxedSwizzle:
5049 return graph()->NewNode(mcgraph()->machine()->I8x16Swizzle(true),
5050 inputs[0], inputs[1]);
5051 case wasm::kExprI8x16RelaxedLaneSelect:
5052 // Relaxed lane select puts the mask as first input (same as S128Select).
5053 return graph()->NewNode(mcgraph()->machine()->I8x16RelaxedLaneSelect(),
5054 inputs[2], inputs[0], inputs[1]);
5055 case wasm::kExprI16x8RelaxedLaneSelect:
5056 return graph()->NewNode(mcgraph()->machine()->I16x8RelaxedLaneSelect(),
5057 inputs[2], inputs[0], inputs[1]);
5058 case wasm::kExprI32x4RelaxedLaneSelect:
5059 return graph()->NewNode(mcgraph()->machine()->I32x4RelaxedLaneSelect(),
5060 inputs[2], inputs[0], inputs[1]);
5061 case wasm::kExprI64x2RelaxedLaneSelect:
5062 return graph()->NewNode(mcgraph()->machine()->I64x2RelaxedLaneSelect(),
5063 inputs[2], inputs[0], inputs[1]);
5064 case wasm::kExprF32x4RelaxedMin:
5065 return graph()->NewNode(mcgraph()->machine()->F32x4RelaxedMin(),
5066 inputs[0], inputs[1]);
5067 case wasm::kExprF32x4RelaxedMax:
5068 return graph()->NewNode(mcgraph()->machine()->F32x4RelaxedMax(),
5069 inputs[0], inputs[1]);
5070 case wasm::kExprF64x2RelaxedMin:
5071 return graph()->NewNode(mcgraph()->machine()->F64x2RelaxedMin(),
5072 inputs[0], inputs[1]);
5073 case wasm::kExprF64x2RelaxedMax:
5074 return graph()->NewNode(mcgraph()->machine()->F64x2RelaxedMax(),
5075 inputs[0], inputs[1]);
5076 case wasm::kExprI32x4RelaxedTruncF64x2SZero:
5077 return graph()->NewNode(
5078 mcgraph()->machine()->I32x4RelaxedTruncF64x2SZero(), inputs[0]);
5079 case wasm::kExprI32x4RelaxedTruncF64x2UZero:
5080 return graph()->NewNode(
5081 mcgraph()->machine()->I32x4RelaxedTruncF64x2UZero(), inputs[0]);
5082 case wasm::kExprI32x4RelaxedTruncF32x4S:
5083 return graph()->NewNode(mcgraph()->machine()->I32x4RelaxedTruncF32x4S(),
5084 inputs[0]);
5085 case wasm::kExprI32x4RelaxedTruncF32x4U:
5086 return graph()->NewNode(mcgraph()->machine()->I32x4RelaxedTruncF32x4U(),
5087 inputs[0]);
5088 default:
5089 FATAL_UNSUPPORTED_OPCODE(opcode);
5090 }
5091}
5092
5093Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
5094 Node* const* inputs) {
5095 has_simd_ = true;
5096 switch (opcode) {
5097 case wasm::kExprF64x2ExtractLane:
5098 return graph()->NewNode(mcgraph()->machine()->F64x2ExtractLane(lane),
5099 inputs[0]);
5100 case wasm::kExprF64x2ReplaceLane:
5101 return graph()->NewNode(mcgraph()->machine()->F64x2ReplaceLane(lane),
5102 inputs[0], inputs[1]);
5103 case wasm::kExprF32x4ExtractLane:
5104 return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
5105 inputs[0]);
5106 case wasm::kExprF32x4ReplaceLane:
5107 return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
5108 inputs[0], inputs[1]);
5109 case wasm::kExprI64x2ExtractLane:
5110 return graph()->NewNode(mcgraph()->machine()->I64x2ExtractLane(lane),
5111 inputs[0]);
5112 case wasm::kExprI64x2ReplaceLane:
5113 return graph()->NewNode(mcgraph()->machine()->I64x2ReplaceLane(lane),
5114 inputs[0], inputs[1]);
5115 case wasm::kExprI32x4ExtractLane:
5116 return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
5117 inputs[0]);
5118 case wasm::kExprI32x4ReplaceLane:
5119 return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
5120 inputs[0], inputs[1]);
5121 case wasm::kExprI16x8ExtractLaneS:
5122 return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLaneS(lane),
5123 inputs[0]);
5124 case wasm::kExprI16x8ExtractLaneU:
5125 return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLaneU(lane),
5126 inputs[0]);
5127 case wasm::kExprI16x8ReplaceLane:
5128 return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
5129 inputs[0], inputs[1]);
5130 case wasm::kExprI8x16ExtractLaneS:
5131 return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLaneS(lane),
5132 inputs[0]);
5133 case wasm::kExprI8x16ExtractLaneU:
5134 return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLaneU(lane),
5135 inputs[0]);
5136 case wasm::kExprI8x16ReplaceLane:
5137 return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
5138 inputs[0], inputs[1]);
5139 default:
5140 FATAL_UNSUPPORTED_OPCODE(opcode);
5141 }
5142}
5143
5144Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
5145 Node* const* inputs) {
5146 has_simd_ = true;
5147 return graph()->NewNode(mcgraph()->machine()->I8x16Shuffle(shuffle),
5148 inputs[0], inputs[1]);
5149}
5150
5151Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
5152 uint32_t alignment, uint64_t offset,
5153 wasm::WasmCodePosition position) {
5154 struct AtomicOpInfo {
5155 enum Type : int8_t {
5156 kNoInput = 0,
5157 kOneInput = 1,
5158 kTwoInputs = 2,
5159 kSpecial
5160 };
5161
5162 using OperatorByType =
5163 const Operator* (MachineOperatorBuilder::*)(MachineType);
5164 using OperatorByRep =
5165 const Operator* (MachineOperatorBuilder::*)(MachineRepresentation);
5166 using OperatorByAtomicLoadRep =
5167 const Operator* (MachineOperatorBuilder::*)(AtomicLoadParameters);
5168 using OperatorByAtomicStoreRep =
5169 const Operator* (MachineOperatorBuilder::*)(AtomicStoreParameters);
5170
5171 const Type type;
5172 const MachineType machine_type;
5173 const OperatorByType operator_by_type = nullptr;
5174 const OperatorByRep operator_by_rep = nullptr;
5175 const OperatorByAtomicLoadRep operator_by_atomic_load_params = nullptr;
5176 const OperatorByAtomicStoreRep operator_by_atomic_store_rep = nullptr;
5177 const wasm::ValueType wasm_type;
5178
5179 constexpr AtomicOpInfo(Type t, MachineType m, OperatorByType o)
5180 : type(t), machine_type(m), operator_by_type(o) {}
5181 constexpr AtomicOpInfo(Type t, MachineType m, OperatorByRep o)
5182 : type(t), machine_type(m), operator_by_rep(o) {}
5183 constexpr AtomicOpInfo(Type t, MachineType m, OperatorByAtomicLoadRep o,
5184 wasm::ValueType v)
5185 : type(t),
5186 machine_type(m),
5187 operator_by_atomic_load_params(o),
5188 wasm_type(v) {}
5189 constexpr AtomicOpInfo(Type t, MachineType m, OperatorByAtomicStoreRep o,
5190 wasm::ValueType v)
5191 : type(t),
5192 machine_type(m),
5193 operator_by_atomic_store_rep(o),
5194 wasm_type(v) {}
5195
5196 // Constexpr, hence just a table lookup in most compilers.
5197 static constexpr AtomicOpInfo Get(wasm::WasmOpcode opcode) {
5198 switch (opcode) {
5199#define CASE(Name, Type, MachType, Op) \
5200 case wasm::kExpr##Name: \
5201 return {Type, MachineType::MachType(), &MachineOperatorBuilder::Op};
5202#define CASE_LOAD_STORE(Name, Type, MachType, Op, WasmType) \
5203 case wasm::kExpr##Name: \
5204 return {Type, MachineType::MachType(), &MachineOperatorBuilder::Op, \
5205 WasmType};
5206
5207 // Binops.
5208 CASE(I32AtomicAdd, kOneInput, Uint32, Word32AtomicAdd)
5209 CASE(I64AtomicAdd, kOneInput, Uint64, Word64AtomicAdd)
5210 CASE(I32AtomicAdd8U, kOneInput, Uint8, Word32AtomicAdd)
5211 CASE(I32AtomicAdd16U, kOneInput, Uint16, Word32AtomicAdd)
5212 CASE(I64AtomicAdd8U, kOneInput, Uint8, Word64AtomicAdd)
5213 CASE(I64AtomicAdd16U, kOneInput, Uint16, Word64AtomicAdd)
5214 CASE(I64AtomicAdd32U, kOneInput, Uint32, Word64AtomicAdd)
5215 CASE(I32AtomicSub, kOneInput, Uint32, Word32AtomicSub)
5216 CASE(I64AtomicSub, kOneInput, Uint64, Word64AtomicSub)
5217 CASE(I32AtomicSub8U, kOneInput, Uint8, Word32AtomicSub)
5218 CASE(I32AtomicSub16U, kOneInput, Uint16, Word32AtomicSub)
5219 CASE(I64AtomicSub8U, kOneInput, Uint8, Word64AtomicSub)
5220 CASE(I64AtomicSub16U, kOneInput, Uint16, Word64AtomicSub)
5221 CASE(I64AtomicSub32U, kOneInput, Uint32, Word64AtomicSub)
5222 CASE(I32AtomicAnd, kOneInput, Uint32, Word32AtomicAnd)
5223 CASE(I64AtomicAnd, kOneInput, Uint64, Word64AtomicAnd)
5224 CASE(I32AtomicAnd8U, kOneInput, Uint8, Word32AtomicAnd)
5225 CASE(I32AtomicAnd16U, kOneInput, Uint16, Word32AtomicAnd)
5226 CASE(I64AtomicAnd8U, kOneInput, Uint8, Word64AtomicAnd)
5227 CASE(I64AtomicAnd16U, kOneInput, Uint16, Word64AtomicAnd)
5228 CASE(I64AtomicAnd32U, kOneInput, Uint32, Word64AtomicAnd)
5229 CASE(I32AtomicOr, kOneInput, Uint32, Word32AtomicOr)
5230 CASE(I64AtomicOr, kOneInput, Uint64, Word64AtomicOr)
5231 CASE(I32AtomicOr8U, kOneInput, Uint8, Word32AtomicOr)
5232 CASE(I32AtomicOr16U, kOneInput, Uint16, Word32AtomicOr)
5233 CASE(I64AtomicOr8U, kOneInput, Uint8, Word64AtomicOr)
5234 CASE(I64AtomicOr16U, kOneInput, Uint16, Word64AtomicOr)
5235 CASE(I64AtomicOr32U, kOneInput, Uint32, Word64AtomicOr)
5236 CASE(I32AtomicXor, kOneInput, Uint32, Word32AtomicXor)
5237 CASE(I64AtomicXor, kOneInput, Uint64, Word64AtomicXor)
5238 CASE(I32AtomicXor8U, kOneInput, Uint8, Word32AtomicXor)
5239 CASE(I32AtomicXor16U, kOneInput, Uint16, Word32AtomicXor)
5240 CASE(I64AtomicXor8U, kOneInput, Uint8, Word64AtomicXor)
5241 CASE(I64AtomicXor16U, kOneInput, Uint16, Word64AtomicXor)
5242 CASE(I64AtomicXor32U, kOneInput, Uint32, Word64AtomicXor)
5243 CASE(I32AtomicExchange, kOneInput, Uint32, Word32AtomicExchange)
5244 CASE(I64AtomicExchange, kOneInput, Uint64, Word64AtomicExchange)
5245 CASE(I32AtomicExchange8U, kOneInput, Uint8, Word32AtomicExchange)
5246 CASE(I32AtomicExchange16U, kOneInput, Uint16, Word32AtomicExchange)
5247 CASE(I64AtomicExchange8U, kOneInput, Uint8, Word64AtomicExchange)
5248 CASE(I64AtomicExchange16U, kOneInput, Uint16, Word64AtomicExchange)
5249 CASE(I64AtomicExchange32U, kOneInput, Uint32, Word64AtomicExchange)
5250
5251 // Compare-exchange.
5252 CASE(I32AtomicCompareExchange, kTwoInputs, Uint32,
5253 Word32AtomicCompareExchange)
5254 CASE(I64AtomicCompareExchange, kTwoInputs, Uint64,
5255 Word64AtomicCompareExchange)
5256 CASE(I32AtomicCompareExchange8U, kTwoInputs, Uint8,
5257 Word32AtomicCompareExchange)
5258 CASE(I32AtomicCompareExchange16U, kTwoInputs, Uint16,
5259 Word32AtomicCompareExchange)
5260 CASE(I64AtomicCompareExchange8U, kTwoInputs, Uint8,
5261 Word64AtomicCompareExchange)
5262 CASE(I64AtomicCompareExchange16U, kTwoInputs, Uint16,
5263 Word64AtomicCompareExchange)
5264 CASE(I64AtomicCompareExchange32U, kTwoInputs, Uint32,
5265 Word64AtomicCompareExchange)
5266
5267 // Load.
5268 CASE_LOAD_STORE(I32AtomicLoad, kNoInput, Uint32, Word32AtomicLoad,
5269 wasm::kWasmI32)
5270 CASE_LOAD_STORE(I64AtomicLoad, kNoInput, Uint64, Word64AtomicLoad,
5271 wasm::kWasmI64)
5272 CASE_LOAD_STORE(I32AtomicLoad8U, kNoInput, Uint8, Word32AtomicLoad,
5273 wasm::kWasmI32)
5274 CASE_LOAD_STORE(I32AtomicLoad16U, kNoInput, Uint16, Word32AtomicLoad,
5275 wasm::kWasmI32)
5276 CASE_LOAD_STORE(I64AtomicLoad8U, kNoInput, Uint8, Word64AtomicLoad,
5277 wasm::kWasmI64)
5278 CASE_LOAD_STORE(I64AtomicLoad16U, kNoInput, Uint16, Word64AtomicLoad,
5279 wasm::kWasmI64)
5280 CASE_LOAD_STORE(I64AtomicLoad32U, kNoInput, Uint32, Word64AtomicLoad,
5281 wasm::kWasmI64)
5282
5283 // Store.
5284 CASE_LOAD_STORE(I32AtomicStore, kOneInput, Uint32, Word32AtomicStore,
5285 wasm::kWasmI32)
5286 CASE_LOAD_STORE(I64AtomicStore, kOneInput, Uint64, Word64AtomicStore,
5287 wasm::kWasmI64)
5288 CASE_LOAD_STORE(I32AtomicStore8U, kOneInput, Uint8, Word32AtomicStore,
5289 wasm::kWasmI32)
5290 CASE_LOAD_STORE(I32AtomicStore16U, kOneInput, Uint16, Word32AtomicStore,
5291 wasm::kWasmI32)
5292 CASE_LOAD_STORE(I64AtomicStore8U, kOneInput, Uint8, Word64AtomicStore,
5293 wasm::kWasmI64)
5294 CASE_LOAD_STORE(I64AtomicStore16U, kOneInput, Uint16, Word64AtomicStore,
5295 wasm::kWasmI64)
5296 CASE_LOAD_STORE(I64AtomicStore32U, kOneInput, Uint32, Word64AtomicStore,
5297 wasm::kWasmI64)
5298
5299#undef CASE
5300#undef CASE_LOAD_STORE
5301
5302 case wasm::kExprAtomicNotify:
5303 return {kSpecial, MachineType::Int32(), OperatorByType{nullptr}};
5304 case wasm::kExprI32AtomicWait:
5305 return {kSpecial, MachineType::Int32(), OperatorByType{nullptr}};
5306 case wasm::kExprI64AtomicWait:
5307 return {kSpecial, MachineType::Int64(), OperatorByType{nullptr}};
5308 default:
5309 UNREACHABLE()V8_Fatal("unreachable code");
5310 }
5311 }
5312 };
5313
5314 AtomicOpInfo info = AtomicOpInfo::Get(opcode);
5315
5316 Node* index = CheckBoundsAndAlignment(info.machine_type.MemSize(), inputs[0],
5317 offset, position);
5318
5319 // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}.
5320 uintptr_t capped_offset = static_cast<uintptr_t>(offset);
5321 if (info.type != AtomicOpInfo::kSpecial) {
5322 const Operator* op;
5323 if (info.operator_by_type) {
5324 op = (mcgraph()->machine()->*info.operator_by_type)(info.machine_type);
5325 } else if (info.operator_by_rep) {
5326 op = (mcgraph()->machine()->*info.operator_by_rep)(
5327 info.machine_type.representation());
5328 } else if (info.operator_by_atomic_load_params) {
5329 op = (mcgraph()->machine()->*info.operator_by_atomic_load_params)(
5330 AtomicLoadParameters(info.machine_type, AtomicMemoryOrder::kSeqCst));
5331 } else {
5332 op = (mcgraph()->machine()->*info.operator_by_atomic_store_rep)(
5333 AtomicStoreParameters(info.machine_type.representation(),
5334 WriteBarrierKind::kNoWriteBarrier,
5335 AtomicMemoryOrder::kSeqCst));
5336 }
5337
5338 Node* input_nodes[6] = {MemBuffer(capped_offset), index};
5339 int num_actual_inputs = info.type;
5340 std::copy_n(inputs + 1, num_actual_inputs, input_nodes + 2);
5341 input_nodes[num_actual_inputs + 2] = effect();
5342 input_nodes[num_actual_inputs + 3] = control();
5343
5344#ifdef V8_TARGET_BIG_ENDIAN
5345 // Reverse the value bytes before storing.
5346 if (info.operator_by_atomic_store_rep) {
5347 input_nodes[num_actual_inputs + 1] = BuildChangeEndiannessStore(
5348 input_nodes[num_actual_inputs + 1],
5349 info.machine_type.representation(), info.wasm_type);
5350 }
5351#endif
5352
5353 Node* result = gasm_->AddNode(
5354 graph()->NewNode(op, num_actual_inputs + 4, input_nodes));
5355
5356#ifdef V8_TARGET_BIG_ENDIAN
5357 // Reverse the value bytes after load.
5358 if (info.operator_by_atomic_load_params) {
5359 result =
5360 BuildChangeEndiannessLoad(result, info.machine_type, info.wasm_type);
5361 }
5362#endif
5363
5364 return result;
5365 }
5366
5367 // After we've bounds-checked, compute the effective offset.
5368 Node* effective_offset =
5369 gasm_->IntAdd(gasm_->UintPtrConstant(capped_offset), index);
5370
5371 switch (opcode) {
5372 case wasm::kExprAtomicNotify:
5373 return gasm_->CallRuntimeStub(wasm::WasmCode::kWasmAtomicNotify,
5374 Operator::kNoThrow, effective_offset,
5375 inputs[1]);
5376
5377 case wasm::kExprI32AtomicWait: {
5378 auto* call_descriptor = GetI32AtomicWaitCallDescriptor();
5379
5380 intptr_t target = mcgraph()->machine()->Is64()
5381 ? wasm::WasmCode::kWasmI32AtomicWait64
5382 : wasm::WasmCode::kWasmI32AtomicWait32;
5383 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
5384 target, RelocInfo::WASM_STUB_CALL);
5385
5386 return gasm_->Call(call_descriptor, call_target, effective_offset,
5387 inputs[1], inputs[2]);
5388 }
5389
5390 case wasm::kExprI64AtomicWait: {
5391 auto* call_descriptor = GetI64AtomicWaitCallDescriptor();
5392
5393 intptr_t target = mcgraph()->machine()->Is64()
5394 ? wasm::WasmCode::kWasmI64AtomicWait64
5395 : wasm::WasmCode::kWasmI64AtomicWait32;
5396 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
5397 target, RelocInfo::WASM_STUB_CALL);
5398
5399 return gasm_->Call(call_descriptor, call_target, effective_offset,
5400 inputs[1], inputs[2]);
5401 }
5402
5403 default:
5404 FATAL_UNSUPPORTED_OPCODE(opcode);
5405 }
5406}
5407
5408void WasmGraphBuilder::AtomicFence() {
5409 SetEffect(graph()->NewNode(mcgraph()->machine()->MemBarrier(), effect(),
5410 control()));
5411}
5412
5413void WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
5414 Node* src, Node* size,
5415 wasm::WasmCodePosition position) {
5416 // The data segment index must be in bounds since it is required by
5417 // validation.
5418 DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments)((void) 0);
5419
5420 Node* function =
5421 gasm_->ExternalConstant(ExternalReference::wasm_memory_init());
5422
5423 MemTypeToUintPtrOrOOBTrap({&dst}, position);
5424
5425 Node* stack_slot = StoreArgsInStackSlot(
5426 {{MachineType::PointerRepresentation(), GetInstance()},
5427 {MachineType::PointerRepresentation(), dst},
5428 {MachineRepresentation::kWord32, src},
5429 {MachineRepresentation::kWord32,
5430 gasm_->Uint32Constant(data_segment_index)},
5431 {MachineRepresentation::kWord32, size}});
5432
5433 auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
5434 .Params(MachineType::Pointer());
5435 Node* call = BuildCCall(&sig, function, stack_slot);
5436 // TODO(manoskouk): Also throw kDataSegmentOutOfBounds.
5437 TrapIfFalse(wasm::kTrapMemOutOfBounds, call, position);
5438}
5439
5440void WasmGraphBuilder::DataDrop(uint32_t data_segment_index,
5441 wasm::WasmCodePosition position) {
5442 DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments)((void) 0);
5443
5444 Node* seg_size_array =
5445 LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
5446 STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2)static_assert(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >>
2, "wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2"
)
;
5447 auto access = ObjectAccess(MachineType::Int32(), kNoWriteBarrier);
5448 gasm_->StoreToObject(access, seg_size_array, data_segment_index << 2,
5449 Int32Constant(0));
5450}
5451
5452Node* WasmGraphBuilder::StoreArgsInStackSlot(
5453 std::initializer_list<std::pair<MachineRepresentation, Node*>> args) {
5454 int slot_size = 0;
5455 for (auto arg : args) {
5456 slot_size += ElementSizeInBytes(arg.first);
5457 }
5458 DCHECK_LT(0, slot_size)((void) 0);
5459 Node* stack_slot =
5460 graph()->NewNode(mcgraph()->machine()->StackSlot(slot_size));
5461
5462 int offset = 0;
5463 for (auto arg : args) {
5464 MachineRepresentation type = arg.first;
5465 Node* value = arg.second;
5466 gasm_->StoreUnaligned(type, stack_slot, Int32Constant(offset), value);
5467 offset += ElementSizeInBytes(type);
5468 }
5469 return stack_slot;
5470}
5471
5472void WasmGraphBuilder::MemTypeToUintPtrOrOOBTrap(
5473 std::initializer_list<Node**> nodes, wasm::WasmCodePosition position) {
5474 if (!env_->module->is_memory64) {
5475 for (Node** node : nodes) {
5476 *node = BuildChangeUint32ToUintPtr(*node);
5477 }
5478 return;
5479 }
5480 if (kSystemPointerSize == kInt64Size) return; // memory64 on 64-bit
5481 Node* any_high_word = nullptr;
5482 for (Node** node : nodes) {
5483 Node* high_word =
5484 gasm_->TruncateInt64ToInt32(gasm_->Word64Shr(*node, Int32Constant(32)));
5485 any_high_word =
5486 any_high_word ? gasm_->Word32Or(any_high_word, high_word) : high_word;
5487 // Only keep the low word as uintptr_t.
5488 *node = gasm_->TruncateInt64ToInt32(*node);
5489 }
5490 TrapIfTrue(wasm::kTrapMemOutOfBounds, any_high_word, position);
5491}
5492
5493void WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
5494 wasm::WasmCodePosition position) {
5495 Node* function =
5496 gasm_->ExternalConstant(ExternalReference::wasm_memory_copy());
5497
5498 MemTypeToUintPtrOrOOBTrap({&dst, &src, &size}, position);
5499
5500 Node* stack_slot = StoreArgsInStackSlot(
5501 {{MachineType::PointerRepresentation(), GetInstance()},
5502 {MachineType::PointerRepresentation(), dst},
5503 {MachineType::PointerRepresentation(), src},
5504 {MachineType::PointerRepresentation(), size}});
5505
5506 auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
5507 .Params(MachineType::Pointer());
5508 Node* call = BuildCCall(&sig, function, stack_slot);
5509 TrapIfFalse(wasm::kTrapMemOutOfBounds, call, position);
5510}
5511
5512void WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
5513 wasm::WasmCodePosition position) {
5514 Node* function =
5515 gasm_->ExternalConstant(ExternalReference::wasm_memory_fill());
5516
5517 MemTypeToUintPtrOrOOBTrap({&dst, &size}, position);
5518
5519 Node* stack_slot = StoreArgsInStackSlot(
5520 {{MachineType::PointerRepresentation(), GetInstance()},
5521 {MachineType::PointerRepresentation(), dst},
5522 {MachineRepresentation::kWord32, value},
5523 {MachineType::PointerRepresentation(), size}});
5524
5525 auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
5526 .Params(MachineType::Pointer());
5527 Node* call = BuildCCall(&sig, function, stack_slot);
5528 TrapIfFalse(wasm::kTrapMemOutOfBounds, call, position);
5529}
5530
5531void WasmGraphBuilder::TableInit(uint32_t table_index,
5532 uint32_t elem_segment_index, Node* dst,
5533 Node* src, Node* size,
5534 wasm::WasmCodePosition position) {
5535 gasm_->CallRuntimeStub(wasm::WasmCode::kWasmTableInit, Operator::kNoThrow,
5536 dst, src, size, gasm_->NumberConstant(table_index),
5537 gasm_->NumberConstant(elem_segment_index));
5538}
5539
5540void WasmGraphBuilder::ElemDrop(uint32_t elem_segment_index,
5541 wasm::WasmCodePosition position) {
5542 // The elem segment index must be in bounds since it is required by
5543 // validation.
5544 DCHECK_LT(elem_segment_index, env_->module->elem_segments.size())((void) 0);
5545
5546 Node* dropped_elem_segments =
5547 LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
5548 auto store_rep =
5549 StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier);
5550 gasm_->Store(store_rep, dropped_elem_segments, elem_segment_index,
5551 Int32Constant(1));
5552}
5553
5554void WasmGraphBuilder::TableCopy(uint32_t table_dst_index,
5555 uint32_t table_src_index, Node* dst, Node* src,
5556 Node* size, wasm::WasmCodePosition position) {
5557 gasm_->CallRuntimeStub(wasm::WasmCode::kWasmTableCopy, Operator::kNoThrow,
5558 dst, src, size, gasm_->NumberConstant(table_dst_index),
5559 gasm_->NumberConstant(table_src_index));
5560}
5561
5562Node* WasmGraphBuilder::TableGrow(uint32_t table_index, Node* value,
5563 Node* delta) {
5564 return BuildChangeSmiToInt32(gasm_->CallRuntimeStub(
5565 wasm::WasmCode::kWasmTableGrow, Operator::kNoThrow,
5566 graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), delta,
5567 value));
5568}
5569
5570Node* WasmGraphBuilder::TableSize(uint32_t table_index) {
5571 Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
5572 Node* table = gasm_->LoadFixedArrayElementAny(tables, table_index);
5573
5574 int length_field_size = WasmTableObject::kCurrentLengthOffsetEnd -
5575 WasmTableObject::kCurrentLengthOffset + 1;
5576 Node* length_smi = gasm_->LoadFromObject(
5577 assert_size(length_field_size, MachineType::TaggedSigned()), table,
5578 wasm::ObjectAccess::ToTagged(WasmTableObject::kCurrentLengthOffset));
5579
5580 return BuildChangeSmiToInt32(length_smi);
5581}
5582
5583void WasmGraphBuilder::TableFill(uint32_t table_index, Node* start, Node* value,
5584 Node* count) {
5585 gasm_->CallRuntimeStub(
5586 wasm::WasmCode::kWasmTableFill, Operator::kNoThrow,
5587 graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), start,
5588 count, value);
5589}
5590
5591Node* WasmGraphBuilder::StructNewWithRtt(uint32_t struct_index,
5592 const wasm::StructType* type,
5593 Node* rtt,
5594 base::Vector<Node*> fields) {
5595 int size = WasmStruct::Size(type);
5596 Node* s = gasm_->Allocate(size);
5597 gasm_->StoreMap(s, rtt);
5598 gasm_->InitializeImmutableInObject(
5599 ObjectAccess(MachineType::TaggedPointer(), kNoWriteBarrier), s,
5600 wasm::ObjectAccess::ToTagged(JSReceiver::kPropertiesOrHashOffset),
5601 LOAD_ROOT(EmptyFixedArray, empty_fixed_array));
5602 for (uint32_t i = 0; i < type->field_count(); i++) {
5603 gasm_->StoreStructField(s, type, i, fields[i]);
5604 }
5605 // If this assert fails then initialization of padding field might be
5606 // necessary.
5607 static_assert(Heap::kMinObjectSizeInTaggedWords == 2 &&
5608 WasmStruct::kHeaderSize == 2 * kTaggedSize,
5609 "empty struct might require initialization of padding field");
5610 return s;
5611}
5612
5613Builtin ChooseArrayAllocationBuiltin(wasm::ValueType element_type,
5614 Node* initial_value) {
5615 if (initial_value != nullptr) {
5616 // {initial_value} will be used for initialization after allocation.
5617 return Builtin::kWasmAllocateArray_Uninitialized;
5618 }
5619 if (element_type.is_reference()) {
5620 return Builtin::kWasmAllocateArray_InitNull;
5621 }
5622 return Builtin::kWasmAllocateArray_InitZero;
5623}
5624
5625Node* WasmGraphBuilder::ArrayNewWithRtt(uint32_t array_index,
5626 const wasm::ArrayType* type,
5627 Node* length, Node* initial_value,
5628 Node* rtt,
5629 wasm::WasmCodePosition position) {
5630 TrapIfFalse(wasm::kTrapArrayTooLarge,
5631 gasm_->Uint32LessThanOrEqual(
5632 length, gasm_->Uint32Constant(WasmArray::MaxLength(type))),
5633 position);
5634 wasm::ValueType element_type = type->element_type();
5635 // TODO(7748): Consider using gasm_->Allocate().
5636 Builtin stub = ChooseArrayAllocationBuiltin(element_type, initial_value);
5637 // Do NOT mark this as Operator::kEliminatable, because that would cause the
5638 // Call node to have no control inputs, which means it could get scheduled
5639 // before the check/trap above.
5640 Node* a =
5641 gasm_->CallBuiltin(stub, Operator::kNoDeopt | Operator::kNoThrow, rtt,
5642 length, Int32Constant(element_type.value_kind_size()));
5643 if (initial_value != nullptr) {
5644 // TODO(manoskouk): If the loop is ever removed here, we have to update
5645 // ArrayNewWithRtt() in graph-builder-interface.cc to not mark the current
5646 // loop as non-innermost.
5647 auto loop = gasm_->MakeLoopLabel(MachineRepresentation::kWord32);
5648 auto done = gasm_->MakeLabel();
5649 Node* start_offset =
5650 Int32Constant(wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize));
5651 Node* element_size = Int32Constant(element_type.value_kind_size());
5652 Node* end_offset =
5653 gasm_->Int32Add(start_offset, gasm_->Int32Mul(element_size, length));
5654 gasm_->Goto(&loop, start_offset);
5655 gasm_->Bind(&loop);
5656 {
5657 Node* offset = loop.PhiAt(0);
5658 Node* check = gasm_->Uint32LessThan(offset, end_offset);
5659 gasm_->GotoIfNot(check, &done);
5660 gasm_->StoreToObject(ObjectAccessForGCStores(type->element_type()), a,
5661 offset, initial_value);
5662 offset = gasm_->Int32Add(offset, element_size);
5663 gasm_->Goto(&loop, offset);
5664 }
5665 gasm_->Bind(&done);
5666 }
5667 return a;
5668}
5669
5670Node* WasmGraphBuilder::ArrayInit(const wasm::ArrayType* type, Node* rtt,
5671 base::Vector<Node*> elements) {
5672 wasm::ValueType element_type = type->element_type();
5673 // TODO(7748): Consider using gasm_->Allocate().
5674 Node* array =
5675 gasm_->CallBuiltin(Builtin::kWasmAllocateArray_Uninitialized,
5676 Operator::kNoDeopt | Operator::kNoThrow, rtt,
5677 Int32Constant(static_cast<int32_t>(elements.size())),
5678 Int32Constant(element_type.value_kind_size()));
5679 for (int i = 0; i < static_cast<int>(elements.size()); i++) {
5680 Node* offset =
5681 gasm_->WasmArrayElementOffset(Int32Constant(i), element_type);
5682 if (type->mutability()) {
5683 gasm_->StoreToObject(ObjectAccessForGCStores(element_type), array, offset,
5684 elements[i]);
5685 } else {
5686 gasm_->InitializeImmutableInObject(ObjectAccessForGCStores(element_type),
5687 array, offset, elements[i]);
5688 }
5689 }
5690 return array;
5691}
5692
5693Node* WasmGraphBuilder::ArrayInitFromData(const wasm::ArrayType* type,
5694 uint32_t data_segment, Node* offset,
5695 Node* length, Node* rtt,
5696 wasm::WasmCodePosition position) {
5697 Node* array = gasm_->CallBuiltin(
5698 Builtin::kWasmArrayInitFromData, Operator::kNoDeopt | Operator::kNoThrow,
5699 gasm_->Uint32Constant(data_segment), offset, length, rtt);
5700 TrapIfTrue(wasm::kTrapArrayTooLarge,
5701 gasm_->TaggedEqual(
5702 array, gasm_->NumberConstant(
5703 wasm::kArrayInitFromDataArrayTooLargeErrorCode)),
5704 position);
5705 TrapIfTrue(
5706 wasm::kTrapDataSegmentOutOfBounds,
5707 gasm_->TaggedEqual(
5708 array, gasm_->NumberConstant(
5709 wasm::kArrayInitFromDataSegmentOutOfBoundsErrorCode)),
5710 position);
5711 return array;
5712}
5713
5714Node* WasmGraphBuilder::RttCanon(uint32_t type_index) {
5715 Node* maps_list =
5716 LOAD_INSTANCE_FIELD(ManagedObjectMaps, MachineType::TaggedPointer());
5717 return gasm_->LoadImmutable(
5718 MachineType::TaggedPointer(), maps_list,
5719 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(type_index));
5720}
5721
5722WasmGraphBuilder::Callbacks WasmGraphBuilder::TestCallbacks(
5723 GraphAssemblerLabel<1>* label) {
5724 return {// succeed_if
5725 [=](Node* condition, BranchHint hint) -> void {
5726 gasm_->GotoIf(condition, label, hint, Int32Constant(1));
5727 },
5728 // fail_if
5729 [=](Node* condition, BranchHint hint) -> void {
5730 gasm_->GotoIf(condition, label, hint, Int32Constant(0));
5731 },
5732 // fail_if_not
5733 [=](Node* condition, BranchHint hint) -> void {
5734 gasm_->GotoIfNot(condition, label, hint, Int32Constant(0));
5735 }};
5736}
5737
5738WasmGraphBuilder::Callbacks WasmGraphBuilder::CastCallbacks(
5739 GraphAssemblerLabel<0>* label, wasm::WasmCodePosition position) {
5740 return {// succeed_if
5741 [=](Node* condition, BranchHint hint) -> void {
5742 gasm_->GotoIf(condition, label, hint);
5743 },
5744 // fail_if
5745 [=](Node* condition, BranchHint hint) -> void {
5746 TrapIfTrue(wasm::kTrapIllegalCast, condition, position);
5747 },
5748 // fail_if_not
5749 [=](Node* condition, BranchHint hint) -> void {
5750 TrapIfFalse(wasm::kTrapIllegalCast, condition, position);
5751 }};
5752}
5753
5754WasmGraphBuilder::Callbacks WasmGraphBuilder::BranchCallbacks(
5755 SmallNodeVector& no_match_controls, SmallNodeVector& no_match_effects,
5756 SmallNodeVector& match_controls, SmallNodeVector& match_effects) {
5757 return {
5758 // succeed_if
5759 [&](Node* condition, BranchHint hint) -> void {
5760 Node* branch = graph()->NewNode(mcgraph()->common()->Branch(hint),
5761 condition, control());
5762 match_controls.emplace_back(
5763 graph()->NewNode(mcgraph()->common()->IfTrue(), branch));
5764 match_effects.emplace_back(effect());
5765 SetControl(graph()->NewNode(mcgraph()->common()->IfFalse(), branch));
5766 },
5767 // fail_if
5768 [&](Node* condition, BranchHint hint) -> void {
5769 Node* branch = graph()->NewNode(mcgraph()->common()->Branch(hint),
5770 condition, control());
5771 no_match_controls.emplace_back(
5772 graph()->NewNode(mcgraph()->common()->IfTrue(), branch));
5773 no_match_effects.emplace_back(effect());
5774 SetControl(graph()->NewNode(mcgraph()->common()->IfFalse(), branch));
5775 },
5776 // fail_if_not
5777 [&](Node* condition, BranchHint hint) -> void {
5778 Node* branch = graph()->NewNode(mcgraph()->common()->Branch(hint),
5779 condition, control());
5780 no_match_controls.emplace_back(
5781 graph()->NewNode(mcgraph()->common()->IfFalse(), branch));
5782 no_match_effects.emplace_back(effect());
5783 SetControl(graph()->NewNode(mcgraph()->common()->IfTrue(), branch));
5784 }};
5785}
5786
5787void WasmGraphBuilder::TypeCheck(
5788 Node* object, Node* rtt, WasmGraphBuilder::ObjectReferenceKnowledge config,
5789 bool null_succeeds, Callbacks callbacks) {
5790 if (config.object_can_be_null) {
5791 (null_succeeds ? callbacks.succeed_if : callbacks.fail_if)(
5792 IsNull(object), BranchHint::kFalse);
5793 }
5794
5795 Node* map = gasm_->LoadMap(object);
5796
5797 // First, check if types happen to be equal. This has been shown to give large
5798 // speedups.
5799 callbacks.succeed_if(gasm_->TaggedEqual(map, rtt), BranchHint::kTrue);
5800
5801 Node* type_info = gasm_->LoadWasmTypeInfo(map);
5802 Node* supertypes = gasm_->LoadSupertypes(type_info);
5803 Node* rtt_depth = gasm_->UintPtrConstant(config.rtt_depth);
5804
5805 // If the depth of the rtt is known to be less that the minimum supertype
5806 // array length, we can access the supertype without bounds-checking the
5807 // supertype array.
5808 if (config.rtt_depth >= wasm::kMinimumSupertypeArraySize) {
5809 Node* supertypes_length =
5810 BuildChangeSmiToIntPtr(gasm_->LoadFixedArrayLengthAsSmi(supertypes));
5811 callbacks.fail_if_not(gasm_->UintLessThan(rtt_depth, supertypes_length),
5812 BranchHint::kTrue);
5813 }
5814 Node* maybe_match = gasm_->LoadImmutableFixedArrayElement(
5815 supertypes, rtt_depth, MachineType::TaggedPointer());
5816
5817 callbacks.fail_if_not(gasm_->TaggedEqual(maybe_match, rtt),
5818 BranchHint::kTrue);
5819}
5820
5821void WasmGraphBuilder::DataCheck(Node* object, bool object_can_be_null,
5822 Callbacks callbacks) {
5823 if (object_can_be_null) {
5824 callbacks.fail_if(IsNull(object), BranchHint::kFalse);
5825 }
5826 callbacks.fail_if(gasm_->IsI31(object), BranchHint::kFalse);
5827 Node* map = gasm_->LoadMap(object);
5828 callbacks.fail_if_not(gasm_->IsDataRefMap(map), BranchHint::kTrue);
5829}
5830
5831void WasmGraphBuilder::ManagedObjectInstanceCheck(Node* object,
5832 bool object_can_be_null,
5833 InstanceType instance_type,
5834 Callbacks callbacks) {
5835 if (object_can_be_null) {
5836 callbacks.fail_if(IsNull(object), BranchHint::kFalse);
5837 }
5838 callbacks.fail_if(gasm_->IsI31(object), BranchHint::kFalse);
5839 callbacks.fail_if_not(gasm_->HasInstanceType(object, instance_type),
5840 BranchHint::kTrue);
5841}
5842
5843void WasmGraphBuilder::BrOnCastAbs(
5844 Node** match_control, Node** match_effect, Node** no_match_control,
5845 Node** no_match_effect, std::function<void(Callbacks)> type_checker) {
5846 SmallNodeVector no_match_controls, no_match_effects, match_controls,
5847 match_effects;
5848
5849 type_checker(BranchCallbacks(no_match_controls, no_match_effects,
5850 match_controls, match_effects));
5851
5852 match_controls.emplace_back(control());
5853 match_effects.emplace_back(effect());
5854
5855 // Wire up the control/effect nodes.
5856 unsigned count = static_cast<unsigned>(match_controls.size());
5857 DCHECK_EQ(match_controls.size(), match_effects.size())((void) 0);
5858 *match_control = Merge(count, match_controls.data());
5859 // EffectPhis need their control dependency as an additional input.
5860 match_effects.emplace_back(*match_control);
5861 *match_effect = EffectPhi(count, match_effects.data());
5862 DCHECK_EQ(no_match_controls.size(), no_match_effects.size())((void) 0);
5863 // Range is 2..4, so casting to unsigned is safe.
5864 count = static_cast<unsigned>(no_match_controls.size());
5865 *no_match_control = Merge(count, no_match_controls.data());
5866 // EffectPhis need their control dependency as an additional input.
5867 no_match_effects.emplace_back(*no_match_control);
5868 *no_match_effect = EffectPhi(count, no_match_effects.data());
5869}
5870
5871Node* WasmGraphBuilder::RefTest(Node* object, Node* rtt,
5872 ObjectReferenceKnowledge config) {
5873 auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
5874 TypeCheck(object, rtt, config, false, TestCallbacks(&done));
5875 gasm_->Goto(&done, Int32Constant(1));
5876 gasm_->Bind(&done);
5877 return done.PhiAt(0);
5878}
5879
5880Node* WasmGraphBuilder::RefCast(Node* object, Node* rtt,
5881 ObjectReferenceKnowledge config,
5882 wasm::WasmCodePosition position) {
5883 if (!FLAG_experimental_wasm_assume_ref_cast_succeeds) {
5884 auto done = gasm_->MakeLabel();
5885 TypeCheck(object, rtt, config, true, CastCallbacks(&done, position));
5886 gasm_->Goto(&done);
5887 gasm_->Bind(&done);
5888 }
5889 return object;
5890}
5891
5892void WasmGraphBuilder::BrOnCast(Node* object, Node* rtt,
5893 ObjectReferenceKnowledge config,
5894 Node** match_control, Node** match_effect,
5895 Node** no_match_control,
5896 Node** no_match_effect) {
5897 BrOnCastAbs(match_control, match_effect, no_match_control, no_match_effect,
5898 [=](Callbacks callbacks) -> void {
5899 return TypeCheck(object, rtt, config, false, callbacks);
5900 });
5901}
5902
5903Node* WasmGraphBuilder::RefIsData(Node* object, bool object_can_be_null) {
5904 auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
5905 DataCheck(object, object_can_be_null, TestCallbacks(&done));
5906 gasm_->Goto(&done, Int32Constant(1));
5907 gasm_->Bind(&done);
5908 return done.PhiAt(0);
5909}
5910
5911Node* WasmGraphBuilder::RefAsData(Node* object, bool object_can_be_null,
5912 wasm::WasmCodePosition position) {
5913 auto done = gasm_->MakeLabel();
5914 DataCheck(object, object_can_be_null, CastCallbacks(&done, position));
5915 gasm_->Goto(&done);
5916 gasm_->Bind(&done);
5917 return object;
5918}
5919
5920void WasmGraphBuilder::BrOnData(Node* object, Node* /*rtt*/,
5921 ObjectReferenceKnowledge config,
5922 Node** match_control, Node** match_effect,
5923 Node** no_match_control,
5924 Node** no_match_effect) {
5925 BrOnCastAbs(match_control, match_effect, no_match_control, no_match_effect,
5926 [=](Callbacks callbacks) -> void {
5927 return DataCheck(object, config.object_can_be_null, callbacks);
5928 });
5929}
5930
5931Node* WasmGraphBuilder::RefIsFunc(Node* object, bool object_can_be_null) {
5932 auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
5933 ManagedObjectInstanceCheck(object, object_can_be_null,
5934 WASM_INTERNAL_FUNCTION_TYPE, TestCallbacks(&done));
5935 gasm_->Goto(&done, Int32Constant(1));
5936 gasm_->Bind(&done);
5937 return done.PhiAt(0);
5938}
5939
5940Node* WasmGraphBuilder::RefAsFunc(Node* object, bool object_can_be_null,
5941 wasm::WasmCodePosition position) {
5942 auto done = gasm_->MakeLabel();
5943 ManagedObjectInstanceCheck(object, object_can_be_null,
5944 WASM_INTERNAL_FUNCTION_TYPE,
5945 CastCallbacks(&done, position));
5946 gasm_->Goto(&done);
5947 gasm_->Bind(&done);
5948 return object;
5949}
5950
5951void WasmGraphBuilder::BrOnFunc(Node* object, Node* /*rtt*/,
5952 ObjectReferenceKnowledge config,
5953 Node** match_control, Node** match_effect,
5954 Node** no_match_control,
5955 Node** no_match_effect) {
5956 BrOnCastAbs(match_control, match_effect, no_match_control, no_match_effect,
5957 [=](Callbacks callbacks) -> void {
5958 return ManagedObjectInstanceCheck(
5959 object, config.object_can_be_null,
5960 WASM_INTERNAL_FUNCTION_TYPE, callbacks);
5961 });
5962}
5963
5964Node* WasmGraphBuilder::RefIsArray(Node* object, bool object_can_be_null) {
5965 auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
5966 ManagedObjectInstanceCheck(object, object_can_be_null, WASM_ARRAY_TYPE,
5967 TestCallbacks(&done));
5968 gasm_->Goto(&done, Int32Constant(1));
5969 gasm_->Bind(&done);
5970 return done.PhiAt(0);
5971}
5972
5973Node* WasmGraphBuilder::RefAsArray(Node* object, bool object_can_be_null,
5974 wasm::WasmCodePosition position) {
5975 auto done = gasm_->MakeLabel();
5976 ManagedObjectInstanceCheck(object, object_can_be_null, WASM_ARRAY_TYPE,
5977 CastCallbacks(&done, position));
5978 gasm_->Goto(&done);
5979 gasm_->Bind(&done);
5980 return object;
5981}
5982
5983void WasmGraphBuilder::BrOnArray(Node* object, Node* /*rtt*/,
5984 ObjectReferenceKnowledge config,
5985 Node** match_control, Node** match_effect,
5986 Node** no_match_control,
5987 Node** no_match_effect) {
5988 BrOnCastAbs(match_control, match_effect, no_match_control, no_match_effect,
5989 [=](Callbacks callbacks) -> void {
5990 return ManagedObjectInstanceCheck(object,
5991 config.object_can_be_null,
5992 WASM_ARRAY_TYPE, callbacks);
5993 });
5994}
5995
5996Node* WasmGraphBuilder::RefIsI31(Node* object) { return gasm_->IsI31(object); }
5997
5998Node* WasmGraphBuilder::RefAsI31(Node* object,
5999 wasm::WasmCodePosition position) {
6000 TrapIfFalse(wasm::kTrapIllegalCast, gasm_->IsI31(object), position);
6001 return object;
6002}
6003
6004void WasmGraphBuilder::BrOnI31(Node* object, Node* /* rtt */,
6005 ObjectReferenceKnowledge /* config */,
6006 Node** match_control, Node** match_effect,
6007 Node** no_match_control,
6008 Node** no_match_effect) {
6009 gasm_->Branch(gasm_->IsI31(object), match_control, no_match_control,
6010 BranchHint::kTrue);
6011
6012 SetControl(*no_match_control);
6013 *match_effect = effect();
6014 *no_match_effect = effect();
6015}
6016
6017Node* WasmGraphBuilder::StructGet(Node* struct_object,
6018 const wasm::StructType* struct_type,
6019 uint32_t field_index, CheckForNull null_check,
6020 bool is_signed,
6021 wasm::WasmCodePosition position) {
6022 if (null_check == kWithNullCheck) {
6023 TrapIfTrue(wasm::kTrapNullDereference, IsNull(struct_object), position);
6024 }
6025 // It is not enough to invoke ValueType::machine_type(), because the
6026 // signedness has to be determined by {is_signed}.
6027 MachineType machine_type = MachineType::TypeForRepresentation(
6028 struct_type->field(field_index).machine_representation(), is_signed);
6029 Node* offset = gasm_->FieldOffset(struct_type, field_index);
6030 return struct_type->mutability(field_index)
6031 ? gasm_->LoadFromObject(machine_type, struct_object, offset)
6032 : gasm_->LoadImmutableFromObject(machine_type, struct_object,
6033 offset);
6034}
6035
6036void WasmGraphBuilder::StructSet(Node* struct_object,
6037 const wasm::StructType* struct_type,
6038 uint32_t field_index, Node* field_value,
6039 CheckForNull null_check,
6040 wasm::WasmCodePosition position) {
6041 if (null_check == kWithNullCheck) {
6042 TrapIfTrue(wasm::kTrapNullDereference, IsNull(struct_object), position);
6043 }
6044 gasm_->StoreStructField(struct_object, struct_type, field_index, field_value);
6045}
6046
6047void WasmGraphBuilder::BoundsCheckArray(Node* array, Node* index,
6048 wasm::WasmCodePosition position) {
6049 if (V8_UNLIKELY(FLAG_experimental_wasm_skip_bounds_checks)(__builtin_expect(!!(FLAG_experimental_wasm_skip_bounds_checks
), 0))
) return;
6050 Node* length = gasm_->LoadWasmArrayLength(array);
6051 TrapIfFalse(wasm::kTrapArrayOutOfBounds, gasm_->Uint32LessThan(index, length),
6052 position);
6053}
6054
6055void WasmGraphBuilder::BoundsCheckArrayCopy(Node* array, Node* index,
6056 Node* length,
6057 wasm::WasmCodePosition position) {
6058 if (V8_UNLIKELY(FLAG_experimental_wasm_skip_bounds_checks)(__builtin_expect(!!(FLAG_experimental_wasm_skip_bounds_checks
), 0))
) return;
6059 Node* array_length = gasm_->LoadWasmArrayLength(array);
6060 Node* range_end = gasm_->Int32Add(index, length);
6061 Node* range_valid = gasm_->Word32And(
6062 gasm_->Uint32LessThanOrEqual(range_end, array_length),
6063 gasm_->Uint32LessThanOrEqual(index, range_end)); // No overflow
6064 TrapIfFalse(wasm::kTrapArrayOutOfBounds, range_valid, position);
6065}
6066
6067Node* WasmGraphBuilder::ArrayGet(Node* array_object,
6068 const wasm::ArrayType* type, Node* index,
6069 CheckForNull null_check, bool is_signed,
6070 wasm::WasmCodePosition position) {
6071 if (null_check == kWithNullCheck) {
6072 TrapIfTrue(wasm::kTrapNullDereference, IsNull(array_object), position);
6073 }
6074 BoundsCheckArray(array_object, index, position);
6075 MachineType machine_type = MachineType::TypeForRepresentation(
6076 type->element_type().machine_representation(), is_signed);
6077 Node* offset = gasm_->WasmArrayElementOffset(index, type->element_type());
6078 return type->mutability()
6079 ? gasm_->LoadFromObject(machine_type, array_object, offset)
6080 : gasm_->LoadImmutableFromObject(machine_type, array_object,
6081 offset);
6082}
6083
6084void WasmGraphBuilder::ArraySet(Node* array_object, const wasm::ArrayType* type,
6085 Node* index, Node* value,
6086 CheckForNull null_check,
6087 wasm::WasmCodePosition position) {
6088 if (null_check == kWithNullCheck) {
6089 TrapIfTrue(wasm::kTrapNullDereference, IsNull(array_object), position);
6090 }
6091 BoundsCheckArray(array_object, index, position);
6092 Node* offset = gasm_->WasmArrayElementOffset(index, type->element_type());
6093 gasm_->StoreToObject(ObjectAccessForGCStores(type->element_type()),
6094 array_object, offset, value);
6095}
6096
6097Node* WasmGraphBuilder::ArrayLen(Node* array_object, CheckForNull null_check,
6098 wasm::WasmCodePosition position) {
6099 if (null_check == kWithNullCheck) {
6100 TrapIfTrue(wasm::kTrapNullDereference, IsNull(array_object), position);
6101 }
6102 return gasm_->LoadWasmArrayLength(array_object);
6103}
6104
6105// TODO(7748): Add an option to copy in a loop for small array sizes. To find
6106// the length limit, run test/mjsunit/wasm/array-copy-benchmark.js.
6107void WasmGraphBuilder::ArrayCopy(Node* dst_array, Node* dst_index,
6108 CheckForNull dst_null_check, Node* src_array,
6109 Node* src_index, CheckForNull src_null_check,
6110 Node* length,
6111 wasm::WasmCodePosition position) {
6112 if (dst_null_check == kWithNullCheck) {
6113 TrapIfTrue(wasm::kTrapNullDereference, IsNull(dst_array), position);
6114 }
6115 if (src_null_check == kWithNullCheck) {
6116 TrapIfTrue(wasm::kTrapNullDereference, IsNull(src_array), position);
6117 }
6118 BoundsCheckArrayCopy(dst_array, dst_index, length, position);
6119 BoundsCheckArrayCopy(src_array, src_index, length, position);
6120
6121 auto skip = gasm_->MakeLabel();
6122
6123 gasm_->GotoIf(gasm_->Word32Equal(length, Int32Constant(0)), &skip,
6124 BranchHint::kFalse);
6125
6126 Node* function =
6127 gasm_->ExternalConstant(ExternalReference::wasm_array_copy());
6128 MachineType arg_types[]{
6129 MachineType::TaggedPointer(), MachineType::TaggedPointer(),
6130 MachineType::Uint32(), MachineType::TaggedPointer(),
6131 MachineType::Uint32(), MachineType::Uint32()};
6132 MachineSignature sig(0, 6, arg_types);
6133 BuildCCall(&sig, function, GetInstance(), dst_array, dst_index, src_array,
6134 src_index, length);
6135 gasm_->Goto(&skip);
6136 gasm_->Bind(&skip);
6137}
6138
6139// 1 bit V8 Smi tag, 31 bits V8 Smi shift, 1 bit i31ref high-bit truncation.
6140constexpr int kI31To32BitSmiShift = 33;
6141
6142Node* WasmGraphBuilder::I31New(Node* input) {
6143 if (SmiValuesAre31Bits()) {
6144 return gasm_->Word32Shl(input, BuildSmiShiftBitsConstant32());
6145 }
6146 DCHECK(SmiValuesAre32Bits())((void) 0);
6147 input = BuildChangeInt32ToIntPtr(input);
6148 return gasm_->WordShl(input, gasm_->IntPtrConstant(kI31To32BitSmiShift));
6149}
6150
6151Node* WasmGraphBuilder::I31GetS(Node* input) {
6152 if (SmiValuesAre31Bits()) {
6153 input = BuildTruncateIntPtrToInt32(input);
6154 return gasm_->Word32SarShiftOutZeros(input, BuildSmiShiftBitsConstant32());
6155 }
6156 DCHECK(SmiValuesAre32Bits())((void) 0);
6157 return BuildTruncateIntPtrToInt32(
6158 gasm_->WordSar(input, gasm_->IntPtrConstant(kI31To32BitSmiShift)));
6159}
6160
6161Node* WasmGraphBuilder::I31GetU(Node* input) {
6162 if (SmiValuesAre31Bits()) {
6163 input = BuildTruncateIntPtrToInt32(input);
6164 return gasm_->Word32Shr(input, BuildSmiShiftBitsConstant32());
6165 }
6166 DCHECK(SmiValuesAre32Bits())((void) 0);
6167 return BuildTruncateIntPtrToInt32(
6168 gasm_->WordShr(input, gasm_->IntPtrConstant(kI31To32BitSmiShift)));
6169}
6170
6171class WasmDecorator final : public GraphDecorator {
6172 public:
6173 explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
6174 : origins_(origins), decoder_(decoder) {}
6175
6176 void Decorate(Node* node) final {
6177 origins_->SetNodeOrigin(
6178 node, NodeOrigin("wasm graph creation", "n/a",
6179 NodeOrigin::kWasmBytecode, decoder_->position()));
6180 }
6181
6182 private:
6183 compiler::NodeOriginTable* origins_;
6184 wasm::Decoder* decoder_;
6185};
6186
6187void WasmGraphBuilder::AddBytecodePositionDecorator(
6188 NodeOriginTable* node_origins, wasm::Decoder* decoder) {
6189 DCHECK_NULL(decorator_)((void) 0);
6190 decorator_ = graph()->zone()->New<WasmDecorator>(node_origins, decoder);
6191 graph()->AddDecorator(decorator_);
6192}
6193
6194void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
6195 DCHECK_NOT_NULL(decorator_)((void) 0);
6196 graph()->RemoveDecorator(decorator_);
6197 decorator_ = nullptr;
6198}
6199
6200namespace {
6201
6202// A non-null {isolate} signifies that the generated code is treated as being in
6203// a JS frame for functions like BuildIsolateRoot().
6204class WasmWrapperGraphBuilder : public WasmGraphBuilder {
6205 public:
6206 WasmWrapperGraphBuilder(Zone* zone, MachineGraph* mcgraph,
6207 const wasm::FunctionSig* sig,
6208 const wasm::WasmModule* module,
6209 Parameter0Mode parameter_mode, Isolate* isolate,
6210 compiler::SourcePositionTable* spt,
6211 StubCallMode stub_mode, wasm::WasmFeatures features)
6212 : WasmGraphBuilder(nullptr, zone, mcgraph, sig, spt, parameter_mode,
6213 isolate),
6214 module_(module),
6215 stub_mode_(stub_mode),
6216 enabled_features_(features) {}
6217
6218 CallDescriptor* GetI64ToBigIntCallDescriptor() {
6219 if (i64_to_bigint_descriptor_) return i64_to_bigint_descriptor_;
6220
6221 i64_to_bigint_descriptor_ =
6222 GetBuiltinCallDescriptor(Builtin::kI64ToBigInt, zone_, stub_mode_);
6223
6224 AddInt64LoweringReplacement(
6225 i64_to_bigint_descriptor_,
6226 GetBuiltinCallDescriptor(Builtin::kI32PairToBigInt, zone_, stub_mode_));
6227 return i64_to_bigint_descriptor_;
6228 }
6229
6230 CallDescriptor* GetBigIntToI64CallDescriptor(bool needs_frame_state) {
6231 if (bigint_to_i64_descriptor_) return bigint_to_i64_descriptor_;
6232
6233 bigint_to_i64_descriptor_ = GetBuiltinCallDescriptor(
6234 Builtin::kBigIntToI64, zone_, stub_mode_, needs_frame_state);
6235
6236 AddInt64LoweringReplacement(
6237 bigint_to_i64_descriptor_,
6238 GetBuiltinCallDescriptor(Builtin::kBigIntToI32Pair, zone_, stub_mode_));
6239 return bigint_to_i64_descriptor_;
6240 }
6241
6242 Node* GetTargetForBuiltinCall(wasm::WasmCode::RuntimeStubId wasm_stub,
6243 Builtin builtin) {
6244 return (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
6245 ? mcgraph()->RelocatableIntPtrConstant(wasm_stub,
6246 RelocInfo::WASM_STUB_CALL)
6247 : gasm_->GetBuiltinPointerTarget(builtin);
6248 }
6249
6250 Node* BuildChangeInt32ToNumber(Node* value) {
6251 // We expect most integers at runtime to be Smis, so it is important for
6252 // wrapper performance that Smi conversion be inlined.
6253 if (SmiValuesAre32Bits()) {
6254 return BuildChangeInt32ToSmi(value);
6255 }
6256 DCHECK(SmiValuesAre31Bits())((void) 0);
6257
6258 auto builtin = gasm_->MakeDeferredLabel();
6259 auto done = gasm_->MakeLabel(MachineRepresentation::kTagged);
6260
6261 // Double value to test if value can be a Smi, and if so, to convert it.
6262 Node* add = gasm_->Int32AddWithOverflow(value, value);
6263 Node* ovf = gasm_->Projection(1, add);
6264 gasm_->GotoIf(ovf, &builtin);
6265
6266 // If it didn't overflow, the result is {2 * value} as pointer-sized value.
6267 Node* smi_tagged = BuildChangeInt32ToIntPtr(gasm_->Projection(0, add));
6268 gasm_->Goto(&done, smi_tagged);
6269
6270 // Otherwise, call builtin, to convert to a HeapNumber.
6271 gasm_->Bind(&builtin);
6272 CommonOperatorBuilder* common = mcgraph()->common();
6273 Node* target =
6274 GetTargetForBuiltinCall(wasm::WasmCode::kWasmInt32ToHeapNumber,
6275 Builtin::kWasmInt32ToHeapNumber);
6276 if (!int32_to_heapnumber_operator_.is_set()) {
6277 auto call_descriptor = Linkage::GetStubCallDescriptor(
6278 mcgraph()->zone(), WasmInt32ToHeapNumberDescriptor(), 0,
6279 CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
6280 int32_to_heapnumber_operator_.set(common->Call(call_descriptor));
6281 }
6282 Node* call =
6283 gasm_->Call(int32_to_heapnumber_operator_.get(), target, value);
6284 gasm_->Goto(&done, call);
6285 gasm_->Bind(&done);
6286 return done.PhiAt(0);
6287 }
6288
6289 Node* BuildChangeTaggedToInt32(Node* value, Node* context,
6290 Node* frame_state) {
6291 // We expect most integers at runtime to be Smis, so it is important for
6292 // wrapper performance that Smi conversion be inlined.
6293 auto builtin = gasm_->MakeDeferredLabel();
6294 auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
6295
6296 gasm_->GotoIfNot(IsSmi(value), &builtin);
6297
6298 // If Smi, convert to int32.
6299 Node* smi = BuildChangeSmiToInt32(value);
6300 gasm_->Goto(&done, smi);
6301
6302 // Otherwise, call builtin which changes non-Smi to Int32.
6303 gasm_->Bind(&builtin);
6304 CommonOperatorBuilder* common = mcgraph()->common();
6305 Node* target =
6306 GetTargetForBuiltinCall(wasm::WasmCode::kWasmTaggedNonSmiToInt32,
6307 Builtin::kWasmTaggedNonSmiToInt32);
6308 if (!tagged_non_smi_to_int32_operator_.is_set()) {
6309 auto call_descriptor = Linkage::GetStubCallDescriptor(
6310 mcgraph()->zone(), WasmTaggedNonSmiToInt32Descriptor(), 0,
6311 frame_state ? CallDescriptor::kNeedsFrameState
6312 : CallDescriptor::kNoFlags,
6313 Operator::kNoProperties, stub_mode_);
6314 tagged_non_smi_to_int32_operator_.set(common->Call(call_descriptor));
6315 }
6316 Node* call = frame_state
6317 ? gasm_->Call(tagged_non_smi_to_int32_operator_.get(),
6318 target, value, context, frame_state)
6319 : gasm_->Call(tagged_non_smi_to_int32_operator_.get(),
6320 target, value, context);
6321 SetSourcePosition(call, 1);
6322 gasm_->Goto(&done, call);
6323 gasm_->Bind(&done);
6324 return done.PhiAt(0);
6325 }
6326
6327 Node* BuildChangeFloat32ToNumber(Node* value) {
6328 CommonOperatorBuilder* common = mcgraph()->common();
6329 Node* target = GetTargetForBuiltinCall(wasm::WasmCode::kWasmFloat32ToNumber,
6330 Builtin::kWasmFloat32ToNumber);
6331 if (!float32_to_number_operator_.is_set()) {
6332 auto call_descriptor = Linkage::GetStubCallDescriptor(
6333 mcgraph()->zone(), WasmFloat32ToNumberDescriptor(), 0,
6334 CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
6335 float32_to_number_operator_.set(common->Call(call_descriptor));
6336 }
6337 return gasm_->Call(float32_to_number_operator_.get(), target, value);
6338 }
6339
6340 Node* BuildChangeFloat64ToNumber(Node* value) {
6341 CommonOperatorBuilder* common = mcgraph()->common();
6342 Node* target = GetTargetForBuiltinCall(wasm::WasmCode::kWasmFloat64ToNumber,
6343 Builtin::kWasmFloat64ToNumber);
6344 if (!float64_to_number_operator_.is_set()) {
6345 auto call_descriptor = Linkage::GetStubCallDescriptor(
6346 mcgraph()->zone(), WasmFloat64ToNumberDescriptor(), 0,
6347 CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
6348 float64_to_number_operator_.set(common->Call(call_descriptor));
6349 }
6350 return gasm_->Call(float64_to_number_operator_.get(), target, value);
6351 }
6352
6353 Node* BuildChangeTaggedToFloat64(Node* value, Node* context,
6354 Node* frame_state) {
6355 CommonOperatorBuilder* common = mcgraph()->common();
6356 Node* target = GetTargetForBuiltinCall(wasm::WasmCode::kWasmTaggedToFloat64,
6357 Builtin::kWasmTaggedToFloat64);
6358 bool needs_frame_state = frame_state != nullptr;
6359 if (!tagged_to_float64_operator_.is_set()) {
6360 auto call_descriptor = Linkage::GetStubCallDescriptor(
6361 mcgraph()->zone(), WasmTaggedToFloat64Descriptor(), 0,
6362 frame_state ? CallDescriptor::kNeedsFrameState
6363 : CallDescriptor::kNoFlags,
6364 Operator::kNoProperties, stub_mode_);
6365 tagged_to_float64_operator_.set(common->Call(call_descriptor));
6366 }
6367 Node* call = needs_frame_state
6368 ? gasm_->Call(tagged_to_float64_operator_.get(), target,
6369 value, context, frame_state)
6370 : gasm_->Call(tagged_to_float64_operator_.get(), target,
6371 value, context);
6372 SetSourcePosition(call, 1);
6373 return call;
6374 }
6375
6376 int AddArgumentNodes(base::Vector<Node*> args, int pos, int param_count,
6377 const wasm::FunctionSig* sig, Node* context) {
6378 // Convert wasm numbers to JS values.
6379 for (int i = 0; i < param_count; ++i) {
6380 Node* param =
6381 Param(i + 1); // Start from index 1 to drop the instance_node.
6382 args[pos++] = ToJS(param, sig->GetParam(i), context);
6383 }
6384 return pos;
6385 }
6386
6387 Node* ToJS(Node* node, wasm::ValueType type, Node* context) {
6388 switch (type.kind()) {
6389 case wasm::kI32:
6390 return BuildChangeInt32ToNumber(node);
6391 case wasm::kI64:
6392 return BuildChangeInt64ToBigInt(node);
6393 case wasm::kF32:
6394 return BuildChangeFloat32ToNumber(node);
6395 case wasm::kF64:
6396 return BuildChangeFloat64ToNumber(node);
6397 case wasm::kRef:
6398 case wasm::kOptRef:
6399 switch (type.heap_representation()) {
6400 case wasm::HeapType::kFunc: {
6401 if (type.kind() == wasm::kOptRef) {
6402 auto done =
6403 gasm_->MakeLabel(MachineRepresentation::kTaggedPointer);
6404 // Do not wrap {null}.
6405 gasm_->GotoIf(IsNull(node), &done, node);
6406 gasm_->Goto(&done,
6407 gasm_->LoadFromObject(
6408 MachineType::TaggedPointer(), node,
6409 wasm::ObjectAccess::ToTagged(
6410 WasmInternalFunction::kExternalOffset)));
6411 gasm_->Bind(&done);
6412 return done.PhiAt(0);
6413 } else {
6414 return gasm_->LoadFromObject(
6415 MachineType::TaggedPointer(), node,
6416 wasm::ObjectAccess::ToTagged(
6417 WasmInternalFunction::kExternalOffset));
6418 }
6419 }
6420 case wasm::HeapType::kEq:
6421 case wasm::HeapType::kData:
6422 case wasm::HeapType::kArray:
6423 case wasm::HeapType::kI31:
6424 // TODO(7748): Update this when JS interop is settled.
6425 if (type.kind() == wasm::kOptRef) {
6426 auto done =
6427 gasm_->MakeLabel(MachineRepresentation::kTaggedPointer);
6428 // Do not wrap {null}.
6429 gasm_->GotoIf(IsNull(node), &done, node);
6430 gasm_->Goto(&done, BuildAllocateObjectWrapper(node, context));
6431 gasm_->Bind(&done);
6432 return done.PhiAt(0);
6433 } else {
6434 return BuildAllocateObjectWrapper(node, context);
6435 }
6436 case wasm::HeapType::kAny: {
6437 if (!enabled_features_.has_gc()) return node;
6438 // Wrap {node} in object wrapper if it is an array/struct.
6439 // Extract external function if this is a WasmInternalFunction.
6440 // Otherwise (i.e. null and external refs), return input.
6441 // Treat i31 as externref because they are indistinguishable from
6442 // Smis.
6443 // TODO(7748): Update this when JS interop is settled.
6444 auto wrap = gasm_->MakeLabel();
6445 auto function = gasm_->MakeLabel();
6446 auto done = gasm_->MakeLabel(MachineRepresentation::kTaggedPointer);
6447 gasm_->GotoIf(IsSmi(node), &done, node);
6448 gasm_->GotoIf(gasm_->IsDataRefMap(gasm_->LoadMap(node)), &wrap);
6449 gasm_->GotoIf(
6450 gasm_->HasInstanceType(node, WASM_INTERNAL_FUNCTION_TYPE),
6451 &function);
6452 // This includes the case where {node == null}.
6453 gasm_->Goto(&done, node);
6454
6455 gasm_->Bind(&wrap);
6456 gasm_->Goto(&done, BuildAllocateObjectWrapper(node, context));
6457
6458 gasm_->Bind(&function);
6459 gasm_->Goto(&done, gasm_->LoadFromObject(
6460 MachineType::TaggedPointer(), node,
6461 wasm::ObjectAccess::ToTagged(
6462 WasmInternalFunction::kExternalOffset)));
6463
6464 gasm_->Bind(&done);
6465 return done.PhiAt(0);
6466 }
6467 default:
6468 DCHECK(type.has_index())((void) 0);
6469 if (module_->has_signature(type.ref_index())) {
6470 // Typed function. Extract the external function.
6471 return gasm_->LoadFromObject(
6472 MachineType::TaggedPointer(), node,
6473 wasm::ObjectAccess::ToTagged(
6474 WasmInternalFunction::kExternalOffset));
6475 }
6476 // If this is reached, then IsJSCompatibleSignature() is too
6477 // permissive.
6478 // TODO(7748): Figure out a JS interop story for arrays and structs.
6479 UNREACHABLE()V8_Fatal("unreachable code");
6480 }
6481 case wasm::kRtt:
6482 case wasm::kI8:
6483 case wasm::kI16:
6484 case wasm::kS128:
6485 case wasm::kVoid:
6486 case wasm::kBottom:
6487 // If this is reached, then IsJSCompatibleSignature() is too permissive.
6488 // TODO(7748): Figure out what to do for RTTs.
6489 UNREACHABLE()V8_Fatal("unreachable code");
6490 }
6491 }
6492
6493 // TODO(7748): Temporary solution to allow round-tripping of Wasm objects
6494 // through JavaScript, where they show up as opaque boxes. This will disappear
6495 // once we have a proper WasmGC <-> JS interaction story.
6496 Node* BuildAllocateObjectWrapper(Node* input, Node* context) {
6497 if (FLAG_wasm_gc_js_interop) return input;
6498 return gasm_->CallBuiltin(Builtin::kWasmAllocateObjectWrapper,
6499 Operator::kEliminatable, input, context);
6500 }
6501
6502 // Assumes {input} has been checked for validity against the target wasm type.
6503 // If {input} is a function, returns the WasmInternalFunction associated with
6504 // it. If {input} has the {wasm_wrapped_object_symbol} property, returns the
6505 // value of that property. Otherwise, returns {input}.
6506 Node* BuildUnpackObjectWrapper(Node* input, Node* context) {
6507 auto not_a_function = gasm_->MakeLabel();
6508 auto end = gasm_->MakeLabel(MachineRepresentation::kTaggedPointer);
6509
6510 gasm_->GotoIfNot(gasm_->HasInstanceType(input, JS_FUNCTION_TYPE),
6511 &not_a_function);
6512
6513 Node* function_data = gasm_->LoadFunctionDataFromJSFunction(input);
6514
6515 // Due to type checking, {function_data} will be a WasmFunctionData.
6516 Node* internal = gasm_->LoadFromObject(
6517 MachineType::TaggedPointer(), function_data,
6518 wasm::ObjectAccess::ToTagged(WasmFunctionData::kInternalOffset));
6519 gasm_->Goto(&end, internal);
6520
6521 gasm_->Bind(&not_a_function);
6522 if (!FLAG_wasm_gc_js_interop) {
6523 Node* obj = gasm_->CallBuiltin(
6524 Builtin::kWasmGetOwnProperty, Operator::kEliminatable, input,
6525 LOAD_ROOT(wasm_wrapped_object_symbol, wasm_wrapped_object_symbol),
6526 context);
6527 // Invalid object wrappers (i.e. any other JS object that doesn't have the
6528 // magic hidden property) will return {undefined}. Map that to {input}.
6529 Node* is_undefined = gasm_->TaggedEqual(obj, UndefinedValue());
6530 gasm_->GotoIf(is_undefined, &end, input);
6531
6532 gasm_->Goto(&end, obj);
6533 } else {
6534 gasm_->Goto(&end, input);
6535 }
6536
6537 gasm_->Bind(&end);
6538
6539 return end.PhiAt(0);
6540 }
6541
6542 Node* BuildChangeInt64ToBigInt(Node* input) {
6543 Node* target;
6544 if (mcgraph()->machine()->Is64()) {
6545 target = GetTargetForBuiltinCall(wasm::WasmCode::kI64ToBigInt,
6546 Builtin::kI64ToBigInt);
6547 } else {
6548 DCHECK(mcgraph()->machine()->Is32())((void) 0);
6549 // On 32-bit platforms we already set the target to the
6550 // I32PairToBigInt builtin here, so that we don't have to replace the
6551 // target in the int64-lowering.
6552 target = GetTargetForBuiltinCall(wasm::WasmCode::kI32PairToBigInt,
6553 Builtin::kI32PairToBigInt);
6554 }
6555 return gasm_->Call(GetI64ToBigIntCallDescriptor(), target, input);
6556 }
6557
6558 Node* BuildChangeBigIntToInt64(Node* input, Node* context,
6559 Node* frame_state) {
6560 Node* target;
6561 if (mcgraph()->machine()->Is64()) {
6562 target = GetTargetForBuiltinCall(wasm::WasmCode::kBigIntToI64,
6563 Builtin::kBigIntToI64);
6564 } else {
6565 DCHECK(mcgraph()->machine()->Is32())((void) 0);
6566 // On 32-bit platforms we already set the target to the
6567 // BigIntToI32Pair builtin here, so that we don't have to replace the
6568 // target in the int64-lowering.
6569 target = GetTargetForBuiltinCall(wasm::WasmCode::kBigIntToI32Pair,
6570 Builtin::kBigIntToI32Pair);
6571 }
6572
6573 return frame_state ? gasm_->Call(GetBigIntToI64CallDescriptor(true), target,
6574 input, context, frame_state)
6575 : gasm_->Call(GetBigIntToI64CallDescriptor(false),
6576 target, input, context);
6577 }
6578
6579 void BuildCheckValidRefValue(Node* input, Node* js_context,
6580 wasm::ValueType type) {
6581 // Make sure ValueType fits in a Smi.
6582 STATIC_ASSERT(wasm::ValueType::kLastUsedBit + 1 <= kSmiValueSize)static_assert(wasm::ValueType::kLastUsedBit + 1 <= kSmiValueSize
, "wasm::ValueType::kLastUsedBit + 1 <= kSmiValueSize")
;
6583 // The instance node is always defined: if an instance is not available, it
6584 // is the undefined value.
6585 Node* inputs[] = {GetInstance(), input,
6586 mcgraph()->IntPtrConstant(
6587 IntToSmi(static_cast<int>(type.raw_bit_field())))};
6588
6589 Node* check = BuildChangeSmiToInt32(BuildCallToRuntimeWithContext(
6590 Runtime::kWasmIsValidRefValue, js_context, inputs, 3));
6591
6592 Diamond type_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
6593 type_check.Chain(control());
6594 SetControl(type_check.if_false);
6595
6596 Node* old_effect = effect();
6597 BuildCallToRuntimeWithContext(Runtime::kWasmThrowJSTypeError, js_context,
6598 nullptr, 0);
6599
6600 SetEffectControl(type_check.EffectPhi(old_effect, effect()),
6601 type_check.merge);
6602 }
6603
6604 Node* FromJS(Node* input, Node* js_context, wasm::ValueType type,
6605 Node* frame_state = nullptr) {
6606 switch (type.kind()) {
6607 case wasm::kRef:
6608 case wasm::kOptRef: {
6609 switch (type.heap_representation()) {
6610 case wasm::HeapType::kAny:
6611 if (!enabled_features_.has_gc()) return input;
6612 // If this is a wrapper for arrays/structs/i31s, unpack it.
6613 // TODO(7748): Update this when JS interop has settled.
6614 return BuildUnpackObjectWrapper(input, js_context);
6615 case wasm::HeapType::kFunc:
6616 BuildCheckValidRefValue(input, js_context, type);
6617 return BuildUnpackObjectWrapper(input, js_context);
6618 case wasm::HeapType::kData:
6619 case wasm::HeapType::kArray:
6620 case wasm::HeapType::kEq:
6621 case wasm::HeapType::kI31:
6622 // TODO(7748): Update this when JS interop has settled.
6623 BuildCheckValidRefValue(input, js_context, type);
6624 // This will just return {input} if the object is not wrapped, i.e.
6625 // if it is null (given the check just above).
6626 return BuildUnpackObjectWrapper(input, js_context);
6627 default:
6628 if (module_->has_signature(type.ref_index())) {
6629 BuildCheckValidRefValue(input, js_context, type);
6630 return BuildUnpackObjectWrapper(input, js_context);
6631 }
6632 // If this is reached, then IsJSCompatibleSignature() is too
6633 // permissive.
6634 UNREACHABLE()V8_Fatal("unreachable code");
6635 }
6636 }
6637 case wasm::kF32:
6638 return gasm_->TruncateFloat64ToFloat32(
6639 BuildChangeTaggedToFloat64(input, js_context, frame_state));
6640
6641 case wasm::kF64:
6642 return BuildChangeTaggedToFloat64(input, js_context, frame_state);
6643
6644 case wasm::kI32:
6645 return BuildChangeTaggedToInt32(input, js_context, frame_state);
6646
6647 case wasm::kI64:
6648 // i64 values can only come from BigInt.
6649 return BuildChangeBigIntToInt64(input, js_context, frame_state);
6650
6651 case wasm::kRtt:
6652 case wasm::kS128:
6653 case wasm::kI8:
6654 case wasm::kI16:
6655 case wasm::kBottom:
6656 case wasm::kVoid:
6657 // If this is reached, then IsJSCompatibleSignature() is too permissive.
6658 // TODO(7748): Figure out what to do for RTTs.
6659 UNREACHABLE()V8_Fatal("unreachable code");
6660 }
6661 }
6662
6663 Node* SmiToFloat32(Node* input) {
6664 return gasm_->RoundInt32ToFloat32(BuildChangeSmiToInt32(input));
6665 }
6666
6667 Node* SmiToFloat64(Node* input) {
6668 return gasm_->ChangeInt32ToFloat64(BuildChangeSmiToInt32(input));
6669 }
6670
6671 Node* HeapNumberToFloat64(Node* input) {
6672 return gasm_->LoadFromObject(
6673 MachineType::Float64(), input,
6674 wasm::ObjectAccess::ToTagged(HeapNumber::kValueOffset));
6675 }
6676
6677 Node* FromJSFast(Node* input, wasm::ValueType type) {
6678 switch (type.kind()) {
6679 case wasm::kI32:
6680 return BuildChangeSmiToInt32(input);
6681 case wasm::kF32: {
6682 auto done = gasm_->MakeLabel(MachineRepresentation::kFloat32);
6683 auto heap_number = gasm_->MakeLabel();
6684 gasm_->GotoIfNot(IsSmi(input), &heap_number);
6685 gasm_->Goto(&done, SmiToFloat32(input));
6686 gasm_->Bind(&heap_number);
6687 Node* value =
6688 gasm_->TruncateFloat64ToFloat32(HeapNumberToFloat64(input));
6689 gasm_->Goto(&done, value);
6690 gasm_->Bind(&done);
6691 return done.PhiAt(0);
6692 }
6693 case wasm::kF64: {
6694 auto done = gasm_->MakeLabel(MachineRepresentation::kFloat64);
6695 auto heap_number = gasm_->MakeLabel();
6696 gasm_->GotoIfNot(IsSmi(input), &heap_number);
6697 gasm_->Goto(&done, SmiToFloat64(input));
6698 gasm_->Bind(&heap_number);
6699 gasm_->Goto(&done, HeapNumberToFloat64(input));
6700 gasm_->Bind(&done);
6701 return done.PhiAt(0);
6702 }
6703 case wasm::kRef:
6704 case wasm::kOptRef:
6705 case wasm::kI64:
6706 case wasm::kRtt:
6707 case wasm::kS128:
6708 case wasm::kI8:
6709 case wasm::kI16:
6710 case wasm::kBottom:
6711 case wasm::kVoid:
6712 UNREACHABLE()V8_Fatal("unreachable code");
6713 }
6714 }
6715
6716 void BuildModifyThreadInWasmFlagHelper(Node* thread_in_wasm_flag_address,
6717 bool new_value) {
6718 if (FLAG_debug_code) {
6719 Node* flag_value = gasm_->LoadFromObject(MachineType::Pointer(),
6720 thread_in_wasm_flag_address, 0);
6721 Node* check =
6722 gasm_->Word32Equal(flag_value, Int32Constant(new_value ? 0 : 1));
6723
6724 Diamond flag_check(graph(), mcgraph()->common(), check,
6725 BranchHint::kTrue);
6726 flag_check.Chain(control());
6727 SetControl(flag_check.if_false);
6728 Node* message_id = gasm_->NumberConstant(static_cast<int32_t>(
6729 new_value ? AbortReason::kUnexpectedThreadInWasmSet
6730 : AbortReason::kUnexpectedThreadInWasmUnset));
6731
6732 Node* old_effect = effect();
6733 Node* call = BuildCallToRuntimeWithContext(
6734 Runtime::kAbort, NoContextConstant(), &message_id, 1);
6735 flag_check.merge->ReplaceInput(1, call);
6736 SetEffectControl(flag_check.EffectPhi(old_effect, effect()),
6737 flag_check.merge);
6738 }
6739
6740 gasm_->StoreToObject(ObjectAccess(MachineType::Int32(), kNoWriteBarrier),
6741 thread_in_wasm_flag_address, 0,
6742 Int32Constant(new_value ? 1 : 0));
6743 }
6744
6745 void BuildModifyThreadInWasmFlag(bool new_value) {
6746 if (!trap_handler::IsTrapHandlerEnabled()) return;
6747 Node* isolate_root = BuildLoadIsolateRoot();
6748
6749 Node* thread_in_wasm_flag_address =
6750 gasm_->LoadFromObject(MachineType::Pointer(), isolate_root,
6751 Isolate::thread_in_wasm_flag_address_offset());
6752
6753 BuildModifyThreadInWasmFlagHelper(thread_in_wasm_flag_address, new_value);
6754 }
6755
6756 class ModifyThreadInWasmFlagScope {
6757 public:
6758 ModifyThreadInWasmFlagScope(
6759 WasmWrapperGraphBuilder* wasm_wrapper_graph_builder,
6760 WasmGraphAssembler* gasm)
6761 : wasm_wrapper_graph_builder_(wasm_wrapper_graph_builder) {
6762 if (!trap_handler::IsTrapHandlerEnabled()) return;
6763 Node* isolate_root = wasm_wrapper_graph_builder_->BuildLoadIsolateRoot();
6764
6765 thread_in_wasm_flag_address_ =
6766 gasm->LoadFromObject(MachineType::Pointer(), isolate_root,
6767 Isolate::thread_in_wasm_flag_address_offset());
6768
6769 wasm_wrapper_graph_builder_->BuildModifyThreadInWasmFlagHelper(
6770 thread_in_wasm_flag_address_, true);
6771 }
6772
6773 ~ModifyThreadInWasmFlagScope() {
6774 if (!trap_handler::IsTrapHandlerEnabled()) return;
6775
6776 wasm_wrapper_graph_builder_->BuildModifyThreadInWasmFlagHelper(
6777 thread_in_wasm_flag_address_, false);
6778 }
6779
6780 private:
6781 WasmWrapperGraphBuilder* wasm_wrapper_graph_builder_;
6782 Node* thread_in_wasm_flag_address_;
6783 };
6784
6785 Node* BuildMultiReturnFixedArrayFromIterable(const wasm::FunctionSig* sig,
6786 Node* iterable, Node* context) {
6787 Node* length = BuildChangeUint31ToSmi(
6788 mcgraph()->Uint32Constant(static_cast<uint32_t>(sig->return_count())));
6789 return gasm_->CallBuiltin(Builtin::kIterableToFixedArrayForWasm,
6790 Operator::kEliminatable, iterable, length,
6791 context);
6792 }
6793
6794 // Generate a call to the AllocateJSArray builtin.
6795 Node* BuildCallAllocateJSArray(Node* array_length, Node* context) {
6796 // Since we don't check that args will fit in an array,
6797 // we make sure this is true based on statically known limits.
6798 STATIC_ASSERT(wasm::kV8MaxWasmFunctionReturns <=static_assert(wasm::kV8MaxWasmFunctionReturns <= JSArray::
kInitialMaxFastElementArray, "wasm::kV8MaxWasmFunctionReturns <= JSArray::kInitialMaxFastElementArray"
)
6799 JSArray::kInitialMaxFastElementArray)static_assert(wasm::kV8MaxWasmFunctionReturns <= JSArray::
kInitialMaxFastElementArray, "wasm::kV8MaxWasmFunctionReturns <= JSArray::kInitialMaxFastElementArray"
)
;
6800 return gasm_->CallBuiltin(Builtin::kWasmAllocateJSArray,
6801 Operator::kEliminatable, array_length, context);
6802 }
6803
6804 Node* BuildCallAndReturn(bool is_import, Node* js_context,
6805 Node* function_data,
6806 base::SmallVector<Node*, 16> args,
6807 const JSWasmCallData* js_wasm_call_data,
6808 Node* frame_state) {
6809 const int rets_count = static_cast<int>(sig_->return_count());
6810 base::SmallVector<Node*, 1> rets(rets_count);
6811
6812 // Set the ThreadInWasm flag before we do the actual call.
6813 {
6814 ModifyThreadInWasmFlagScope modify_thread_in_wasm_flag_builder(
6815 this, gasm_.get());
6816
6817 if (is_import) {
6818 // Call to an imported function.
6819 // Load function index from {WasmExportedFunctionData}.
6820 Node* function_index = BuildChangeSmiToInt32(
6821 gasm_->LoadExportedFunctionIndexAsSmi(function_data));
6822 BuildImportCall(sig_, base::VectorOf(args), base::VectorOf(rets),
6823 wasm::kNoCodePosition, function_index, kCallContinues);
6824 } else {
6825 // Call to a wasm function defined in this module.
6826 // The (cached) call target is the jump table slot for that function.
6827 Node* internal = gasm_->LoadFromObject(
6828 MachineType::TaggedPointer(), function_data,
6829 wasm::ObjectAccess::ToTagged(WasmFunctionData::kInternalOffset));
6830 args[0] = BuildLoadExternalPointerFromObject(
6831 internal, WasmInternalFunction::kForeignAddressOffset);
6832 Node* instance_node = gasm_->LoadFromObject(
6833 MachineType::TaggedPointer(), internal,
6834 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset));
6835 BuildWasmCall(sig_, base::VectorOf(args), base::VectorOf(rets),
6836 wasm::kNoCodePosition, instance_node, frame_state);
6837 }
6838 }
6839
6840 Node* jsval;
6841 if (sig_->return_count() == 0) {
6842 jsval = UndefinedValue();
6843 } else if (sig_->return_count() == 1) {
6844 jsval = js_wasm_call_data && !js_wasm_call_data->result_needs_conversion()
6845 ? rets[0]
6846 : ToJS(rets[0], sig_->GetReturn(), js_context);
6847 } else {
6848 int32_t return_count = static_cast<int32_t>(sig_->return_count());
6849 Node* size = gasm_->NumberConstant(return_count);
6850
6851 jsval = BuildCallAllocateJSArray(size, js_context);
6852
6853 Node* fixed_array = gasm_->LoadJSArrayElements(jsval);
6854
6855 for (int i = 0; i < return_count; ++i) {
6856 Node* value = ToJS(rets[i], sig_->GetReturn(i), js_context);
6857 gasm_->StoreFixedArrayElementAny(fixed_array, i, value);
6858 }
6859 }
6860 return jsval;
6861 }
6862
6863 bool QualifiesForFastTransform(const wasm::FunctionSig*) {
6864 const int wasm_count = static_cast<int>(sig_->parameter_count());
6865 for (int i = 0; i < wasm_count; ++i) {
6866 wasm::ValueType type = sig_->GetParam(i);
6867 switch (type.kind()) {
6868 case wasm::kRef:
6869 case wasm::kOptRef:
6870 case wasm::kI64:
6871 case wasm::kRtt:
6872 case wasm::kS128:
6873 case wasm::kI8:
6874 case wasm::kI16:
6875 case wasm::kBottom:
6876 case wasm::kVoid:
6877 return false;
6878 case wasm::kI32:
6879 case wasm::kF32:
6880 case wasm::kF64:
6881 break;
6882 }
6883 }
6884 return true;
6885 }
6886
6887 Node* IsSmi(Node* input) {
6888 return gasm_->Word32Equal(
6889 gasm_->Word32And(BuildTruncateIntPtrToInt32(input),
6890 Int32Constant(kSmiTagMask)),
6891 Int32Constant(kSmiTag));
6892 }
6893
6894 void CanTransformFast(
6895 Node* input, wasm::ValueType type,
6896 v8::internal::compiler::GraphAssemblerLabel<0>* slow_path) {
6897 switch (type.kind()) {
6898 case wasm::kI32: {
6899 gasm_->GotoIfNot(IsSmi(input), slow_path);
6900 return;
6901 }
6902 case wasm::kF32:
6903 case wasm::kF64: {
6904 auto done = gasm_->MakeLabel();
6905 gasm_->GotoIf(IsSmi(input), &done);
6906 Node* map = gasm_->LoadMap(input);
6907 Node* heap_number_map = LOAD_ROOT(HeapNumberMap, heap_number_map);
6908#if V8_MAP_PACKING
6909 Node* is_heap_number = gasm_->WordEqual(heap_number_map, map);
6910#else
6911 Node* is_heap_number = gasm_->TaggedEqual(heap_number_map, map);
6912#endif
6913 gasm_->GotoIf(is_heap_number, &done);
6914 gasm_->Goto(slow_path);
6915 gasm_->Bind(&done);
6916 return;
6917 }
6918 case wasm::kRef:
6919 case wasm::kOptRef:
6920 case wasm::kI64:
6921 case wasm::kRtt:
6922 case wasm::kS128:
6923 case wasm::kI8:
6924 case wasm::kI16:
6925 case wasm::kBottom:
6926 case wasm::kVoid:
6927 UNREACHABLE()V8_Fatal("unreachable code");
6928 }
6929 }
6930
6931 void BuildJSToWasmWrapper(bool is_import,
6932 const JSWasmCallData* js_wasm_call_data = nullptr,
6933 Node* frame_state = nullptr) {
6934 const int wasm_param_count = static_cast<int>(sig_->parameter_count());
6935
6936 // Build the start and the JS parameter nodes.
6937 Start(wasm_param_count + 5);
6938
6939 // Create the js_closure and js_context parameters.
6940 Node* js_closure = Param(Linkage::kJSCallClosureParamIndex, "%closure");
6941 Node* js_context = Param(
6942 Linkage::GetJSCallContextParamIndex(wasm_param_count + 1), "%context");
6943 Node* function_data = gasm_->LoadFunctionDataFromJSFunction(js_closure);
6944
6945 if (!wasm::IsJSCompatibleSignature(sig_, module_, enabled_features_)) {
6946 // Throw a TypeError. Use the js_context of the calling javascript
6947 // function (passed as a parameter), such that the generated code is
6948 // js_context independent.
6949 BuildCallToRuntimeWithContext(Runtime::kWasmThrowJSTypeError, js_context,
6950 nullptr, 0);
6951 TerminateThrow(effect(), control());
6952 return;
6953 }
6954
6955 const int args_count = wasm_param_count + 1; // +1 for wasm_code.
6956
6957 // Check whether the signature of the function allows for a fast
6958 // transformation (if any params exist that need transformation).
6959 // Create a fast transformation path, only if it does.
6960 bool include_fast_path = !js_wasm_call_data && wasm_param_count > 0 &&
6961 QualifiesForFastTransform(sig_);
6962
6963 // Prepare Param() nodes. Param() nodes can only be created once,
6964 // so we need to use the same nodes along all possible transformation paths.
6965 base::SmallVector<Node*, 16> params(args_count);
6966 for (int i = 0; i < wasm_param_count; ++i) params[i + 1] = Param(i + 1);
6967
6968 auto done = gasm_->MakeLabel(MachineRepresentation::kTagged);
6969 if (include_fast_path) {
6970 auto slow_path = gasm_->MakeDeferredLabel();
6971 // Check if the params received on runtime can be actually transformed
6972 // using the fast transformation. When a param that cannot be transformed
6973 // fast is encountered, skip checking the rest and fall back to the slow
6974 // path.
6975 for (int i = 0; i < wasm_param_count; ++i) {
6976 CanTransformFast(params[i + 1], sig_->GetParam(i), &slow_path);
6977 }
6978 // Convert JS parameters to wasm numbers using the fast transformation
6979 // and build the call.
6980 base::SmallVector<Node*, 16> args(args_count);
6981 for (int i = 0; i < wasm_param_count; ++i) {
6982 Node* wasm_param = FromJSFast(params[i + 1], sig_->GetParam(i));
6983 args[i + 1] = wasm_param;
6984 }
6985 Node* jsval = BuildCallAndReturn(is_import, js_context, function_data,
6986 args, js_wasm_call_data, frame_state);
6987 gasm_->Goto(&done, jsval);
6988 gasm_->Bind(&slow_path);
6989 }
6990 // Convert JS parameters to wasm numbers using the default transformation
6991 // and build the call.
6992 base::SmallVector<Node*, 16> args(args_count);
6993 for (int i = 0; i < wasm_param_count; ++i) {
6994 bool do_conversion =
6995 !js_wasm_call_data || js_wasm_call_data->arg_needs_conversion(i);
6996 if (do_conversion) {
6997 args[i + 1] =
6998 FromJS(params[i + 1], js_context, sig_->GetParam(i), frame_state);
6999 } else {
7000 Node* wasm_param = params[i + 1];
7001
7002 // For Float32 parameters
7003 // we set UseInfo::CheckedNumberOrOddballAsFloat64 in
7004 // simplified-lowering and we need to add here a conversion from Float64
7005 // to Float32.
7006 if (sig_->GetParam(i).kind() == wasm::kF32) {
7007 wasm_param = gasm_->TruncateFloat64ToFloat32(wasm_param);
7008 }
7009
7010 args[i + 1] = wasm_param;
7011 }
7012 }
7013 Node* jsval = BuildCallAndReturn(is_import, js_context, function_data, args,
7014 js_wasm_call_data, frame_state);
7015 // If both the default and a fast transformation paths are present,
7016 // get the return value based on the path used.
7017 if (include_fast_path) {
7018 gasm_->Goto(&done, jsval);
7019 gasm_->Bind(&done);
7020 Return(done.PhiAt(0));
7021 } else {
7022 Return(jsval);
7023 }
7024 if (ContainsInt64(sig_)) LowerInt64(kCalledFromJS);
7025 }
7026
7027 Node* BuildReceiverNode(Node* callable_node, Node* native_context,
7028 Node* undefined_node) {
7029 // Check function strict bit.
7030 Node* shared_function_info = gasm_->LoadSharedFunctionInfo(callable_node);
7031 Node* flags = gasm_->LoadFromObject(
7032 MachineType::Int32(), shared_function_info,
7033 wasm::ObjectAccess::FlagsOffsetInSharedFunctionInfo());
7034 Node* strict_check =
7035 Binop(wasm::kExprI32And, flags,
7036 Int32Constant(SharedFunctionInfo::IsNativeBit::kMask |
7037 SharedFunctionInfo::IsStrictBit::kMask));
7038
7039 // Load global receiver if sloppy else use undefined.
7040 Diamond strict_d(graph(), mcgraph()->common(), strict_check,
7041 BranchHint::kNone);
7042 Node* old_effect = effect();
7043 SetControl(strict_d.if_false);
7044 Node* global_proxy = gasm_->LoadFixedArrayElementPtr(
7045 native_context, Context::GLOBAL_PROXY_INDEX);
7046 SetEffectControl(strict_d.EffectPhi(old_effect, global_proxy),
7047 strict_d.merge);
7048 return strict_d.Phi(MachineRepresentation::kTagged, undefined_node,
7049 global_proxy);
7050 }
7051
7052 Node* BuildSuspend(Node* value, Node* api_function_ref,
7053 MachineRepresentation rep) {
7054 // If value is a promise, suspend to the js-to-wasm prompt, and resume later
7055 // with the promise's resolved value.
7056 auto resume = gasm_->MakeLabel(rep);
7057 gasm_->GotoIf(IsSmi(value), &resume, value);
7058 gasm_->GotoIfNot(gasm_->HasInstanceType(value, JS_PROMISE_TYPE), &resume,
7059 BranchHint::kTrue, value);
7060 Node* suspender = gasm_->Load(
7061 MachineType::TaggedPointer(), api_function_ref,
7062 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kSuspenderOffset));
7063 Node* native_context = gasm_->Load(
7064 MachineType::TaggedPointer(), api_function_ref,
7065 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kNativeContextOffset));
7066 auto* call_descriptor = GetBuiltinCallDescriptor(
7067 Builtin::kWasmSuspend, zone_, StubCallMode::kCallWasmRuntimeStub);
7068 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
7069 wasm::WasmCode::kWasmSuspend, RelocInfo::WASM_STUB_CALL);
7070 Node* args[] = {value, suspender};
7071 Node* chained_promise = BuildCallToRuntimeWithContext(
7072 Runtime::kWasmCreateResumePromise, native_context, args, 2);
7073 Node* resolved =
7074 gasm_->Call(call_descriptor, call_target, chained_promise, suspender);
7075 gasm_->Goto(&resume, resolved);
7076 gasm_->Bind(&resume);
7077 return resume.PhiAt(0);
7078 }
7079
7080 // For wasm-to-js wrappers, parameter 0 is a WasmApiFunctionRef.
7081 bool BuildWasmToJSWrapper(WasmImportCallKind kind, int expected_arity,
7082 wasm::Suspend suspend) {
7083 int wasm_count = static_cast<int>(sig_->parameter_count());
7084
7085 // Build the start and the parameter nodes.
7086 Start(wasm_count + 3);
7087
7088 Node* native_context = gasm_->Load(
7089 MachineType::TaggedPointer(), Param(0),
7090 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kNativeContextOffset));
7091
7092 if (kind == WasmImportCallKind::kRuntimeTypeError) {
7093 // =======================================================================
7094 // === Runtime TypeError =================================================
7095 // =======================================================================
7096 BuildCallToRuntimeWithContext(Runtime::kWasmThrowJSTypeError,
7097 native_context, nullptr, 0);
7098 TerminateThrow(effect(), control());
7099 return false;
7100 }
7101
7102 Node* callable_node = gasm_->Load(
7103 MachineType::TaggedPointer(), Param(0),
7104 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kCallableOffset));
7105
7106 Node* undefined_node = UndefinedValue();
7107
7108 Node* call = nullptr;
7109
7110 // Clear the ThreadInWasm flag.
7111 BuildModifyThreadInWasmFlag(false);
7112
7113 switch (kind) {
7114 // =======================================================================
7115 // === JS Functions with matching arity ==================================
7116 // =======================================================================
7117 case WasmImportCallKind::kJSFunctionArityMatch: {
7118 base::SmallVector<Node*, 16> args(wasm_count + 7);
7119 int pos = 0;
7120 Node* function_context =
7121 gasm_->LoadContextFromJSFunction(callable_node);
7122 args[pos++] = callable_node; // target callable.
7123
7124 // Determine receiver at runtime.
7125 args[pos++] =
7126 BuildReceiverNode(callable_node, native_context, undefined_node);
7127
7128 auto call_descriptor = Linkage::GetJSCallDescriptor(
7129 graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
7130
7131 // Convert wasm numbers to JS values.
7132 pos = AddArgumentNodes(base::VectorOf(args), pos, wasm_count, sig_,
7133 native_context);
7134
7135 args[pos++] = undefined_node; // new target
7136 args[pos++] =
7137 Int32Constant(JSParameterCount(wasm_count)); // argument count
7138 args[pos++] = function_context;
7139 args[pos++] = effect();
7140 args[pos++] = control();
7141
7142 DCHECK_EQ(pos, args.size())((void) 0);
7143 call = gasm_->Call(call_descriptor, pos, args.begin());
7144 if (suspend == wasm::kSuspend) {
7145 MachineRepresentation rep =
7146 sig_->return_count() >= 1
7147 ? sig_->GetReturn(0).machine_representation()
7148 : MachineRepresentation::kNone;
7149 call = BuildSuspend(call, Param(0), rep);
7150 }
7151 break;
7152 }
7153 // =======================================================================
7154 // === JS Functions with mismatching arity ===============================
7155 // =======================================================================
7156 case WasmImportCallKind::kJSFunctionArityMismatch: {
7157 int pushed_count = std::max(expected_arity, wasm_count);
7158 base::SmallVector<Node*, 16> args(pushed_count + 7);
7159 int pos = 0;
7160
7161 args[pos++] = callable_node; // target callable.
7162 // Determine receiver at runtime.
7163 args[pos++] =
7164 BuildReceiverNode(callable_node, native_context, undefined_node);
7165
7166 // Convert wasm numbers to JS values.
7167 pos = AddArgumentNodes(base::VectorOf(args), pos, wasm_count, sig_,
7168 native_context);
7169 for (int i = wasm_count; i < expected_arity; ++i) {
7170 args[pos++] = undefined_node;
7171 }
7172 args[pos++] = undefined_node; // new target
7173 args[pos++] =
7174 Int32Constant(JSParameterCount(wasm_count)); // argument count
7175
7176 Node* function_context =
7177 gasm_->LoadContextFromJSFunction(callable_node);
7178 args[pos++] = function_context;
7179 args[pos++] = effect();
7180 args[pos++] = control();
7181 DCHECK_EQ(pos, args.size())((void) 0);
7182
7183 auto call_descriptor = Linkage::GetJSCallDescriptor(
7184 graph()->zone(), false, pushed_count + 1, CallDescriptor::kNoFlags);
7185 call = gasm_->Call(call_descriptor, pos, args.begin());
7186 // TODO(12191): Handle suspending wrapper.
7187 break;
7188 }
7189 // =======================================================================
7190 // === General case of unknown callable ==================================
7191 // =======================================================================
7192 case WasmImportCallKind::kUseCallBuiltin: {
7193 base::SmallVector<Node*, 16> args(wasm_count + 7);
7194 int pos = 0;
7195 args[pos++] =
7196 gasm_->GetBuiltinPointerTarget(Builtin::kCall_ReceiverIsAny);
7197 args[pos++] = callable_node;
7198 args[pos++] =
7199 Int32Constant(JSParameterCount(wasm_count)); // argument count
7200 args[pos++] = undefined_node; // receiver
7201
7202 auto call_descriptor = Linkage::GetStubCallDescriptor(
7203 graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
7204 CallDescriptor::kNoFlags, Operator::kNoProperties,
7205 StubCallMode::kCallBuiltinPointer);
7206
7207 // Convert wasm numbers to JS values.
7208 pos = AddArgumentNodes(base::VectorOf(args), pos, wasm_count, sig_,
7209 native_context);
7210
7211 // The native_context is sufficient here, because all kind of callables
7212 // which depend on the context provide their own context. The context
7213 // here is only needed if the target is a constructor to throw a
7214 // TypeError, if the target is a native function, or if the target is a
7215 // callable JSObject, which can only be constructed by the runtime.
7216 args[pos++] = native_context;
7217 args[pos++] = effect();
7218 args[pos++] = control();
7219
7220 DCHECK_EQ(pos, args.size())((void) 0);
7221 call = gasm_->Call(call_descriptor, pos, args.begin());
7222 // TODO(12191): Handle suspending wrapper.
7223 break;
7224 }
7225 default:
7226 UNREACHABLE()V8_Fatal("unreachable code");
7227 }
7228 DCHECK_NOT_NULL(call)((void) 0);
7229
7230 SetSourcePosition(call, 0);
7231
7232 // Convert the return value(s) back.
7233 if (sig_->return_count() <= 1) {
7234 Node* val = sig_->return_count() == 0
7235 ? Int32Constant(0)
7236 : FromJS(call, native_context, sig_->GetReturn());
7237 BuildModifyThreadInWasmFlag(true);
7238 Return(val);
7239 } else {
7240 Node* fixed_array =
7241 BuildMultiReturnFixedArrayFromIterable(sig_, call, native_context);
7242 base::SmallVector<Node*, 8> wasm_values(sig_->return_count());
7243 for (unsigned i = 0; i < sig_->return_count(); ++i) {
7244 wasm_values[i] = FromJS(gasm_->LoadFixedArrayElementAny(fixed_array, i),
7245 native_context, sig_->GetReturn(i));
7246 }
7247 BuildModifyThreadInWasmFlag(true);
7248 Return(base::VectorOf(wasm_values));
7249 }
7250
7251 if (ContainsInt64(sig_)) LowerInt64(kCalledFromWasm);
7252 return true;
7253 }
7254
7255 void BuildCapiCallWrapper() {
7256 // Set up the graph start.
7257 Start(static_cast<int>(sig_->parameter_count()) +
7258 1 /* offset for first parameter index being -1 */ +
7259 1 /* WasmApiFunctionRef */);
7260 // Store arguments on our stack, then align the stack for calling to C.
7261 int param_bytes = 0;
7262 for (wasm::ValueType type : sig_->parameters()) {
7263 param_bytes += type.value_kind_size();
7264 }
7265 int return_bytes = 0;
7266 for (wasm::ValueType type : sig_->returns()) {
7267 return_bytes += type.value_kind_size();
7268 }
7269
7270 int stack_slot_bytes = std::max(param_bytes, return_bytes);
7271 Node* values = stack_slot_bytes == 0
7272 ? mcgraph()->IntPtrConstant(0)
7273 : graph()->NewNode(mcgraph()->machine()->StackSlot(
7274 stack_slot_bytes, kDoubleAlignment));
7275
7276 int offset = 0;
7277 int param_count = static_cast<int>(sig_->parameter_count());
7278 for (int i = 0; i < param_count; ++i) {
7279 wasm::ValueType type = sig_->GetParam(i);
7280 // Start from the parameter with index 1 to drop the instance_node.
7281 // TODO(jkummerow): When a values is a reference type, we should pass it
7282 // in a GC-safe way, not just as a raw pointer.
7283 SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), values,
7284 Int32Constant(offset), Param(i + 1), effect(),
7285 control()));
7286 offset += type.value_kind_size();
7287 }
7288
7289 Node* function_node = gasm_->Load(
7290 MachineType::TaggedPointer(), Param(0),
7291 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kCallableOffset));
7292 Node* sfi_data = gasm_->LoadFunctionDataFromJSFunction(function_node);
7293 Node* host_data_foreign =
7294 gasm_->Load(MachineType::AnyTagged(), sfi_data,
7295 wasm::ObjectAccess::ToTagged(
7296 WasmCapiFunctionData::kEmbedderDataOffset));
7297
7298 BuildModifyThreadInWasmFlag(false);
7299 Node* isolate_root = BuildLoadIsolateRoot();
7300 Node* fp_value = graph()->NewNode(mcgraph()->machine()->LoadFramePointer());
7301 gasm_->Store(StoreRepresentation(MachineType::PointerRepresentation(),
7302 kNoWriteBarrier),
7303 isolate_root, Isolate::c_entry_fp_offset(), fp_value);
7304
7305 Node* function = BuildLoadCallTargetFromExportedFunctionData(sfi_data);
7306
7307 // Parameters: Address host_data_foreign, Address arguments.
7308 MachineType host_sig_types[] = {
7309 MachineType::Pointer(), MachineType::Pointer(), MachineType::Pointer()};
7310 MachineSignature host_sig(1, 2, host_sig_types);
7311 Node* return_value =
7312 BuildCCall(&host_sig, function, host_data_foreign, values);
7313
7314 BuildModifyThreadInWasmFlag(true);
7315
7316 Node* old_effect = effect();
7317 Node* exception_branch = graph()->NewNode(
7318 mcgraph()->common()->Branch(BranchHint::kTrue),
7319 gasm_->WordEqual(return_value, mcgraph()->IntPtrConstant(0)),
7320 control());
7321 SetControl(
7322 graph()->NewNode(mcgraph()->common()->IfFalse(), exception_branch));
7323 WasmRethrowExplicitContextDescriptor interface_descriptor;
7324 auto call_descriptor = Linkage::GetStubCallDescriptor(
7325 mcgraph()->zone(), interface_descriptor,
7326 interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
7327 Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
7328 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
7329 wasm::WasmCode::kWasmRethrowExplicitContext, RelocInfo::WASM_STUB_CALL);
7330 Node* context = gasm_->Load(
7331 MachineType::TaggedPointer(), Param(0),
7332 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kNativeContextOffset));
7333 gasm_->Call(call_descriptor, call_target, return_value, context);
7334 TerminateThrow(effect(), control());
7335
7336 SetEffectControl(old_effect, graph()->NewNode(mcgraph()->common()->IfTrue(),
7337 exception_branch));
7338 DCHECK_LT(sig_->return_count(), wasm::kV8MaxWasmFunctionReturns)((void) 0);
7339 size_t return_count = sig_->return_count();
7340 if (return_count == 0) {
7341 Return(Int32Constant(0));
7342 } else {
7343 base::SmallVector<Node*, 8> returns(return_count);
7344 offset = 0;
7345 for (size_t i = 0; i < return_count; ++i) {
7346 wasm::ValueType type = sig_->GetReturn(i);
7347 Node* val = SetEffect(
7348 graph()->NewNode(GetSafeLoadOperator(offset, type), values,
7349 Int32Constant(offset), effect(), control()));
7350 returns[i] = val;
7351 offset += type.value_kind_size();
7352 }
7353 Return(base::VectorOf(returns));
7354 }
7355
7356 if (ContainsInt64(sig_)) LowerInt64(kCalledFromWasm);
7357 }
7358
7359 void BuildJSFastApiCallWrapper(Handle<JSFunction> target) {
7360 // Here 'callable_node' must be equal to 'target' but we cannot pass a
7361 // HeapConstant(target) because WasmCode::Validate() fails with
7362 // Unexpected mode: FULL_EMBEDDED_OBJECT.
7363 Node* callable_node = gasm_->Load(
7364 MachineType::TaggedPointer(), Param(0),
7365 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kCallableOffset));
7366 Node* native_context = gasm_->Load(
7367 MachineType::TaggedPointer(), Param(0),
7368 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kNativeContextOffset));
7369 Node* undefined_node = UndefinedValue();
7370 Node* receiver_node =
7371 BuildReceiverNode(callable_node, native_context, undefined_node);
7372
7373 SharedFunctionInfo shared = target->shared();
7374 FunctionTemplateInfo api_func_data = shared.get_api_func_data();
7375 const Address c_address = api_func_data.GetCFunction(0);
7376 const v8::CFunctionInfo* c_signature = api_func_data.GetCSignature(0);
7377 int c_arg_count = c_signature->ArgumentCount();
7378
7379 BuildModifyThreadInWasmFlag(false);
7380
7381#ifdef V8_USE_SIMULATOR_WITH_GENERIC_C_CALLS
7382 Address c_functions[] = {c_address};
7383 const v8::CFunctionInfo* const c_signatures[] = {c_signature};
7384 target->GetIsolate()->simulator_data()->RegisterFunctionsAndSignatures(
7385 c_functions, c_signatures, 1);
7386#endif // V8_USE_SIMULATOR_WITH_GENERIC_C_CALLS
7387
7388 MachineSignature::Builder builder(graph()->zone(), 1, c_arg_count);
7389 builder.AddReturn(MachineType::TypeForCType(c_signature->ReturnInfo()));
7390 for (int i = 0; i < c_arg_count; i += 1) {
7391 builder.AddParam(MachineType::TypeForCType(c_signature->ArgumentInfo(i)));
7392 }
7393
7394 base::SmallVector<Node*, 16> args(c_arg_count + 3);
7395 int pos = 0;
7396
7397 args[pos++] = mcgraph()->ExternalConstant(
7398 ExternalReference::Create(c_address, ExternalReference::FAST_C_CALL));
7399
7400 auto store_stack = [this](Node* node) -> Node* {
7401 constexpr int kAlign = alignof(uintptr_t);
7402 constexpr int kSize = sizeof(uintptr_t);
7403 Node* stack_slot = gasm_->StackSlot(kSize, kAlign);
7404
7405 gasm_->Store(StoreRepresentation(MachineType::PointerRepresentation(),
7406 kNoWriteBarrier),
7407 stack_slot, 0, node);
7408 return stack_slot;
7409 };
7410
7411 // Set receiver.
7412 args[pos++] = store_stack(receiver_node);
7413
7414 for (int i = 1; i < c_arg_count; i += 1) {
7415 switch (c_signature->ArgumentInfo(i).GetType()) {
7416 case CTypeInfo::Type::kV8Value:
7417 args[pos++] = store_stack(Param(i));
7418 break;
7419 default:
7420 args[pos++] = Param(i);
7421 break;
7422 }
7423 }
7424 DCHECK(!c_signature->HasOptions())((void) 0);
7425 args[pos++] = effect();
7426 args[pos++] = control();
7427
7428 auto call_descriptor =
7429 Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build());
7430
7431 // CPU profiler support.
7432 Node* target_address = gasm_->ExternalConstant(
7433 ExternalReference::fast_api_call_target_address(target->GetIsolate()));
7434 gasm_->Store(StoreRepresentation(MachineType::PointerRepresentation(),
7435 kNoWriteBarrier),
7436 target_address, 0, gasm_->IntPtrConstant(c_address));
7437
7438 // Disable JS execution.
7439 Node* javascript_execution_assert = gasm_->ExternalConstant(
7440 ExternalReference::javascript_execution_assert(target->GetIsolate()));
7441 gasm_->Store(
7442 StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
7443 javascript_execution_assert, 0, gasm_->Int32Constant(0));
7444
7445 // Execute the fast call API.
7446 Node* call = gasm_->Call(call_descriptor, pos, args.begin());
7447
7448 // Reenable JS execution.
7449 gasm_->Store(
7450 StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
7451 javascript_execution_assert, 0, gasm_->Int32Constant(1));
7452
7453 // Reset the CPU profiler target address.
7454 gasm_->Store(StoreRepresentation(MachineType::PointerRepresentation(),
7455 kNoWriteBarrier),
7456 target_address, 0, gasm_->IntPtrConstant(0));
7457
7458 BuildModifyThreadInWasmFlag(true);
7459
7460 Return(call);
7461 }
7462
7463 void BuildJSToJSWrapper() {
7464 int wasm_count = static_cast<int>(sig_->parameter_count());
7465
7466 // Build the start and the parameter nodes.
7467 int param_count = 1 /* closure */ + 1 /* receiver */ + wasm_count +
7468 1 /* new.target */ + 1 /* #arg */ + 1 /* context */;
7469 Start(param_count);
7470 Node* closure = Param(Linkage::kJSCallClosureParamIndex);
7471 Node* context = Param(Linkage::GetJSCallContextParamIndex(wasm_count + 1));
7472
7473 // Throw a TypeError if the signature is incompatible with JavaScript.
7474 if (!wasm::IsJSCompatibleSignature(sig_, module_, enabled_features_)) {
7475 BuildCallToRuntimeWithContext(Runtime::kWasmThrowJSTypeError, context,
7476 nullptr, 0);
7477 TerminateThrow(effect(), control());
7478 return;
7479 }
7480
7481 // Load the original callable from the closure.
7482 Node* func_data = gasm_->LoadFunctionDataFromJSFunction(closure);
7483 Node* internal = gasm_->LoadFromObject(
7484 MachineType::AnyTagged(), func_data,
7485 wasm::ObjectAccess::ToTagged(WasmFunctionData::kInternalOffset));
7486 Node* ref = gasm_->LoadFromObject(
7487 MachineType::AnyTagged(), internal,
7488 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset));
7489 Node* callable = gasm_->LoadFromObject(
7490 MachineType::AnyTagged(), ref,
7491 wasm::ObjectAccess::ToTagged(WasmApiFunctionRef::kCallableOffset));
7492
7493 // Call the underlying closure.
7494 base::SmallVector<Node*, 16> args(wasm_count + 7);
7495 int pos = 0;
7496 args[pos++] = gasm_->GetBuiltinPointerTarget(Builtin::kCall_ReceiverIsAny);
7497 args[pos++] = callable;
7498 args[pos++] =
7499 Int32Constant(JSParameterCount(wasm_count)); // argument count
7500 args[pos++] = UndefinedValue(); // receiver
7501
7502 auto call_descriptor = Linkage::GetStubCallDescriptor(
7503 graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
7504 CallDescriptor::kNoFlags, Operator::kNoProperties,
7505 StubCallMode::kCallBuiltinPointer);
7506
7507 // Convert parameter JS values to wasm numbers and back to JS values.
7508 for (int i = 0; i < wasm_count; ++i) {
7509 Node* param = Param(i + 1); // Start from index 1 to skip receiver.
7510 args[pos++] = ToJS(FromJS(param, context, sig_->GetParam(i)),
7511 sig_->GetParam(i), context);
7512 }
7513
7514 args[pos++] = context;
7515 args[pos++] = effect();
7516 args[pos++] = control();
7517
7518 DCHECK_EQ(pos, args.size())((void) 0);
7519 Node* call = gasm_->Call(call_descriptor, pos, args.begin());
7520
7521 // Convert return JS values to wasm numbers and back to JS values.
7522 Node* jsval;
7523 if (sig_->return_count() == 0) {
7524 jsval = UndefinedValue();
7525 } else if (sig_->return_count() == 1) {
7526 jsval = ToJS(FromJS(call, context, sig_->GetReturn()), sig_->GetReturn(),
7527 context);
7528 } else {
7529 Node* fixed_array =
7530 BuildMultiReturnFixedArrayFromIterable(sig_, call, context);
7531 int32_t return_count = static_cast<int32_t>(sig_->return_count());
7532 Node* size = gasm_->NumberConstant(return_count);
7533 jsval = BuildCallAllocateJSArray(size, context);
7534 Node* result_fixed_array = gasm_->LoadJSArrayElements(jsval);
7535 for (unsigned i = 0; i < sig_->return_count(); ++i) {
7536 const auto& type = sig_->GetReturn(i);
7537 Node* elem = gasm_->LoadFixedArrayElementAny(fixed_array, i);
7538 Node* cast = ToJS(FromJS(elem, context, type), type, context);
7539 gasm_->StoreFixedArrayElementAny(result_fixed_array, i, cast);
7540 }
7541 }
7542 Return(jsval);
7543 }
7544
7545 void BuildCWasmEntry() {
7546 // +1 offset for first parameter index being -1.
7547 Start(CWasmEntryParameters::kNumParameters + 1);
7548
7549 Node* code_entry = Param(CWasmEntryParameters::kCodeEntry);
7550 Node* object_ref = Param(CWasmEntryParameters::kObjectRef);
7551 Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer);
7552 Node* c_entry_fp = Param(CWasmEntryParameters::kCEntryFp);
7553
7554 Node* fp_value = graph()->NewNode(mcgraph()->machine()->LoadFramePointer());
7555 gasm_->Store(StoreRepresentation(MachineType::PointerRepresentation(),
7556 kNoWriteBarrier),
7557 fp_value, TypedFrameConstants::kFirstPushedFrameValueOffset,
7558 c_entry_fp);
7559
7560 int wasm_arg_count = static_cast<int>(sig_->parameter_count());
7561 base::SmallVector<Node*, 16> args(wasm_arg_count + 4);
7562
7563 int pos = 0;
7564 args[pos++] = code_entry;
7565 args[pos++] = object_ref;
7566
7567 int offset = 0;
7568 for (wasm::ValueType type : sig_->parameters()) {
7569 Node* arg_load = SetEffect(
7570 graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
7571 Int32Constant(offset), effect(), control()));
7572 args[pos++] = arg_load;
7573 offset += type.value_kind_size();
7574 }
7575
7576 args[pos++] = effect();
7577 args[pos++] = control();
7578
7579 // Call the wasm code.
7580 auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
7581
7582 DCHECK_EQ(pos, args.size())((void) 0);
7583 Node* call = gasm_->Call(call_descriptor, pos, args.begin());
7584
7585 Node* if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), call);
7586 Node* if_exception =
7587 graph()->NewNode(mcgraph()->common()->IfException(), call, call);
7588
7589 // Handle exception: return it.
7590 SetControl(if_exception);
7591 Return(if_exception);
7592
7593 // Handle success: store the return value(s).
7594 SetControl(if_success);
7595 pos = 0;
7596 offset = 0;
7597 for (wasm::ValueType type : sig_->returns()) {
7598 Node* value = sig_->return_count() == 1
7599 ? call
7600 : graph()->NewNode(mcgraph()->common()->Projection(pos),
7601 call, control());
7602 SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
7603 Int32Constant(offset), value, effect(),
7604 control()));
7605 offset += type.value_kind_size();
7606 pos++;
7607 }
7608
7609 Return(mcgraph()->IntPtrConstant(0));
7610
7611 if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
7612 // No special lowering should be requested in the C entry.
7613 DCHECK_NULL(lowering_special_case_)((void) 0);
7614
7615 MachineRepresentation sig_reps[] = {
7616 MachineType::PointerRepresentation(), // return value
7617 MachineType::PointerRepresentation(), // target
7618 MachineRepresentation::kTagged, // object_ref
7619 MachineType::PointerRepresentation(), // argv
7620 MachineType::PointerRepresentation() // c_entry_fp
7621 };
7622 Signature<MachineRepresentation> c_entry_sig(1, 4, sig_reps);
7623 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
7624 mcgraph()->common(), gasm_->simplified(),
7625 mcgraph()->zone(), &c_entry_sig);
7626 r.LowerGraph();
7627 }
7628 }
7629
7630 private:
7631 const wasm::WasmModule* module_;
7632 StubCallMode stub_mode_;
7633 SetOncePointer<const Operator> int32_to_heapnumber_operator_;
7634 SetOncePointer<const Operator> tagged_non_smi_to_int32_operator_;
7635 SetOncePointer<const Operator> float32_to_number_operator_;
7636 SetOncePointer<const Operator> float64_to_number_operator_;
7637 SetOncePointer<const Operator> tagged_to_float64_operator_;
7638 wasm::WasmFeatures enabled_features_;
7639 CallDescriptor* bigint_to_i64_descriptor_ = nullptr;
7640 CallDescriptor* i64_to_bigint_descriptor_ = nullptr;
7641};
7642
7643} // namespace
7644
7645void BuildInlinedJSToWasmWrapper(
7646 Zone* zone, MachineGraph* mcgraph, const wasm::FunctionSig* signature,
7647 const wasm::WasmModule* module, Isolate* isolate,
7648 compiler::SourcePositionTable* spt, StubCallMode stub_mode,
7649 wasm::WasmFeatures features, const JSWasmCallData* js_wasm_call_data,
7650 Node* frame_state) {
7651 WasmWrapperGraphBuilder builder(zone, mcgraph, signature, module,
7652 WasmGraphBuilder::kNoSpecialParameterMode,
7653 isolate, spt, stub_mode, features);
7654 builder.BuildJSToWasmWrapper(false, js_wasm_call_data, frame_state);
7655}
7656
7657std::unique_ptr<TurbofanCompilationJob> NewJSToWasmCompilationJob(
7658 Isolate* isolate, const wasm::FunctionSig* sig,
7659 const wasm::WasmModule* module, bool is_import,
7660 const wasm::WasmFeatures& enabled_features) {
7661 //----------------------------------------------------------------------------
7662 // Create the Graph.
7663 //----------------------------------------------------------------------------
7664 std::unique_ptr<Zone> zone = std::make_unique<Zone>(
7665 wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
7666 Graph* graph = zone->New<Graph>(zone.get());
7667 CommonOperatorBuilder* common = zone->New<CommonOperatorBuilder>(zone.get());
7668 MachineOperatorBuilder* machine = zone->New<MachineOperatorBuilder>(
7669 zone.get(), MachineType::PointerRepresentation(),
7670 InstructionSelector::SupportedMachineOperatorFlags(),
7671 InstructionSelector::AlignmentRequirements());
7672 MachineGraph* mcgraph = zone->New<MachineGraph>(graph, common, machine);
7673
7674 WasmWrapperGraphBuilder builder(
7675 zone.get(), mcgraph, sig, module,
7676 WasmGraphBuilder::kNoSpecialParameterMode, isolate, nullptr,
7677 StubCallMode::kCallBuiltinPointer, enabled_features);
7678 builder.BuildJSToWasmWrapper(is_import);
7679
7680 //----------------------------------------------------------------------------
7681 // Create the compilation job.
7682 //----------------------------------------------------------------------------
7683 std::unique_ptr<char[]> debug_name = WasmExportedFunction::GetDebugName(sig);
7684
7685 int params = static_cast<int>(sig->parameter_count());
7686 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
7687 zone.get(), false, params + 1, CallDescriptor::kNoFlags);
7688
7689 return Pipeline::NewWasmHeapStubCompilationJob(
7690 isolate, incoming, std::move(zone), graph, CodeKind::JS_TO_WASM_FUNCTION,
7691 std::move(debug_name), WasmAssemblerOptions());
7692}
7693
7694static MachineRepresentation NormalizeFastApiRepresentation(
7695 const CTypeInfo& info) {
7696 MachineType t = MachineType::TypeForCType(info);
7697 // Wasm representation of bool is i32 instead of i1.
7698 if (t.semantic() == MachineSemantic::kBool) {
7699 return MachineRepresentation::kWord32;
7700 }
7701 return t.representation();
7702}
7703
7704static bool IsSupportedWasmFastApiFunction(
7705 const wasm::FunctionSig* expected_sig, Handle<SharedFunctionInfo> shared) {
7706 if (!shared->IsApiFunction()) {
7707 return false;
7708 }
7709 if (shared->get_api_func_data().GetCFunctionsCount() == 0) {
7710 return false;
7711 }
7712 if (!shared->get_api_func_data().accept_any_receiver()) {
7713 return false;
7714 }
7715 if (!shared->get_api_func_data().signature().IsUndefined()) {
7716 // TODO(wasm): CFunctionInfo* signature check.
7717 return false;
7718 }
7719 const CFunctionInfo* info = shared->get_api_func_data().GetCSignature(0);
7720 if (!fast_api_call::CanOptimizeFastSignature(info)) {
7721 return false;
7722 }
7723 // Options are not supported yet.
7724 if (info->HasOptions()) {
7725 return false;
7726 }
7727
7728 const auto log_imported_function_mismatch = [&shared]() {
7729 if (FLAG_trace_opt) {
7730 CodeTracer::Scope scope(shared->GetIsolate()->GetCodeTracer());
7731 PrintF(scope.file(), "[disabled optimization for ");
7732 shared->ShortPrint(scope.file());
7733 PrintF(scope.file(),
7734 ", reason: the signature of the imported function in the Wasm "
7735 "module doesn't match that of the Fast API function]\n");
7736 }
7737 };
7738
7739 // C functions only have one return value.
7740 if (expected_sig->return_count() > 1) {
7741 // Here and below, we log when the function we call is declared as an Api
7742 // function but we cannot optimize the call, which might be unxepected. In
7743 // that case we use the "slow" path making a normal Wasm->JS call and
7744 // calling the "slow" callback specified in FunctionTemplate::New().
7745 log_imported_function_mismatch();
7746 return false;
7747 }
7748 CTypeInfo return_info = info->ReturnInfo();
7749 // Unsupported if return type doesn't match.
7750 if (expected_sig->return_count() == 0 &&
7751 return_info.GetType() != CTypeInfo::Type::kVoid) {
7752 log_imported_function_mismatch();
7753 return false;
7754 }
7755 // Unsupported if return type doesn't match.
7756 if (expected_sig->return_count() == 1) {
7757 if (return_info.GetType() == CTypeInfo::Type::kVoid) {
7758 log_imported_function_mismatch();
7759 return false;
7760 }
7761 if (NormalizeFastApiRepresentation(return_info) !=
7762 expected_sig->GetReturn(0).machine_type().representation()) {
7763 log_imported_function_mismatch();
7764 return false;
7765 }
7766 }
7767 // Unsupported if arity doesn't match.
7768 if (expected_sig->parameter_count() != info->ArgumentCount() - 1) {
7769 log_imported_function_mismatch();
7770 return false;
7771 }
7772 // Unsupported if any argument types don't match.
7773 for (unsigned int i = 0; i < expected_sig->parameter_count(); i += 1) {
7774 // Arg 0 is the receiver, skip over it since wasm doesn't
7775 // have a concept of receivers.
7776 CTypeInfo arg = info->ArgumentInfo(i + 1);
7777 if (NormalizeFastApiRepresentation(arg) !=
7778 expected_sig->GetParam(i).machine_type().representation()) {
7779 log_imported_function_mismatch();
7780 return false;
7781 }
7782 }
7783 return true;
7784}
7785
7786WasmImportData ResolveWasmImportCall(
7787 Handle<JSReceiver> callable, const wasm::FunctionSig* expected_sig,
7788 const wasm::WasmModule* module,
7789 const wasm::WasmFeatures& enabled_features) {
7790 Isolate* isolate = callable->GetIsolate();
7791 Handle<HeapObject> no_suspender;
7792 if (WasmExportedFunction::IsWasmExportedFunction(*callable)) {
7793 auto imported_function = Handle<WasmExportedFunction>::cast(callable);
7794 if (!imported_function->MatchesSignature(module, expected_sig)) {
7795 return {WasmImportCallKind::kLinkError, callable, no_suspender};
7796 }
7797 uint32_t func_index =
7798 static_cast<uint32_t>(imported_function->function_index());
7799 if (func_index >=
7800 imported_function->instance().module()->num_imported_functions) {
7801 return {WasmImportCallKind::kWasmToWasm, callable, no_suspender};
7802 }
7803 // Resolve the shortcut to the underlying callable and continue.
7804 Handle<WasmInstanceObject> instance(imported_function->instance(), isolate);
7805 ImportedFunctionEntry entry(instance, func_index);
7806 callable = handle(entry.callable(), isolate);
7807 }
7808 Handle<HeapObject> suspender = isolate->factory()->undefined_value();
7809 if (WasmJSFunction::IsWasmJSFunction(*callable)) {
7810 auto js_function = Handle<WasmJSFunction>::cast(callable);
7811 suspender = handle(js_function->GetSuspender(), isolate);
7812 if ((!suspender->IsUndefined() &&
7813 !js_function->MatchesSignatureForSuspend(expected_sig)) ||
7814 (suspender->IsUndefined() &&
7815 !js_function->MatchesSignature(expected_sig))) {
7816 return {WasmImportCallKind::kLinkError, callable, no_suspender};
7817 }
7818 // Resolve the short-cut to the underlying callable and continue.
7819 callable = handle(js_function->GetCallable(), isolate);
7820 }
7821 if (WasmCapiFunction::IsWasmCapiFunction(*callable)) {
7822 auto capi_function = Handle<WasmCapiFunction>::cast(callable);
7823 if (!capi_function->MatchesSignature(expected_sig)) {
7824 return {WasmImportCallKind::kLinkError, callable, no_suspender};
7825 }
7826 return {WasmImportCallKind::kWasmToCapi, callable, no_suspender};
7827 }
7828 // Assuming we are calling to JS, check whether this would be a runtime error.
7829 if (!wasm::IsJSCompatibleSignature(expected_sig, module, enabled_features)) {
7830 return {WasmImportCallKind::kRuntimeTypeError, callable, no_suspender};
7831 }
7832 // Check if this can be a JS fast API call.
7833 if (FLAG_turbo_fast_api_calls &&
7834 (callable->IsJSFunction() || callable->IsJSBoundFunction())) {
7835 Handle<JSFunction> target;
7836 if (callable->IsJSBoundFunction()) {
7837 Handle<JSBoundFunction> bound_target =
7838 Handle<JSBoundFunction>::cast(callable);
7839 // Nested bound functions and arguments not supported yet.
7840 if (bound_target->bound_arguments().length() == 0 &&
7841 !bound_target->bound_target_function().IsJSBoundFunction()) {
7842 Handle<JSReceiver> bound_target_function = handle(
7843 bound_target->bound_target_function(), callable->GetIsolate());
7844 if (bound_target_function->IsJSFunction()) {
7845 target = Handle<JSFunction>::cast(bound_target_function);
7846 }
7847 }
7848 } else {
7849 DCHECK(callable->IsJSFunction())((void) 0);
7850 target = Handle<JSFunction>::cast(callable);
7851 }
7852
7853 if (!target.is_null()) {
7854 Handle<SharedFunctionInfo> shared(target->shared(), target->GetIsolate());
7855
7856 if (IsSupportedWasmFastApiFunction(expected_sig, shared)) {
7857 return {WasmImportCallKind::kWasmToJSFastApi, target, no_suspender};
7858 }
7859 }
7860 }
7861 // For JavaScript calls, determine whether the target has an arity match.
7862 if (callable->IsJSFunction()) {
7863 Handle<JSFunction> function = Handle<JSFunction>::cast(callable);
7864 Handle<SharedFunctionInfo> shared(function->shared(),
7865 function->GetIsolate());
7866
7867// Check for math intrinsics.
7868#define COMPARE_SIG_FOR_BUILTIN(name) \
7869 { \
7870 const wasm::FunctionSig* sig = \
7871 wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
7872 if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name); \
7873 DCHECK_NOT_NULL(sig)((void) 0); \
7874 if (*expected_sig == *sig) { \
7875 return {WasmImportCallKind::k##name, callable, no_suspender}; \
7876 } \
7877 }
7878#define COMPARE_SIG_FOR_BUILTIN_F64(name) \
7879 case Builtin::kMath##name: \
7880 COMPARE_SIG_FOR_BUILTIN(F64##name); \
7881 break;
7882#define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
7883 case Builtin::kMath##name: \
7884 COMPARE_SIG_FOR_BUILTIN(F64##name); \
7885 COMPARE_SIG_FOR_BUILTIN(F32##name); \
7886 break;
7887
7888 if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
7889 switch (shared->builtin_id()) {
7890 COMPARE_SIG_FOR_BUILTIN_F64(Acos);
7891 COMPARE_SIG_FOR_BUILTIN_F64(Asin);
7892 COMPARE_SIG_FOR_BUILTIN_F64(Atan);
7893 COMPARE_SIG_FOR_BUILTIN_F64(Cos);
7894 COMPARE_SIG_FOR_BUILTIN_F64(Sin);
7895 COMPARE_SIG_FOR_BUILTIN_F64(Tan);
7896 COMPARE_SIG_FOR_BUILTIN_F64(Exp);
7897 COMPARE_SIG_FOR_BUILTIN_F64(Log);
7898 COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
7899 COMPARE_SIG_FOR_BUILTIN_F64(Pow);
7900 COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
7901 COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
7902 COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
7903 COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
7904 COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
7905 COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
7906 case Builtin::kMathFround:
7907 COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
7908 break;
7909 default:
7910 break;
7911 }
7912 }
7913
7914#undef COMPARE_SIG_FOR_BUILTIN
7915#undef COMPARE_SIG_FOR_BUILTIN_F64
7916#undef COMPARE_SIG_FOR_BUILTIN_F32_F64
7917
7918 if (IsClassConstructor(shared->kind())) {
7919 // Class constructor will throw anyway.
7920 return {WasmImportCallKind::kUseCallBuiltin, callable, suspender};
7921 }
7922
7923 if (shared->internal_formal_parameter_count_without_receiver() ==
7924 expected_sig->parameter_count()) {
7925 return {WasmImportCallKind::kJSFunctionArityMatch, callable, suspender};
7926 }
7927
7928 // If function isn't compiled, compile it now.
7929 Isolate* isolate = callable->GetIsolate();
7930 IsCompiledScope is_compiled_scope(shared->is_compiled_scope(isolate));
7931 if (!is_compiled_scope.is_compiled()) {
7932 Compiler::Compile(isolate, function, Compiler::CLEAR_EXCEPTION,
7933 &is_compiled_scope);
7934 }
7935
7936 return {WasmImportCallKind::kJSFunctionArityMismatch, callable, suspender};
7937 }
7938 // Unknown case. Use the call builtin.
7939 return {WasmImportCallKind::kUseCallBuiltin, callable, suspender};
7940}
7941
7942namespace {
7943
7944wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
7945 const char** name_ptr) {
7946#define CASE(name) \
7947 case WasmImportCallKind::k##name: \
7948 *name_ptr = "WasmMathIntrinsic:" #name; \
7949 return wasm::kExpr##name
7950 switch (kind) {
7951 CASE(F64Acos);
7952 CASE(F64Asin);
7953 CASE(F64Atan);
7954 CASE(F64Cos);
7955 CASE(F64Sin);
7956 CASE(F64Tan);
7957 CASE(F64Exp);
7958 CASE(F64Log);
7959 CASE(F64Atan2);
7960 CASE(F64Pow);
7961 CASE(F64Ceil);
7962 CASE(F64Floor);
7963 CASE(F64Sqrt);
7964 CASE(F64Min);
7965 CASE(F64Max);
7966 CASE(F64Abs);
7967 CASE(F32Min);
7968 CASE(F32Max);
7969 CASE(F32Abs);
7970 CASE(F32Ceil);
7971 CASE(F32Floor);
7972 CASE(F32Sqrt);
7973 CASE(F32ConvertF64);
7974 default:
7975 UNREACHABLE()V8_Fatal("unreachable code");
7976 }
7977#undef CASE
7978}
7979
7980wasm::WasmCompilationResult CompileWasmMathIntrinsic(
7981 WasmImportCallKind kind, const wasm::FunctionSig* sig) {
7982 DCHECK_EQ(1, sig->return_count())((void) 0);
7983
7984 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),static v8::base::AtomicWord trace_event_unique_atomic7985 = 0
; const uint8_t* trace_event_unique_category_group_enabled7985
; trace_event_unique_category_group_enabled7985 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic7985
))); if (!trace_event_unique_category_group_enabled7985) { trace_event_unique_category_group_enabled7985
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic7985
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled7985
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer7985
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled7985
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled7985, "wasm.CompileWasmMathIntrinsic"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer7985 .Initialize(trace_event_unique_category_group_enabled7985
, "wasm.CompileWasmMathIntrinsic", h); }
7985 "wasm.CompileWasmMathIntrinsic")static v8::base::AtomicWord trace_event_unique_atomic7985 = 0
; const uint8_t* trace_event_unique_category_group_enabled7985
; trace_event_unique_category_group_enabled7985 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic7985
))); if (!trace_event_unique_category_group_enabled7985) { trace_event_unique_category_group_enabled7985
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic7985
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled7985
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer7985
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled7985
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled7985, "wasm.CompileWasmMathIntrinsic"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer7985 .Initialize(trace_event_unique_category_group_enabled7985
, "wasm.CompileWasmMathIntrinsic", h); }
;
7986
7987 Zone zone(wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
7988
7989 // Compile a Wasm function with a single bytecode and let TurboFan
7990 // generate either inlined machine code or a call to a helper.
7991 SourcePositionTable* source_positions = nullptr;
7992 MachineGraph* mcgraph = zone.New<MachineGraph>(
7993 zone.New<Graph>(&zone), zone.New<CommonOperatorBuilder>(&zone),
7994 zone.New<MachineOperatorBuilder>(
7995 &zone, MachineType::PointerRepresentation(),
7996 InstructionSelector::SupportedMachineOperatorFlags(),
7997 InstructionSelector::AlignmentRequirements()));
7998
7999 wasm::CompilationEnv env(
8000 nullptr, wasm::kNoBoundsChecks,
8001 wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
8002 wasm::WasmFeatures::All(), wasm::DynamicTiering::kDisabled);
8003
8004 WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
8005 source_positions);
8006
8007 // Set up the graph start.
8008 builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
8009
8010 // Generate either a unop or a binop.
8011 Node* node = nullptr;
8012 const char* debug_name = "WasmMathIntrinsic";
8013 auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
8014 switch (sig->parameter_count()) {
8015 case 1:
8016 node = builder.Unop(opcode, builder.Param(1));
8017 break;
8018 case 2:
8019 node = builder.Binop(opcode, builder.Param(1), builder.Param(2));
8020 break;
8021 default:
8022 UNREACHABLE()V8_Fatal("unreachable code");
8023 }
8024
8025 builder.Return(node);
8026
8027 // Run the compiler pipeline to generate machine code.
8028 auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
8029 if (mcgraph->machine()->Is32()) {
8030 call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
8031 }
8032
8033 // The code does not call to JS, but conceptually it is an import wrapper,
8034 // hence use {WASM_TO_JS_FUNCTION} here.
8035 // TODO(wasm): Rename this to {WASM_IMPORT_CALL}?
8036 return Pipeline::GenerateCodeForWasmNativeStub(
8037 call_descriptor, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, debug_name,
8038 WasmStubAssemblerOptions(), source_positions);
8039}
8040
8041} // namespace
8042
8043wasm::WasmCompilationResult CompileWasmImportCallWrapper(
8044 wasm::CompilationEnv* env, WasmImportCallKind kind,
8045 const wasm::FunctionSig* sig, bool source_positions, int expected_arity,
8046 wasm::Suspend suspend) {
8047 DCHECK_NE(WasmImportCallKind::kLinkError, kind)((void) 0);
8048 DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind)((void) 0);
8049 DCHECK_NE(WasmImportCallKind::kWasmToJSFastApi, kind)((void) 0);
8050
8051 // Check for math intrinsics first.
8052 if (FLAG_wasm_math_intrinsics &&
8053 kind >= WasmImportCallKind::kFirstMathIntrinsic &&
8054 kind <= WasmImportCallKind::kLastMathIntrinsic) {
8055 return CompileWasmMathIntrinsic(kind, sig);
8056 }
8057
8058 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),static v8::base::AtomicWord trace_event_unique_atomic8059 = 0
; const uint8_t* trace_event_unique_category_group_enabled8059
; trace_event_unique_category_group_enabled8059 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8059
))); if (!trace_event_unique_category_group_enabled8059) { trace_event_unique_category_group_enabled8059
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8059
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8059
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8059
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8059
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8059, "wasm.CompileWasmImportCallWrapper"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8059 .Initialize(trace_event_unique_category_group_enabled8059
, "wasm.CompileWasmImportCallWrapper", h); }
8059 "wasm.CompileWasmImportCallWrapper")static v8::base::AtomicWord trace_event_unique_atomic8059 = 0
; const uint8_t* trace_event_unique_category_group_enabled8059
; trace_event_unique_category_group_enabled8059 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8059
))); if (!trace_event_unique_category_group_enabled8059) { trace_event_unique_category_group_enabled8059
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8059
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8059
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8059
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8059
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8059, "wasm.CompileWasmImportCallWrapper"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8059 .Initialize(trace_event_unique_category_group_enabled8059
, "wasm.CompileWasmImportCallWrapper", h); }
;
8060 base::TimeTicks start_time;
8061 if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)(__builtin_expect(!!(FLAG_trace_wasm_compilation_times), 0))) {
8062 start_time = base::TimeTicks::Now();
8063 }
8064
8065 //----------------------------------------------------------------------------
8066 // Create the Graph
8067 //----------------------------------------------------------------------------
8068 Zone zone(wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8069 Graph* graph = zone.New<Graph>(&zone);
8070 CommonOperatorBuilder* common = zone.New<CommonOperatorBuilder>(&zone);
8071 MachineOperatorBuilder* machine = zone.New<MachineOperatorBuilder>(
8072 &zone, MachineType::PointerRepresentation(),
8073 InstructionSelector::SupportedMachineOperatorFlags(),
8074 InstructionSelector::AlignmentRequirements());
8075 MachineGraph* mcgraph = zone.New<MachineGraph>(graph, common, machine);
8076
8077 SourcePositionTable* source_position_table =
8078 source_positions ? zone.New<SourcePositionTable>(graph) : nullptr;
8079
8080 WasmWrapperGraphBuilder builder(
8081 &zone, mcgraph, sig, env->module,
8082 WasmGraphBuilder::kWasmApiFunctionRefMode, nullptr, source_position_table,
8083 StubCallMode::kCallWasmRuntimeStub, env->enabled_features);
8084 builder.BuildWasmToJSWrapper(kind, expected_arity, suspend);
8085
8086 // Build a name in the form "wasm-to-js-<kind>-<signature>".
8087 constexpr size_t kMaxNameLen = 128;
8088 char func_name[kMaxNameLen];
8089 int name_prefix_len = SNPrintF(base::VectorOf(func_name, kMaxNameLen),
8090 "wasm-to-js-%d-", static_cast<int>(kind));
8091 PrintSignature(base::VectorOf(func_name, kMaxNameLen) + name_prefix_len, sig,
8092 '-');
8093
8094 // Schedule and compile to machine code.
8095 CallDescriptor* incoming =
8096 GetWasmCallDescriptor(&zone, sig, WasmCallKind::kWasmImportWrapper);
8097 if (machine->Is32()) {
8098 incoming = GetI32WasmCallDescriptor(&zone, incoming);
8099 }
8100 wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
8101 incoming, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, func_name,
8102 WasmStubAssemblerOptions(), source_position_table);
8103
8104 if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)(__builtin_expect(!!(FLAG_trace_wasm_compilation_times), 0))) {
8105 base::TimeDelta time = base::TimeTicks::Now() - start_time;
8106 int codesize = result.code_desc.body_size();
8107 StdoutStream{} << "Compiled WasmToJS wrapper " << func_name << ", took "
8108 << time.InMilliseconds() << " ms; codesize " << codesize
8109 << std::endl;
8110 }
8111
8112 return result;
8113}
8114
8115wasm::WasmCode* CompileWasmCapiCallWrapper(wasm::NativeModule* native_module,
8116 const wasm::FunctionSig* sig) {
8117 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),static v8::base::AtomicWord trace_event_unique_atomic8118 = 0
; const uint8_t* trace_event_unique_category_group_enabled8118
; trace_event_unique_category_group_enabled8118 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8118
))); if (!trace_event_unique_category_group_enabled8118) { trace_event_unique_category_group_enabled8118
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8118
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8118
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8118
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8118
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8118, "wasm.CompileWasmCapiFunction"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8118 .Initialize(trace_event_unique_category_group_enabled8118
, "wasm.CompileWasmCapiFunction", h); }
8118 "wasm.CompileWasmCapiFunction")static v8::base::AtomicWord trace_event_unique_atomic8118 = 0
; const uint8_t* trace_event_unique_category_group_enabled8118
; trace_event_unique_category_group_enabled8118 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8118
))); if (!trace_event_unique_category_group_enabled8118) { trace_event_unique_category_group_enabled8118
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8118
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8118
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8118
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8118
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8118, "wasm.CompileWasmCapiFunction"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8118 .Initialize(trace_event_unique_category_group_enabled8118
, "wasm.CompileWasmCapiFunction", h); }
;
8119
8120 Zone zone(wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8121
8122 // TODO(jkummerow): Extract common code into helper method.
8123 SourcePositionTable* source_positions = nullptr;
8124 MachineGraph* mcgraph = zone.New<MachineGraph>(
8125 zone.New<Graph>(&zone), zone.New<CommonOperatorBuilder>(&zone),
8126 zone.New<MachineOperatorBuilder>(
8127 &zone, MachineType::PointerRepresentation(),
8128 InstructionSelector::SupportedMachineOperatorFlags(),
8129 InstructionSelector::AlignmentRequirements()));
8130
8131 WasmWrapperGraphBuilder builder(
8132 &zone, mcgraph, sig, native_module->module(),
8133 WasmGraphBuilder::kWasmApiFunctionRefMode, nullptr, source_positions,
8134 StubCallMode::kCallWasmRuntimeStub, native_module->enabled_features());
8135
8136 builder.BuildCapiCallWrapper();
8137
8138 // Run the compiler pipeline to generate machine code.
8139 CallDescriptor* call_descriptor =
8140 GetWasmCallDescriptor(&zone, sig, WasmCallKind::kWasmCapiFunction);
8141 if (mcgraph->machine()->Is32()) {
8142 call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
8143 }
8144
8145 const char* debug_name = "WasmCapiCall";
8146 wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
8147 call_descriptor, mcgraph, CodeKind::WASM_TO_CAPI_FUNCTION, debug_name,
8148 WasmStubAssemblerOptions(), source_positions);
8149 wasm::WasmCode* published_code;
8150 {
8151 wasm::CodeSpaceWriteScope code_space_write_scope(native_module);
8152 std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
8153 wasm::kAnonymousFuncIndex, result.code_desc, result.frame_slot_count,
8154 result.tagged_parameter_slots,
8155 result.protected_instructions_data.as_vector(),
8156 result.source_positions.as_vector(), wasm::WasmCode::kWasmToCapiWrapper,
8157 wasm::ExecutionTier::kNone, wasm::kNoDebugging);
8158 published_code = native_module->PublishCode(std::move(wasm_code));
8159 }
8160 return published_code;
8161}
8162
8163wasm::WasmCode* CompileWasmJSFastCallWrapper(wasm::NativeModule* native_module,
8164 const wasm::FunctionSig* sig,
8165 Handle<JSFunction> target) {
8166 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),static v8::base::AtomicWord trace_event_unique_atomic8167 = 0
; const uint8_t* trace_event_unique_category_group_enabled8167
; trace_event_unique_category_group_enabled8167 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8167
))); if (!trace_event_unique_category_group_enabled8167) { trace_event_unique_category_group_enabled8167
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8167
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8167
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8167
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8167
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8167, "wasm.CompileWasmJSFastCallWrapper"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8167 .Initialize(trace_event_unique_category_group_enabled8167
, "wasm.CompileWasmJSFastCallWrapper", h); }
8167 "wasm.CompileWasmJSFastCallWrapper")static v8::base::AtomicWord trace_event_unique_atomic8167 = 0
; const uint8_t* trace_event_unique_category_group_enabled8167
; trace_event_unique_category_group_enabled8167 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8167
))); if (!trace_event_unique_category_group_enabled8167) { trace_event_unique_category_group_enabled8167
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8167
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8167
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8167
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8167
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8167, "wasm.CompileWasmJSFastCallWrapper"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0))); trace_event_unique_tracer8167 .Initialize(trace_event_unique_category_group_enabled8167
, "wasm.CompileWasmJSFastCallWrapper", h); }
;
8168
8169 Zone zone(wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8170
8171 // TODO(jkummerow): Extract common code into helper method.
8172 SourcePositionTable* source_positions = nullptr;
8173 MachineGraph* mcgraph = zone.New<MachineGraph>(
8174 zone.New<Graph>(&zone), zone.New<CommonOperatorBuilder>(&zone),
8175 zone.New<MachineOperatorBuilder>(
8176 &zone, MachineType::PointerRepresentation(),
8177 InstructionSelector::SupportedMachineOperatorFlags(),
8178 InstructionSelector::AlignmentRequirements()));
8179
8180 WasmWrapperGraphBuilder builder(
8181 &zone, mcgraph, sig, native_module->module(),
8182 WasmGraphBuilder::kWasmApiFunctionRefMode, nullptr, source_positions,
8183 StubCallMode::kCallWasmRuntimeStub, native_module->enabled_features());
8184
8185 // Set up the graph start.
8186 int param_count = static_cast<int>(sig->parameter_count()) +
8187 1 /* offset for first parameter index being -1 */ +
8188 1 /* Wasm instance */ + 1 /* kExtraCallableParam */;
8189 builder.Start(param_count);
8190 builder.BuildJSFastApiCallWrapper(target);
8191
8192 // Run the compiler pipeline to generate machine code.
8193 CallDescriptor* call_descriptor =
8194 GetWasmCallDescriptor(&zone, sig, WasmCallKind::kWasmImportWrapper);
8195 if (mcgraph->machine()->Is32()) {
8196 call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
8197 }
8198
8199 const char* debug_name = "WasmJSFastApiCall";
8200 wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
8201 call_descriptor, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, debug_name,
8202 WasmStubAssemblerOptions(), source_positions);
8203 {
8204 wasm::CodeSpaceWriteScope code_space_write_scope(native_module);
8205 std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
8206 wasm::kAnonymousFuncIndex, result.code_desc, result.frame_slot_count,
8207 result.tagged_parameter_slots,
8208 result.protected_instructions_data.as_vector(),
8209 result.source_positions.as_vector(), wasm::WasmCode::kWasmToJsWrapper,
8210 wasm::ExecutionTier::kNone, wasm::kNoDebugging);
8211 return native_module->PublishCode(std::move(wasm_code));
8212 }
8213}
8214
8215MaybeHandle<Code> CompileWasmToJSWrapper(Isolate* isolate,
8216 const wasm::FunctionSig* sig,
8217 WasmImportCallKind kind,
8218 int expected_arity,
8219 wasm::Suspend suspend) {
8220 std::unique_ptr<Zone> zone = std::make_unique<Zone>(
8221 isolate->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8222
8223 // Create the Graph
8224 Graph* graph = zone->New<Graph>(zone.get());
8225 CommonOperatorBuilder* common = zone->New<CommonOperatorBuilder>(zone.get());
8226 MachineOperatorBuilder* machine = zone->New<MachineOperatorBuilder>(
8227 zone.get(), MachineType::PointerRepresentation(),
8228 InstructionSelector::SupportedMachineOperatorFlags(),
8229 InstructionSelector::AlignmentRequirements());
8230 MachineGraph* mcgraph = zone->New<MachineGraph>(graph, common, machine);
8231
8232 WasmWrapperGraphBuilder builder(zone.get(), mcgraph, sig, nullptr,
8233 WasmGraphBuilder::kWasmApiFunctionRefMode,
8234 nullptr, nullptr,
8235 StubCallMode::kCallBuiltinPointer,
8236 wasm::WasmFeatures::FromIsolate(isolate));
8237 builder.BuildWasmToJSWrapper(kind, expected_arity, suspend);
8238
8239 // Build a name in the form "wasm-to-js-<kind>-<signature>".
8240 constexpr size_t kMaxNameLen = 128;
8241 constexpr size_t kNamePrefixLen = 11;
8242 auto name_buffer = std::unique_ptr<char[]>(new char[kMaxNameLen]);
8243 memcpy(name_buffer.get(), "wasm-to-js:", kNamePrefixLen);
8244 PrintSignature(
8245 base::VectorOf(name_buffer.get(), kMaxNameLen) + kNamePrefixLen, sig);
8246
8247 // Generate the call descriptor.
8248 CallDescriptor* incoming =
8249 GetWasmCallDescriptor(zone.get(), sig, WasmCallKind::kWasmImportWrapper);
8250
8251 // Run the compilation job synchronously.
8252 std::unique_ptr<TurbofanCompilationJob> job(
8253 Pipeline::NewWasmHeapStubCompilationJob(
8254 isolate, incoming, std::move(zone), graph,
8255 CodeKind::WASM_TO_JS_FUNCTION, std::move(name_buffer),
8256 AssemblerOptions::Default(isolate)));
8257
8258 // Compile the wrapper
8259 if (job->ExecuteJob(isolate->counters()->runtime_call_stats()) ==
8260 CompilationJob::FAILED ||
8261 job->FinalizeJob(isolate) == CompilationJob::FAILED) {
8262 return Handle<Code>();
8263 }
8264 Handle<Code> code = job->compilation_info()->code();
8265 return code;
8266}
8267
8268MaybeHandle<Code> CompileJSToJSWrapper(Isolate* isolate,
8269 const wasm::FunctionSig* sig,
8270 const wasm::WasmModule* module) {
8271 std::unique_ptr<Zone> zone = std::make_unique<Zone>(
8272 isolate->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8273 Graph* graph = zone->New<Graph>(zone.get());
8274 CommonOperatorBuilder* common = zone->New<CommonOperatorBuilder>(zone.get());
8275 MachineOperatorBuilder* machine = zone->New<MachineOperatorBuilder>(
8276 zone.get(), MachineType::PointerRepresentation(),
8277 InstructionSelector::SupportedMachineOperatorFlags(),
8278 InstructionSelector::AlignmentRequirements());
8279 MachineGraph* mcgraph = zone->New<MachineGraph>(graph, common, machine);
8280
8281 WasmWrapperGraphBuilder builder(zone.get(), mcgraph, sig, module,
8282 WasmGraphBuilder::kNoSpecialParameterMode,
8283 isolate, nullptr,
8284 StubCallMode::kCallBuiltinPointer,
8285 wasm::WasmFeatures::FromIsolate(isolate));
8286 builder.BuildJSToJSWrapper();
8287
8288 int wasm_count = static_cast<int>(sig->parameter_count());
8289 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
8290 zone.get(), false, wasm_count + 1, CallDescriptor::kNoFlags);
8291
8292 // Build a name in the form "js-to-js:<params>:<returns>".
8293 constexpr size_t kMaxNameLen = 128;
8294 constexpr size_t kNamePrefixLen = 9;
8295 auto name_buffer = std::unique_ptr<char[]>(new char[kMaxNameLen]);
8296 memcpy(name_buffer.get(), "js-to-js:", kNamePrefixLen);
8297 PrintSignature(
8298 base::VectorOf(name_buffer.get(), kMaxNameLen) + kNamePrefixLen, sig);
8299
8300 // Run the compilation job synchronously.
8301 std::unique_ptr<TurbofanCompilationJob> job(
8302 Pipeline::NewWasmHeapStubCompilationJob(
8303 isolate, incoming, std::move(zone), graph,
8304 CodeKind::JS_TO_JS_FUNCTION, std::move(name_buffer),
8305 AssemblerOptions::Default(isolate)));
8306
8307 if (job->ExecuteJob(isolate->counters()->runtime_call_stats()) ==
8308 CompilationJob::FAILED ||
8309 job->FinalizeJob(isolate) == CompilationJob::FAILED) {
8310 return {};
8311 }
8312 Handle<Code> code = job->compilation_info()->code();
8313
8314 return code;
8315}
8316
8317Handle<CodeT> CompileCWasmEntry(Isolate* isolate, const wasm::FunctionSig* sig,
8318 const wasm::WasmModule* module) {
8319 std::unique_ptr<Zone> zone = std::make_unique<Zone>(
8320 isolate->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8321 Graph* graph = zone->New<Graph>(zone.get());
8322 CommonOperatorBuilder* common = zone->New<CommonOperatorBuilder>(zone.get());
8323 MachineOperatorBuilder* machine = zone->New<MachineOperatorBuilder>(
8324 zone.get(), MachineType::PointerRepresentation(),
8325 InstructionSelector::SupportedMachineOperatorFlags(),
8326 InstructionSelector::AlignmentRequirements());
8327 MachineGraph* mcgraph = zone->New<MachineGraph>(graph, common, machine);
8328
8329 WasmWrapperGraphBuilder builder(zone.get(), mcgraph, sig, module,
8330 WasmGraphBuilder::kWasmApiFunctionRefMode,
8331 nullptr, nullptr,
8332 StubCallMode::kCallBuiltinPointer,
8333 wasm::WasmFeatures::FromIsolate(isolate));
8334 builder.BuildCWasmEntry();
8335
8336 // Schedule and compile to machine code.
8337 MachineType sig_types[] = {MachineType::Pointer(), // return
8338 MachineType::Pointer(), // target
8339 MachineType::AnyTagged(), // object_ref
8340 MachineType::Pointer(), // argv
8341 MachineType::Pointer()}; // c_entry_fp
8342 MachineSignature incoming_sig(1, 4, sig_types);
8343 // Traps need the root register, for TailCallRuntime to call
8344 // Runtime::kThrowWasmError.
8345 CallDescriptor::Flags flags = CallDescriptor::kInitializeRootRegister;
8346 CallDescriptor* incoming =
8347 Linkage::GetSimplifiedCDescriptor(zone.get(), &incoming_sig, flags);
8348
8349 // Build a name in the form "c-wasm-entry:<params>:<returns>".
8350 constexpr size_t kMaxNameLen = 128;
8351 constexpr size_t kNamePrefixLen = 13;
8352 auto name_buffer = std::unique_ptr<char[]>(new char[kMaxNameLen]);
8353 memcpy(name_buffer.get(), "c-wasm-entry:", kNamePrefixLen);
8354 PrintSignature(
8355 base::VectorOf(name_buffer.get(), kMaxNameLen) + kNamePrefixLen, sig);
8356
8357 // Run the compilation job synchronously.
8358 std::unique_ptr<TurbofanCompilationJob> job(
8359 Pipeline::NewWasmHeapStubCompilationJob(
8360 isolate, incoming, std::move(zone), graph, CodeKind::C_WASM_ENTRY,
8361 std::move(name_buffer), AssemblerOptions::Default(isolate)));
8362
8363 CHECK_NE(job->ExecuteJob(isolate->counters()->runtime_call_stats(), nullptr),do { bool _cmp = ::v8::base::CmpNEImpl< typename ::v8::base
::pass_value_or_ref<decltype(job->ExecuteJob(isolate->
counters()->runtime_call_stats(), nullptr))>::type, typename
::v8::base::pass_value_or_ref<decltype(CompilationJob::FAILED
)>::type>((job->ExecuteJob(isolate->counters()->
runtime_call_stats(), nullptr)), (CompilationJob::FAILED)); do
{ if ((__builtin_expect(!!(!(_cmp)), 0))) { V8_Fatal("Check failed: %s."
, "job->ExecuteJob(isolate->counters()->runtime_call_stats(), nullptr)"
" " "!=" " " "CompilationJob::FAILED"); } } while (false); }
while (false)
8364 CompilationJob::FAILED)do { bool _cmp = ::v8::base::CmpNEImpl< typename ::v8::base
::pass_value_or_ref<decltype(job->ExecuteJob(isolate->
counters()->runtime_call_stats(), nullptr))>::type, typename
::v8::base::pass_value_or_ref<decltype(CompilationJob::FAILED
)>::type>((job->ExecuteJob(isolate->counters()->
runtime_call_stats(), nullptr)), (CompilationJob::FAILED)); do
{ if ((__builtin_expect(!!(!(_cmp)), 0))) { V8_Fatal("Check failed: %s."
, "job->ExecuteJob(isolate->counters()->runtime_call_stats(), nullptr)"
" " "!=" " " "CompilationJob::FAILED"); } } while (false); }
while (false)
;
8365 CHECK_NE(job->FinalizeJob(isolate), CompilationJob::FAILED)do { bool _cmp = ::v8::base::CmpNEImpl< typename ::v8::base
::pass_value_or_ref<decltype(job->FinalizeJob(isolate))
>::type, typename ::v8::base::pass_value_or_ref<decltype
(CompilationJob::FAILED)>::type>((job->FinalizeJob(isolate
)), (CompilationJob::FAILED)); do { if ((__builtin_expect(!!(
!(_cmp)), 0))) { V8_Fatal("Check failed: %s.", "job->FinalizeJob(isolate)"
" " "!=" " " "CompilationJob::FAILED"); } } while (false); }
while (false)
;
8366
8367 return ToCodeT(job->compilation_info()->code(), isolate);
8368}
8369
8370namespace {
8371
8372bool BuildGraphForWasmFunction(wasm::CompilationEnv* env,
8373 const wasm::FunctionBody& func_body,
8374 int func_index, wasm::WasmFeatures* detected,
8375 MachineGraph* mcgraph,
8376 std::vector<compiler::WasmLoopInfo>* loop_infos,
8377 NodeOriginTable* node_origins,
8378 SourcePositionTable* source_positions) {
8379 // Create a TF graph during decoding.
8380 WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
8381 source_positions);
8382 auto* allocator = wasm::GetWasmEngine()->allocator();
8383 wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
8384 allocator, env->enabled_features, env->module, &builder, detected,
8385 func_body, loop_infos, node_origins, func_index, wasm::kRegularFunction);
8386 if (graph_construction_result.failed()) {
8387 if (FLAG_trace_wasm_compiler) {
8388 StdoutStream{} << "Compilation failed: "
8389 << graph_construction_result.error().message()
8390 << std::endl;
8391 }
8392 return false;
8393 }
8394
8395 auto sig = CreateMachineSignature(mcgraph->zone(), func_body.sig,
8396 WasmGraphBuilder::kCalledFromWasm);
8397 builder.LowerInt64(sig);
8398
8399 return true;
8400}
8401
8402base::Vector<const char> GetDebugName(Zone* zone,
8403 const wasm::WasmModule* module,
8404 const wasm::WireBytesStorage* wire_bytes,
8405 int index) {
8406 base::Optional<wasm::ModuleWireBytes> module_bytes =
8407 wire_bytes->GetModuleBytes();
8408 if (module_bytes.has_value() &&
8409 (FLAG_trace_turbo || FLAG_trace_turbo_scheduled ||
8410 FLAG_trace_turbo_graph || FLAG_print_wasm_code)) {
8411 wasm::WireBytesRef name = module->lazily_generated_names.LookupFunctionName(
8412 module_bytes.value(), index);
8413 if (!name.is_empty()) {
8414 int name_len = name.length();
8415 char* index_name = zone->NewArray<char>(name_len);
8416 memcpy(index_name, module_bytes->start() + name.offset(), name_len);
8417 return base::Vector<const char>(index_name, name_len);
8418 }
8419 }
8420
8421 constexpr int kBufferLength = 24;
8422
8423 base::EmbeddedVector<char, kBufferLength> name_vector;
8424 int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
8425 DCHECK(name_len > 0 && name_len < name_vector.length())((void) 0);
8426
8427 char* index_name = zone->NewArray<char>(name_len);
8428 memcpy(index_name, name_vector.begin(), name_len);
8429 return base::Vector<const char>(index_name, name_len);
8430}
8431
8432} // namespace
8433
8434wasm::WasmCompilationResult ExecuteTurbofanWasmCompilation(
8435 wasm::CompilationEnv* env, const wasm::WireBytesStorage* wire_byte_storage,
8436 const wasm::FunctionBody& func_body, int func_index, Counters* counters,
8437 wasm::WasmFeatures* detected) {
8438 // Check that we do not accidentally compile a Wasm function to TurboFan if
8439 // --liftoff-only is set.
8440 DCHECK(!FLAG_liftoff_only)((void) 0);
8441
8442 TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),static v8::base::AtomicWord trace_event_unique_atomic8444 = 0
; const uint8_t* trace_event_unique_category_group_enabled8444
; trace_event_unique_category_group_enabled8444 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8444
))); if (!trace_event_unique_category_group_enabled8444) { trace_event_unique_category_group_enabled8444
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8444
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8444
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8444
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8444
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0)), "func_index", func_index, "body_size", func_body
.end - func_body.start); trace_event_unique_tracer8444 .Initialize
(trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, h); }
8443 "wasm.CompileTopTier", "func_index", func_index, "body_size",static v8::base::AtomicWord trace_event_unique_atomic8444 = 0
; const uint8_t* trace_event_unique_category_group_enabled8444
; trace_event_unique_category_group_enabled8444 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8444
))); if (!trace_event_unique_category_group_enabled8444) { trace_event_unique_category_group_enabled8444
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8444
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8444
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8444
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8444
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0)), "func_index", func_index, "body_size", func_body
.end - func_body.start); trace_event_unique_tracer8444 .Initialize
(trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, h); }
8444 func_body.end - func_body.start)static v8::base::AtomicWord trace_event_unique_atomic8444 = 0
; const uint8_t* trace_event_unique_category_group_enabled8444
; trace_event_unique_category_group_enabled8444 = reinterpret_cast
<const uint8_t*>(v8::base::Relaxed_Load(&(trace_event_unique_atomic8444
))); if (!trace_event_unique_category_group_enabled8444) { trace_event_unique_category_group_enabled8444
= v8::internal::tracing::TraceEventHelper::GetTracingController
() ->GetCategoryGroupEnabled("disabled-by-default-" "v8.wasm.detailed"
); v8::base::Relaxed_Store(&(trace_event_unique_atomic8444
), (reinterpret_cast<v8::base::AtomicWord>( trace_event_unique_category_group_enabled8444
))); };; v8::internal::tracing::ScopedTracer trace_event_unique_tracer8444
; if (v8::base::Relaxed_Load(reinterpret_cast<const v8::base
::Atomic8*>( trace_event_unique_category_group_enabled8444
)) & (kEnabledForRecording_CategoryGroupEnabledFlags | kEnabledForEventCallback_CategoryGroupEnabledFlags
)) { uint64_t h = v8::internal::tracing::AddTraceEvent( ('X')
, trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, v8::internal::tracing::kGlobalScope, v8::internal::tracing::
kNoId, v8::internal::tracing::kNoId, (static_cast<unsigned
int>(0)), "func_index", func_index, "body_size", func_body
.end - func_body.start); trace_event_unique_tracer8444 .Initialize
(trace_event_unique_category_group_enabled8444, "wasm.CompileTopTier"
, h); }
;
8445 Zone zone(wasm::GetWasmEngine()->allocator(), ZONE_NAME__func__, kCompressGraphZone);
8446 MachineGraph* mcgraph = zone.New<MachineGraph>(
8447 zone.New<Graph>(&zone), zone.New<CommonOperatorBuilder>(&zone),
8448 zone.New<MachineOperatorBuilder>(
8449 &zone, MachineType::PointerRepresentation(),
8450 InstructionSelector::SupportedMachineOperatorFlags(),
8451 InstructionSelector::AlignmentRequirements()));
8452
8453 OptimizedCompilationInfo info(
8454 GetDebugName(&zone, env->module, wire_byte_storage, func_index), &zone,
8455 CodeKind::WASM_FUNCTION);
8456 if (env->runtime_exception_support) {
8457 info.set_wasm_runtime_exception_support();
8458 }
8459
8460 if (FLAG_experimental_wasm_gc) info.set_allocation_folding();
8461
8462 if (info.trace_turbo_json()) {
8463 TurboCfgFile tcf;
8464 tcf << AsC1VCompilation(&info);
8465 }
8466
8467 NodeOriginTable* node_origins =
8468 info.trace_turbo_json() ? zone.New<NodeOriginTable>(mcgraph->graph())
8469 : nullptr;
8470 SourcePositionTable* source_positions =
8471 mcgraph->zone()->New<SourcePositionTable>(mcgraph->graph());
8472
8473 std::vector<WasmLoopInfo> loop_infos;
8474
8475 wasm::WasmFeatures unused_detected_features;
8476 if (!detected) detected = &unused_detected_features;
8477 if (!BuildGraphForWasmFunction(env, func_body, func_index, detected, mcgraph,
8478 &loop_infos, node_origins, source_positions)) {
8479 return wasm::WasmCompilationResult{};
8480 }
8481
8482 if (node_origins) {
8483 node_origins->AddDecorator();
8484 }
8485
8486 // Run the compiler pipeline to generate machine code.
8487 auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
8488 if (mcgraph->machine()->Is32()) {
8489 call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
8490 }
8491
8492 if (ContainsSimd(func_body.sig) && !CpuFeatures::SupportsWasmSimd128()) {
8493 // Fail compilation if hardware does not support SIMD.
8494 return wasm::WasmCompilationResult{};
8495 }
8496
8497 Pipeline::GenerateCodeForWasmFunction(
8498 &info, env, wire_byte_storage, mcgraph, call_descriptor, source_positions,
8499 node_origins, func_body, env->module, func_index, &loop_infos);
8500
8501 if (counters) {
8502 int zone_bytes =
8503 static_cast<int>(mcgraph->graph()->zone()->allocation_size());
8504 counters->wasm_compile_function_peak_memory_bytes()->AddSample(zone_bytes);
8505 if (func_body.end - func_body.start >= 100 * KB) {
8506 counters->wasm_compile_huge_function_peak_memory_bytes()->AddSample(
8507 zone_bytes);
8508 }
8509 }
8510 // If we tiered up only one function for debugging, dump statistics
8511 // immediately.
8512 if (V8_UNLIKELY(FLAG_turbo_stats_wasm && FLAG_wasm_tier_up_filter >= 0)(__builtin_expect(!!(FLAG_turbo_stats_wasm && FLAG_wasm_tier_up_filter
>= 0), 0))
) {
8513 wasm::GetWasmEngine()->DumpTurboStatistics();
8514 }
8515 auto result = info.ReleaseWasmCompilationResult();
8516 CHECK_NOT_NULL(result)do { if ((__builtin_expect(!!(!((result) != nullptr)), 0))) {
V8_Fatal("Check failed: %s.", "(result) != nullptr"); } } while
(false)
; // Compilation expected to succeed.
8517 DCHECK_EQ(wasm::ExecutionTier::kTurbofan, result->result_tier)((void) 0);
8518 return std::move(*result);
8519}
8520
8521namespace {
8522// Helper for allocating either an GP or FP reg, or the next stack slot.
8523class LinkageLocationAllocator {
8524 public:
8525 template <size_t kNumGpRegs, size_t kNumFpRegs>
8526 constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
8527 const DoubleRegister (&fp)[kNumFpRegs],
8528 int slot_offset)
8529 : allocator_(wasm::LinkageAllocator(gp, fp)), slot_offset_(slot_offset) {}
8530
8531 LinkageLocation Next(MachineRepresentation rep) {
8532 MachineType type = MachineType::TypeForRepresentation(rep);
8533 if (IsFloatingPoint(rep)) {
8534 if (allocator_.CanAllocateFP(rep)) {
8535 int reg_code = allocator_.NextFpReg(rep);
8536 return LinkageLocation::ForRegister(reg_code, type);
8537 }
8538 } else if (allocator_.CanAllocateGP()) {
8539 int reg_code = allocator_.NextGpReg();
8540 return LinkageLocation::ForRegister(reg_code, type);
8541 }
8542 // Cannot use register; use stack slot.
8543 int index = -1 - (slot_offset_ + allocator_.NextStackSlot(rep));
8544 return LinkageLocation::ForCallerFrameSlot(index, type);
8545 }
8546
8547 int NumStackSlots() const { return allocator_.NumStackSlots(); }
8548 void EndSlotArea() { allocator_.EndSlotArea(); }
8549
8550 private:
8551 wasm::LinkageAllocator allocator_;
8552 // Since params and returns are in different stack frames, we must allocate
8553 // them separately. Parameter slots don't need an offset, but return slots
8554 // must be offset to just before the param slots, using this |slot_offset_|.
8555 int slot_offset_;
8556};
8557
8558LocationSignature* BuildLocations(Zone* zone, const wasm::FunctionSig* fsig,
8559 bool extra_callable_param,
8560 int* parameter_slots, int* return_slots) {
8561 int extra_params = extra_callable_param ? 2 : 1;
8562 LocationSignature::Builder locations(zone, fsig->return_count(),
8563 fsig->parameter_count() + extra_params);
8564
8565 // Add register and/or stack parameter(s).
8566 LinkageLocationAllocator params(
8567 wasm::kGpParamRegisters, wasm::kFpParamRegisters, 0 /* no slot offset */);
8568
8569 // The instance object.
8570 locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
8571 const size_t param_offset = 1; // Actual params start here.
8572
8573 // Parameters are separated into two groups (first all untagged, then all
8574 // tagged parameters). This allows for easy iteration of tagged parameters
8575 // during frame iteration.
8576 const size_t parameter_count = fsig->parameter_count();
8577 for (size_t i = 0; i < parameter_count; i++) {
8578 MachineRepresentation param = fsig->GetParam(i).machine_representation();
8579 // Skip tagged parameters (e.g. any-ref).
8580 if (IsAnyTagged(param)) continue;
8581 auto l = params.Next(param);
8582 locations.AddParamAt(i + param_offset, l);
8583 }
8584
8585 // End the untagged area, so tagged slots come after.
8586 params.EndSlotArea();
8587
8588 for (size_t i = 0; i < parameter_count; i++) {
8589 MachineRepresentation param = fsig->GetParam(i).machine_representation();
8590 // Skip untagged parameters.
8591 if (!IsAnyTagged(param)) continue;
8592 auto l = params.Next(param);
8593 locations.AddParamAt(i + param_offset, l);
8594 }
8595
8596 // Import call wrappers have an additional (implicit) parameter, the callable.
8597 // For consistency with JS, we use the JSFunction register.
8598 if (extra_callable_param) {
8599 locations.AddParam(LinkageLocation::ForRegister(
8600 kJSFunctionRegister.code(), MachineType::TaggedPointer()));
8601 }
8602
8603 *parameter_slots = AddArgumentPaddingSlots(params.NumStackSlots());
8604
8605 // Add return location(s).
8606 LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
8607 wasm::kFpReturnRegisters, *parameter_slots);
8608
8609 const size_t return_count = locations.return_count_;
8610 for (size_t i = 0; i < return_count; i++) {
8611 MachineRepresentation ret = fsig->GetReturn(i).machine_representation();
8612 locations.AddReturn(rets.Next(ret));
8613 }
8614
8615 *return_slots = rets.NumStackSlots();
8616
8617 return locations.Build();
8618}
8619} // namespace
8620
8621// General code uses the above configuration data.
8622CallDescriptor* GetWasmCallDescriptor(Zone* zone, const wasm::FunctionSig* fsig,
8623 WasmCallKind call_kind,
8624 bool need_frame_state) {
8625 // The extra here is to accomodate the instance object as first parameter
8626 // and, when specified, the additional callable.
8627 bool extra_callable_param =
8628 call_kind == kWasmImportWrapper || call_kind == kWasmCapiFunction;
8629
8630 int parameter_slots;
8631 int return_slots;
8632 LocationSignature* location_sig = BuildLocations(
8633 zone, fsig, extra_callable_param, &parameter_slots, &return_slots);
8634
8635 const RegList kCalleeSaveRegisters;
8636 const DoubleRegList kCalleeSaveFPRegisters;
8637
8638 // The target for wasm calls is always a code object.
8639 MachineType target_type = MachineType::Pointer();
8640 LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
8641
8642 CallDescriptor::Kind descriptor_kind;
8643 if (call_kind == kWasmFunction) {
8644 descriptor_kind = CallDescriptor::kCallWasmFunction;
8645 } else if (call_kind == kWasmImportWrapper) {
8646 descriptor_kind = CallDescriptor::kCallWasmImportWrapper;
8647 } else {
8648 DCHECK_EQ(call_kind, kWasmCapiFunction)((void) 0);
8649 descriptor_kind = CallDescriptor::kCallWasmCapiFunction;
8650 }
8651
8652 CallDescriptor::Flags flags = need_frame_state
8653 ? CallDescriptor::kNeedsFrameState
8654 : CallDescriptor::kNoFlags;
8655 return zone->New<CallDescriptor>( // --
8656 descriptor_kind, // kind
8657 target_type, // target MachineType
8658 target_loc, // target location
8659 location_sig, // location_sig
8660 parameter_slots, // parameter slot count
8661 compiler::Operator::kNoProperties, // properties
8662 kCalleeSaveRegisters, // callee-saved registers
8663 kCalleeSaveFPRegisters, // callee-saved fp regs
8664 flags, // flags
8665 "wasm-call", // debug name
8666 StackArgumentOrder::kDefault, // order of the arguments in the stack
8667 fsig, // signature
8668 RegList{}, // allocatable registers
8669 return_slots); // return slot count
8670}
8671
8672namespace {
8673const wasm::FunctionSig* ReplaceTypeInSig(Zone* zone,
8674 const wasm::FunctionSig* sig,
8675 wasm::ValueType from,
8676 wasm::ValueType to,
8677 size_t num_replacements) {
8678 size_t param_occurences =
8679 std::count(sig->parameters().begin(), sig->parameters().end(), from);
8680 size_t return_occurences =
8681 std::count(sig->returns().begin(), sig->returns().end(), from);
8682 if (param_occurences == 0 && return_occurences == 0) return sig;
8683
8684 wasm::FunctionSig::Builder builder(
8685 zone, sig->return_count() + return_occurences * (num_replacements - 1),
8686 sig->parameter_count() + param_occurences * (num_replacements - 1));
8687
8688 for (wasm::ValueType ret : sig->returns()) {
8689 if (ret == from) {
8690 for (size_t i = 0; i < num_replacements; i++) builder.AddReturn(to);
8691 } else {
8692 builder.AddReturn(ret);
8693 }
8694 }
8695
8696 for (wasm::ValueType param : sig->parameters()) {
8697 if (param == from) {
8698 for (size_t i = 0; i < num_replacements; i++) builder.AddParam(to);
8699 } else {
8700 builder.AddParam(param);
8701 }
8702 }
8703
8704 return builder.Build();
8705}
8706
8707CallDescriptor* ReplaceTypeInCallDescriptorWith(
8708 Zone* zone, const CallDescriptor* call_descriptor, size_t num_replacements,
8709 wasm::ValueType input_type, wasm::ValueType output_type) {
8710 if (call_descriptor->wasm_sig() == nullptr) {
8711 // This happens for builtins calls. They need no replacements anyway.
8712#if DEBUG
8713 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
8714 DCHECK_NE(call_descriptor->GetParameterType(i),((void) 0)
8715 input_type.machine_type())((void) 0);
8716 }
8717 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
8718 DCHECK_NE(call_descriptor->GetReturnType(i), input_type.machine_type())((void) 0);
8719 }
8720#endif
8721 return const_cast<CallDescriptor*>(call_descriptor);
8722 }
8723 const wasm::FunctionSig* sig =
8724 ReplaceTypeInSig(zone, call_descriptor->wasm_sig(), input_type,
8725 output_type, num_replacements);
8726 // If {ReplaceTypeInSig} took the early fast path, there's nothing to do.
8727 if (sig == call_descriptor->wasm_sig()) {
8728 return const_cast<CallDescriptor*>(call_descriptor);
8729 }
8730
8731 // The last parameter may be the special callable parameter. In that case we
8732 // have to preserve it as the last parameter, i.e. we allocate it in the new
8733 // location signature again in the same register.
8734 bool extra_callable_param =
8735 (call_descriptor->GetInputLocation(call_descriptor->InputCount() - 1) ==
8736 LinkageLocation::ForRegister(kJSFunctionRegister.code(),
8737 MachineType::TaggedPointer()));
8738
8739 int parameter_slots;
8740 int return_slots;
8741 LocationSignature* location_sig = BuildLocations(
8742 zone, sig, extra_callable_param, &parameter_slots, &return_slots);
8743
8744 return zone->New<CallDescriptor>( // --
8745 call_descriptor->kind(), // kind
8746 call_descriptor->GetInputType(0), // target MachineType
8747 call_descriptor->GetInputLocation(0), // target location
8748 location_sig, // location_sig
8749 parameter_slots, // parameter slot count
8750 call_descriptor->properties(), // properties
8751 call_descriptor->CalleeSavedRegisters(), // callee-saved registers
8752 call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
8753 call_descriptor->flags(), // flags
8754 call_descriptor->debug_name(), // debug name
8755 call_descriptor->GetStackArgumentOrder(), // stack order
8756 sig, // signature
8757 call_descriptor->AllocatableRegisters(), // allocatable registers
8758 return_slots); // return slot count
8759}
8760} // namespace
8761
8762// static
8763const wasm::FunctionSig* WasmGraphBuilder::Int64LoweredSig(
8764 Zone* zone, const wasm::FunctionSig* sig) {
8765 return (kSystemPointerSize == 4)
8766 ? ReplaceTypeInSig(zone, sig, wasm::kWasmI64, wasm::kWasmI32, 2)
8767 : sig;
8768}
8769
8770CallDescriptor* GetI32WasmCallDescriptor(
8771 Zone* zone, const CallDescriptor* call_descriptor) {
8772 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
8773 wasm::kWasmI64, wasm::kWasmI32);
8774}
8775
8776AssemblerOptions WasmAssemblerOptions() {
8777 AssemblerOptions options;
8778 // Relocation info required to serialize {WasmCode} for proper functions.
8779 options.record_reloc_info_for_serialization = true;
8780 options.enable_root_relative_access = false;
8781 return options;
8782}
8783
8784AssemblerOptions WasmStubAssemblerOptions() {
8785 AssemblerOptions options;
8786 // Relocation info not necessary because stubs are not serialized.
8787 options.record_reloc_info_for_serialization = false;
8788 options.enable_root_relative_access = false;
8789 return options;
8790}
8791
8792#undef FATAL_UNSUPPORTED_OPCODE
8793#undef WASM_INSTANCE_OBJECT_SIZE
8794#undef LOAD_INSTANCE_FIELD
8795#undef LOAD_MUTABLE_INSTANCE_FIELD
8796#undef LOAD_ROOT
8797
8798} // namespace compiler
8799} // namespace internal
8800} // namespace v8