wasm.tq

// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include 'src/builtins/builtins-wasm-gen.h'
#include 'src/builtins/builtins-call-gen.h'

namespace runtime {
extern runtime WasmMemoryGrow(Context, WasmTrustedInstanceData, Smi, Smi): Smi;
extern runtime WasmRefFunc(Context, WasmTrustedInstanceData, Smi): JSAny;
extern runtime WasmInternalFunctionCreateExternal(
    Context, WasmInternalFunction): JSFunction;
extern runtime WasmTableInit(
    Context, WasmTrustedInstanceData, Object, Object, Smi, Smi, Smi): JSAny;
extern runtime WasmTableCopy(
    Context, WasmTrustedInstanceData, Object, Object, Smi, Smi, Smi): JSAny;
extern runtime WasmTableFill(
    Context, WasmTrustedInstanceData, Smi, Smi, Object, Smi): JSAny;
extern runtime WasmTableGrow(
    Context, WasmTrustedInstanceData, Smi, Object, Smi): Smi;
extern runtime WasmFunctionTableGet(Context, WasmTrustedInstanceData, Smi, Smi):
    JSAny;
extern runtime WasmFunctionTableSet(
    Context, WasmTrustedInstanceData, Smi, Smi, Object): JSAny;
extern runtime ThrowRangeError(Context, Smi): never;
extern runtime ThrowWasmError(Context, Smi): never;
extern runtime WasmThrowRangeError(Context, Smi): never;
extern runtime TrapHandlerThrowWasmError(Context): never;
extern runtime WasmThrowTypeError(Context, Smi, JSAny): never;
extern runtime WasmThrowDataViewTypeError(Context, Smi, JSAny): never;
extern runtime WasmThrowDataViewDetachedError(Context, Smi): never;
extern runtime WasmThrow(Context, Object, FixedArray): JSAny;
extern runtime WasmReThrow(Context, Object): JSAny;
extern runtime WasmTriggerTierUp(Context, WasmTrustedInstanceData): JSAny;
extern runtime WasmStackGuard(Context): JSAny;
extern runtime ThrowWasmStackOverflow(Context): JSAny;
extern runtime WasmTraceMemory(Context, Smi): JSAny;
extern runtime WasmTraceEnter(Context): JSAny;
extern runtime WasmTraceExit(Context, Smi): JSAny;
extern runtime WasmI32AtomicWait(
    Context, WasmTrustedInstanceData, Smi, Number, Number, BigInt): Smi;
extern runtime WasmI64AtomicWait(
    Context, WasmTrustedInstanceData, Smi, Number, BigInt, BigInt): Smi;
extern runtime WasmArrayCopy(Context, WasmArray, Smi, WasmArray, Smi, Smi):
    JSAny;
extern runtime WasmArrayNewSegment(
    Context, WasmTrustedInstanceData, Smi, Smi, Smi, Map): Object;
extern runtime WasmStringNewSegmentWtf8(
    Context, WasmTrustedInstanceData, Smi, Smi, Smi, Smi): String;
extern runtime WasmArrayInitSegment(
    Context, WasmTrustedInstanceData, Smi, WasmArray, Smi, Smi, Smi): JSAny;
extern runtime WasmStringNewWtf8(
    Context, WasmTrustedInstanceData, Smi, Smi, Number, Number): String|Null;
extern runtime WasmStringNewWtf8Array(Context, Smi, WasmArray, Smi, Smi): String
    |Null;
extern runtime WasmStringNewWtf16(
    Context, WasmTrustedInstanceData, Smi, Number, Number): String;
extern runtime WasmStringNewWtf16Array(Context, WasmArray, Smi, Smi): String;
extern runtime WasmStringConst(Context, WasmTrustedInstanceData, Smi): String;
extern runtime WasmStringMeasureUtf8(Context, String): Number;
extern runtime WasmStringMeasureWtf8(Context, String): Number;
extern runtime WasmStringEncodeWtf8(
    Context, WasmTrustedInstanceData, Smi, Smi, String, Number): Number;
extern runtime WasmStringEncodeWtf8Array(
    Context, Smi, String, WasmArray, Number): Number;
extern runtime WasmStringToUtf8Array(Context, String): WasmArray;
extern runtime WasmStringEncodeWtf16(
    Context, WasmTrustedInstanceData, Smi, String, Number, Smi, Smi): JSAny;
extern runtime WasmStringAsWtf8(Context, String): ByteArray;
extern runtime WasmStringViewWtf8Encode(
    Context, WasmTrustedInstanceData, Smi, ByteArray, Number, Number,
    Number): JSAny;
extern runtime WasmStringViewWtf8Slice(Context, ByteArray, Number, Number):
    String;
extern runtime WasmStringFromCodePoint(Context, Number): String;
extern runtime WasmStringHash(NoContext, String): Smi;
extern runtime WasmSubstring(Context, String, Smi, Smi): String;
extern runtime WasmJSToWasmObject(Context, JSAny, Smi): JSAny;
}

namespace unsafe {
extern macro Allocate(intptr): HeapObject;
extern macro Allocate(intptr, constexpr AllocationFlag): HeapObject;
}

macro NumberToInt32(input: Number): int32 {
  return Convert<int32>(input);
}
macro NumberToUint32(input: Number): uint32 {
  return Unsigned(Convert<int32>(input));
}

namespace wasm {
const kAnyType: constexpr int31
    generates 'wasm::kWasmAnyRef.raw_bit_field()';
const kMaxPolymorphism:
    constexpr int31 generates 'wasm::kMaxPolymorphism';

extern macro WasmBuiltinsAssembler::LoadTrustedDataFromInstance(
    WasmInstanceObject): WasmTrustedInstanceData;

extern macro WasmBuiltinsAssembler::LoadInstanceDataFromFrame():
    WasmTrustedInstanceData;

// WasmTrustedInstanceData has a field layout that Torque can't handle yet.
// TODO(bbudge) Eliminate these functions when Torque is ready.
extern macro WasmBuiltinsAssembler::LoadContextFromInstanceData(
    WasmTrustedInstanceData): NativeContext;
extern macro WasmBuiltinsAssembler::LoadTablesFromInstanceData(
    WasmTrustedInstanceData): FixedArray;
extern macro WasmBuiltinsAssembler::LoadFuncRefsFromInstanceData(
    WasmTrustedInstanceData): FixedArray;
extern macro WasmBuiltinsAssembler::LoadManagedObjectMapsFromInstanceData(
    WasmTrustedInstanceData): FixedArray;
extern macro WasmBuiltinsAssembler::LoadContextFromWasmOrJsFrame():
    NativeContext;

extern macro WasmBuiltinsAssembler::StringToFloat64(String): float64;

macro LoadContextFromFrame(): NativeContext {
  return LoadContextFromInstanceData(LoadInstanceDataFromFrame());
}

builtin WasmInt32ToHeapNumber(val: int32): HeapNumber {
  return AllocateHeapNumberWithValue(Convert<float64>(val));
}

builtin WasmFuncRefToJS(
    implicit context: Context)(val: WasmFuncRef|WasmNull): JSFunction|Null {
  typeswitch (val) {
    case (WasmNull): {
      return Null;
    }
    case (func: WasmFuncRef): {
      const internal: WasmInternalFunction = func.internal;
      const maybeExternal: Object = internal.external;
      if (maybeExternal != Undefined) {
        return %RawDownCast<JSFunction>(maybeExternal);
      }
      tail runtime::WasmInternalFunctionCreateExternal(context, internal);
    }
  }
}

builtin WasmTaggedNonSmiToInt32(implicit context: Context)(val: HeapObject):
    int32 {
  return ChangeTaggedNonSmiToInt32(val);
}

builtin WasmTaggedToFloat64(implicit context: Context)(val: JSAny): float64 {
  return ChangeTaggedToFloat64(val);
}

builtin WasmTaggedToFloat32(implicit context: Context)(val: JSAny): float32 {
  return TruncateFloat64ToFloat32(ChangeTaggedToFloat64(val));
}

builtin WasmMemoryGrow(memIndex: int32, numPages: int32): int32 {
  dcheck(IsValidPositiveSmi(ChangeInt32ToIntPtr(memIndex)));
  if (!IsValidPositiveSmi(ChangeInt32ToIntPtr(numPages)))
    return Int32Constant(-1);
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const context: NativeContext = LoadContextFromInstanceData(trustedData);
  const result: Smi = runtime::WasmMemoryGrow(
      context, trustedData, SmiFromInt32(memIndex), SmiFromInt32(numPages));
  return SmiToInt32(result);
}

builtin WasmTableInit(
    dstRaw: uint32, srcRaw: uint32, sizeRaw: uint32, tableIndex: Smi,
    segmentIndex: Smi): JSAny {
  try {
    const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
    const dst: Smi = Convert<PositiveSmi>(dstRaw) otherwise TableOutOfBounds;
    const src: Smi = Convert<PositiveSmi>(srcRaw) otherwise TableOutOfBounds;
    const size: Smi = Convert<PositiveSmi>(sizeRaw) otherwise TableOutOfBounds;
    tail runtime::WasmTableInit(
        LoadContextFromInstanceData(trustedData), trustedData, tableIndex,
        segmentIndex, dst, src, size);
  } label TableOutOfBounds deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

builtin WasmTableCopy(
    dstRaw: uint32, srcRaw: uint32, sizeRaw: uint32, dstTable: Smi,
    srcTable: Smi): JSAny {
  try {
    const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
    const dst: Smi = Convert<PositiveSmi>(dstRaw) otherwise TableOutOfBounds;
    const src: Smi = Convert<PositiveSmi>(srcRaw) otherwise TableOutOfBounds;
    const size: Smi = Convert<PositiveSmi>(sizeRaw) otherwise TableOutOfBounds;
    tail runtime::WasmTableCopy(
        LoadContextFromInstanceData(trustedData), trustedData, dstTable,
        srcTable, dst, src, size);
  } label TableOutOfBounds deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

builtin WasmTableFill(
    table: Smi, startRaw: uint32, countRaw: uint32, value: Object): JSAny {
  try {
    const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
    const start: Smi =
        Convert<PositiveSmi>(startRaw) otherwise TableOutOfBounds;
    const count: Smi =
        Convert<PositiveSmi>(countRaw) otherwise TableOutOfBounds;
    tail runtime::WasmTableFill(
        LoadContextFromInstanceData(trustedData), trustedData, table, start,
        value, count);
  } label TableOutOfBounds deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

builtin WasmTableGrow(table: Smi, deltaRaw: uint32, value: Object): Smi {
  try {
    const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
    const delta: Smi =
        Convert<PositiveSmi>(deltaRaw) otherwise TableOutOfBounds;
    tail runtime::WasmTableGrow(
        LoadContextFromInstanceData(trustedData), trustedData, table, value,
        delta);
  } label TableOutOfBounds deferred {
    return -1;
  }
}

builtin WasmTableGet(tableIndex: intptr, index: uint32): Object {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const entryIndex: intptr = Signed(ChangeUint32ToWord(index));
  try {
    dcheck(IsValidPositiveSmi(tableIndex));

    const tables: FixedArray = LoadTablesFromInstanceData(trustedData);
    const table: WasmTableObject = %RawDownCast<WasmTableObject>(
        LoadFixedArrayElement(tables, tableIndex));
    const entriesCount: uint32 = Unsigned(SmiToInt32(table.current_length));
    if (index >= entriesCount) goto IndexOutOfRange;

    const entries: FixedArray = table.entries;
    const entry: Object = LoadFixedArrayElement(entries, entryIndex);
    return entry;
  } label IndexOutOfRange deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

builtin WasmTableSet(tableIndex: intptr, index: uint32, value: Object):
    Object {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const entryIndex: intptr = Signed(ChangeUint32ToWord(index));
  try {
    dcheck(IsValidPositiveSmi(tableIndex));

    const tables: FixedArray = LoadTablesFromInstanceData(trustedData);
    const table: WasmTableObject = %RawDownCast<WasmTableObject>(
        LoadFixedArrayElement(tables, tableIndex));

    const entriesCount: uint32 = Unsigned(SmiToInt32(table.current_length));
    if (index >= entriesCount) goto IndexOutOfRange;

    const entries: FixedArray = table.entries;
    StoreFixedArrayElement(entries, entryIndex, value);
    return Undefined;
  } label IndexOutOfRange deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

// Returns WasmFuncRef or WasmNull, or throws an exception.
builtin WasmTableGetFuncRef(tableIndex: intptr, index: uint32): Object {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const entryIndex: intptr = Signed(ChangeUint32ToWord(index));
  try {
    dcheck(IsValidPositiveSmi(tableIndex));

    const tables: FixedArray = LoadTablesFromInstanceData(trustedData);
    const table: WasmTableObject = %RawDownCast<WasmTableObject>(
        LoadFixedArrayElement(tables, tableIndex));
    const entriesCount: uint32 = Unsigned(SmiToInt32(table.current_length));
    if (index >= entriesCount) goto IndexOutOfRange;

    const entries: FixedArray = table.entries;
    const entry: HeapObject =
        UnsafeCast<HeapObject>(LoadFixedArrayElement(entries, entryIndex));

    dcheck(Is<WasmFuncRef>(entry) || Is<WasmNull>(entry) || Is<Tuple2>(entry));
    if (IsTuple2Map(entry.map)) goto CallRuntime;
    if (Is<WasmNull>(entry)) return entry;
    dcheck(Is<WasmFuncRef>(entry));
    return entry;
  } label CallRuntime deferred {
    tail runtime::WasmFunctionTableGet(
        LoadContextFromInstanceData(trustedData), trustedData,
        SmiFromIntPtr(tableIndex), SmiFromIntPtr(entryIndex));
  } label IndexOutOfRange deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

// Stub to wrap around the slow path (runtime call) of table.get for funcref
// tables for which the reference is not yet initialized.
builtin WasmFunctionTableGet(tableIndex: intptr, index: int32): Object {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  tail runtime::WasmFunctionTableGet(
      LoadContextFromInstanceData(trustedData), trustedData,
      SmiFromIntPtr(tableIndex), SmiFromInt32(index));
}

builtin WasmTableSetFuncRef(
    tableIndex: intptr, index: uint32, value: WasmFuncRef): Object {
  dcheck(Is<WasmFuncRef>(value) || Is<WasmNull>(value));
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const entryIndex: intptr = Signed(ChangeUint32ToWord(index));
  try {
    dcheck(IsValidPositiveSmi(tableIndex));

    const tables: FixedArray = LoadTablesFromInstanceData(trustedData);
    const table: WasmTableObject = %RawDownCast<WasmTableObject>(
        LoadFixedArrayElement(tables, tableIndex));

    const entriesCount: uint32 = Unsigned(SmiToInt32(table.current_length));
    if (index >= entriesCount) goto IndexOutOfRange;

    tail runtime::WasmFunctionTableSet(
        LoadContextFromInstanceData(trustedData), trustedData,
        SmiFromIntPtr(tableIndex), SmiFromIntPtr(entryIndex), value);
  } label IndexOutOfRange deferred {
    tail ThrowWasmTrapTableOutOfBounds();
  }
}

builtin WasmRefFunc(index: uint32): Object {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  try {
    const funcRefs: FixedArray = LoadFuncRefsFromInstanceData(trustedData);
    const funcref: Object = funcRefs.objects[index];
    // {funcref} is either a WasmFuncRef or Smi::zero(). A Smi check is the
    // fastest way to distinguish these two cases.
    if (TaggedIsSmi(funcref)) goto CallRuntime;
    dcheck(Is<WasmFuncRef>(funcref));
    return funcref;
  } label CallRuntime deferred {
    tail runtime::WasmRefFunc(
        LoadContextFromInstanceData(trustedData), trustedData,
        SmiFromUint32(index));
  }
}

builtin WasmInternalFunctionCreateExternal(
    context: Context, func: WasmInternalFunction): JSFunction {
  return runtime::WasmInternalFunctionCreateExternal(context, func);
}

builtin WasmAllocateZeroedFixedArray(size: intptr): FixedArray {
  if (size == 0) return kEmptyFixedArray;
  const result = UnsafeCast<FixedArray>(AllocateFixedArray(
      ElementsKind::PACKED_ELEMENTS, size, AllocationFlag::kNone));
  FillEntireFixedArrayWithSmiZero(ElementsKind::PACKED_ELEMENTS, result, size);
  return result;
}

builtin WasmAllocateFixedArray(size: intptr): FixedArray {
  if (size == 0) return kEmptyFixedArray;
  return UnsafeCast<FixedArray>(AllocateFixedArray(
      ElementsKind::PACKED_ELEMENTS, size, AllocationFlag::kNone));
}

builtin WasmThrow(tag: Object, values: FixedArray): JSAny {
  tail runtime::WasmThrow(LoadContextFromFrame(), tag, values);
}

builtin WasmRethrow(exception: Object): JSAny {
  dcheck(exception != kWasmNull);
  tail runtime::WasmReThrow(LoadContextFromFrame(), exception);
}

builtin WasmThrowRef(exception: Object): JSAny {
  if (exception == kWasmNull) tail ThrowWasmTrapRethrowNull();
  tail runtime::WasmReThrow(LoadContextFromFrame(), exception);
}

// We need this for frames that do not have the instance in the parameters.
// Currently, this is CapiCallWrapper frames.
builtin WasmRethrowExplicitContext(
    exception: Object, explicitContext: Context): JSAny {
  if (exception == Null) tail ThrowWasmTrapRethrowNull();
  tail runtime::WasmReThrow(explicitContext, exception);
}

builtin WasmTriggerTierUp(): JSAny {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  tail runtime::WasmTriggerTierUp(LoadContextFromFrame(), trustedData);
}

builtin WasmStackGuard(): JSAny {
  tail runtime::WasmStackGuard(LoadContextFromFrame());
}

builtin WasmStackOverflow(): JSAny {
  tail runtime::ThrowWasmStackOverflow(LoadContextFromFrame());
}

builtin WasmTraceMemory(info: Smi): JSAny {
  tail runtime::WasmTraceMemory(LoadContextFromFrame(), info);
}

builtin WasmTraceEnter(): JSAny {
  tail runtime::WasmTraceEnter(LoadContextFromFrame());
}

builtin WasmTraceExit(info: Smi): JSAny {
  tail runtime::WasmTraceExit(LoadContextFromFrame(), info);
}

builtin WasmAllocateJSArray(implicit context: Context)(size: Smi): JSArray {
  const map: Map = GetFastPackedElementsJSArrayMap();
  return AllocateJSArray(ElementsKind::PACKED_ELEMENTS, map, size, size);
}

builtin WasmAllocateStructWithRtt(rtt: Map, instanceSize: int32): HeapObject {
  const result: HeapObject = unsafe::Allocate(Convert<intptr>(instanceSize));
  *UnsafeConstCast(&result.map) = rtt;
  // TODO(ishell): consider removing properties_or_hash field from WasmObjects.
  %RawDownCast<WasmStruct>(result).properties_or_hash = kEmptyFixedArray;
  return result;
}

builtin WasmAllocateArray_Uninitialized(
    rtt: Map, length: uint32, elementSize: uint32): WasmArray {
  // instanceSize = RoundUp(elementSize * length, kObjectAlignment)
  //              + WasmArray::kHeaderSize
  const instanceSize: intptr =
      torque_internal::AlignTagged(
          Convert<intptr>(length) * Convert<intptr>(elementSize)) +
      Convert<intptr>(kWasmArrayHeaderSize);
  const result: HeapObject = unsafe::Allocate(instanceSize);
  *UnsafeConstCast(&result.map) = rtt;
  // TODO(ishell): consider removing properties_or_hash field from WasmObjects.
  %RawDownCast<WasmArray>(result).properties_or_hash = kEmptyFixedArray;
  %RawDownCast<WasmArray>(result).length = length;
  return %RawDownCast<WasmArray>(result);
}

builtin WasmArrayNewSegment(
    segmentIndex: uint32, offset: uint32, length: uint32, isElement: Smi,
    rtt: Map): Object {
  const trustedData = LoadInstanceDataFromFrame();
  try {
    const smiOffset = Convert<PositiveSmi>(offset) otherwise SegmentOutOfBounds;
    const smiLength = Convert<PositiveSmi>(length) otherwise ArrayTooLarge;
    tail runtime::WasmArrayNewSegment(
        LoadContextFromInstanceData(trustedData), trustedData,
        SmiFromUint32(segmentIndex), smiOffset, smiLength, rtt);
  } label SegmentOutOfBounds {
    if (isElement == SmiConstant(0)) {
      tail ThrowWasmTrapDataSegmentOutOfBounds();
    } else {
      tail ThrowWasmTrapElementSegmentOutOfBounds();
    }
  } label ArrayTooLarge {
    tail ThrowWasmTrapArrayTooLarge();
  }
}

// {segmentIndex} has to be tagged as a possible stack parameter.
builtin WasmArrayInitSegment(
    arrayIndex: uint32, segmentOffset: uint32, length: uint32,
    segmentIndex: Smi, isElement: Smi, arrayRaw: HeapObject): JSAny {
  const trustedData = LoadInstanceDataFromFrame();
  if (arrayRaw == kWasmNull) {
    tail ThrowWasmTrapNullDereference();
  }
  const array = %RawDownCast<WasmArray>(arrayRaw);
  try {
    const smiArrayIndex =
        Convert<PositiveSmi>(arrayIndex) otherwise ArrayOutOfBounds;
    const smiOffset =
        Convert<PositiveSmi>(segmentOffset) otherwise SegmentOutOfBounds;
    const smiLength = Convert<PositiveSmi>(length) otherwise ArrayOutOfBounds;

    tail runtime::WasmArrayInitSegment(
        LoadContextFromInstanceData(trustedData), trustedData, segmentIndex,
        array, smiArrayIndex, smiOffset, smiLength);
  } label SegmentOutOfBounds {
    if (isElement == SmiConstant(0)) {
      tail ThrowWasmTrapDataSegmentOutOfBounds();
    } else {
      tail ThrowWasmTrapElementSegmentOutOfBounds();
    }
  } label ArrayOutOfBounds {
    tail ThrowWasmTrapArrayOutOfBounds();
  }
}

// We put all uint32 parameters at the beginning so that they are assigned to
// registers.
builtin WasmArrayCopyWithChecks(
    dstIndex: uint32, srcIndex: uint32, length: uint32, dstObject: Object,
    srcObject: Object): JSAny {
  if (dstObject == kWasmNull) tail ThrowWasmTrapNullDereference();
  if (srcObject == kWasmNull) tail ThrowWasmTrapNullDereference();
  const dstArray = UnsafeCast<WasmArray>(dstObject);
  const srcArray = UnsafeCast<WasmArray>(srcObject);
  // Check that the end of the copying range is in-bounds and that the range
  // does not overflow.
  if (dstIndex + length > dstArray.length || dstIndex + length < dstIndex ||
      srcIndex + length > srcArray.length || srcIndex + length < srcIndex) {
    tail ThrowWasmTrapArrayOutOfBounds();
  }
  if (length == 0) return Undefined;
  tail runtime::WasmArrayCopy(
      LoadContextFromFrame(), dstArray, SmiFromUint32(dstIndex), srcArray,
      SmiFromUint32(srcIndex), SmiFromUint32(length));
}

builtin WasmArrayCopy(
    dstIndex: uint32, srcIndex: uint32, length: uint32, dstObject: Object,
    srcObject: Object): JSAny {
  if (dstObject == kWasmNull) tail ThrowWasmTrapNullDereference();
  if (srcObject == kWasmNull) tail ThrowWasmTrapNullDereference();
  if (length == 0) return Undefined;
  tail runtime::WasmArrayCopy(
      LoadContextFromFrame(), UnsafeCast<WasmArray>(dstObject),
      SmiFromUint32(dstIndex), UnsafeCast<WasmArray>(srcObject),
      SmiFromUint32(srcIndex), SmiFromUint32(length));
}

builtin WasmUint32ToNumber(value: uint32): Number {
  return ChangeUint32ToTagged(value);
}

builtin UintPtr53ToNumber(value: uintptr): Number {
  if (value <= kSmiMaxValue) return Convert<Smi>(Convert<intptr>(value));
  const valueFloat = ChangeUintPtrToFloat64(value);
  // Values need to be within [0..2^53], such that they can be represented as
  // float64.
  dcheck(ChangeFloat64ToUintPtr(valueFloat) == value);
  return AllocateHeapNumberWithValue(valueFloat);
}

extern builtin I64ToBigInt(intptr): BigInt;
extern builtin I32PairToBigInt(/*low*/ intptr, /*high*/ intptr): BigInt;

builtin WasmI32AtomicWait(
    memIndex: int32, offset: uintptr, expectedValue: int32,
    timeout: BigInt): uint32 {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const result: Smi = runtime::WasmI32AtomicWait(
      LoadContextFromInstanceData(trustedData), trustedData,
      SmiFromInt32(memIndex), UintPtr53ToNumber(offset),
      ChangeInt32ToTagged(expectedValue), timeout);
  return Unsigned(SmiToInt32(result));
}

builtin WasmI64AtomicWait(
    memIndex: int32, offset: uintptr, expectedValue: BigInt,
    timeout: BigInt): uint32 {
  const trustedData: WasmTrustedInstanceData = LoadInstanceDataFromFrame();
  const result: Smi = runtime::WasmI64AtomicWait(
      LoadContextFromInstanceData(trustedData), trustedData,
      SmiFromInt32(memIndex), UintPtr53ToNumber(offset), expectedValue,
      timeout);
  return Unsigned(SmiToInt32(result));
}

// Type feedback collection support for `call_ref`.

extern macro LoadWasmInternalFunctionInstructionStart(WasmInternalFunction):
    RawPtr;

// Vector format:
// Two slots per call_ref instruction. These slots' values can be:
// - uninitialized: (0, <unused>). Note: we use {0} as the sentinel because
//   it also works as default for vector slots used as counts.
// - monomorphic: (funcref, count (smi)). The second slot is a counter for how
//   often the funcref in the first slot has been seen.
// - polymorphic: (fixed_array, <unused>). In this case, the array
//   contains 2..kMaxPolymorphism pairs (funcref, count (smi))
// - megamorphic: ("megamorphic" sentinel, <unused>)
//
// TODO(rstz): The counter might overflow if it exceeds the range of a Smi.
// This can lead to incorrect inlining decisions.
macro UpdateIC(vector: FixedArray, index: intptr, funcref: WasmFuncRef):
    void {
  const value = vector.objects[index];
  if (value == funcref) {
    // Monomorphic hit. Check for this case first to maximize its performance.
    const count = UnsafeCast<Smi>(vector.objects[index + 1]) + SmiConstant(1);
    vector.objects[index + 1] = count;
    return;
  }
  // Check for polymorphic hit; its performance is second-most-important.
  if (Is<FixedArray>(value)) {
    const entries = UnsafeCast<FixedArray>(value);
    for (let i: intptr = 0; i < entries.length_intptr; i += 2) {
      if (entries.objects[i] == funcref) {
        // Polymorphic hit.
        const count = UnsafeCast<Smi>(entries.objects[i + 1]) + SmiConstant(1);
        entries.objects[i + 1] = count;
        return;
      }
    }
  }
  // All other cases are some sort of miss.
  if (TaggedEqual(value, SmiConstant(0))) {
    // Was uninitialized.
    vector.objects[index] = funcref;
    vector.objects[index + 1] = SmiConstant(1);
  } else if (Is<FixedArray>(value)) {
    // Polymorphic miss.
    const entries = UnsafeCast<FixedArray>(value);
    const kMaxSlots = kMaxPolymorphism * 2;  // 2 slots per entry.
    if (entries.length == SmiConstant(kMaxSlots)) {
      vector.objects[index] = ic::kMegamorphicSymbol;
      vector.objects[index + 1] = ic::kMegamorphicSymbol;
    } else {
      const newEntries = UnsafeCast<FixedArray>(AllocateFixedArray(
          ElementsKind::PACKED_ELEMENTS, entries.length_intptr + 2,
          AllocationFlag::kNone));
      for (let i: intptr = 0; i < entries.length_intptr; i++) {
        newEntries.objects[i] = entries.objects[i];
      }
      const newIndex = entries.length_intptr;
      newEntries.objects[newIndex] = funcref;
      newEntries.objects[newIndex + 1] = SmiConstant(1);
      vector.objects[index] = newEntries;
    }
  } else if (Is<WasmFuncRef>(value)) {
    // Monomorphic miss.
    const newEntries = UnsafeCast<FixedArray>(AllocateFixedArray(
        ElementsKind::PACKED_ELEMENTS, 4, AllocationFlag::kNone));
    newEntries.objects[0] = value;
    newEntries.objects[1] = vector.objects[index + 1];
    newEntries.objects[2] = funcref;
    newEntries.objects[3] = SmiConstant(1);
    vector.objects[index] = newEntries;
    // Clear the first entry's counter; the specific value we write doesn't
    // matter.
    vector.objects[index + 1] = Undefined;
  } else {
    // The "ic::IsMegamorphic(value)" case doesn't need to do anything.
    dcheck(value == ic::kMegamorphicSymbol);
  }
}

// This is the return type of "CallRefIC". The type is not used anywhere else;
// Liftoff uses the two returned values directly.
struct TargetAndRef {
  target: RawPtr;
  ref: WasmTrustedInstanceData|WasmApiFunctionRef;
}

builtin CallRefIC(vector: FixedArray, index: intptr, funcref: WasmFuncRef):
    TargetAndRef {
  dcheck(Is<WasmFuncRef>(funcref));
  UpdateIC(vector, index, funcref);

  const internal = funcref.internal;
  let target = internal.call_target_ptr;
  if (Signed(target) == IntPtrConstant(0)) {
    target = LoadWasmInternalFunctionInstructionStart(internal);
  }
  return TargetAndRef{target: target, ref: internal.ref};
}

extern macro TryHasOwnProperty(HeapObject, Map, InstanceType, Name): never
    labels Found, NotFound, Bailout;
type OnNonExistent constexpr 'OnNonExistent';
const kReturnUndefined: constexpr OnNonExistent
    generates 'OnNonExistent::kReturnUndefined';
extern macro SmiConstant(constexpr OnNonExistent): Smi;
extern transitioning builtin GetPropertyWithReceiver(
    implicit context: Context)(JSAny, Name, JSAny, Smi): JSAny;

transitioning builtin WasmGetOwnProperty(
    implicit context: Context)(object: Object, uniqueName: Name): JSAny {
  try {
    const heapObject: HeapObject =
        TaggedToHeapObject(object) otherwise NotFound;
    const receiver: JSReceiver =
        Cast<JSReceiver>(heapObject) otherwise NotFound;
    try {
      TryHasOwnProperty(
          receiver, receiver.map, receiver.instanceType, uniqueName)
          otherwise Found, NotFound, NotFound;
    } label Found {
      tail GetPropertyWithReceiver(
          receiver, uniqueName, receiver, SmiConstant(kReturnUndefined));
    }
  } label NotFound deferred {
    return Undefined;
  }
}

// Trap builtins.

builtin WasmTrap(error: Smi): JSAny {
  tail runtime::ThrowWasmError(LoadContextFromWasmOrJsFrame(), error);
}

builtin ThrowWasmTrapUnreachable(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapUnreachable));
}

builtin WasmTrapHandlerThrowTrap(): JSAny {
  tail runtime::TrapHandlerThrowWasmError(LoadContextFromWasmOrJsFrame());
}

builtin ThrowWasmTrapMemOutOfBounds(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapMemOutOfBounds));
}

builtin ThrowWasmTrapUnalignedAccess(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapUnalignedAccess));
}

builtin ThrowWasmTrapDivByZero(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapDivByZero));
}

builtin ThrowWasmTrapDivUnrepresentable(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapDivUnrepresentable));
}

builtin ThrowWasmTrapRemByZero(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapRemByZero));
}

builtin ThrowWasmTrapFloatUnrepresentable(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapFloatUnrepresentable));
}

builtin ThrowWasmTrapFuncSigMismatch(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapFuncSigMismatch));
}

builtin ThrowWasmTrapDataSegmentOutOfBounds(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapDataSegmentOutOfBounds));
}

builtin ThrowWasmTrapElementSegmentOutOfBounds(): JSAny {
  tail WasmTrap(
      SmiConstant(MessageTemplate::kWasmTrapElementSegmentOutOfBounds));
}

builtin ThrowWasmTrapTableOutOfBounds(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapTableOutOfBounds));
}

builtin ThrowWasmTrapRethrowNull(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapRethrowNull));
}

builtin ThrowWasmTrapNullDereference(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapNullDereference));
}

builtin ThrowWasmTrapIllegalCast(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapIllegalCast));
}

builtin ThrowWasmTrapArrayOutOfBounds(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapArrayOutOfBounds));
}

builtin ThrowWasmTrapArrayTooLarge(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapArrayTooLarge));
}

builtin ThrowWasmTrapStringOffsetOutOfBounds(): JSAny {
  tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapStringOffsetOutOfBounds));
}

macro GetRefAt<T: type, From: type>(base: From, offset: intptr): &T {
  return torque_internal::unsafe::NewOffHeapReference<T>(
      %RawDownCast<RawPtr<T>>(base + offset));
}

extern macro LoadPointerFromRootRegister(intptr): RawPtr;

const kThreadInWasmFlagAddressOffset: constexpr intptr
    generates 'Isolate::thread_in_wasm_flag_address_offset()';

const kActiveSuspenderOffset: constexpr intptr
    generates 'IsolateData::root_slot_offset(RootIndex::kActiveSuspender)';

macro ModifyThreadInWasmFlag(newValue: int32): void {
  const threadInWasmFlagAddress =
      LoadPointerFromRootRegister(kThreadInWasmFlagAddressOffset);
  const threadInWasmFlagRef = GetRefAt<int32>(threadInWasmFlagAddress, 0);
  *threadInWasmFlagRef = newValue;
}

builtin WasmStringNewWtf8(
    offset: uint32, size: uint32, memory: Smi, utf8Variant: Smi): String|Null {
  const trustedData = LoadInstanceDataFromFrame();
  tail runtime::WasmStringNewWtf8(
      LoadContextFromInstanceData(trustedData), trustedData, memory,
      utf8Variant, WasmUint32ToNumber(offset), WasmUint32ToNumber(size));
}
builtin WasmStringNewWtf8Array(
    start: uint32, end: uint32, array: WasmArray, utf8Variant: Smi): String
    |Null {
  // This can be called from Wasm and from "JS String Builtins".
  const context = LoadContextFromWasmOrJsFrame();
  try {
    if (array.length < end) goto OffsetOutOfRange;
    if (end < start) goto OffsetOutOfRange;
    tail runtime::WasmStringNewWtf8Array(
        context, utf8Variant, array, SmiFromUint32(start), SmiFromUint32(end));
  } label OffsetOutOfRange deferred {
    const error = MessageTemplate::kWasmTrapArrayOutOfBounds;
    runtime::ThrowWasmError(context, SmiConstant(error));
  }
}
builtin WasmStringNewWtf16(memory: uint32, offset: uint32, size: uint32):
    String {
  const trustedData = LoadInstanceDataFromFrame();
  tail runtime::WasmStringNewWtf16(
      LoadContextFromInstanceData(trustedData), trustedData,
      SmiFromUint32(memory), WasmUint32ToNumber(offset),
      WasmUint32ToNumber(size));
}

struct TwoByteToOneByteIterator {
  macro Next(): char8 labels NoMore {
    if (this.start == this.end) goto NoMore;
    const raw: char16 = *torque_internal::unsafe::NewReference<char16>(
        this.object, this.start);
    const result: char8 = %RawDownCast<char8>(raw & 0xFF);
    this.start += 2;
    return result;
  }

  object: HeapObject|TaggedZeroPattern;
  start: intptr;
  end: intptr;
}

macro StringFromTwoByteSlice(length: uint32, slice: ConstSlice<char16>):
    String {
  // Ideas for additional future improvements:
  // (1) We could add a fast path for very short strings, e.g. <= 8 chars,
  //     and just allocate two-byte strings for them. That would save time
  //     here, and would only waste a couple of bytes at most. A concern is
  //     that such strings couldn't take one-byte fast paths later on, e.g.
  //     in toLower/toUpper case conversions.
  // (2) We could load more than one array element at a time, e.g. using
  //     intptr-wide loads, or possibly even wider SIMD instructions. We'd
  //     have to make sure that non-aligned start offsets are handled,
  //     and the implementation would become more platform-specific.
  // (3) We could shift the problem around by allocating two-byte strings
  //     here and checking whether they're one-byte-compatible later, e.g.
  //     when promoting them from new to old space. Drawback: rewriting
  //     strings to different maps isn't great for optimized code that's
  //     based on collected type feedback, or that wants to elide duplicate
  //     map checks within the function.
  // (4) We could allocate space for a two-byte string, then optimistically
  //     start writing one-byte characters into it, and then either restart
  //     in two-byte mode if needed, or return the over-allocated bytes to
  //     the allocator in the end.
  // (5) We could standardize a `string.new_ascii_array` instruction, which
  //     could safely produce one-byte strings without checking characters.
  //     See https://github.com/WebAssembly/stringref/issues/53.

  try {
    // To reduce the amount of branching, check 8 code units at a time. The
    // tradeoff for choosing 8 is that we want to check for early termination
    // of the loop often (to avoid unnecessary work) but not too often
    // (because each check has a cost).
    let i: intptr = 0;
    const intptrLength = slice.length;
    const eightElementLoopEnd = intptrLength - 8;
    while (i <= eightElementLoopEnd) {
      const bits = Convert<uint32>(*slice.UncheckedAtIndex(i)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 1)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 2)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 3)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 4)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 5)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 6)) |
          Convert<uint32>(*slice.UncheckedAtIndex(i + 7));
      if (bits > 0xFF) goto TwoByte;
      i += 8;
    }
    let bits: uint32 = 0;
    while (i < intptrLength) {
      bits |= Convert<uint32>(*slice.UncheckedAtIndex(i));
      i += 1;
    }
    if (bits > 0xFF) goto TwoByte;
  } label TwoByte {
    return AllocateSeqTwoByteString(length, slice.Iterator());
  }

  const end = slice.offset + torque_internal::TimesSizeOf<char16>(slice.length);
  return AllocateNonEmptySeqOneByteString(length, TwoByteToOneByteIterator{
    object: slice.object,
    start: slice.offset,
    end: end
  });
}

builtin WasmStringNewWtf16Array(array: WasmArray, start: uint32, end: uint32):
    String {
  try {
    if (array.length < end) goto OffsetOutOfRange;
    if (end < start) goto OffsetOutOfRange;
    const length: uint32 = end - start;
    if (length == 0) return kEmptyString;
    if (length == 1) {
      const offset = kWasmArrayHeaderSize +
          torque_internal::TimesSizeOf<char16>(Convert<intptr>(start));
      const code: char16 = *torque_internal::unsafe::NewReference<char16>(
          array, offset);
      // This makes sure we check the SingleCharacterStringTable.
      return StringFromSingleCharCode(code);
    }
    // Calling into the runtime has overhead, but once we're there it's faster,
    // so it pays off for long strings. The threshold has been determined
    // experimentally.
    if (length >= 32) goto Runtime;
    const intptrLength = Convert<intptr>(length);
    const arrayContent = torque_internal::unsafe::NewConstSlice<char16>(
        array, kWasmArrayHeaderSize, Convert<intptr>(array.length));
    const substring =
        Subslice(arrayContent, Convert<intptr>(start), intptrLength)
        otherwise goto OffsetOutOfRange;

    return StringFromTwoByteSlice(length, substring);
  } label OffsetOutOfRange deferred {
    // This can be called from Wasm and from "JS String Builtins".
    const context = LoadContextFromWasmOrJsFrame();
    const error = MessageTemplate::kWasmTrapArrayOutOfBounds;
    runtime::ThrowWasmError(context, SmiConstant(error));
  } label Runtime deferred {
    const context = LoadContextFromWasmOrJsFrame();
    tail runtime::WasmStringNewWtf16Array(
        context, array, SmiFromUint32(start), SmiFromUint32(end));
  }
}

// For imports based string constants.
// Always returns a String if it didn't trap; typed "JSAny" to satisfy
// Torque's type checker for tail calls.
builtin WasmStringFromDataSegment(
    segmentLength: uint32, arrayStart: uint32, arrayEnd: uint32,
    segmentIndex: Smi, segmentOffset: Smi, variant: Smi): JSAny {
  const trustedData = LoadInstanceDataFromFrame();
  try {
    const segmentOffsetU: uint32 = Unsigned(SmiToInt32(segmentOffset));
    if (segmentLength > Convert<uint32>(kSmiMax) - segmentOffsetU) {
      goto SegmentOOB;
    }
    if (arrayStart > segmentLength) goto ArrayOutOfBounds;
    if (arrayEnd < arrayStart) goto ArrayOutOfBounds;
    const arrayLength = arrayEnd - arrayStart;
    if (arrayLength > segmentLength - arrayStart) goto ArrayOutOfBounds;
    const smiOffset = Convert<PositiveSmi>(segmentOffsetU + arrayStart)
        otherwise SegmentOOB;
    const smiLength = Convert<PositiveSmi>(arrayLength) otherwise SegmentOOB;
    tail runtime::WasmStringNewSegmentWtf8(
        LoadContextFromInstanceData(trustedData), trustedData, segmentIndex,
        smiOffset, smiLength, variant);
  } label SegmentOOB deferred {
    tail ThrowWasmTrapElementSegmentOutOfBounds();
  } label ArrayOutOfBounds deferred {
    tail ThrowWasmTrapArrayOutOfBounds();
  }
}

// Contract: input is any string, output is a string that the TF operator
// "StringPrepareForGetCodeunit" can handle.
builtin WasmStringAsWtf16(str: String): String {
  const cons = Cast<ConsString>(str) otherwise return str;
  return Flatten(cons);
}

builtin WasmStringConst(index: uint32): String {
  const trustedData = LoadInstanceDataFromFrame();
  tail runtime::WasmStringConst(
      LoadContextFromInstanceData(trustedData), trustedData,
      SmiFromUint32(index));
}
builtin WasmStringMeasureUtf8(string: String): int32 {
  const result = runtime::WasmStringMeasureUtf8(LoadContextFromFrame(), string);
  return NumberToInt32(result);
}
builtin WasmStringMeasureWtf8(string: String): int32 {
  const result = runtime::WasmStringMeasureWtf8(LoadContextFromFrame(), string);
  return NumberToInt32(result);
}
builtin WasmStringEncodeWtf8(
    string: String, offset: uint32, memory: Smi, utf8Variant: Smi): uint32 {
  const trustedData = LoadInstanceDataFromFrame();
  const result = runtime::WasmStringEncodeWtf8(
      LoadContextFromInstanceData(trustedData), trustedData, memory,
      utf8Variant, string, WasmUint32ToNumber(offset));
  return NumberToUint32(result);
}
builtin WasmStringEncodeWtf8Array(
    string: String, array: WasmArray, start: uint32, utf8Variant: Smi): uint32 {
  const trustedData = LoadInstanceDataFromFrame();
  const result = runtime::WasmStringEncodeWtf8Array(
      LoadContextFromInstanceData(trustedData), utf8Variant, string, array,
      WasmUint32ToNumber(start));
  return NumberToUint32(result);
}
builtin WasmStringToUtf8Array(string: String): WasmArray {
  return runtime::WasmStringToUtf8Array(LoadContextFromFrame(), string);
}
builtin WasmStringEncodeWtf16(string: String, offset: uint32, memory: Smi):
    uint32 {
  const trustedData = LoadInstanceDataFromFrame();
  runtime::WasmStringEncodeWtf16(
      LoadContextFromInstanceData(trustedData), trustedData, memory, string,
      WasmUint32ToNumber(offset), SmiConstant(0), SmiFromInt32(string.length));
  return Unsigned(string.length);
}
builtin WasmStringEncodeWtf16Array(
    string: String, array: WasmArray, start: uint32): uint32 {
  try {
    if (start > array.length) goto OffsetOutOfRange;
    if (array.length - start < Unsigned(string.length)) goto OffsetOutOfRange;

    const byteOffset: intptr = kWasmArrayHeaderSize +
        torque_internal::TimesSizeOf<char16>(Convert<intptr>(start));
    const arrayContent = torque_internal::unsafe::NewMutableSlice<char16>(
        array, byteOffset, string.length_intptr);
    try {
      StringToSlice(string) otherwise OneByte, TwoByte;
    } label OneByte(slice: ConstSlice<char8>) {
      let fromIt = slice.Iterator();
      let toIt = arrayContent.Iterator();
      while (true) {
        let toRef = toIt.NextReference() otherwise break;
        *toRef = %RawDownCast<char16>(Convert<uint16>(fromIt.NextNotEmpty()));
      }
    } label TwoByte(slice: ConstSlice<char16>) {
      let fromIt = slice.Iterator();
      let toIt = arrayContent.Iterator();
      while (true) {
        let toRef = toIt.NextReference() otherwise break;
        *toRef = fromIt.NextNotEmpty();
      }
    }
    return Unsigned(string.length);
  } label OffsetOutOfRange deferred {
    const error = MessageTemplate::kWasmTrapArrayOutOfBounds;
    runtime::ThrowWasmError(LoadContextFromWasmOrJsFrame(), SmiConstant(error));
  }
}

builtin ThrowToLowerCaseCalledOnNull(): JSAny {
  const context = LoadContextFromFrame();
  const error = MessageTemplate::kCalledOnNullOrUndefined;
  const name = StringConstant('String.prototype.toLowerCase');
  runtime::WasmThrowTypeError(context, SmiConstant(error), name);
}

builtin ThrowIndexOfCalledOnNull(): JSAny {
  const context = LoadContextFromFrame();
  const error = MessageTemplate::kCalledOnNullOrUndefined;
  const name = StringConstant('String.prototype.indexOf');
  runtime::WasmThrowTypeError(context, SmiConstant(error), name);
}

builtin ThrowDataViewTypeError(value: JSAny): JSAny {
  const context = LoadContextFromFrame();
  const error = MessageTemplate::kIncompatibleMethodReceiver;
  runtime::WasmThrowDataViewTypeError(context, SmiConstant(error), value);
}

builtin ThrowDataViewDetachedError(): JSAny {
  const context = LoadContextFromFrame();
  const error = MessageTemplate::kDetachedOperation;
  runtime::WasmThrowDataViewDetachedError(context, SmiConstant(error));
}

builtin ThrowDataViewOutOfBounds(): JSAny {
  const context = LoadContextFromFrame();
  const error = MessageTemplate::kInvalidDataViewAccessorOffset;
  runtime::WasmThrowRangeError(context, SmiConstant(error));
}

builtin WasmStringConcat(a: String, b: String): String {
  const context = LoadContextFromFrame();
  tail StringAdd_CheckNone(a, b);
}

extern builtin StringEqual(NoContext, String, String, intptr): Boolean;

builtin WasmStringEqual(a: String, b: String): int32 {
  if (TaggedEqual(a, b)) return 1;
  if (a.length != b.length) return 0;
  if (StringEqual(kNoContext, a, b, a.length_intptr) == True) {
    return 1;
  }
  return 0;
}

builtin WasmStringIsUSVSequence(str: String): int32 {
  if (IsOneByteStringInstanceType(str.instanceType)) return 1;
  const length = runtime::WasmStringMeasureUtf8(LoadContextFromFrame(), str);
  if (NumberToInt32(length) < 0) return 0;
  return 1;
}

builtin WasmStringAsWtf8(str: String): ByteArray {
  tail runtime::WasmStringAsWtf8(LoadContextFromFrame(), str);
}

macro IsWtf8CodepointStart(view: ByteArray, pos: uint32): bool {
  // We're already at the start of a codepoint if the current byte
  // doesn't start with 0b10xxxxxx.
  return (view.bytes[Convert<uintptr>(pos)] & 0xc0) != 0x80;
}
macro AlignWtf8PositionForward(view: ByteArray, pos: uint32): uint32 {
  const length = Unsigned(SmiToInt32(view.length));
  if (pos >= length) return length;

  if (IsWtf8CodepointStart(view, pos)) return pos;

  // Otherwise `pos` is part of a multibyte codepoint, and is not the
  // leading byte.  The next codepoint will start at pos + 1, pos + 2,
  // or pos + 3.
  if (pos + 1 == length) return length;
  if (IsWtf8CodepointStart(view, pos + 1)) return pos + 1;

  if (pos + 2 == length) return length;
  if (IsWtf8CodepointStart(view, pos + 2)) return pos + 2;

  return pos + 3;
}
macro AlignWtf8PositionBackward(view: ByteArray, pos: uint32): uint32 {
  // Return the highest offset that starts a codepoint which is not
  // greater than pos.  Preconditions: pos in [0, view.length), view
  // contains well-formed WTF-8.
  if (IsWtf8CodepointStart(view, pos)) return pos;
  if (IsWtf8CodepointStart(view, pos - 1)) return pos - 1;
  if (IsWtf8CodepointStart(view, pos - 2)) return pos - 2;
  return pos - 3;
}
builtin WasmStringViewWtf8Advance(view: ByteArray, pos: uint32, bytes: uint32):
    uint32 {
  const clampedPos = AlignWtf8PositionForward(view, pos);
  if (bytes == 0) return clampedPos;
  const length = Unsigned(SmiToInt32(view.length));
  if (bytes >= length - clampedPos) return length;
  return AlignWtf8PositionBackward(view, clampedPos + bytes);
}
struct NewPositionAndBytesWritten {
  newPosition: uintptr;
  bytesWritten: uintptr;
}
builtin WasmStringViewWtf8Encode(
    addr: uint32, pos: uint32, bytes: uint32, view: ByteArray, memory: Smi,
    utf8Variant: Smi): NewPositionAndBytesWritten {
  const start = WasmStringViewWtf8Advance(view, pos, 0);
  const end = WasmStringViewWtf8Advance(view, start, bytes);
  const trustedData = LoadInstanceDataFromFrame();
  const context = LoadContextFromInstanceData(trustedData);

  // kMaxArgs in code-assembler.cc:CallRunTimeImpl is currently limited
  // to 6 arguments when calling a runtime function.  Throw away the
  // memory argument for now; when we need multi-memory we can bump
  // kMaxArgs.
  dcheck(memory == SmiFromInt32(0));

  // Always call out to run-time, to catch invalid addr.
  runtime::WasmStringViewWtf8Encode(
      context, trustedData, utf8Variant, view, WasmUint32ToNumber(addr),
      WasmUint32ToNumber(start), WasmUint32ToNumber(end));

  return NewPositionAndBytesWritten{
    newPosition: Convert<uintptr>(end),
    bytesWritten: Convert<uintptr>(end - start)
  };
}
builtin WasmStringViewWtf8Slice(view: ByteArray, start: uint32, end: uint32):
    String {
  const start = WasmStringViewWtf8Advance(view, start, 0);
  const end = WasmStringViewWtf8Advance(view, end, 0);

  if (end <= start) return kEmptyString;

  tail runtime::WasmStringViewWtf8Slice(
      LoadContextFromFrame(), view, WasmUint32ToNumber(start),
      WasmUint32ToNumber(end));
}
transitioning builtin WasmStringViewWtf16GetCodeUnit(
    string: String, offset: uint32): uint32 {
  try {
    if (Unsigned(string.length) <= offset) goto OffsetOutOfRange;
    const code: char16 = StringCharCodeAt(string, Convert<uintptr>(offset));
    return Convert<uint32>(code);
  } label OffsetOutOfRange deferred {
    const error = MessageTemplate::kWasmTrapStringOffsetOutOfBounds;
    runtime::ThrowWasmError(LoadContextFromFrame(), SmiConstant(error));
  }
}
builtin WasmStringViewWtf16Encode(
    offset: uint32, start: uint32, length: uint32, string: String,
    memory: Smi): uint32 {
  const trustedData = LoadInstanceDataFromFrame();
  const clampedStart =
      start < Unsigned(string.length) ? start : Unsigned(string.length);
  const maxLength = Unsigned(string.length) - clampedStart;
  const clampedLength = length < maxLength ? length : maxLength;
  runtime::WasmStringEncodeWtf16(
      LoadContextFromInstanceData(trustedData), trustedData, memory, string,
      WasmUint32ToNumber(offset), SmiFromUint32(clampedStart),
      SmiFromUint32(clampedLength));
  return clampedLength;
}
transitioning builtin WasmStringViewWtf16Slice(
    string: String, start: uint32, end: uint32): String {
  const length = Unsigned(string.length);
  if (start >= length) return kEmptyString;
  if (end <= start) return kEmptyString;

  // On a high level, the intended logic is:
  // (1) If start == 0 && end == string.length, return string.
  // (2) If clampedLength == 1, use a cached single-character string.
  // (3) If clampedLength < SlicedString::kMinLength, make a copy.
  // (4) If clampedLength < string.length / 2, make a copy.
  // (5) Else, create a slice.
  // The reason for having case (4) is that case (5) has the risk of keeping
  // huge parent strings alive unnecessarily, and Wasm currently doesn't have a
  // way to control that behavior, so we have to be careful.
  // The reason for having case (5) is that case (4) would lead to quadratic
  // overall behavior if code repeatedly chops off a few characters of a long
  // string, which we want to avoid.
  // The string::SubString implementation can handle cases (1), (2), (3),
  // and (5). The inline code here handles case (4), and doesn't mind if it
  // also catches some of case (3).
  const clampedEnd = end <= length ? end : length;
  const clampedLength = clampedEnd - start;
  if (clampedLength > 1 && clampedLength < length / 2) {
    try {
      // Calling into the runtime has overhead, but once we're there it's
      // faster, so it pays off for long strings.
      if (clampedLength > 32) goto Runtime;
      StringToSlice(string) otherwise OneByte, TwoByte;
    } label OneByte(slice: ConstSlice<char8>) {
      let subslice = Subslice(
          slice, Convert<intptr>(start), Convert<intptr>(clampedLength))
          otherwise unreachable;
      return AllocateNonEmptySeqOneByteString(
          clampedLength, subslice.Iterator());
    } label TwoByte(slice: ConstSlice<char16>) {
      let subslice = Subslice(
          slice, Convert<intptr>(start), Convert<intptr>(clampedLength))
          otherwise unreachable;
      return StringFromTwoByteSlice(clampedLength, subslice);
    } label Runtime deferred {
      const context = LoadContextFromWasmOrJsFrame();
      tail runtime::WasmSubstring(
          context, string, SmiFromUint32(start), SmiFromUint32(clampedLength));
    }
  }
  return string::SubString(
      string, Convert<uintptr>(start), Convert<uintptr>(clampedEnd));
}
builtin WasmStringAsIter(string: String): WasmStringViewIter {
  return new WasmStringViewIter{string: string, offset: 0, optional_padding: 0};
}
macro IsLeadSurrogate(code: char16): bool {
  return (code & 0xfc00) == 0xd800;
}
macro IsTrailSurrogate(code: char16): bool {
  return (code & 0xfc00) == 0xdc00;
}
macro CombineSurrogatePair(lead: char16, trail: char16): int32 {
  const lead32 = Convert<uint32>(lead);
  const trail32 = Convert<uint32>(trail);
  // Surrogate pairs encode codepoints in the range
  // [0x010000, 0x10FFFF].  Each surrogate has 10 bits of information in
  // the low bits.  We can combine them together with a shift-and-add,
  // then add a bias of 0x010000 - 0xD800<<10 - 0xDC00 = 0xFCA02400.
  const surrogateBias: uint32 = 0xFCA02400;
  return Signed((lead32 << 10) + trail32 + surrogateBias);
}

builtin WasmStringCodePointAt(string: String, offset: uint32): uint32 {
  try {
    if (Unsigned(string.length) <= offset) goto OffsetOutOfRange;
    const lead: char16 = StringCharCodeAt(string, Convert<uintptr>(offset));
    if (!IsLeadSurrogate(lead)) return Convert<uint32>(lead);
    const trailOffset = offset + 1;
    if (Unsigned(string.length) <= trailOffset) return Convert<uint32>(lead);
    const trail: char16 =
        StringCharCodeAt(string, Convert<uintptr>(trailOffset));
    if (!IsTrailSurrogate(trail)) return Convert<uint32>(lead);
    return Unsigned(CombineSurrogatePair(lead, trail));
  } label OffsetOutOfRange deferred {
    const error = MessageTemplate::kWasmTrapStringOffsetOutOfBounds;
    runtime::ThrowWasmError(LoadContextFromFrame(), SmiConstant(error));
  }
}

builtin WasmStringViewIterNext(view: WasmStringViewIter): int32 {
  const string = view.string;
  const offset = view.offset;
  if (offset >= Unsigned(string.length)) return -1;
  const code: char16 = StringCharCodeAt(string, Convert<uintptr>(offset));
  try {
    if (IsLeadSurrogate(code) && offset + 1 < Unsigned(string.length)) {
      goto CheckForSurrogatePair;
    }
  } label CheckForSurrogatePair deferred {
    const code2: char16 =
        StringCharCodeAt(string, Convert<uintptr>(offset + 1));
    if (IsTrailSurrogate(code2)) {
      view.offset = offset + 2;
      return CombineSurrogatePair(code, code2);
    }
  }
  view.offset = offset + 1;
  return Signed(Convert<uint32>(code));
}
builtin WasmStringViewIterAdvance(
    view: WasmStringViewIter, codepoints: uint32): uint32 {
  const string = view.string;
  let offset = view.offset;
  let advanced: uint32 = 0;
  while (advanced < codepoints) {
    if (offset == Unsigned(string.length)) break;
    advanced = advanced + 1;
    if (offset + 1 < Unsigned(string.length) &&
        IsLeadSurrogate(StringCharCodeAt(string, Convert<uintptr>(offset))) &&
        IsTrailSurrogate(
            StringCharCodeAt(string, Convert<uintptr>(offset + 1)))) {
      offset = offset + 2;
    } else {
      offset = offset + 1;
    }
  }
  view.offset = offset;
  return advanced;
}
builtin WasmStringViewIterRewind(view: WasmStringViewIter, codepoints: uint32):
    uint32 {
  const string = view.string;
  let offset = view.offset;
  let rewound: uint32 = 0;
  if (string.length == 0) return 0;
  while (rewound < codepoints) {
    if (offset == 0) break;
    rewound = rewound + 1;
    if (offset >= 2 &&
        IsTrailSurrogate(
            StringCharCodeAt(string, Convert<uintptr>(offset - 1))) &&
        IsLeadSurrogate(
            StringCharCodeAt(string, Convert<uintptr>(offset - 2)))) {
      offset = offset - 2;
    } else {
      offset = offset - 1;
    }
  }
  view.offset = offset;
  return rewound;
}
builtin WasmStringViewIterSlice(view: WasmStringViewIter, codepoints: uint32):
    String {
  const string = view.string;
  const start = view.offset;
  let end = view.offset;
  let advanced: uint32 = 0;
  while (advanced < codepoints) {
    if (end == Unsigned(string.length)) break;
    advanced = advanced + 1;
    if (end + 1 < Unsigned(string.length) &&
        IsLeadSurrogate(StringCharCodeAt(string, Convert<uintptr>(end))) &&
        IsTrailSurrogate(StringCharCodeAt(string, Convert<uintptr>(end + 1)))) {
      end = end + 2;
    } else {
      end = end + 1;
    }
  }
  return (start == end) ?
      kEmptyString :
      string::SubString(string, Convert<uintptr>(start), Convert<uintptr>(end));
}

builtin WasmIntToString(x: int32, radix: int32): String {
  if (radix == 10) {
    const smi = SmiFromInt32(x);
    const untagged = SmiToInt32(smi);
    if (x == untagged) {
      // Queries and populates the NumberToStringCache, but needs tagged
      // inputs, so only call this for Smis.
      return NumberToString(smi);
    }
    return number::IntToDecimalString(x);
  }

  // Pretend that Number.prototype.toString was called.
  if (radix < 2 || radix > 36) {
    runtime::ThrowRangeError(
        LoadContextFromInstanceData(LoadInstanceDataFromFrame()),
        SmiConstant(MessageTemplate::kToRadixFormatRange));
  }
  return number::IntToString(x, Unsigned(radix));
}

builtin WasmStringToDouble(s: String): float64 {
  const hash: NameHash = s.raw_hash_field;
  if (IsIntegerIndex(hash) &&
      hash.array_index_length < kMaxCachedArrayIndexLength) {
    const arrayIndex: uint32 = DecodeArrayIndexFromHashField(hash);
    return Convert<float64>(Signed(arrayIndex));
  }
  return StringToFloat64(Flatten(s));
}

builtin WasmStringFromCodePoint(codePoint: uint32): String {
  tail runtime::WasmStringFromCodePoint(
      LoadContextFromFrame(), WasmUint32ToNumber(codePoint));
}

builtin WasmStringHash(string: String): int32 {
  const result = runtime::WasmStringHash(kNoContext, string);
  return SmiToInt32(result);
}

builtin WasmAnyConvertExtern(externObject: JSAny): JSAny {
  const trustedData = LoadInstanceDataFromFrame();
  const context = LoadContextFromInstanceData(trustedData);

  tail runtime::WasmJSToWasmObject(
      context, externObject, SmiConstant(kAnyType));
}

extern macro CallOrConstructBuiltinsAssembler::GetCompatibleReceiver(
    JSReceiver, HeapObject, Context): JSReceiver;

builtin WasmFastApiCallTypeCheckAndUpdateIC(
    implicit context: Context)(data: WasmFastApiCallData,
    receiver: JSAny): Smi {
  try {
    const rec = Cast<JSReceiver>(receiver) otherwise goto IllegalCast;
    ModifyThreadInWasmFlag(0);
    // We don't care about the actual compatible receiver; we just rely
    // on this helper throwing an exception when there isn't one.
    GetCompatibleReceiver(rec, data.signature, context);
    ModifyThreadInWasmFlag(1);
    data.cached_map = StrongToWeak(rec.map);
    return 1;
  } label IllegalCast {
    const error = MessageTemplate::kIllegalInvocation;
    runtime::WasmThrowTypeError(context, SmiConstant(error), Convert<Smi>(0));
  }
}
}  // namespace wasm