diff options
Diffstat (limited to 'js/src/jit/arm64/Assembler-arm64.h')
| -rw-r--r-- | js/src/jit/arm64/Assembler-arm64.h | 557 |
1 files changed, 557 insertions, 0 deletions
diff --git a/js/src/jit/arm64/Assembler-arm64.h b/js/src/jit/arm64/Assembler-arm64.h new file mode 100644 index 0000000000..287ab23b3d --- /dev/null +++ b/js/src/jit/arm64/Assembler-arm64.h @@ -0,0 +1,557 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef A64_ASSEMBLER_A64_H_ +#define A64_ASSEMBLER_A64_H_ + +#include "jit/arm64/vixl/Assembler-vixl.h" + +#include "jit/JitCompartment.h" + +namespace js { +namespace jit { + +// VIXL imports. +typedef vixl::Register ARMRegister; +typedef vixl::FPRegister ARMFPRegister; +using vixl::ARMBuffer; +using vixl::Instruction; + +static const uint32_t AlignmentAtPrologue = 0; +static const uint32_t AlignmentMidPrologue = 8; +static const Scale ScalePointer = TimesEight; + +// The MacroAssembler uses scratch registers extensively and unexpectedly. +// For safety, scratch registers should always be acquired using +// vixl::UseScratchRegisterScope. +static constexpr Register ScratchReg = { Registers::ip0 }; +static constexpr ARMRegister ScratchReg64 = { ScratchReg, 64 }; + +static constexpr Register ScratchReg2 = { Registers::ip1 }; +static constexpr ARMRegister ScratchReg2_64 = { ScratchReg2, 64 }; + +static constexpr FloatRegister ScratchDoubleReg = { FloatRegisters::d31, FloatRegisters::Double }; +static constexpr FloatRegister ReturnDoubleReg = { FloatRegisters::d0, FloatRegisters::Double }; + +static constexpr FloatRegister ReturnFloat32Reg = { FloatRegisters::s0, FloatRegisters::Single }; +static constexpr FloatRegister ScratchFloat32Reg = { FloatRegisters::s31, FloatRegisters::Single }; + +static constexpr Register InvalidReg = { Registers::invalid_reg }; +static constexpr FloatRegister InvalidFloatReg = { FloatRegisters::invalid_fpreg, FloatRegisters::Single }; + +static constexpr Register OsrFrameReg = { Registers::x3 }; +static constexpr Register ArgumentsRectifierReg = { Registers::x8 }; +static constexpr Register CallTempReg0 = { Registers::x9 }; +static constexpr Register CallTempReg1 = { Registers::x10 }; +static constexpr Register CallTempReg2 = { Registers::x11 }; +static constexpr Register CallTempReg3 = { Registers::x12 }; +static constexpr Register CallTempReg4 = { Registers::x13 }; +static constexpr Register CallTempReg5 = { Registers::x14 }; + +static constexpr Register PreBarrierReg = { Registers::x1 }; + +static constexpr Register ReturnReg = { Registers::x0 }; +static constexpr Register64 ReturnReg64(ReturnReg); +static constexpr Register JSReturnReg = { Registers::x2 }; +static constexpr Register FramePointer = { Registers::fp }; +static constexpr Register ZeroRegister = { Registers::sp }; +static constexpr ARMRegister ZeroRegister64 = { Registers::sp, 64 }; +static constexpr ARMRegister ZeroRegister32 = { Registers::sp, 32 }; + +static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg; +static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg; + +// StackPointer is intentionally undefined on ARM64 to prevent misuse: +// using sp as a base register is only valid if sp % 16 == 0. +static constexpr Register RealStackPointer = { Registers::sp }; + +static constexpr Register PseudoStackPointer = { Registers::x28 }; +static constexpr ARMRegister PseudoStackPointer64 = { Registers::x28, 64 }; +static constexpr ARMRegister PseudoStackPointer32 = { Registers::x28, 32 }; + +// StackPointer for use by irregexp. +static constexpr Register RegExpStackPointer = PseudoStackPointer; + +static constexpr Register IntArgReg0 = { Registers::x0 }; +static constexpr Register IntArgReg1 = { Registers::x1 }; +static constexpr Register IntArgReg2 = { Registers::x2 }; +static constexpr Register IntArgReg3 = { Registers::x3 }; +static constexpr Register IntArgReg4 = { Registers::x4 }; +static constexpr Register IntArgReg5 = { Registers::x5 }; +static constexpr Register IntArgReg6 = { Registers::x6 }; +static constexpr Register IntArgReg7 = { Registers::x7 }; +static constexpr Register GlobalReg = { Registers::x20 }; +static constexpr Register HeapReg = { Registers::x21 }; +static constexpr Register HeapLenReg = { Registers::x22 }; + +// Define unsized Registers. +#define DEFINE_UNSIZED_REGISTERS(N) \ +static constexpr Register r##N = { Registers::x##N }; +REGISTER_CODE_LIST(DEFINE_UNSIZED_REGISTERS) +#undef DEFINE_UNSIZED_REGISTERS +static constexpr Register ip0 = { Registers::x16 }; +static constexpr Register ip1 = { Registers::x16 }; +static constexpr Register fp = { Registers::x30 }; +static constexpr Register lr = { Registers::x30 }; +static constexpr Register rzr = { Registers::xzr }; + +// Import VIXL registers into the js::jit namespace. +#define IMPORT_VIXL_REGISTERS(N) \ +static constexpr ARMRegister w##N = vixl::w##N; \ +static constexpr ARMRegister x##N = vixl::x##N; +REGISTER_CODE_LIST(IMPORT_VIXL_REGISTERS) +#undef IMPORT_VIXL_REGISTERS +static constexpr ARMRegister wzr = vixl::wzr; +static constexpr ARMRegister xzr = vixl::xzr; +static constexpr ARMRegister wsp = vixl::wsp; +static constexpr ARMRegister sp = vixl::sp; + +// Import VIXL VRegisters into the js::jit namespace. +#define IMPORT_VIXL_VREGISTERS(N) \ +static constexpr ARMFPRegister s##N = vixl::s##N; \ +static constexpr ARMFPRegister d##N = vixl::d##N; +REGISTER_CODE_LIST(IMPORT_VIXL_VREGISTERS) +#undef IMPORT_VIXL_VREGISTERS + +static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg); + +// Registers used in the GenerateFFIIonExit Enable Activation block. +static constexpr Register WasmIonExitRegCallee = r8; +static constexpr Register WasmIonExitRegE0 = r0; +static constexpr Register WasmIonExitRegE1 = r1; + +// Registers used in the GenerateFFIIonExit Disable Activation block. +// None of these may be the second scratch register. +static constexpr Register WasmIonExitRegReturnData = r2; +static constexpr Register WasmIonExitRegReturnType = r3; +static constexpr Register WasmIonExitRegD0 = r0; +static constexpr Register WasmIonExitRegD1 = r1; +static constexpr Register WasmIonExitRegD2 = r4; + +// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand). +static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; +static constexpr Register RegExpMatcherStringReg = CallTempReg1; +static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; + +// Registerd used in RegExpTester instruction (do not use ReturnReg). +static constexpr Register RegExpTesterRegExpReg = CallTempReg0; +static constexpr Register RegExpTesterStringReg = CallTempReg1; +static constexpr Register RegExpTesterLastIndexReg = CallTempReg2; + +static constexpr Register JSReturnReg_Type = r3; +static constexpr Register JSReturnReg_Data = r2; + +static constexpr FloatRegister NANReg = { FloatRegisters::d14, FloatRegisters::Single }; +// N.B. r8 isn't listed as an aapcs temp register, but we can use it as such because we never +// use return-structs. +static constexpr Register CallTempNonArgRegs[] = { r8, r9, r10, r11, r12, r13, r14, r15 }; +static const uint32_t NumCallTempNonArgRegs = + mozilla::ArrayLength(CallTempNonArgRegs); + +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +// This boolean indicates whether we support SIMD instructions flavoured for +// this architecture or not. Rather than a method in the LIRGenerator, it is +// here such that it is accessible from the entire codebase. Once full support +// for SIMD is reached on all tier-1 platforms, this constant can be deleted. +static constexpr bool SupportsSimd = false; +static constexpr uint32_t SimdMemoryAlignment = 16; + +static_assert(CodeAlignment % SimdMemoryAlignment == 0, + "Code alignment should be larger than any of the alignments which are used for " + "the constant sections of the code buffer. Thus it should be larger than the " + "alignment for SIMD constants."); + +static const uint32_t WasmStackAlignment = SimdMemoryAlignment; +static const int32_t WasmGlobalRegBias = 1024; + +// Does this architecture support SIMD conversions between Uint32x4 and Float32x4? +static constexpr bool SupportsUint32x4FloatConversions = false; + +// Does this architecture support comparisons of unsigned integer vectors? +static constexpr bool SupportsUint8x16Compares = false; +static constexpr bool SupportsUint16x8Compares = false; +static constexpr bool SupportsUint32x4Compares = false; + +class Assembler : public vixl::Assembler +{ + public: + Assembler() + : vixl::Assembler() + { } + + typedef vixl::Condition Condition; + + void finish(); + bool asmMergeWith(const Assembler& other) { + MOZ_CRASH("NYI"); + } + void trace(JSTracer* trc); + + // Emit the jump table, returning the BufferOffset to the first entry in the table. + BufferOffset emitExtendedJumpTable(); + BufferOffset ExtendedJumpTable_; + void executableCopy(uint8_t* buffer); + + BufferOffset immPool(ARMRegister dest, uint8_t* value, vixl::LoadLiteralOp op, + ARMBuffer::PoolEntry* pe = nullptr); + BufferOffset immPool64(ARMRegister dest, uint64_t value, ARMBuffer::PoolEntry* pe = nullptr); + BufferOffset immPool64Branch(RepatchLabel* label, ARMBuffer::PoolEntry* pe, vixl::Condition c); + BufferOffset fImmPool(ARMFPRegister dest, uint8_t* value, vixl::LoadLiteralOp op); + BufferOffset fImmPool64(ARMFPRegister dest, double value); + BufferOffset fImmPool32(ARMFPRegister dest, float value); + + void bind(Label* label) { bind(label, nextOffset()); } + void bind(Label* label, BufferOffset boff); + void bind(RepatchLabel* label); + void bindLater(Label* label, wasm::TrapDesc target) { + MOZ_CRASH("NYI"); + } + + bool oom() const { + return AssemblerShared::oom() || + armbuffer_.oom() || + jumpRelocations_.oom() || + dataRelocations_.oom() || + preBarriers_.oom(); + } + + void copyJumpRelocationTable(uint8_t* dest) const { + if (jumpRelocations_.length()) + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } + void copyDataRelocationTable(uint8_t* dest) const { + if (dataRelocations_.length()) + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } + void copyPreBarrierTable(uint8_t* dest) const { + if (preBarriers_.length()) + memcpy(dest, preBarriers_.buffer(), preBarriers_.length()); + } + + size_t jumpRelocationTableBytes() const { + return jumpRelocations_.length(); + } + size_t dataRelocationTableBytes() const { + return dataRelocations_.length(); + } + size_t preBarrierTableBytes() const { + return preBarriers_.length(); + } + size_t bytesNeeded() const { + return SizeOfCodeGenerated() + + jumpRelocationTableBytes() + + dataRelocationTableBytes() + + preBarrierTableBytes(); + } + + void processCodeLabels(uint8_t* rawCode) { + for (size_t i = 0; i < codeLabels_.length(); i++) { + CodeLabel label = codeLabels_[i]; + Bind(rawCode, label.patchAt(), rawCode + label.target()->offset()); + } + } + + void Bind(uint8_t* rawCode, CodeOffset* label, const void* address) { + *reinterpret_cast<const void**>(rawCode + label->offset()) = address; + } + + void retarget(Label* cur, Label* next); + + // The buffer is about to be linked. Ensure any constant pools or + // excess bookkeeping has been flushed to the instruction stream. + void flush() { + armbuffer_.flushPool(); + } + + void comment(const char* msg) { + // This is not implemented because setPrinter() is not implemented. + // TODO spew("; %s", msg); + } + + int actualIndex(int curOffset) { + ARMBuffer::PoolEntry pe(curOffset); + return armbuffer_.poolEntryOffset(pe); + } + size_t labelToPatchOffset(CodeOffset label) { + return label.offset(); + } + static uint8_t* PatchableJumpAddress(JitCode* code, uint32_t index) { + return code->raw() + index; + } + void setPrinter(Sprinter* sp) { + } + + static bool SupportsFloatingPoint() { return true; } + static bool SupportsUnalignedAccesses() { return true; } + static bool SupportsSimd() { return js::jit::SupportsSimd; } + + // Tracks a jump that is patchable after finalization. + void addJumpRelocation(BufferOffset src, Relocation::Kind reloc); + + protected: + // Add a jump whose target is unknown until finalization. + // The jump may not be patched at runtime. + void addPendingJump(BufferOffset src, ImmPtr target, Relocation::Kind kind); + + // Add a jump whose target is unknown until finalization, and may change + // thereafter. The jump is patchable at runtime. + size_t addPatchableJump(BufferOffset src, Relocation::Kind kind); + + public: + static uint32_t PatchWrite_NearCallSize() { + return 4; + } + + static uint32_t NopSize() { + return 4; + } + + static void PatchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall) { + Instruction* dest = (Instruction*)start.raw(); + //printf("patching %p with call to %p\n", start.raw(), toCall.raw()); + bl(dest, ((Instruction*)toCall.raw() - dest)>>2); + + } + static void PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expected); + + static void PatchDataWithValueCheck(CodeLocationLabel label, + ImmPtr newValue, + ImmPtr expected); + + static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) { + // Raw is going to be the return address. + uint32_t* raw = (uint32_t*)label.raw(); + // Overwrite the 4 bytes before the return address, which will end up being + // the call instruction. + *(raw - 1) = imm.value; + } + static uint32_t AlignDoubleArg(uint32_t offset) { + MOZ_CRASH("AlignDoubleArg()"); + } + static uintptr_t GetPointer(uint8_t* ptr) { + Instruction* i = reinterpret_cast<Instruction*>(ptr); + uint64_t ret = i->Literal64(); + return ret; + } + + // Toggle a jmp or cmp emitted by toggledJump(). + static void ToggleToJmp(CodeLocationLabel inst_); + static void ToggleToCmp(CodeLocationLabel inst_); + static void ToggleCall(CodeLocationLabel inst_, bool enabled); + + static void TraceJumpRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader); + static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader); + + static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm); + + static void FixupNurseryObjects(JSContext* cx, JitCode* code, CompactBufferReader& reader, + const ObjectVector& nurseryObjects); + + public: + // A Jump table entry is 2 instructions, with 8 bytes of raw data + static const size_t SizeOfJumpTableEntry = 16; + + struct JumpTableEntry + { + uint32_t ldr; + uint32_t br; + void* data; + + Instruction* getLdr() { + return reinterpret_cast<Instruction*>(&ldr); + } + }; + + // Offset of the patchable target for the given entry. + static const size_t OffsetOfJumpTableEntryPointer = 8; + + public: + void writeCodePointer(AbsoluteLabel* absoluteLabel) { + MOZ_ASSERT(!absoluteLabel->bound()); + uintptr_t x = LabelBase::INVALID_OFFSET; + BufferOffset off = EmitData(&x, sizeof(uintptr_t)); + + // The x86/x64 makes general use of AbsoluteLabel and weaves a linked list + // of uses of an AbsoluteLabel through the assembly. ARM only uses labels + // for the case statements of switch jump tables. Thus, for simplicity, we + // simply treat the AbsoluteLabel as a label and bind it to the offset of + // the jump table entry that needs to be patched. + LabelBase* label = absoluteLabel; + label->bind(off.getOffset()); + } + + void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, + const Disassembler::HeapAccess& heapAccess) + { + MOZ_CRASH("verifyHeapAccessDisassembly"); + } + + protected: + // Because jumps may be relocated to a target inaccessible by a short jump, + // each relocatable jump must have a unique entry in the extended jump table. + // Valid relocatable targets are of type Relocation::JITCODE. + struct JumpRelocation + { + BufferOffset jump; // Offset to the short jump, from the start of the code buffer. + uint32_t extendedTableIndex; // Unique index within the extended jump table. + + JumpRelocation(BufferOffset jump, uint32_t extendedTableIndex) + : jump(jump), extendedTableIndex(extendedTableIndex) + { } + }; + + // Structure for fixing up pc-relative loads/jumps when the machine + // code gets moved (executable copy, gc, etc.). + struct RelativePatch + { + BufferOffset offset; + void* target; + Relocation::Kind kind; + + RelativePatch(BufferOffset offset, void* target, Relocation::Kind kind) + : offset(offset), target(target), kind(kind) + { } + }; + + // List of jumps for which the target is either unknown until finalization, + // or cannot be known due to GC. Each entry here requires a unique entry + // in the extended jump table, and is patched at finalization. + js::Vector<RelativePatch, 8, SystemAllocPolicy> pendingJumps_; + + // Final output formatters. + CompactBufferWriter jumpRelocations_; + CompactBufferWriter dataRelocations_; + CompactBufferWriter preBarriers_; +}; + +static const uint32_t NumIntArgRegs = 8; +static const uint32_t NumFloatArgRegs = 8; + +class ABIArgGenerator +{ + public: + ABIArgGenerator() + : intRegIndex_(0), + floatRegIndex_(0), + stackOffset_(0), + current_() + { } + + ABIArg next(MIRType argType); + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + + protected: + unsigned intRegIndex_; + unsigned floatRegIndex_; + uint32_t stackOffset_; + ABIArg current_; +}; + +static constexpr Register ABINonArgReg0 = r8; +static constexpr Register ABINonArgReg1 = r9; +static constexpr Register ABINonArgReg2 = r10; +static constexpr Register ABINonArgReturnReg0 = r8; +static constexpr Register ABINonArgReturnReg1 = r9; + +// TLS pointer argument register for WebAssembly functions. This must not alias +// any other register used for passing function arguments or return values. +// Preserved by WebAssembly functions. +static constexpr Register WasmTlsReg = { Registers::x17 }; + +// Registers used for wasm table calls. These registers must be disjoint +// from the ABI argument registers, WasmTlsReg and each other. +static constexpr Register WasmTableCallScratchReg = ABINonArgReg0; +static constexpr Register WasmTableCallSigReg = ABINonArgReg1; +static constexpr Register WasmTableCallIndexReg = ABINonArgReg2; + +static inline bool +GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, Register* out) +{ + if (usedIntArgs >= NumIntArgRegs) + return false; + *out = Register::FromCode(usedIntArgs); + return true; +} + +static inline bool +GetFloatArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, FloatRegister* out) +{ + if (usedFloatArgs >= NumFloatArgRegs) + return false; + *out = FloatRegister::FromCode(usedFloatArgs); + return true; +} + +// Get a register in which we plan to put a quantity that will be used as an +// integer argument. This differs from GetIntArgReg in that if we have no more +// actual argument registers to use we will fall back on using whatever +// CallTempReg* don't overlap the argument registers, and only fail once those +// run out too. +static inline bool +GetTempRegForIntArg(uint32_t usedIntArgs, uint32_t usedFloatArgs, Register* out) +{ + if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) + return true; + // Unfortunately, we have to assume things about the point at which + // GetIntArgReg returns false, because we need to know how many registers it + // can allocate. + usedIntArgs -= NumIntArgRegs; + if (usedIntArgs >= NumCallTempNonArgRegs) + return false; + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +inline Imm32 +Imm64::firstHalf() const +{ + return low(); +} + +inline Imm32 +Imm64::secondHalf() const +{ + return hi(); +} + +void PatchJump(CodeLocationJump& jump_, CodeLocationLabel label, + ReprotectCode reprotect = DontReprotect); + +static inline void +PatchBackedge(CodeLocationJump& jump_, CodeLocationLabel label, JitRuntime::BackedgeTarget target) +{ + PatchJump(jump_, label); +} + +// Forbids pool generation during a specified interval. Not nestable. +class AutoForbidPools +{ + Assembler* asm_; + + public: + AutoForbidPools(Assembler* asm_, size_t maxInst) + : asm_(asm_) + { + asm_->enterNoPool(maxInst); + } + + ~AutoForbidPools() { + asm_->leaveNoPool(); + } +}; + +} // namespace jit +} // namespace js + +#endif // A64_ASSEMBLER_A64_H_ |