x86_64 compiler backend wip

libs/ast/include/pslang/ast/builtins.hpp

@@ -12,7 +12,7 @@ namespace pslang::ast
 	{
 		expression_ptr expression;
 		type_ptr type;
-		location location;
+		ast::location location;
 		types::type_ptr inferred_source_type = nullptr;
 		types::type_ptr inferred_type = nullptr;
 	};
@@ -21,9 +21,9 @@ namespace pslang::ast
 	{
 		expression_ptr expression;
 		type_ptr type;
-		location location;
+		ast::location location;
 		types::type_ptr inferred_source_type = nullptr;
 		types::type_ptr inferred_type = nullptr;
 	};
 
-}
+}

libs/ast/include/pslang/ast/control.hpp

@@ -43,8 +43,8 @@ namespace pslang::ast
 		{
 			expression_ptr condition;
 			statement_list_ptr statements;
-			location prelude_location;
-			location location;
+			ast::location prelude_location;
+			ast::location location;
 		};
 
 		std::vector<block> blocks;
@@ -55,18 +55,18 @@ namespace pslang::ast
 	{
 		expression_ptr condition;
 		statement_list_ptr statements;
-		location prelude_location;
-		location location;
+		ast::location prelude_location;
+		ast::location location;
 	};
 
 	struct break_statement
 	{
-		location location;
+		ast::location location;
 	};
 
 	struct continue_statement
 	{
-		location location;
+		ast::location location;
 	};
 
 }

libs/interpreter/include/pslang/interpreter/context.hpp

@@ -50,7 +50,7 @@ namespace pslang::interpreter
 	struct context
 	{
 		bool trace = false;
-		foreign_resolver foreign_resolver;
+		interpreter::foreign_resolver foreign_resolver;
 		std::vector<frame> frame_stack;
 	};
 

libs/ir/include/pslang/ir/node.hpp

@@ -152,7 +152,7 @@ namespace pslang::ir
 
 	struct node
 	{
-		instruction instruction;
+		ir::instruction instruction;
 
 		// Null for non-value instructions
 		types::type_ptr inferred_type = nullptr;

libs/jit/include/pslang/jit/abi.hpp

@@ -1,5 +1,9 @@
 #pragma once
 
+// 'linux' macro is not standard-conforming, but can still be present
+// with -std=gnu++XX or absence of -pedantic. Just undefine it in any case.
+#undef linux
+
 namespace pslang::jit
 {
 

libs/jit/include/pslang/jit/arch/linux_x86_64/compiler.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <pslang/jit/jit.hpp>
+#include <pslang/ir/compiler.hpp>
+
+namespace pslang::jit::linux_x86_64
+{
+
+	void compile(program_context & pcontext, ir::module_context const & mcontext);
+
+}

libs/jit/include/pslang/jit/arch/linux_x86_64/instruction_builder.hpp

@@ -0,0 +1,344 @@
+#pragma once
+
+#include <vector>
+#include <cstdint>
+
+namespace pslang::jit::linux_x86_64
+{
+
+	enum class reg : std::uint8_t
+	{
+		rax = 0,
+		rcx = 1,
+		rdx = 2,
+		rbx = 3,
+		rsp = 4,
+		rbp = 5,
+		rsi = 6,
+		rdi = 7,
+		r8  = 8,
+		r9  = 9,
+		r10 = 10,
+		r11 = 11,
+		r12 = 12,
+		r13 = 13,
+		r14 = 14,
+		r15 = 15,
+
+		xmm0  = 0,
+		xmm1  = 1,
+		xmm2  = 2,
+		xmm3  = 3,
+		xmm4  = 4,
+		xmm5  = 5,
+		xmm6  = 6,
+		xmm7  = 7,
+		xmm8  = 8,
+		xmm9  = 9,
+		xmm10 = 10,
+		xmm11 = 11,
+		xmm12 = 12,
+		xmm13 = 13,
+		xmm14 = 14,
+		xmm15 = 15,
+	};
+
+	struct instruction_builder
+	{
+		std::vector<std::uint8_t> & code;
+
+		void nop();
+
+		// Return from procedure: pop the 64-bit return address from the top of the stack,
+		// optionally pop extra @count bytes from the stack, then transfer execution
+		// to the return address
+		void ret(std::uint16_t count = 0);
+
+		// Add a signed 32-bit @value to the register @reg
+		void add_imm(reg reg, std::int32_t value);
+
+		// Subtract a signed 32-bit @value from the register @reg
+		void sub_imm(reg reg, std::int32_t value);
+
+		// Store the zero-extended 32-bit @value in register @reg_dst
+		void movzd(std::uint32_t value, reg reg_dst);
+
+		// Store the sign-extended 32-bit @value in register @reg_dst
+		void movsd(std::int32_t value, reg reg_dst);
+
+		// Store the zero-extended 8-bit GPR @reg_src in 64-bit register @reg_dst
+		void movzx_8(reg reg_src, reg reg_dst);
+
+		// Store the zero-extended 16-bit GPR @reg_src in 64-bit register @reg_dst
+		void movzx_16(reg reg_src, reg reg_dst);
+
+		// Store the sign-extended 8-bit GPR @reg_src in 64-bit register @reg_dst
+		void movsx_8(reg reg_src, reg reg_dst);
+
+		// Store the sign-extended 16-bit GPR @reg_src in 64-bit register @reg_dst
+		void movsx_16(reg reg_src, reg reg_dst);
+
+		// Store the sign-extended 32-bit GPR @reg_src in 64-bit register @reg_dst
+		void movsx_32(reg reg_src, reg reg_dst);
+
+		// Store the 64-bit @value in register @reg_dst
+		void mov(std::uint64_t value, reg reg_dst);
+
+		// Copy the 64-bit GPR @reg_src to 64-bit GPR @reg_dst
+		void mov(reg reg_src, reg reg_dst);
+
+		// Copy the 64-bit value of GPR @reg_src into XMM @reg_dst
+		void mov_gpr_to_xmm(reg reg_src, reg reg_dst);
+
+		// Copy the 32-bit value of GPR @reg_src into XMM @reg_dst
+		void mov_gpr_to_xmm_32(reg reg_src, reg reg_dst);
+
+		// Read the 64-bit value from @reg_addr + @offset into @reg_dst
+		void mov_read(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Read the 32-bit value from @reg_addr + @offset into @reg_dst, and zero-extend @reg_dst
+		void mov_read_32(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Read the 16-bit value from @reg_addr + @offset into @reg_dst
+		void mov_read_16(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Read the 8-bit value from @reg_addr + @offset into @reg_dst
+		void mov_read_8(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Read the 64-bit value from RIP + @offset into @reg_dst
+		// NB: RIP holds the address of the _next_ instruction
+		void mov_read_rip(std::int32_t offset, reg reg_dst);
+
+		// Read the 64-bit value from RIP + @offset into @reg_dst
+		// NB: this function compensates for MOV instruction size and behaves
+		// as if RIP pointed to the _current_ instruction (i.e. MOV itself)
+		void mov_read_rip_prev(std::int32_t offset, reg reg_dst);
+
+		// Write the 64-bit value from @reg_src to @reg_addr + @offset
+		void mov_write(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Write the 32-bit value from @reg_src to @reg_addr + @offset
+		void mov_write_32(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Write the 16-bit value from @reg_src to @reg_addr + @offset
+		void mov_write_16(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Write the 8-bit value from @reg_src to @reg_addr + @offset
+		void mov_write_8(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Write the 64-bit value from @reg_src to RIP + @offset
+		// NB: RIP holds the address of the _next_ instruction
+		void mov_write_rip(reg reg_src, std::int32_t offset);
+
+		// Write the 64-bit value from @reg_src to RIP + @offset
+		// NB: RIP holds the address of the _next_ instruction
+		// NB: this function compensates for MOV instruction size and behaves
+		// as if RIP pointed to the _current_ instruction (i.e. MOV itself)
+		void mov_write_rip_prev(reg reg_src, std::int32_t offset);
+
+		// Read the 64-bit value from GPR @reg_addr + @offset into XMM @reg_dst
+		void mov_read_xmm(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Write the 64-bit value from XMM @reg_src to GPR @reg_addr + @offset
+		void mov_write_xmm(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Read the 32-bit value from GPR @reg_addr + @offset into XMM @reg_dst
+		void mov_read_xmm_32(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Write the 32-bit value from XMM @reg_src to GPR @reg_addr + @offset
+		void mov_write_xmm_32(reg reg_src, reg reg_addr, std::int32_t offset);
+
+		// Add the 32-bit signed @offset to 64-bit GPR @reg_addr and store the result in GPR @reg_dst
+		void lea(reg reg_addr, std::int32_t offset, reg reg_dst);
+
+		// Add the 32-bit signed @offset to RIP and store the result in GPR @reg_dst
+		// NB: RIP holds the address of the _next_ instruction
+		void lea_rip(std::int32_t offset, reg reg_dst);
+
+		// Add the 32-bit signed @offset to RIP and store the result in GPR @reg_dst
+		// NB: this function compensates for LEA instruction size and behaves
+		// as if RIP pointed to the _current_ instruction (i.e. LEA itself)
+		void lea_rip_prev(std::int32_t offset, reg reg_dst);
+
+		// Push the 64-bit GPR @reg onto the stack (also subtracts 8 from RSP)
+		void push(reg reg);
+
+		// Pop the 64-bit GPR @reg from the stack (also adds 8 to RSP)
+		void pop(reg reg);
+
+		// Negate the 64-bit GPR @reg
+		void neg(reg reg);
+
+		// Bitwise not the 64-bit GPR @reg
+		void not_(reg reg);
+
+		// Bitwise-and 64-bit GPR @reg1 and @reg2 and only compute the flags (without writing the result anywhere)
+		void test(reg reg1, reg reg2);
+
+		// Add the 64-bit GPR @reg_dst to the 64-bit GPR @reg_src and store the result in @reg_dst
+		void add(reg reg_src, reg reg_dst);
+
+		// Subtract the 64-bit GPR @reg_src from the 64-bit GPR @reg_dst and store the result in @reg_dst
+		void sub(reg reg_src, reg reg_dst);
+
+		// Multiply the 64-bit GPR @reg_dst with the 64-bit GPR @reg_src and store the result in @reg_dst
+		void mul(reg reg_src, reg reg_dst);
+
+		// Divide the unsigned 128-bit value of RDX:RAX by unsigned GPR @reg_src,
+		// and store the quotient in RAX and the remainder in RDX
+		void udiv(reg reg_src);
+
+		// Divide the signed 128-bit value of RDX:RAX by signed GPR @reg_src,
+		// and store the quotient in RAX and the remainder in RDX
+		void idiv(reg reg_src);
+
+		// Sign-extend RAX into RDX (needed to prepare RDX before signed division)
+		void cqo();
+
+		// Bitwise and the 64-bit GPR @reg_src with the 64-bit GPR @reg_dst and store the result in @reg_dst
+		void and_(reg reg_src, reg reg_dst);
+
+		// Bitwise or the 64-bit GPR @reg_src with the 64-bit GPR @reg_dst and store the result in @reg_dst
+		void or_(reg reg_src, reg reg_dst);
+
+		// Bitwise xor the 64-bit GPR @reg_src with the 64-bit GPR @reg_dst and store the result in @reg_dst
+		void xor_(reg reg_src, reg reg_dst);
+
+		// Left-shift the 64-bit GPR @reg_dst by the value of the low 6 bits of RCX and store the result in @reg_dst
+		void shl(reg reg_dst);
+
+		// Right-shift with zero-filling the 64-bit GPR @reg_dst by the value of the low 6 bits of RCX and store the result in @reg_dst
+		void shr(reg reg_dst);
+
+		// Right-shift with sign-extending the 64-bit GPR @reg_dst by the value of the low 6 bits of RCX and store the result in @reg_dst
+		void sar(reg reg_dst);
+
+		// Compare the 64-bit GPR @reg1 and @reg2 and only compute the flags (without writing the result anywhere)
+		// Effectively computes (@reg1 - @reg2)
+		void cmp(reg reg1, reg reg2);
+
+		// If ZF=1, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovz(reg reg_src, reg reg_dst);
+
+		// If ZF=0, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovnz(reg reg_src, reg reg_dst);
+
+		// If CF=1, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovb(reg reg_src, reg reg_dst);
+
+		// If CF=0, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovnb(reg reg_src, reg reg_dst);
+
+		// If SF != OF, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovl(reg reg_src, reg reg_dst);
+
+		// If SF == OF, copy the 64-bit GPR @reg_src into @reg_dst, otherwise do nothing
+		void cmovnl(reg reg_src, reg reg_dst);
+
+		// Add the 64-bit XMM @reg_dst to the 64-bit XMM @reg_src and store the result in @reg_dst
+		void add_xmm(reg reg_src, reg reg_dst);
+
+		// Add the 32-bit XMM @reg_dst to the 32-bit XMM @reg_src and store the result in @reg_dst
+		void add_xmm_32(reg reg_src, reg reg_dst);
+
+		// Subtract the 64-bit XMM @reg_src from the 64-bit XMM @reg_dst and store the result in @reg_dst
+		void sub_xmm(reg reg_src, reg reg_dst);
+
+		// Subtract the 32-bit XMM @reg_src from the 32-bit XMM @reg_dst and store the result in @reg_dst
+		void sub_xmm_32(reg reg_src, reg reg_dst);
+
+		// Multiply the 64-bit XMM @reg_dst with the 64-bit XMM @reg_src and store the result in @reg_dst
+		void mul_xmm(reg reg_src, reg reg_dst);
+
+		// Multiply the 32-bit XMM @reg_dst with the 32-bit XMM @reg_src and store the result in @reg_dst
+		void mul_xmm_32(reg reg_src, reg reg_dst);
+
+		// Divide the 64-bit XMM @reg_dst by the 64-bit XMM @reg_src and store the result in @reg_dst
+		void div_xmm(reg reg_src, reg reg_dst);
+
+		// Divide the 32-bit XMM @reg_dst by the 32-bit XMM @reg_src and store the result in @reg_dst
+		void div_xmm_32(reg reg_src, reg reg_dst);
+
+		// Bitwise-xor the 64-bit XMM @reg_dst with the 64-bit XMM @reg_src and store the result in @reg_dst
+		void xor_xmm(reg reg_src, reg reg_dst);
+
+		// Bitwise-xor the 32-bit XMM @reg_dst with the 32-bit XMM @reg_src and store the result in @reg_dst
+		void xor_xmm_32(reg reg_src, reg reg_dst);
+
+		// Compare the 64-bit XMM @reg1 and @reg2 and only compute the flags (without writing the result anywhere)
+		void cmp_xmm(reg reg1, reg reg2);
+
+		// Compare the 32-bit XMM @reg1 and @reg2 and only compute the flags (without writing the result anywhere)
+		void cmp_xmm_32(reg reg1, reg reg2);
+
+		// Convert the 64-bit signed value in GPR @reg_src into the 64-bit XMM @reg_dst
+		void gpr_to_xmm(reg reg_src, reg reg_dst);
+
+		// Convert the 64-bit signed value in GPR @reg_src into the 32-bit XMM @reg_dst
+		void gpr_to_xmm_32(reg reg_src, reg reg_dst);
+
+		// Convert the 64-bit in XMM @reg_src into the signed 64-bit XMM @reg_dst
+		void xmm_to_gpr(reg reg_src, reg reg_dst);
+
+		// Convert the 32-bit in XMM @reg_src into the signed 64-bit XMM @reg_dst
+		void xmm_32_to_gpr(reg reg_src, reg reg_dst);
+
+		// Convert the low 4x16-bit values from XMM @reg_src into 4x32-bit values in XMM @reg_dst
+		void xmm_16_to_32(reg reg_src, reg reg_dst);
+
+		// Convert the 4x32-bit values from XMM @reg_src into the low 4x16-bit values in XMM @reg_dst
+		void xmm_32_to_16(reg reg_src, reg reg_dst);
+
+		// Convert the low 2x32-bit values from XMM @reg_src into 2x64-bit values in XMM @reg_dst
+		void xmm_32_to_64(reg reg_src, reg reg_dst);
+
+		// Convert the 2x64-bit values from XMM @reg_src into the low 2x32-bit values in XMM @reg_dst
+		void xmm_64_to_32(reg reg_src, reg reg_dst);
+
+		// Jump to 32-bit signed @offset relative to RIP
+		// NB: RIP holds the address of the _next_ instruction
+		void jump(std::int32_t offset);
+
+		// Conditional jump to 32-bit signed @offset relative to RIP
+		// if ZF flag is set
+		// NB: RIP holds the address of the _next_ instruction
+		void jump_if_zero(std::int32_t offset);
+
+		// Conditional jump to 32-bit signed @offset relative to RIP
+		// if ZF flag is NOT set
+		// NB: RIP holds the address of the _next_ instruction
+		void jump_if_nonzero(std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a JUMP instruction,
+		// replace its 32-bit jump offset with @offset
+		void jump_inject(std::uint8_t * opcode, std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a JUMP instruction,
+		// replace its 32-bit jump offset with @offset, compensating for the size
+		// of the JUMP instruction itself
+		void jump_inject_prev(std::uint8_t * opcode, std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a conditional JUMP instruction
+		// (JZ or JNZ), replace its 32-bit jump offset with @offset
+		void cjump_inject(std::uint8_t * opcode, std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a conditional JUMP instruction
+		// (JZ or JNZ), replace its 32-bit jump offset with @offset, compensating for the size
+		// of the JUMP instruction itself
+		void cjump_inject_prev(std::uint8_t * opcode, std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a RIP-relative LEA instruction,
+		// replace its 32-bit jump offset with @offset
+		void lea_rip_inject(std::uint8_t * opcode, std::int32_t offset);
+
+		// Assuming that @opcode refers to the location of a RIP-relative LEA instruction,
+		// replace its 32-bit jump offset with @offset, compensating for the size
+		// of the LEA instruction itself
+		void lea_rip_inject_prev(std::uint8_t * opcode, std::int32_t offset);
+
+	private:
+		template <typename ... Args>
+		void do_push(Args ... values);
+	};
+
+}

libs/jit/include/pslang/jit/helpers.hpp

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+#include <stdexcept>
+#include <format>
+
+namespace pslang::jit
+{
+
+	inline std::uint8_t operator""_ub(unsigned long long arg)
+	{
+		return (std::uint8_t)arg;
+	}
+
+	template <typename T>
+	static void check_bits(T value, std::uint32_t bits, char const * message)
+	{
+		auto mask = (T(1) << bits) - T(1);
+		bool good = true;
+		if (value >= T(0))
+		{
+			good = value == (value & mask);
+		}
+		else
+		{
+			good = (value & (~mask)) == (T(-1) & (~mask));
+		}
+		if (!good)
+		{
+			T min, max;
+			if (std::is_unsigned_v<T>)
+			{
+				min = 0;
+				max = mask;
+			}
+			else
+			{
+				max = (T(1) << (bits - 1)) - 1;
+				min = - (max + 1);
+			}
+			throw std::runtime_error(std::format("{}: {} not in range [{}..{})", message, value, min, max));
+		}
+	}
+
+	template <typename ... Args>
+	void push_bytes(std::vector<std::uint8_t> & code, Args ... args)
+	{
+		(code.insert(code.end(), (std::uint8_t const *)(&args), (std::uint8_t const *)(&args) + sizeof(args)), ...);
+	}
+
+}

libs/jit/source/arch/linux_x86_64/instruction_builder.cpp

@@ -0,0 +1,609 @@
+#include <pslang/jit/arch/linux_x86_64/instruction_builder.hpp>
+#include <pslang/jit/helpers.hpp>
+
+namespace pslang::jit::linux_x86_64
+{
+
+	namespace
+	{
+
+		// Construct the REX prefix:
+		// W (0/1) - enable 64-bit operand
+		// R (0/1) - extend ModR/M reg field
+		// X (0/1) - extend the SIB index field
+		// B (0/1) - extend the base or opcode register field
+		std::uint8_t rex(std::uint8_t W, std::uint8_t R, std::uint8_t X, std::uint8_t B)
+		{
+			check_bits(W, 1, "REX prefix W");
+			check_bits(R, 1, "REX prefix R");
+			check_bits(X, 1, "REX prefix X");
+			check_bits(B, 1, "REX prefix B");
+			return 0x40u | (W << 3) | (R << 2) | (X << 1) | B;
+		}
+
+		// Same, but using `reg` enum instead - passes the highest (#3) bit of
+		// the register number to rex()
+		std::uint8_t rex(std::uint8_t W, reg R, reg X, reg B)
+		{
+			return rex(W, std::uint8_t(R) >> 3, std::uint8_t(X) >> 3, std::uint8_t(B) >> 3);
+		}
+
+		// Memory
+		static constexpr std::uint8_t MOD_MEM = 0;
+		// Memory + 8-bit offset
+		static constexpr std::uint8_t MOD_MEM8 = 1;
+		// Memory + 32-bit offset
+		static constexpr std::uint8_t MOD_MEM32 = 2;
+		// Register
+		static constexpr std::uint8_t MOD_REG = 3;
+
+		std::uint8_t modrm(std::uint8_t mod, std::uint8_t reg, std::uint8_t rm)
+		{
+			check_bits(mod, 2, "ModR/M mod");
+			check_bits(reg, 3, "ModR/M reg");
+			check_bits(rm, 3, "ModR/M rm");
+			return (mod << 6) | (reg << 3) | rm;
+		}
+
+		std::uint8_t modrm(std::uint8_t mod, reg reg, std::uint8_t rm)
+		{
+			return modrm(mod, (std::uint8_t)(reg) & 0b111, rm);
+		}
+
+		std::uint8_t modrm(std::uint8_t mod, std::uint8_t reg_, reg rm)
+		{
+			return modrm(mod, reg_, (std::uint8_t)(rm) & 0b111);
+		}
+
+		std::uint8_t modrm(std::uint8_t mod, reg reg_, reg rm)
+		{
+			return modrm(mod, (std::uint8_t)(reg_) & 0b111, (std::uint8_t)(rm) & 0b111);
+		}
+
+		static constexpr std::uint8_t SIB_NOINDEX = 4;
+
+		std::uint8_t sib(std::uint8_t scale, std::uint8_t index, std::uint8_t base)
+		{
+			check_bits(scale, 2, "SIB scale");
+			check_bits(index, 3, "SIB index");
+			check_bits(base, 3, "SIB base");
+			return (scale << 6) | (index << 3) | base;
+		}
+
+		std::uint8_t sib(std::uint8_t scale, reg reg_, std::uint8_t base)
+		{
+			return sib(scale, (std::uint8_t)(reg_) & 0b111, base);
+		}
+
+		std::uint8_t sib(std::uint8_t scale, std::uint8_t reg_, reg base)
+		{
+			return sib(scale, reg_, (std::uint8_t)(base) & 0b111);
+		}
+
+		std::uint8_t sib(std::uint8_t scale, reg reg_, reg base)
+		{
+			return sib(scale, (std::uint8_t)(reg_) & 0b111, (std::uint8_t)(base) & 0b111);
+		}
+
+		std::uint8_t opcode_add_reg(std::uint8_t opcode, reg reg)
+		{
+			return opcode + ((std::uint8_t)(reg) & 0b111);
+		}
+
+		bool need_sib(reg reg)
+		{
+			return ((std::uint8_t)(reg) & 0b111) == 4;
+		}
+
+		std::uint8_t vex(std::uint8_t R, std::uint8_t X, std::uint8_t B, std::uint8_t MMMMM)
+		{
+			check_bits(R, 1, "VEX prefix R");
+			check_bits(X, 1, "VEX prefix X");
+			check_bits(B, 1, "VEX prefix B");
+			check_bits(MMMMM, 1, "VEX prefix m-mmmm");
+			return ((1 - R) << 7) | ((1 - X) << 6) | ((1 - B) << 5) | MMMMM;
+		}
+
+		std::uint8_t vex(reg R, reg X, reg B, std::uint8_t MMMMM)
+		{
+			return rex(std::uint8_t(R) >> 3, std::uint8_t(X) >> 3, std::uint8_t(B) >> 3, MMMMM);
+		}
+
+	}
+
+	void instruction_builder::nop()
+	{
+		do_push(0x90_ub);
+	}
+
+	void instruction_builder::ret(std::uint16_t count)
+	{
+		if (count == 0)
+			do_push(0xc3_ub);
+		else
+			do_push(0xc2_ub, count);
+	}
+
+	void instruction_builder::add_imm(reg reg, std::int32_t value)
+	{
+		if (value >= -128 && value <= 127)
+			do_push(rex(1, {}, {}, reg), 0x83_ub, modrm(MOD_REG, 0, reg), (std::int8_t)value);
+		else
+			do_push(rex(1, {}, {}, reg), 0x81_ub, modrm(MOD_REG, 0, reg), value);
+	}
+
+	void instruction_builder::sub_imm(reg reg, std::int32_t value)
+	{
+		if (value >= -128 && value <= 127)
+			do_push(rex(1, {}, {}, reg), 0x83_ub, modrm(MOD_REG, 5, reg), (std::int8_t)value);
+		else
+			do_push(rex(1, {}, {}, reg), 0x81_ub, modrm(MOD_REG, 5, reg), value);
+	}
+
+	void instruction_builder::movzd(std::uint32_t value, reg reg_dst)
+	{
+		if (((std::uint8_t)(reg_dst) & 0b1000) != 0)
+			do_push(rex(0, {}, {}, reg_dst), opcode_add_reg(0xb8, reg_dst), value);
+		else
+			do_push(opcode_add_reg(0xb8, reg_dst), value);
+	}
+
+	void instruction_builder::movsd(std::int32_t value, reg reg_dst)
+	{
+		do_push(rex(1, {}, {}, reg_dst), 0xc7_ub, modrm(MOD_REG, 0, reg_dst), value);
+	}
+
+	void instruction_builder::movzx_8(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0xb6_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::movzx_16(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0xb7_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::movsx_8(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0xbe_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::movsx_16(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0xbf_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::movsx_32(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x63_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mov(std::uint64_t value, reg reg_dst)
+	{
+		do_push(rex(1, {}, {}, reg_dst), opcode_add_reg(0xb8, reg_dst), value);
+	}
+
+	void instruction_builder::mov(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x8b_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mov_gpr_to_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0x66_ub, rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x6e_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mov_gpr_to_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0x66_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x6e_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mov_read(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(1, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(1, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_read_32(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(0, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(0, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_read_16(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(0x66_ub, rex(0, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0x66_ub, rex(0, reg_dst, {}, reg_addr), 0x8b_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_read_8(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(0, reg_dst, {}, reg_addr), 0x8a_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(0, reg_dst, {}, reg_addr), 0x8a_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_read_rip(std::int32_t offset, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, {}), 0x8b_ub, modrm(0, reg_dst, 0b101), offset);
+	}
+
+	void instruction_builder::mov_read_rip_prev(std::int32_t offset, reg reg_dst)
+	{
+		mov_read_rip(offset - 7, reg_dst);
+	}
+
+	void instruction_builder::mov_write(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(1, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(1, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_32(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(0, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(0, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_16(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(0x66_ub, rex(0, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0x66_ub, rex(0, reg_src, {}, reg_addr), 0x89_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_8(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(0, reg_src, {}, reg_addr), 0x88_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(0, reg_src, {}, reg_addr), 0x88_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_rip(reg reg_src, std::int32_t offset)
+	{
+		do_push(rex(1, reg_src, {}, {}), 0x89_ub, modrm(0, reg_src, 0b101), offset);
+	}
+
+	void instruction_builder::mov_write_rip_prev(reg reg_src, std::int32_t offset)
+	{
+		mov_write_rip(reg_src, offset - 7);
+	}
+
+	void instruction_builder::mov_read_xmm(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(0x66_ub, rex(1, reg_dst, {}, reg_addr), 0x0f_ub, 0x6e_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0x66_ub, rex(1, reg_dst, {}, reg_addr), 0x0f_ub, 0x6e_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_xmm(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(0x66_ub, rex(0, reg_src, {}, reg_addr), 0x0f_ub, 0xd6_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0x66_ub, rex(0, reg_src, {}, reg_addr), 0x0f_ub, 0xd6_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_read_xmm_32(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(0xf3_ub, rex(0, reg_dst, {}, reg_addr), 0x0f_ub, 0x10_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0xf3_ub, rex(0, reg_dst, {}, reg_addr), 0x0f_ub, 0x10_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::mov_write_xmm_32(reg reg_src, reg reg_addr, std::int32_t offset)
+	{
+		if (need_sib(reg_addr))
+			do_push(0xf3_ub, rex(0, reg_src, {}, reg_addr), 0x0f_ub, 0x11_ub, modrm(MOD_MEM32, reg_src, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(0xf3_ub, rex(0, reg_src, {}, reg_addr), 0x0f_ub, 0x11_ub, modrm(MOD_MEM32, reg_src, reg_addr), offset);
+	}
+
+	void instruction_builder::lea(reg reg_addr, std::int32_t offset, reg reg_dst)
+	{
+		if (need_sib(reg_addr))
+			do_push(rex(1, reg_dst, {}, reg_addr), 0x8d_ub, modrm(MOD_MEM32, reg_dst, reg_addr), sib(0, SIB_NOINDEX, reg_addr), offset);
+		else
+			do_push(rex(1, reg_dst, {}, reg_addr), 0x8d_ub, modrm(MOD_MEM32, reg_dst, reg_addr), offset);
+	}
+
+	void instruction_builder::lea_rip(std::int32_t offset, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, {}), 0x8d_ub, modrm(0, reg_dst, 0b101), offset);
+	}
+
+	void instruction_builder::lea_rip_prev(std::int32_t offset, reg reg_dst)
+	{
+		lea_rip(offset - 7, reg_dst);
+	}
+
+	void instruction_builder::push(reg reg)
+	{
+		do_push(rex(0, {}, {}, reg), opcode_add_reg(0x50, reg));
+	}
+
+	void instruction_builder::pop(reg reg)
+	{
+		do_push(rex(0, {}, {}, reg), opcode_add_reg(0x58, reg));
+	}
+
+	void instruction_builder::neg(reg reg)
+	{
+		do_push(rex(1, {}, {}, reg), 0xf7_ub, modrm(MOD_REG, 3, reg));
+	}
+
+	void instruction_builder::not_(reg reg)
+	{
+		do_push(rex(1, {}, {}, reg), 0xf7_ub, modrm(MOD_REG, 2, reg));
+	}
+
+	void instruction_builder::test(reg reg1, reg reg2)
+	{
+		do_push(rex(1, reg1, {}, reg2), 0x85_ub, modrm(MOD_REG, reg1, reg2));
+	}
+
+	void instruction_builder::add(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x03_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::sub(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x2b_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mul(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0xaf_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::udiv(reg reg_src)
+	{
+		do_push(rex(1, {}, {}, reg_src), 0xf7_ub, modrm(MOD_REG, 0b110, reg_src));
+	}
+
+	void instruction_builder::idiv(reg reg_src)
+	{
+		do_push(rex(1, {}, {}, reg_src), 0xf7_ub, modrm(MOD_REG, 0b111, reg_src));
+	}
+
+	void instruction_builder::cqo()
+	{
+		do_push(0x48_ub, 0x99_ub);
+	}
+
+	void instruction_builder::and_(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x23_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::or_(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0b_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xor_(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x33_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::shl(reg reg_dst)
+	{
+		do_push(rex(1, {}, {}, reg_dst), 0xd3_ub, modrm(MOD_REG, 0b100, reg_dst));
+	}
+
+	void instruction_builder::shr(reg reg_dst)
+	{
+		do_push(rex(1, {}, {}, reg_dst), 0xd3_ub, modrm(MOD_REG, 0b101, reg_dst));
+	}
+
+	void instruction_builder::sar(reg reg_dst)
+	{
+		do_push(rex(1, {}, {}, reg_dst), 0xd3_ub, modrm(MOD_REG, 0b111, reg_dst));
+	}
+
+	void instruction_builder::cmp(reg reg1, reg reg2)
+	{
+		do_push(rex(1, reg1, {}, reg2), 0x3b_ub, modrm(MOD_REG, reg1, reg2));
+	}
+
+	void instruction_builder::cmovz(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x44_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmovnz(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x45_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmovb(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x42_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmovnb(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x43_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmovl(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x4c_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmovnl(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x4d_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::add_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x58_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::add_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x58_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::sub_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5c_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::sub_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5c_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mul_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x59_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::mul_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x59_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::div_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5e_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::div_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5e_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xor_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0x66_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x57_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xor_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x57_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::cmp_xmm(reg reg1, reg reg2)
+	{
+		do_push(0x66_ub, rex(0, reg1, {}, reg2), 0x0f_ub, 0x2e_ub, modrm(MOD_REG, reg1, reg2));
+	}
+
+	void instruction_builder::cmp_xmm_32(reg reg1, reg reg2)
+	{
+		do_push(rex(0, reg1, {}, reg2), 0x0f_ub, 0x2e_ub, modrm(MOD_REG, reg1, reg2));
+	}
+
+	void instruction_builder::gpr_to_xmm(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x2a_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::gpr_to_xmm_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x2a_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xmm_to_gpr(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x2d_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xmm_32_to_gpr(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(1, reg_dst, {}, reg_src), 0x0f_ub, 0x2d_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xmm_16_to_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xc4_ub, vex(reg_dst, {}, reg_src, 0b00010), 0b01111001_ub, 0x13_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xmm_32_to_16(reg reg_src, reg reg_dst)
+	{
+		do_push(0xc4_ub, vex(reg_dst, {}, reg_src, 0b00011), std::uint8_t(1 | ~(std::uint8_t(reg_src) << 3)), 0x1d_ub, modrm(MOD_REG, reg_dst, 0));
+	}
+
+	void instruction_builder::xmm_32_to_64(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf3_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5a_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::xmm_64_to_32(reg reg_src, reg reg_dst)
+	{
+		do_push(0xf2_ub, rex(0, reg_dst, {}, reg_src), 0x0f_ub, 0x5a_ub, modrm(MOD_REG, reg_dst, reg_src));
+	}
+
+	void instruction_builder::jump(std::int32_t offset)
+	{
+		do_push(0xe9_ub, offset);
+	}
+
+	void instruction_builder::jump_if_zero(std::int32_t offset)
+	{
+		do_push(0x0f_ub, 0x84_ub, offset);
+	}
+
+	void instruction_builder::jump_if_nonzero(std::int32_t offset)
+	{
+		do_push(0x0f_ub, 0x85_ub, offset);
+	}
+
+	void instruction_builder::jump_inject(std::uint8_t * opcode, std::int32_t offset)
+	{
+		auto src = (std::uint8_t const *)(&offset);
+		std::copy(src, src + 4, opcode + 1);
+	}
+
+	void instruction_builder::jump_inject_prev(std::uint8_t * opcode, std::int32_t offset)
+	{
+		jump_inject(opcode, offset - 5);
+	}
+
+	void instruction_builder::cjump_inject(std::uint8_t * opcode, std::int32_t offset)
+	{
+		auto src = (std::uint8_t const *)(&offset);
+		std::copy(src, src + 4, opcode + 2);
+	}
+
+	void instruction_builder::cjump_inject_prev(std::uint8_t * opcode, std::int32_t offset)
+	{
+		cjump_inject(opcode, offset - 6);
+	}
+
+	void instruction_builder::lea_rip_inject(std::uint8_t * opcode, std::int32_t offset)
+	{
+		auto src = (std::uint8_t const *)(&offset);
+		std::copy(src, src + 4, opcode + 3);
+	}
+
+	void instruction_builder::lea_rip_inject_prev(std::uint8_t * opcode, std::int32_t offset)
+	{
+		lea_rip_inject(opcode, offset - 7);
+	}
+
+	template <typename ... Args>
+	void instruction_builder::do_push(Args ... values)
+	{
+		push_bytes(code, values...);
+	}
+
+}

libs/jit/source/arch/macos_aarch64/compiler.cpp

@@ -7,8 +7,6 @@
 #include <pslang/ast/function.hpp>
 #include <pslang/types/type_visitor.hpp>
 
-#include <sstream>
-
 namespace pslang::jit::macos_aarch64
 {
 
@@ -342,8 +340,7 @@ namespace pslang::jit::macos_aarch64
 
 			void apply(ir::node_ref it, ir::global const & node, types::type_ptr const &)
 			{
-				// TODO
-				throw std::runtime_error("Not implemented");
+				// Globals are added in a separate pass before code
 			}
 
 			void apply(ir::node_ref it, ir::copy const & node, types::type_ptr const & type)
@@ -356,9 +353,19 @@ namespace pslang::jit::macos_aarch64
 				auto src_address = node_address(node.source, 1);
 				for (auto field_id : node.path)
 				{
-					auto const & field = std::get<types::struct_type>(*src_type).node->fields[field_id];
-					src_type = field.inferred_type;
-					src_address.offset += field.layout.offset;
+					if (auto struct_type = std::get_if<types::struct_type>(src_type.get()))
+					{
+						auto struct_node = struct_type->node;
+						src_type = struct_node->fields[field_id].inferred_type;
+						src_address.offset += struct_node->fields[field_id].layout.offset;
+					}
+					else if (auto array_type = std::get_if<types::array_type>(src_type.get()))
+					{
+						src_type = array_type->element_type;
+						src_address.offset += field_id * ast::type_size(*array_type->element_type);
+					}
+					else
+						throw std::runtime_error("Unknown object type for field copy");
 				}
 
 				copy_memory(src_address.reg, src_address.offset, dst_address.reg, dst_address.offset, size, 2);
@@ -611,12 +618,6 @@ namespace pslang::jit::macos_aarch64
 							builder.csetm(0, 0b1101);
 					}
 					break;
-				default:
-					{
-						std::ostringstream os;
-						os << "binary operation " << node.type << " is not implemented";
-						throw std::runtime_error(os.str());
-					}
 				}
 
 				if (result_is_fp)
@@ -1010,6 +1011,10 @@ namespace pslang::jit::macos_aarch64
 					{
 						stack_position[it] = argument_position[argument->index];
 					}
+					else if (std::holds_alternative<ir::global>(it->instruction))
+					{
+						// stack position doesn't make sense for globals
+					}
 					else if (ir::is_value_instruction(it->instruction))
 					{
 						auto size = ast::type_size(*it->inferred_type);
@@ -1098,8 +1103,10 @@ namespace pslang::jit::macos_aarch64
 					// Globals are added in a separate pass before code
 					if (lcontext.nodes.contains(it))
 						continue;
+
 					// Uncomment to debug per-node instruction generation:
-					builder.nop();
+					// builder.nop();
+
 					lcontext.nodes[it] = pcontext.storage.size();
 					std::visit([&](auto const & instruction){ apply(it, instruction, it->inferred_type); }, it->instruction);
 				}
@@ -1219,10 +1226,10 @@ namespace pslang::jit::macos_aarch64
 				std::visit([&](auto const & instruction){ visitor.apply(it, instruction, it->inferred_type); }, it->instruction);
 		}
 		auto data_end = pcontext.storage.align();
-		pcontext.storage.data.push_back({data_begin, data_end});
+		if (data_begin != data_end)
+			pcontext.storage.data.push_back({data_begin, data_end});
 
 		auto code_begin = data_end;
-		instruction_builder builder{pcontext.storage.storage};
 
 		{
 			populate_const_data_visitor visitor{pcontext, lcontext};
@@ -1230,6 +1237,8 @@ namespace pslang::jit::macos_aarch64
 				std::visit([&](auto const & instruction){ visitor.apply(instruction, it->inferred_type); }, it->instruction);
 		}
 
+		instruction_builder builder{pcontext.storage.storage};
+
 		for (auto const & symbol : mcontext.symbols)
 		{
 			pcontext.symbols[symbol.first] = pcontext.storage.size();
@@ -1248,7 +1257,7 @@ namespace pslang::jit::macos_aarch64
 		for (auto const & resolve : lcontext.adr_resolve)
 			builder.adr_inject(pcontext.storage.storage.data() + resolve.offset, lcontext.nodes.at(resolve.target) - resolve.offset);
 
-			auto code_end = pcontext.storage.align();
+		auto code_end = pcontext.storage.align();
 		pcontext.storage.code.push_back({code_begin, code_end});
 	}
 

libs/jit/source/arch/macos_aarch64/instruction_builder.cpp

@@ -1,40 +1,9 @@
 #include <pslang/jit/arch/macos_aarch64/instruction_builder.hpp>
-
-#include <format>
+#include <pslang/jit/helpers.hpp>
 
 namespace pslang::jit::macos_aarch64
 {
 
-	template <typename T>
-	static void check_bits(T value, std::uint32_t bits, char const * message)
-	{
-		auto mask = (T(1) << bits) - T(1);
-		bool good = true;
-		if (value >= T(0))
-		{
-			good = value == (value & mask);
-		}
-		else
-		{
-			good = (value & (~mask)) == (T(-1) & (~mask));
-		}
-		if (!good)
-		{
-			T min, max;
-			if (std::is_unsigned_v<T>)
-			{
-				min = 0;
-				max = mask;
-			}
-			else
-			{
-				max = (T(1) << (bits - 1)) - 1;
-				min = - (max + 1);
-			}
-			throw std::runtime_error(std::format("{}: {} not in range [{}..{})", message, value, min, max));
-		}
-	}
-
 	static constexpr std::uint32_t REG_MASK = 0x1fu;
 
 	void instruction_builder::nop()

libs/jit/source/jit.cpp

@@ -1,5 +1,6 @@
 #include <pslang/jit/jit.hpp>
 #include <pslang/jit/arch/macos_aarch64/compiler.hpp>
+#include <pslang/jit/arch/linux_x86_64/compiler.hpp>
 
 #include <stdexcept>
 
@@ -11,6 +12,15 @@ namespace pslang::jit
 		switch (pcontext.abi.platform)
 		{
 		case platform::linux:
+			switch (pcontext.abi.isa)
+			{
+			case isa::x86_64:
+				linux_x86_64::compile(pcontext, mcontext);
+				break;
+			case isa::aarch64:
+				throw std::runtime_error("Linux aarch64 JIT compilation not supported");
+			}
+			break;
 			throw std::runtime_error("Linux JIT compilation not supported");
 		case platform::windows:
 			throw std::runtime_error("Windows JIT compilation not supported");

libs/types/include/pslang/types/half_float.hpp

@@ -7,7 +7,9 @@
 namespace pslang::types
 {
 
-	// TODO: actual half-float operations? Maybe C++23 std::float16_t?
+	// TODO: actual 16-bit half-float
+	// C++23 std::float16_t doesn't cut it, because it isn't
+	// exposed if the host platform doesn't support float16 natively
 
 	struct half_float
 	{
@@ -42,4 +44,4 @@ namespace pslang::types
 		return {f1.repr / f2.repr};
 	}
 
-}
+}

libs/types/source/type.cpp

@@ -160,6 +160,9 @@ namespace pslang::types
 		if (std::get_if<unit_type>(&type))
 			return 0;
 
+		if (std::get_if<pointer_type>(&type))
+			return 8;
+
 		if (std::get_if<function_type>(&type))
 			return 8;