hook_usercall_win32.h File Reference

#include <bit>
#include <functional>
#include <optional>
#include <typeindex>
#include <type_traits>
#include <vector>
#include "../thirdparty/Detours/src/detours.h"
#include "host/common.h"
#include "mem/mem.h"

Go to the source code of this file.

Classes
struct	hedgedev::csl::hook::usercall_info

Namespaces
namespace	hedgedev
namespace	hedgedev::csl
namespace	hedgedev::csl::hook

Macros
#define	__usercall   0
#define	__userpurge   1
#define	USER_REGISTER(REGISTER)
#define	USER_RETURN(REGISTER)
#define	USER_RETURN_VOID   USER_RETURN(none)
#define	USER_PARAM(INDEX, REGISTER)
#define	USER_FUNCTION_PTR(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
#define	USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
#define	STATIC_USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
#define	INSTALL_USER_HOOK(FUNCTION_NAME)
#define	INSTALL_USER_HOOK_EXPLICIT(FUNCTION_NAME, ADDRESS)
#define	UNINSTALL_USER_HOOK(FUNCTION_NAME)

Enumerations
enum class	hedgedev::csl::hook::user_register : uint64_t {   hedgedev::csl::hook::none , hedgedev::csl::hook::EAX , hedgedev::csl::hook::ECX , hedgedev::csl::hook::EDX ,   hedgedev::csl::hook::EBX , hedgedev::csl::hook::ESP , hedgedev::csl::hook::EBP , hedgedev::csl::hook::ESI ,   hedgedev::csl::hook::EDI , hedgedev::csl::hook::ST0 , hedgedev::csl::hook::ST1 , hedgedev::csl::hook::ST2 ,   hedgedev::csl::hook::ST3 , hedgedev::csl::hook::ST4 , hedgedev::csl::hook::ST5 , hedgedev::csl::hook::ST6 ,   hedgedev::csl::hook::ST7 , hedgedev::csl::hook::XMM0 , hedgedev::csl::hook::XMM1 , hedgedev::csl::hook::XMM2 ,   hedgedev::csl::hook::XMM3 , hedgedev::csl::hook::XMM4 , hedgedev::csl::hook::XMM5 , hedgedev::csl::hook::XMM6 ,   hedgedev::csl::hook::XMM7 , hedgedev::csl::hook::count }
enum class	hedgedev::csl::hook::user_register_alias : uint64_t {   hedgedev::csl::hook::none = user_register::none , hedgedev::csl::hook::AL = user_register::EAX , hedgedev::csl::hook::AX = user_register::EAX , hedgedev::csl::hook::EAX = user_register::EAX ,   hedgedev::csl::hook::CL = user_register::ECX , hedgedev::csl::hook::CX = user_register::ECX , hedgedev::csl::hook::ECX = user_register::ECX , hedgedev::csl::hook::DL = user_register::EDX ,   hedgedev::csl::hook::DX = user_register::EDX , hedgedev::csl::hook::EDX = user_register::EDX , hedgedev::csl::hook::BL = user_register::EBX , hedgedev::csl::hook::BX = user_register::EBX ,   hedgedev::csl::hook::EBX = user_register::EBX , hedgedev::csl::hook::SPL = user_register::ESP , hedgedev::csl::hook::SP = user_register::ESP , hedgedev::csl::hook::ESP = user_register::ESP ,   hedgedev::csl::hook::BPL = user_register::EBP , hedgedev::csl::hook::BP = user_register::EBP , hedgedev::csl::hook::EBP = user_register::EBP , hedgedev::csl::hook::SIL = user_register::ESI ,   hedgedev::csl::hook::SI = user_register::ESI , hedgedev::csl::hook::ESI = user_register::ESI , hedgedev::csl::hook::DIL = user_register::EDI , hedgedev::csl::hook::DI = user_register::EDI ,   hedgedev::csl::hook::EDI = user_register::EDI , hedgedev::csl::hook::ST0 = user_register::ST0 , hedgedev::csl::hook::ST1 = user_register::ST1 , hedgedev::csl::hook::ST2 = user_register::ST2 ,   hedgedev::csl::hook::ST3 = user_register::ST3 , hedgedev::csl::hook::ST4 = user_register::ST4 , hedgedev::csl::hook::ST5 = user_register::ST5 , hedgedev::csl::hook::ST6 = user_register::ST6 ,   hedgedev::csl::hook::ST7 = user_register::ST7 , hedgedev::csl::hook::XMM0 = user_register::XMM0 , hedgedev::csl::hook::XMM1 = user_register::XMM1 , hedgedev::csl::hook::XMM2 = user_register::XMM2 ,   hedgedev::csl::hook::XMM3 = user_register::XMM3 , hedgedev::csl::hook::XMM4 = user_register::XMM4 , hedgedev::csl::hook::XMM5 = user_register::XMM5 , hedgedev::csl::hook::XMM6 = user_register::XMM6 ,   hedgedev::csl::hook::XMM7 = user_register::XMM7 }
enum class	hedgedev::csl::hook::user_register_family : uint64_t { hedgedev::csl::hook::none , hedgedev::csl::hook::GPR = user_register::EAX , hedgedev::csl::hook::FPU = user_register::ST0 , hedgedev::csl::hook::SSE = user_register::XMM0 }

Functions
user_register_family	hedgedev::csl::hook::get_register_family (user_register in_register)
int	hedgedev::csl::hook::get_register_id (user_register in_register)
bool	hedgedev::csl::hook::is_protected_register (user_register in_register)
void *	hedgedev::csl::hook::emit_user_trampoline (const usercall_info &in_info, void *in_address, bool in_to_original=false)

Macro Definition Documentation

__usercall

#define __usercall   0

A custom calling convention that stores arguments in optimised locations, before falling back to using the stack, similar to __fastcall.

This calling convention puts the responsibility of cleaning up stack arguments on the caller.

This identifier cannot be used as a standard calling convention.

__userpurge

#define __userpurge   1

A custom calling convention that stores arguments in optimised locations, before falling back to using the stack, similar to __fastcall.

This calling convention puts the responsibility of cleaning up stack arguments on the callee.

This identifier cannot be used as a standard calling convention.

INSTALL_USER_HOOK

#define INSTALL_USER_HOOK

(

FUNCTION_NAME

)

Value:

INSTALL_USER_HOOK_EXPLICIT(FUNCTION_NAME, g_info_##FUNCTION_NAME.detour_address)

Installs a hook defined with USER_HOOK.

Parameters

FUNCTION_NAME

The name of the function to call before the original.

Returns

true if the installation succeeeded, or if the hook was already installed. Otherwise, false.

INSTALL_USER_HOOK_EXPLICIT

#define INSTALL_USER_HOOK_EXPLICIT	(		FUNCTION_NAME,
			ADDRESS )

Value:

std::invoke([&]() -> bool                                                                                         \
{                                                                                                                 \
    auto& info = g_info_##FUNCTION_NAME;                                                                          \
                                                                                                                  \
    if (!info.detour_address && !(ADDRESS))                                                                       \
        return false;                                                                                             \
                                                                                                                  \
    *reinterpret_cast<void**>(&info.detour_address) = reinterpret_cast<void*>(ADDRESS);                           \
                                                                                                                  \
    DetourTransactionBegin();                                                                                     \
    DetourUpdateThread(GetCurrentThread());                                                                       \
    DetourAttach(reinterpret_cast<void**>(&info.detour_address), g_trampoline_to_##FUNCTION_NAME##_hook);         \
                                                                                                                  \
    const auto result = DetourTransactionCommit() == NO_ERROR;                                                    \
                                                                                                                  \
    if (result)                                                                                                   \
    {                                                                                                             \
        *reinterpret_cast<void**>(&original_##FUNCTION_NAME) =                                                    \
            hedgedev::csl::hook::emit_user_trampoline(info, reinterpret_cast<void*>(&info.detour_address), true); \
                                                                                                                  \
        g_address_post_hook_##FUNCTION_NAME =                                                                     \
            hedgedev::csl::hook::_get_post_hook_address(reinterpret_cast<void*>(g_address_##FUNCTION_NAME));      \
    }                                                                                                             \
                                                                                                                  \
    return result;                                                                                                \
})

Installs a hook defined with USER_HOOK at an explicit address.

Parameters

FUNCTION_NAME	The name of the function to call before the original.
ADDRESS	The address of the function to hook.

Returns

true if the installation succeeeded, or if the hook was already installed. Otherwise, false.

STATIC_USER_HOOK

#define STATIC_USER_HOOK	(		RETURN_TYPE,
			CALLING_CONVENTION,
			FUNCTION_NAME,
			ADDRESS,
			REGISTERS,
			PARAM_COUNT,
			... )

Value:

USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, __VA_ARGS__);   \
__CMNLIB_INTERNAL_STATIC_HOOK_IMPL(FUNCTION_NAME, ADDRESS, INSTALL_USER_HOOK)                              \
RETURN_TYPE __cdecl impl_##FUNCTION_NAME(__VA_ARGS__)

Defines the body of a hook for a function with custom calling convention in memory, and installs it upon initialisation.

Parameters

RETURN_TYPE	The return type of the function.
CALLING_CONVENTION	The calling convention of the function, such as __usercall or __userpurge.
FUNCTION_NAME	The name of the function.
ADDRESS	The address of the function.
REGISTERS	The registers used by the return value and parameters. If a parameter has an unspecified register, it'll automatically be put on the stack.
PARAM_COUNT	The total number of parameters in __VA_ARGS__.
__VA_ARGS__	The parameters of the function.

Examples

For an optimised function with the following prototype:

int __usercall my_optimised_function@<eax>(int in_first_param@<eax>, float in_second_param@<xmm0>, int in_third_param)

A hook should be set up like the following:

USER_HOOK(int, __usercall, my_optimised_function_hook, 0xDEADBEEF, USER_RETURN(EAX) | USER_PARAM(0, EAX) | USER_PARAM(1, XMM0), 3, int in_first_param, float in_second_param, int in_third_param)
{
    return original_my_optimised_function_hook(in_first_param, in_second_param, in_third_param);
}

Returns

Use GET_STATIC_HOOK_RESULT.

UNINSTALL_USER_HOOK

#define UNINSTALL_USER_HOOK

(

FUNCTION_NAME

)

Value:

std::invoke([&]() -> bool                                                                                 \
{                                                                                                         \
    auto& info = g_info_##FUNCTION_NAME;                                                                  \
                                                                                                          \
    if (info.original_address == info.detour_address)                                                     \
        return true;                                                                                      \
                                                                                                          \
    DetourTransactionBegin();                                                                             \
    DetourUpdateThread(GetCurrentThread());                                                               \
    DetourDetach(reinterpret_cast<void**>(&info.detour_address), g_trampoline_to_##FUNCTION_NAME##_hook); \
                                                                                                          \
    return DetourTransactionCommit() == NO_ERROR;                                                         \
})

Uninstalls a hook installed with INSTALL_USER_HOOK.

Parameters

FUNCTION_NAME

The name of the function to unhook.

Returns

true if the uninstallation succeeeded, or if the hook was already uninstalled. Otherwise, false.

USER_FUNCTION_PTR

#define USER_FUNCTION_PTR	(		RETURN_TYPE,
			CALLING_CONVENTION,
			FUNCTION_NAME,
			ADDRESS,
			REGISTERS,
			PARAM_COUNT,
			... )

Value:

hedgedev::csl::hook::usercall_info g_info_##FUNCTION_NAME##_fp                                                                                                                                \
    { typeid(RETURN_TYPE), __CMNLIB_INTERNAL_USER_SIZE_OF(RETURN_TYPE), CALLING_CONVENTION, reinterpret_cast<void*>(ADDRESS), reinterpret_cast<void*>(ADDRESS), REGISTERS, PARAM_COUNT };     \
void* g_trampoline_to_##FUNCTION_NAME = hedgedev::csl::hook::emit_user_trampoline(g_info_##FUNCTION_NAME##_fp, reinterpret_cast<void*>(&g_info_##FUNCTION_NAME##_fp.original_address), true); \
FUNCTION_PTR(RETURN_TYPE, __cdecl, FUNCTION_NAME, g_trampoline_to_##FUNCTION_NAME, __VA_ARGS__)

Declares a pointer to a function with custom calling convention in memory.

Parameters

RETURN_TYPE	The return type of the function.
CALLING_CONVENTION	The calling convention of the function, such as __usercall or __userpurge.
FUNCTION_NAME	The name of the function.
ADDRESS	The address of the function.
REGISTERS	The registers used by the return value and parameters. If a parameter has an unspecified register, it'll automatically be put on the stack.
PARAM_COUNT	The total number of parameters in __VA_ARGS__.
__VA_ARGS__	The parameters of the function.

Examples

For an optimised function with the following prototype:

int __usercall my_optimised_function@<eax>(int in_first_param@<eax>, float in_second_param@<xmm0>, int in_third_param)

A function pointer should be set up like the following:

USER_FUNCTION_PTR(int, __usercall, my_optimised_function, 0xDEADBEEF, USER_RETURN(EAX) | USER_PARAM(0, EAX) | USER_PARAM(1, XMM0), 3, int in_first_param, float in_second_param, int in_third_param);

USER_HOOK

#define USER_HOOK	(		RETURN_TYPE,
			CALLING_CONVENTION,
			FUNCTION_NAME,
			ADDRESS,
			REGISTERS,
			PARAM_COUNT,
			... )

Value:

hedgedev::csl::hook::usercall_info g_info_##FUNCTION_NAME                                                                                                                                 \
    { typeid(RETURN_TYPE), __CMNLIB_INTERNAL_USER_SIZE_OF(RETURN_TYPE), CALLING_CONVENTION, reinterpret_cast<void*>(ADDRESS), reinterpret_cast<void*>(ADDRESS), REGISTERS, PARAM_COUNT }; \
RETURN_TYPE __cdecl impl_##FUNCTION_NAME(__VA_ARGS__);                                                                                                                                    \
void* g_trampoline_to_##FUNCTION_NAME##_hook = hedgedev::csl::hook::emit_user_trampoline(g_info_##FUNCTION_NAME, reinterpret_cast<void*>(&impl_##FUNCTION_NAME));                         \
HOOK(RETURN_TYPE, __cdecl, FUNCTION_NAME, nullptr, __VA_ARGS__)

Defines the body of a hook for a function with custom calling convention in memory.

Parameters

RETURN_TYPE	The return type of the function.
CALLING_CONVENTION	The calling convention of the function, such as __usercall or __userpurge.
FUNCTION_NAME	The name of the function.
ADDRESS	The address of the function.
REGISTERS	The registers used by the return value and parameters. If a parameter has an unspecified register, it'll automatically be put on the stack.
PARAM_COUNT	The total number of parameters in __VA_ARGS__.
__VA_ARGS__	The parameters of the function.

Examples

For an optimised function with the following prototype:

int __usercall my_optimised_function@<eax>(int in_first_param@<eax>, float in_second_param@<xmm0>, int in_third_param)

A hook should be set up like the following:

USER_HOOK(int, __usercall, my_optimised_function_hook, 0xDEADBEEF, USER_RETURN(EAX) | USER_PARAM(0, EAX) | USER_PARAM(1, XMM0), 3, int in_first_param, float in_second_param, int in_third_param)
{
    return original_my_optimised_function_hook(in_first_param, in_second_param, in_third_param);
}

USER_PARAM

#define USER_PARAM	(		INDEX,
			REGISTER )

Value:

(USER_REGISTER(REGISTER) << ((INDEX + 1) * hedgedev::csl::hook::k_user_register_size))

Specify the register to use for a specific parameter.

Parameters

INDEX	The index of the parameter.
REGISTER	The register to use for the parameter.

USER_REGISTER

#define USER_REGISTER

(

REGISTER

)

Value:

uint64_t(hedgedev::csl::hook::user_register(hedgedev::csl::hook::user_register_alias::REGISTER))

USER_RETURN

#define USER_RETURN

(

REGISTER

)

Value:

USER_REGISTER(REGISTER)

Specify the register to use for the return value.

Parameters

REGISTER

The register to use for the return value.

USER_RETURN_VOID

#define USER_RETURN_VOID   USER_RETURN(none)

Specifies the hook has no return value.