CommonLib
Loading...
Searching...
No Matches
HookingUserCall.h
Go to the documentation of this file.
1#pragma once
2
3#include <bit>
4#include <optional>
5#include <typeindex>
6#include <vector>
7
8///
9/// A custom calling convention that stores arguments in optimised locations
10/// before falling back to using the stack, similar to `__fastcall`.
11///
12/// This calling convention puts the responsibility of cleaning up stack
13/// arguments on the caller.
14///
15/// This identifier cannot be used as a standard calling convention.
16///
17#define __usercall 0
18
19///
20/// A custom calling convention that stores arguments in optimised locations
21/// before falling back to using the stack, similar to `__fastcall`.
22///
23/// This calling convention puts the responsibility of cleaning up stack
24/// arguments on the callee.
25///
26/// This identifier cannot be used as a standard calling convention.
27///
28#define __userpurge 1
29
30#define USER_REGISTER(REGISTER)
31 ((uint64_t)(hedgedev::csl::hook::UserRegister(hedgedev::csl::hook::Register::REGISTER)))
32
33///
34/// Specifies the hook has no return value.
35///
36#define USER_RETURN_VOID
37 USER_REGISTER(None)
38
39///
40/// Specify the register to use for the return value.
41///
42/// \param REGISTER The register to use for the return value.
43///
44#define USER_RETURN(REGISTER)
45 USER_REGISTER(REGISTER)
46
47///
48/// Specify the register to use for a specific parameter.
49///
50/// \param INDEX The index of the parameter.
51/// \param REGISTER The register to use for the parameter.
52///
53#define USER_PARAM(INDEX, REGISTER)
54 (USER_REGISTER(REGISTER) << ((INDEX + 1) * hedgedev::csl::hook::g_kUserRegisterSize))
55
56///
57/// Declares a pointer to a function with custom calling convention in memory.
58///
59/// \param RETURN_TYPE The return type of the function.
60/// \param CALLING_CONVENTION The calling convention of the function, such as `__usercall` or `__userpurge`.
61/// \param FUNCTION_NAME The name of the function.
62/// \param ADDRESS The address of the function.
63/// \param REGISTERS The registers used by the return value and parameters.
64/// \param PARAM_COUNT The total number of parameters in `__VA_ARGS__`.
65/// \param __VA_ARGS__ The parameters of the function.
66///
67#define USER_FUNCTION_PTR(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
68 hedgedev::csl::hook::UserCallInfo info_##FUNCTION_NAME { typeid(RETURN_TYPE), sizeof(RETURN_TYPE), CALLING_CONVENTION, (void*)(ADDRESS), (void*)(ADDRESS), REGISTERS, PARAM_COUNT };
69 void* trampoline_##FUNCTION_NAME = hedgedev::csl::hook::EmitUserTrampoline(info_##FUNCTION_NAME, size_t(&info_##FUNCTION_NAME.fpOriginal), true);
70 FUNCTION_PTR(RETURN_TYPE, __cdecl, FUNCTION_NAME, trampoline_##FUNCTION_NAME, __VA_ARGS__)
71
72///
73/// Defines the body of a hook for a function with custom calling convention in memory.
74///
75/// \param RETURN_TYPE The return type of the function.
76/// \param CALLING_CONVENTION The calling convention of the function, such as `__usercall` or `__userpurge`.
77/// \param FUNCTION_NAME The name of the function.
78/// \param ADDRESS The address of the function.
79/// \param REGISTERS The registers used by the return value and parameters.
80/// \param PARAM_COUNT The total number of parameters in `__VA_ARGS__`.
81/// \param __VA_ARGS__ The parameters of the function.
82///
83#define USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
84 hedgedev::csl::hook::UserCallInfo info_##FUNCTION_NAME { typeid(RETURN_TYPE), sizeof(RETURN_TYPE), CALLING_CONVENTION, (void*)(ADDRESS), (void*)(ADDRESS), REGISTERS, PARAM_COUNT };
85 RETURN_TYPE __cdecl impl_##FUNCTION_NAME(__VA_ARGS__);
86 void* trampolineToHook_##FUNCTION_NAME = hedgedev::csl::hook::EmitUserTrampoline(info_##FUNCTION_NAME, size_t(&impl_##FUNCTION_NAME));
87 HOOK(RETURN_TYPE, __cdecl, FUNCTION_NAME, nullptr, __VA_ARGS__)
88
89///
90/// Defines the body of a hook for a function with custom calling convention in memory, and installs it upon initialisation.
91///
92/// \param RETURN_TYPE The return type of the function.
93/// \param CALLING_CONVENTION The calling convention of the function, such as `__usercall` or `__userpurge`.
94/// \param FUNCTION_NAME The name of the function.
95/// \param ADDRESS The address of the function.
96/// \param REGISTERS The registers used by the return value and parameters.
97/// \param PARAM_COUNT The total number of parameters in `__VA_ARGS__`.
98/// \param __VA_ARGS__ The parameters of the function.
99///
100#define STATIC_USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, ...)
101 USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT, __VA_ARGS__);
102 __CMNLIB_INTERNAL_STATIC_HOOK_IMPL(FUNCTION_NAME, ADDRESS, INSTALL_USER_HOOK)
103 RETURN_TYPE __cdecl impl_##FUNCTION_NAME(__VA_ARGS__)
104
105///
106/// Installs a hook defined with \ref USER_HOOK.
107///
108/// \param FUNCTION_NAME The name of the function to call before the original.
109///
110/// \returns `true` if the installation succeeeded, or if the hook was already installed. Otherwise, `false`.
111///
112#define INSTALL_USER_HOOK(FUNCTION_NAME)
113 INSTALL_USER_HOOK_EXPLICIT(FUNCTION_NAME, info_##FUNCTION_NAME.fpDetour)
114
115///
116/// Installs a hook defined with \ref USER_HOOK at an explicit address.
117///
118/// \param FUNCTION_NAME The name of the function to call before the original.
119/// \param ADDRESS The address of the function to hook.
120///
121/// \returns `true` if the installation succeeeded, or if the hook was already installed. Otherwise, `false`.
122///
123#define INSTALL_USER_HOOK_EXPLICIT(FUNCTION_NAME, ADDRESS)
124 std::invoke([&]()
125 {
126 const auto& rInfo = info_##FUNCTION_NAME;
127
128 if (!rInfo.fpDetour && !(ADDRESS))
129 return false;
130
131 *(void**)&rInfo.fpDetour = (void*)(ADDRESS);
132
133 DetourTransactionBegin();
134 DetourUpdateThread(GetCurrentThread());
135 DetourAttach((void**)&rInfo.fpDetour, trampolineToHook_##FUNCTION_NAME);
136
137 const auto result = DetourTransactionCommit() == NO_ERROR;
138
139 *(void**)&original_##FUNCTION_NAME = hedgedev::csl::hook::EmitUserTrampoline(rInfo, size_t(&rInfo.fpDetour), true);
140
141 return result;
142 })
143
144///
145/// Uninstalls a hook installed with \ref INSTALL_USER_HOOK.
146///
147/// \param FUNCTION_NAME The name of the function to unhook.
148///
149/// \returns `true` if the uninstallation succeeeded, or if the hook was already uninstalled. Otherwise, `false`.
150///
151#define UNINSTALL_USER_HOOK(FUNCTION_NAME)
152 std::invoke([&]()
153 {
154 const auto& rInfo = info_##FUNCTION_NAME;
155
156 if (rInfo.fpOriginal == rInfo.fpDetour)
157 return true;
158
159 DetourTransactionBegin();
160 DetourUpdateThread(GetCurrentThread());
161 DetourDetach((void**)&rInfo.fpDetour, trampolineToHook_##FUNCTION_NAME);
162
163 return DetourTransactionCommit() == NO_ERROR;
164 })
165
166namespace hedgedev::csl::hook
167{
168 ///
169 /// Registers used for `__usercall`/`__userpurge` calling conventions.
170 ///
171 /// These registers have been ordered in an specific way that line up with
172 /// register IDs used for encoding instructions.
173 ///
207
264
272
273 ///
274 /// The total number of bits in the register flags.
275 ///
276 inline static constexpr size_t g_kUserRegisterBitsLength = sizeof(uint64_t) * sizeof(uint64_t);
277
278 ///
279 /// The total number of bits per register in the register flags.
280 ///
281 inline static constexpr size_t g_kUserRegisterSize = std::bit_width(uint64_t(UserRegister::Count));
282
283 ///
284 /// The maximum number of registers that can fit in the register flags.
285 ///
286 inline static constexpr size_t g_kUserRegisterMax = g_kUserRegisterBitsLength / g_kUserRegisterSize;
287
288 ///
289 /// The number of remaining bits when all slots are used up in the register flags.
290 ///
291 inline static constexpr size_t g_kUserRegisterBitsRemainder = g_kUserRegisterBitsLength % g_kUserRegisterSize;
292
293 ///
294 /// The mask for getting the register bits.
295 ///
296 inline static constexpr uint64_t g_kUserRegisterMask = 0x1F;
297
299 {
300 std::type_index ReturnType;
303 void* fpOriginal{};
304 void* fpDetour{};
307
309 {
310 return size_t(std::bit_width(Registers) + g_kUserRegisterBitsRemainder) / g_kUserRegisterSize;
311 }
312
314 {
315 const auto registerCount = GetRegisterCount();
316
317 if (registerCount <= 1)
318 return 0;
319
320 return registerCount - 1;
321 }
322
324 {
325 return ParamCount - GetRegisterParamCount();
326 }
327
329 {
330 return GetRegisterParamCount() + GetStackParamCount();
331 }
332
333 UserRegister GetRegister(int in_index) const
334 {
335 return UserRegister(Registers >> (g_kUserRegisterSize * in_index) & g_kUserRegisterMask);
336 }
337
339 {
340 return GetRegister(0);
341 }
342
343 UserRegister GetParamRegister(int in_index) const
344 {
345 return GetRegister(in_index + 1);
346 }
347
348 bool IsProtectedRegister(UserRegister in_register) const
349 {
350 return in_register == UserRegister::EBX ||
351 in_register == UserRegister::EBP ||
352 in_register == UserRegister::ESI ||
353 in_register == UserRegister::EDI;
354 }
355
356 template <typename T>
357 bool IsReturnType() const
358 {
359 return typeid(T) == ReturnType;
360 }
361
362 bool IsStackParam(int in_index) const
363 {
364 return GetRegisterParamCount() < size_t(in_index + 1);
365 }
366 };
367
368 ///
369 /// Gets the family a register belongs to.
370 ///
371 /// \param in_register The register to check.
372 ///
374 {
375 if (in_register >= UserRegister::EAX && in_register <= UserRegister::EDI)
376 {
378 }
379 else if (in_register >= UserRegister::ST0 && in_register <= UserRegister::ST7)
380 {
382 }
383 else if (in_register >= UserRegister::XMM0 && in_register <= UserRegister::XMM7)
384 {
386 }
387
389 }
390
391 ///
392 /// Gets the ID of a register.
393 ///
394 /// \param in_register The register to get the ID for.
395 ///
396 inline int GetRegisterID(UserRegister in_register)
397 {
398 return int(in_register) - int(GetRegisterFamily(in_register));
399 }
400
401 ///
402 /// Emits a trampoline that forwards arguments from a function with custom calling convention to a
403 /// `__cdecl` function in a hook defined with \ref USER_HOOK.
404 ///
405 /// \param in_rInfo The info about the hook.
406 /// \param in_address The address of the `__cdecl` function to trampoline to.
407 /// \param in_isToOriginal Determines whether this trampoline is going back to the original function.
408 ///
409 /// \returns A pointer to the trampoline that was emitted.
410 ///
411 inline void* EmitUserTrampoline(const UserCallInfo& in_rInfo, const size_t in_address, bool in_isToOriginal = false)
412 {
413 std::vector<uint8_t> result{};
414
415 const auto emitImm8 = [&](int in_value)
416 {
417 auto pos = result.size();
418
419 result.push_back(in_value & 0xFF);
420
421 return pos;
422 };
423
424 const auto emitImm16 = [&](int in_value)
425 {
426 auto pos = emitImm8(in_value);
427
428 result.push_back((in_value >> 8) & 0xFF);
429
430 return pos;
431 };
432
433 const auto emitImm32 = [&](int in_value)
434 {
435 auto pos = emitImm16(in_value);
436
437 result.push_back((in_value >> 16) & 0xFF);
438 result.push_back((in_value >> 24) & 0xFF);
439
440 return pos;
441 };
442
443 const auto emitPush = [&](UserRegister in_register, std::optional<int> in_offset = {})
444 {
445 const auto registerId = GetRegisterID(in_register);
446
447 if (in_offset.has_value())
448 {
449 const auto offset = in_offset.value();
450 const auto emitEbpNoOffset = in_register == UserRegister::EBP && offset == 0;
451
452 result.push_back(0xFF);
453
454 if (!offset && !emitEbpNoOffset)
455 {
456 // push [{register}]
457 result.push_back(uint8_t(0x30 | registerId));
458 }
459 else if (offset <= 0x7F || emitEbpNoOffset)
460 {
461 // push [{register} + {imm8}]
462 result.push_back(uint8_t(0x70 | registerId));
463 }
464 else
465 {
466 // push [{register} + {imm32}]
467 result.push_back(uint8_t(0xB0 | registerId));
468 }
469
470 if (in_register == UserRegister::ESP)
471 result.push_back(0x24);
472
473 if (!offset && !emitEbpNoOffset)
474 return;
475
476 if ((offset > 0 && offset <= 0x7F) || emitEbpNoOffset)
477 {
478 emitImm8(offset);
479 }
480 else if (offset > 0x7F)
481 {
482 emitImm32(offset);
483 }
484
485 return;
486 }
487
488 // push {register}
489 result.push_back(uint8_t(0x50 | registerId));
490 };
491
492 const auto emitPop = [&](UserRegister in_register)
493 {
494 // pop {register}
495 result.push_back(uint8_t(0x58 | GetRegisterID(in_register)));
496 };
497
498 const auto emitAddSubtract = [&](UserRegister in_register, int in_value)
499 {
500 if (!in_value)
501 return;
502
503 const auto registerId = GetRegisterID(in_register);
504 const auto isAdd = in_value > 0;
505 const auto valueAbs = std::abs(in_value);
506
507 if (valueAbs == 1)
508 {
509 auto opcode = 0x40;
510
511 if (!isAdd)
512 opcode |= 8;
513
514 // inc/dec {register}
515 result.push_back(uint8_t(opcode | registerId));
516 }
517 else
518 {
519 auto modifier = 0;
520
521 if (!isAdd)
522 modifier = 0x28;
523
524 if (valueAbs > 0 && valueAbs <= 0x7F)
525 {
526 // add/sub {register}, {imm8}
527 result.push_back(0x83);
528 result.push_back(uint8_t(0xC0 | modifier | registerId));
529 emitImm8(valueAbs);
530 }
531 else if (valueAbs > 0x7F)
532 {
533 // add/sub {register}, {imm32}
534 if (in_register == UserRegister::EAX)
535 {
536 result.push_back(uint8_t(0x05 | modifier));
537 }
538 else
539 {
540 result.push_back(0x81);
541 result.push_back(uint8_t(0xC0 | modifier | registerId));
542 }
543
544 emitImm32(valueAbs);
545 }
546 }
547 };
548
549 const auto emitLoadStoreSSE = [&](UserRegister in_dst, UserRegister in_src, int in_offset = 0, int in_floatSize = sizeof(float), bool in_isStore = false)
550 {
551 const auto dstRegisterId = GetRegisterID(in_dst);
552 const auto srcRegisterId = GetRegisterID(in_src);
553
554 const auto ptrRegister = in_isStore ? in_dst : in_src;
555 const auto ptrRegisterId = in_isStore ? dstRegisterId : srcRegisterId;
556
557 const auto dataRegister = in_isStore ? in_src : in_dst;
558 const auto dataRegisterId = in_isStore ? srcRegisterId : dstRegisterId;
559
560 const auto emitEbpNoOffset = ptrRegister == UserRegister::EBP && in_offset == 0;
561
562 // movss dword ptr
563 auto opcode = 0xF3;
564
565 // movsd qword ptr
566 if (in_floatSize == sizeof(double))
567 opcode = 0xF2;
568
569 result.push_back(opcode);
570 result.push_back(0x0F);
571 result.push_back(in_isStore ? 0x11 : 0x10);
572
573 auto modifier = 0;
574
575 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
576 {
577 modifier = 0x40;
578 }
579 else if (in_offset > 0x7F)
580 {
581 modifier = 0x80;
582 }
583
584 if (ptrRegister == UserRegister::ESP)
585 {
586 result.push_back(uint8_t(0x04 | modifier | (dataRegisterId << 3)));
587 result.push_back(0x24);
588 }
589 else
590 {
591 result.push_back(uint8_t(modifier | (ptrRegisterId << 3) | dataRegisterId));
592 }
593
594 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
595 {
596 // movss/movsd {dst}, dword/qword ptr [{src} + {imm8}]
597 // movss/movsd dword/qword ptr [{dst} + {imm8}], {src}
598 emitImm8(in_offset);
599 }
600 else if (in_offset > 0x7F)
601 {
602 // movss/movsd {dst}, dword/qword ptr [{src} + {imm32}]
603 // movss/movsd dword/qword ptr [{dst} + {imm32}], {src}
604 emitImm32(in_offset);
605 }
606 };
607
608 const auto emitExchangeRegisterFPU = [&](UserRegister in_register)
609 {
611 return;
612
613 // fxch {register}
614 result.push_back(0xD9);
615 result.push_back(uint8_t(0xC8 | GetRegisterID(in_register)));
616 };
617
618 const auto emitLoadStoreFPU = [&](UserRegister in_register, int in_offset = 0, int in_floatSize = sizeof(float), bool in_isStore = false)
619 {
620 const auto registerId = GetRegisterID(in_register);
621
622 switch (GetRegisterFamily(in_register))
623 {
625 {
626 const auto emitEbpNoOffset = in_register == UserRegister::EBP && in_offset == 0;
627
628 // fld dword ptr
629 auto opcode = 0xD9;
630
631 // fld qword ptr
632 if (in_floatSize == sizeof(double))
633 opcode |= 4;
634
635 result.push_back(opcode);
636
637 auto modifier = 0;
638
639 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
640 {
641 modifier = 0x40;
642 }
643 else if (in_offset > 0x7F)
644 {
645 modifier = 0x80;
646 }
647
648 // fstp
649 if (in_isStore)
650 modifier |= 0x18;
651
652 if (in_register == UserRegister::ESP)
653 modifier |= 4;
654
655 result.push_back(uint8_t(modifier | registerId));
656
657 if (in_register == UserRegister::ESP)
658 result.push_back(0x24);
659
660 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
661 {
662 // fld/fstp dword/qword ptr [{register} + {imm8}]
663 emitImm8(in_offset);
664 }
665 else if (in_offset > 0x7F)
666 {
667 // fld/fstp dword/qword ptr [{register} + {imm32}]
668 emitImm32(in_offset);
669 }
670
671 break;
672 }
673
674 // fstp {dst}
676 result.push_back(0xDD);
677 result.push_back(uint8_t(0xD8 | registerId));
678 break;
679 }
680 };
681
682 const auto emitLoadStore = [&](UserRegister in_dst, UserRegister in_src, int in_offset = 0, bool in_isStore = false)
683 {
684 const auto dstRegisterId = GetRegisterID(in_dst);
685 const auto srcRegisterId = GetRegisterID(in_src);
686
687 const auto ptrRegister = in_isStore ? in_dst : in_src;
688 const auto ptrRegisterId = in_isStore ? dstRegisterId : srcRegisterId;
689
690 const auto dataRegister = in_isStore ? in_src : in_dst;
691 const auto dataRegisterId = in_isStore ? srcRegisterId : dstRegisterId;
692
693 const auto emitEbpNoOffset = ptrRegister == UserRegister::EBP && in_offset == 0;
694
695 // mov [{dst}], {src}
696 auto opcode = 0x89;
697
698 // mov {dst}, [{src}]
699 if (!in_isStore)
700 opcode |= 2;
701
702 result.push_back(opcode);
703
704 auto modifier = 0;
705
706 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
707 {
708 modifier = 0x40;
709 }
710 else if (in_offset > 0x7F)
711 {
712 modifier = 0x80;
713 }
714
715 if (ptrRegister == UserRegister::ESP)
716 {
717 result.push_back(uint8_t(0x04 | modifier | (dataRegisterId << 3)));
718 result.push_back(0x24);
719 }
720 else
721 {
722 result.push_back(uint8_t(modifier | (ptrRegisterId << 3) | dataRegisterId));
723 }
724
725 if ((in_offset > 0 && in_offset <= 0x7F) || emitEbpNoOffset)
726 {
727 // mov {dst}, [{src} + {imm8}]
728 // mov [{dst} + {imm8}], {src}
729 emitImm8(in_offset);
730 }
731 else if (in_offset > 0x7F)
732 {
733 // mov {dst}, [{src} + {imm32}]
734 // mov [{dst} + {imm32}], {src}
735 emitImm32(in_offset);
736 }
737 };
738
739 const auto emitMoveRegisterFPU = [&](UserRegister in_dst, UserRegister in_src, int in_floatSize = sizeof(float))
740 {
741 const auto dstRegisterId = GetRegisterID(in_dst);
742 const auto srcRegisterId = GetRegisterID(in_src);
743 const auto dstRegisterFamily = GetRegisterFamily(in_dst);
744 const auto srcRegisterFamily = GetRegisterFamily(in_src);
745
746 if (dstRegisterFamily == UserRegisterFamily::FPU)
747 {
748 if (dstRegisterFamily == srcRegisterFamily)
749 {
750 if (in_dst == in_src)
751 return;
752
753 // Exchange ST(0) with source register.
754 emitExchangeRegisterFPU(in_src);
755
756 // Copy ST(0) to destination register.
757 emitLoadStoreFPU(in_dst);
758
759 // Restore ST(0).
760 emitExchangeRegisterFPU(in_src);
761 }
762 else
763 {
764 // Allocate stack space for register.
765 emitAddSubtract(UserRegister::ESP, -in_floatSize);
766
767 if (srcRegisterFamily == UserRegisterFamily::GPR)
768 {
769 // Copy GPR register into stack.
770 emitLoadStore(UserRegister::ESP, in_src, 0, true);
771 }
772 else if (srcRegisterFamily == UserRegisterFamily::SSE)
773 {
774 // Copy SSE register into stack.
775 emitLoadStoreSSE(UserRegister::ESP, in_src, 0, in_floatSize, true);
776 }
777
778 // Exchange ST(0) with destination register.
779 if (in_dst != UserRegister::ST0)
780 emitExchangeRegisterFPU(in_dst);
781
782 // Load stack value into FPU register.
783 emitLoadStoreFPU(UserRegister::ESP, 0, in_floatSize);
784
785 // Restore ST(0).
786 if (in_dst != UserRegister::ST0)
787 emitExchangeRegisterFPU(in_dst);
788
789 // Deallocate stack space.
790 emitAddSubtract(UserRegister::ESP, in_floatSize);
791 }
792 }
793 else
794 {
795 // Allocate stack space for FPU register.
796 emitAddSubtract(UserRegister::ESP, -in_floatSize);
797
798 // Exchange ST(0) with source register.
799 if (in_src != UserRegister::ST0)
800 emitExchangeRegisterFPU(in_src);
801
802 // Copy ST(0) into stack.
803 emitLoadStoreFPU(UserRegister::ESP, 0, in_floatSize, true);
804
805 // Restore ST(0).
806 if (in_src != UserRegister::ST0)
807 emitExchangeRegisterFPU(in_src);
808
809 if (dstRegisterFamily == UserRegisterFamily::GPR)
810 {
811 // Load stack value into GPR register.
812 emitLoadStore(in_dst, UserRegister::ESP, 0);
813 }
814 else if (dstRegisterFamily == UserRegisterFamily::SSE)
815 {
816 // Load stack value into SSE register.
817 emitLoadStoreSSE(in_dst, UserRegister::ESP, 0, in_floatSize);
818 }
819
820 // Deallocate stack space.
821 emitAddSubtract(UserRegister::ESP, in_floatSize);
822 }
823 };
824
825 const auto emitMoveRegisterSSE = [&](UserRegister in_dst, UserRegister in_src, int in_floatSize = sizeof(float))
826 {
827 const auto dstRegisterId = GetRegisterID(in_dst);
828 const auto srcRegisterId = GetRegisterID(in_src);
829 const auto dstRegisterFamily = GetRegisterFamily(in_dst);
830 const auto srcRegisterFamily = GetRegisterFamily(in_src);
831
832 if (dstRegisterFamily == srcRegisterFamily && dstRegisterFamily == UserRegisterFamily::SSE)
833 {
834 // movss {dst}, {src}
835 auto opcode = 0xF3;
836
837 // movsd {dst}, {src}
838 if (in_floatSize == sizeof(double))
839 opcode = 0xF2;
840
841 result.push_back(opcode);
842 result.push_back(0x0F);
843 result.push_back(0x10);
844 result.push_back(uint8_t(0xC0 | (srcRegisterId << 3) | dstRegisterId));
845 }
846 else if ((dstRegisterFamily == UserRegisterFamily::GPR || dstRegisterFamily == UserRegisterFamily::SSE) &&
847 (srcRegisterFamily == UserRegisterFamily::GPR || srcRegisterFamily == UserRegisterFamily::SSE))
848 {
849 auto modifier = 0x6E;
850
851 if (dstRegisterFamily == UserRegisterFamily::GPR)
852 modifier |= 0x10;
853
854 // movd {dst}, {src}
855 result.push_back(0x66);
856 result.push_back(0x0F);
857 result.push_back(modifier);
858 result.push_back(uint8_t(0xC0 | (srcRegisterId << 3) | dstRegisterId));
859 }
860 else
861 {
862 emitMoveRegisterFPU(in_dst, in_src, in_floatSize);
863 }
864 };
865
866 const auto emitMoveRegister = [&](UserRegister in_dst, UserRegister in_src, int in_floatSize = sizeof(float))
867 {
868 const auto dstRegisterFamily = GetRegisterFamily(in_dst);
869 const auto srcRegisterFamily = GetRegisterFamily(in_src);
870
871 if (dstRegisterFamily == srcRegisterFamily)
872 {
873 switch (dstRegisterFamily)
874 {
875 // mov {dst}, {src}
877 result.push_back(0x89);
878 result.push_back(uint8_t(0xC0 | (GetRegisterID(in_src) << 3) | GetRegisterID(in_dst)));
879 break;
880
882 emitMoveRegisterFPU(in_dst, in_src, in_floatSize);
883 break;
884
886 emitMoveRegisterSSE(in_dst, in_src, in_floatSize);
887 break;
888 }
889 }
890 else
891 {
892 emitMoveRegisterSSE(in_dst, in_src, in_floatSize);
893 }
894 };
895
896 const auto emitBranch = [&](bool in_isCall, bool in_isFar)
897 {
898 if (in_isFar)
899 {
900 // jmp/call [{imm32}]
901 result.push_back(0xFF);
902 result.push_back(in_isCall ? 0x15 : 0x25);
903 }
904 else
905 {
906 // jmp/call {imm32}
907 result.push_back(in_isCall ? 0xE8 : 0xE9);
908 }
909
910 return emitImm32(0);
911 };
912
913 const auto emitBranchAddr = [&](void* in_pStart, size_t in_branchOffset, size_t in_target, bool in_isAbsolute)
914 {
915 const auto offset = size_t(in_pStart) + in_branchOffset;
916
917 if (in_isAbsolute)
918 {
919 *(uint32_t*)offset = uint32_t(in_target);
920 }
921 else
922 {
923 *(uint32_t*)offset = uint32_t((in_target - offset) - sizeof(uint32_t));
924 }
925 };
926
927 const auto emitReturn = [&](size_t in_size = 0)
928 {
929 auto opcode = 0xC2;
930
931 if (!in_size)
932 opcode |= 1;
933
934 // ret
935 result.push_back(opcode);
936
937 if (!in_size)
938 return;
939
940 // ret {imm16}
941 emitImm16(in_size);
942 };
943
944 const auto registerCount = in_rInfo.GetRegisterCount();
945 const auto stackParamCount = in_rInfo.GetStackParamCount();
946 const auto paramCount = in_rInfo.GetParamCount();
947
948 auto stackOffset = in_isToOriginal
949 ? paramCount * 4
950 : stackParamCount * 4;
951
952 if (in_isToOriginal)
953 {
954 // Back up protected registers.
955 for (size_t i = 0; i < registerCount; i++)
956 {
957 const auto currentRegister = in_rInfo.GetRegister(i);
958
959 if (!in_rInfo.IsProtectedRegister(currentRegister))
960 continue;
961
962 emitPush(currentRegister);
963 stackOffset += 4;
964 }
965 }
966
967 // Set up arguments.
968 for (auto i = paramCount; i-- > 0;)
969 {
970 if (in_rInfo.IsStackParam(i))
971 {
972 emitPush(UserRegister::ESP, stackOffset);
973 }
974 else
975 {
976 const auto currentRegister = in_rInfo.GetParamRegister(i);
977
978 if (in_isToOriginal)
979 {
980 // Move __cdecl arguments into __usercall registers.
981 emitLoadStore(currentRegister, UserRegister::ESP, stackOffset);
982 stackOffset -= 4;
983 }
984 else
985 {
986 // Push __usercall registers into __cdecl arguments.
987 emitPush(currentRegister);
988 }
989 }
990 }
991
992 // Reserve call to hook/original function.
993 // Uses a far call to return to the original function so we can mutate
994 // the return address (e.g. user uninstalls the hook inside the hook).
995 auto branchOffset = emitBranch(true, in_isToOriginal);
996
997 if (in_isToOriginal)
998 {
999 // Clean up __usercall stack arguments.
1000 if (!in_rInfo.IsUserPurge && stackParamCount > 0)
1001 emitAddSubtract(UserRegister::ESP, stackParamCount * 4);
1002
1003 // Restore protected registers.
1004 for (auto i = registerCount; i-- > 0;)
1005 {
1006 const auto currentRegister = in_rInfo.GetRegister(i);
1007
1008 if (!in_rInfo.IsProtectedRegister(currentRegister))
1009 continue;
1010
1011 emitPop(currentRegister);
1012 }
1013 }
1014
1015 const auto returnRegister = in_rInfo.GetReturnRegister();
1016
1017 if (returnRegister != UserRegister::None)
1018 {
1019 // Move __cdecl return value into __usercall return register and vice-versa.
1020 if ((in_rInfo.IsReturnType<float>() || in_rInfo.IsReturnType<double>()))
1021 {
1022 if (returnRegister != UserRegister::ST0)
1023 {
1024 if (in_isToOriginal)
1025 {
1026 emitMoveRegister(UserRegister::ST0, returnRegister, in_rInfo.ReturnTypeSize);
1027 }
1028 else
1029 {
1030 emitMoveRegister(returnRegister, UserRegister::ST0, in_rInfo.ReturnTypeSize);
1031 }
1032 }
1033 }
1034 else if (returnRegister != UserRegister::EAX)
1035 {
1036 if (in_isToOriginal)
1037 {
1038 emitMoveRegister(UserRegister::EAX, returnRegister);
1039 }
1040 else
1041 {
1042 emitMoveRegister(returnRegister, UserRegister::EAX);
1043 }
1044 }
1045 }
1046
1047 if (in_isToOriginal)
1048 {
1049 emitReturn();
1050 }
1051 else
1052 {
1053 // Pop register arguments.
1054 for (size_t i = 0; i < paramCount; i++)
1055 {
1056 if (in_rInfo.IsStackParam(i))
1057 continue;
1058
1059 const auto currentRegister = in_rInfo.GetParamRegister(i);
1060
1061 if (currentRegister == returnRegister)
1062 {
1063 emitAddSubtract(UserRegister::ESP, 4);
1064 continue;
1065 }
1066
1067 emitPop(currentRegister);
1068 }
1069
1070 if (stackParamCount > 0)
1071 {
1072 stackOffset = stackParamCount * 4;
1073
1074 // Restore stack pointer for original call.
1075 emitAddSubtract(UserRegister::ESP, stackOffset);
1076
1077 // Pop stack arguments in __userpurge.
1078 emitReturn(in_rInfo.IsUserPurge ? stackOffset : 0);
1079 }
1080 else
1081 {
1082 emitReturn();
1083 }
1084 }
1085
1086 const auto size = result.size();
1087
1088 auto pTrampoline = _aligned_malloc(size, sizeof(void*));
1089 memcpy_s(pTrampoline, size, result.data(), size);
1090 emitBranchAddr(pTrampoline, branchOffset, in_address, in_isToOriginal);
1091
1092 hedgedev::csl::mem::Protect(pTrampoline, size, hedgedev::csl::mem::GetProtectionFlags(hedgedev::csl::mem::PageProtection::RWX));
1093
1094 return pTrampoline;
1095 }
1096}
#define USER_HOOK(RETURN_TYPE, CALLING_CONVENTION, FUNCTION_NAME, ADDRESS, REGISTERS, PARAM_COUNT,...)
Definition HookingUserCall.h:83
#define INSTALL_USER_HOOK(FUNCTION_NAME)
Definition HookingUserCall.h:112
#define USER_REGISTER(REGISTER)
Definition HookingUserCall.h:30
#define INSTALL_USER_HOOK_EXPLICIT(FUNCTION_NAME, ADDRESS)
Definition HookingUserCall.h:123
Definition Hooking.h:321
UserRegisterFamily
Definition HookingUserCall.h:266
@ FPU
Definition HookingUserCall.h:269
@ SSE
Definition HookingUserCall.h:270
@ None
Definition HookingUserCall.h:267
@ GPR
Definition HookingUserCall.h:268
UserRegister
Definition HookingUserCall.h:175
@ ST1
Definition HookingUserCall.h:188
@ XMM5
Definition HookingUserCall.h:201
@ XMM7
Definition HookingUserCall.h:203
@ EDX
Definition HookingUserCall.h:180
@ ST5
Definition HookingUserCall.h:192
@ ST3
Definition HookingUserCall.h:190
@ XMM0
Definition HookingUserCall.h:196
@ ST6
Definition HookingUserCall.h:193
@ ESP
Definition HookingUserCall.h:182
@ EDI
Definition HookingUserCall.h:185
@ EBP
Definition HookingUserCall.h:183
@ XMM3
Definition HookingUserCall.h:199
@ XMM4
Definition HookingUserCall.h:200
@ None
Definition HookingUserCall.h:176
@ ST2
Definition HookingUserCall.h:189
@ EBX
Definition HookingUserCall.h:181
@ XMM1
Definition HookingUserCall.h:197
@ XMM6
Definition HookingUserCall.h:202
@ ST4
Definition HookingUserCall.h:191
@ ST7
Definition HookingUserCall.h:194
@ ST0
Definition HookingUserCall.h:187
@ ESI
Definition HookingUserCall.h:184
@ XMM2
Definition HookingUserCall.h:198
@ Count
Definition HookingUserCall.h:205
@ EAX
Definition HookingUserCall.h:178
@ ECX
Definition HookingUserCall.h:179
int GetRegisterID(UserRegister in_register)
Definition HookingUserCall.h:396
UserRegisterFamily GetRegisterFamily(UserRegister in_register)
Definition HookingUserCall.h:373
void * EmitUserTrampoline(const UserCallInfo &in_rInfo, const size_t in_address, bool in_isToOriginal=false)
Definition HookingUserCall.h:411
Register
Definition HookingUserCall.h:209
@ ST1
Definition HookingUserCall.h:245
@ XMM5
Definition HookingUserCall.h:258
@ SPL
Definition HookingUserCall.h:228
@ XMM7
Definition HookingUserCall.h:260
@ EDX
Definition HookingUserCall.h:222
@ BPL
Definition HookingUserCall.h:232
@ ST5
Definition HookingUserCall.h:249
@ ST3
Definition HookingUserCall.h:247
@ XMM0
Definition HookingUserCall.h:253
@ ST6
Definition HookingUserCall.h:250
@ ESP
Definition HookingUserCall.h:230
@ EDI
Definition HookingUserCall.h:242
@ EBP
Definition HookingUserCall.h:234
@ XMM3
Definition HookingUserCall.h:256
@ XMM4
Definition HookingUserCall.h:257
@ AX
Definition HookingUserCall.h:213
@ CL
Definition HookingUserCall.h:216
@ SP
Definition HookingUserCall.h:229
@ None
Definition HookingUserCall.h:210
@ DX
Definition HookingUserCall.h:221
@ ST2
Definition HookingUserCall.h:246
@ CX
Definition HookingUserCall.h:217
@ EBX
Definition HookingUserCall.h:226
@ SIL
Definition HookingUserCall.h:236
@ BP
Definition HookingUserCall.h:233
@ XMM1
Definition HookingUserCall.h:254
@ DIL
Definition HookingUserCall.h:240
@ XMM6
Definition HookingUserCall.h:259
@ DI
Definition HookingUserCall.h:241
@ BL
Definition HookingUserCall.h:224
@ ST4
Definition HookingUserCall.h:248
@ BX
Definition HookingUserCall.h:225
@ ST7
Definition HookingUserCall.h:251
@ SI
Definition HookingUserCall.h:237
@ ST0
Definition HookingUserCall.h:244
@ ESI
Definition HookingUserCall.h:238
@ XMM2
Definition HookingUserCall.h:255
@ AL
Definition HookingUserCall.h:212
@ Count
Definition HookingUserCall.h:262
@ EAX
Definition HookingUserCall.h:214
@ ECX
Definition HookingUserCall.h:218
@ DL
Definition HookingUserCall.h:220
Definition HookingUserCall.h:299
std::type_index ReturnType
Definition HookingUserCall.h:300
bool IsReturnType() const
Definition HookingUserCall.h:357
size_t GetStackParamCount() const
Definition HookingUserCall.h:323
size_t GetRegisterParamCount() const
Definition HookingUserCall.h:313
bool IsProtectedRegister(UserRegister in_register) const
Definition HookingUserCall.h:348
bool IsStackParam(int in_index) const
Definition HookingUserCall.h:362
uint8_t ParamCount
Definition HookingUserCall.h:306
UserRegister GetParamRegister(int in_index) const
Definition HookingUserCall.h:343
void * fpDetour
Definition HookingUserCall.h:304
void * fpOriginal
Definition HookingUserCall.h:303
size_t ReturnTypeSize
Definition HookingUserCall.h:301
UserRegister GetRegister(int in_index) const
Definition HookingUserCall.h:333
uint64_t Registers
Definition HookingUserCall.h:305
UserRegister GetReturnRegister() const
Definition HookingUserCall.h:338
bool IsUserPurge
Definition HookingUserCall.h:302
size_t GetParamCount() const
Definition HookingUserCall.h:328
size_t GetRegisterCount() const
Definition HookingUserCall.h:308