Skip to content

Instantly share code, notes, and snippets.

@mrexodia
Created May 31, 2026 19:00
Show Gist options
  • Select an option

  • Save mrexodia/42c1fa6eb21b705a007c612adbbda7c8 to your computer and use it in GitHub Desktop.

Select an option

Save mrexodia/42c1fa6eb21b705a007c612adbbda7c8 to your computer and use it in GitHub Desktop.
Supporting files: Analyzing memory access patterns is easier than ever

x64dbg trace-memory crackmes

These programs are small inputs for the trace dump, trace Xref, and trace pattern-search features.

Build

From this directory in Git Bash:

mkdir -p build
"/c/Program Files/LLVM/bin/clang.exe" -O0 -gcodeview -fuse-ld=lld -Xlinker -debug -Xlinker -dynamicbase:no -Xlinker -highentropyva:no -o build/trace_rc4.exe trace_rc4.c
"/c/Program Files/LLVM/bin/clang.exe" -O0 -gcodeview -fuse-ld=lld -Xlinker -debug -Xlinker -dynamicbase:no -Xlinker -highentropyva:no -o build/trace_overflow.exe trace_overflow.c

The linker flags -dynamicbase:no -highentropyva:no are the lld-link equivalents of /DYNAMICBASE:NO /HIGHENTROPYVA:NO. They keep module addresses stable across runs.

The examples export marker functions for convenient breakpoints. trace_rc4.exe uses trace_begin and trace_end. trace_overflow.exe uses only trace_begin; the access violation stops the trace.

Example 1: trace_rc4.exe

Correct serial:

x64dbg{trace-memory}

Suggested run:

trace_rc4.exe wrong

Trace workflow:

  1. Open trace_rc4.exe in x64dbg.
  2. Set the command line to trace_rc4.exe wrong if needed.
  3. Open the Symbols view for the main module.
  4. Set breakpoints on the exported functions trace_begin and trace_end.
  5. Run to trace_begin.
  6. Start Trace into from trace_begin.
  7. The trace stops when trace_end is reached.
  8. Open the Trace view, select the last traced instruction, and load the trace dump if x64dbg has not loaded it automatically.

Things to show:

  • The console prints the stack addresses of S and expected.
  • In the trace dump, go to the expected address and step through the trace rows around rc4_crypt; the buffer changes into the plaintext serial.
  • Search the trace dump for this ASCII pattern:
78 36 34 64 62 67 7B

That is the byte pattern for x64dbg{. It is not stored as plaintext in the program; it appears only after the RC4 decrypt loop writes it.

  • Search for this RC4 initialization pattern:
00 01 02 03 04 05 06 07

The result points at the temporary S[256] table created during rc4_init.

  • Select a byte in the expected buffer and use the trace dump Xref action. The useful references are the RC4 write, the compare read, and the wipe write.

Example 2: trace_overflow.exe

Trace workflow:

  1. Open trace_overflow.exe in x64dbg.
  2. Set a breakpoint on the exported function trace_begin.
  3. Run to trace_begin.
  4. Start Trace into from trace_begin. Set the max trace count to 100000 if x64dbg asks.
  5. The trace stops on the access violation caused by call_callback.
  6. Open the Trace view, select the last trace row, and load the trace dump if needed.

Things to show:

  • The console prints the addresses of overflow, greeting, and the callback field.
  • The crash happens because g_arena.callback was overwritten with 43 43 43 43 43 43 43 43, the ASCII bytes for CCCCCCCC.
  • In the trace dump, go to the printed callback field address, select the first byte, and use the trace dump Xref action.
  • Jump to the last write before the crash. It lands inside copy_unbounded(g_arena.overflow, ...), after the copy has crossed the end of overflow.
  • Repeat the same action on the printed greeting address. Its bytes changed to greeting_is_gone, showing that the same overflow corrupted neighboring data before it corrupted the function pointer.

This is the cleanest motivating example for trace Xref: it answers "who last wrote the byte that caused this crash?" without needing a hardware breakpoint before the bug happens.

// trace_overflow.c
// A tiny crash-investigation demo for x64dbg trace dump and trace xrefs.
// Build: clang.exe -O0 -gcodeview -fuse-ld=lld -Xlinker -debug -Xlinker -dynamicbase:no -Xlinker -highentropyva:no -o trace_overflow.exe trace_overflow.c
#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
#if defined(_MSC_VER) || defined(__clang__)
#define NOINLINE __declspec(noinline)
#define EXPORT __declspec(dllexport)
#else
#define NOINLINE __attribute__((noinline))
#define EXPORT
#endif
typedef void (*Callback)(void);
typedef struct Arena {
char overflow[16];
char greeting[16];
Callback callback;
char scratch[8];
} Arena;
static Arena g_arena;
EXPORT NOINLINE void trace_begin(void)
{
// Set a breakpoint here in x64dbg, then start Trace Into.
// The access violation stops the trace; this demo does not need trace_end.
}
NOINLINE void safe_callback(void)
{
puts("safe callback");
}
NOINLINE void zero_arena(Arena *arena)
{
volatile uint8_t *p = (volatile uint8_t *)arena;
size_t i;
for (i = 0; i < sizeof(*arena); i++)
p[i] = 0;
}
NOINLINE void copy_unbounded(char *dst, const char *src)
{
while ((*dst++ = *src++) != 0) {
// Deliberately no bounds check.
}
}
NOINLINE void call_callback(void)
{
g_arena.callback();
}
int main(void)
{
setvbuf(stdout, NULL, _IONBF, 0);
zero_arena(&g_arena);
copy_unbounded(g_arena.greeting, "Hello!");
g_arena.callback = safe_callback;
printf("trace_overflow: crash after a buffer overflow\n");
printf(" overflow buffer: %p (16 bytes)\n", (void *)g_arena.overflow);
printf(" greeting buffer: %p (16 bytes)\n", (void *)g_arena.greeting);
printf(" callback field: %p (8 bytes)\n", (void *)&g_arena.callback);
printf(" callback value: %p\n", (void *)g_arena.callback);
printf(" greeting before overflow: %s\n", g_arena.greeting);
printf("Set a breakpoint on trace_begin, then trace until the access violation.\n");
trace_begin();
copy_unbounded(g_arena.overflow,
"AAAAAAAAAAAAAAAA"
"greeting_is_gone"
"CCCCCCCC");
call_callback();
return 0;
}
// trace_rc4.c
// A tiny crackme for x64dbg trace dump, trace xrefs, and trace pattern search.
// Build: clang.exe -O0 -gcodeview -fuse-ld=lld -Xlinker -debug -Xlinker -dynamicbase:no -Xlinker -highentropyva:no -o trace_rc4.exe trace_rc4.c
#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#if defined(_MSC_VER) || defined(__clang__)
#define NOINLINE __declspec(noinline)
#define EXPORT __declspec(dllexport)
#else
#define NOINLINE __attribute__((noinline))
#define EXPORT
#endif
#define SECRET_LEN 20
// Encrypted RC4 output for the plaintext serial used by this demo.
// The plaintext does not appear as a string in the executable.
static const uint8_t encrypted_serial[SECRET_LEN] = {
0x7F, 0x20, 0xC2, 0xBC, 0xB3, 0xF4, 0x87, 0x9F, 0xFE, 0xA6,
0x06, 0x34, 0x6E, 0x18, 0x62, 0x54, 0x42, 0xC4, 0x73, 0x27
};
static const uint8_t rc4_key[] = "x64dbg-trace-demo";
EXPORT NOINLINE void trace_begin(void)
{
// Set a breakpoint here in x64dbg, then start Trace Into.
}
EXPORT NOINLINE void trace_end(void)
{
// Set a breakpoint here in x64dbg, so tracing stops after the demo.
}
NOINLINE void rc4_init(uint8_t s[256], const uint8_t *key, size_t key_len)
{
uint32_t i;
uint8_t j = 0;
for (i = 0; i < 256; i++)
s[i] = (uint8_t)i;
for (i = 0; i < 256; i++) {
uint8_t tmp;
j = (uint8_t)(j + s[i] + key[i % key_len]);
tmp = s[i];
s[i] = s[j];
s[j] = tmp;
}
}
NOINLINE void rc4_crypt(uint8_t s[256], const uint8_t *input, uint8_t *output, size_t len)
{
size_t n;
uint8_t i = 0;
uint8_t j = 0;
for (n = 0; n < len; n++) {
uint8_t tmp;
uint8_t k;
i = (uint8_t)(i + 1);
j = (uint8_t)(j + s[i]);
tmp = s[i];
s[i] = s[j];
s[j] = tmp;
k = s[(uint8_t)(s[i] + s[j])];
output[n] = (uint8_t)(input[n] ^ k);
}
}
NOINLINE size_t bounded_strlen(const char *s, size_t max_len)
{
size_t n = 0;
while (n <= max_len && s[n] != 0)
n++;
return n;
}
NOINLINE int slow_equals(const char *user, const uint8_t *expected, size_t expected_len)
{
size_t i;
size_t user_len = bounded_strlen(user, expected_len);
uint8_t diff = 0;
for (i = 0; i < expected_len; i++) {
uint8_t c = 0;
if (i < user_len)
c = (uint8_t)user[i];
diff |= (uint8_t)(c ^ expected[i]);
}
return diff == 0 && user_len == expected_len;
}
NOINLINE void wipe_bytes(void *ptr, size_t len)
{
volatile uint8_t *p = (volatile uint8_t *)ptr;
size_t i;
for (i = 0; i < len; i++)
p[i] = 0;
}
int main(int argc, char **argv)
{
uint8_t s[256];
uint8_t expected[SECRET_LEN + 1];
const char *user = argc > 1 ? argv[1] : "wrong";
int ok;
size_t i;
for (i = 0; i < sizeof(expected); i++)
expected[i] = 0;
printf("trace_rc4: hidden serial check\n");
printf(" S table: %p (256 bytes)\n", (void *)s);
printf(" expected serial: %p (%u bytes)\n", (void *)expected, (unsigned)SECRET_LEN);
printf(" user input: %s\n", user);
printf("Set breakpoints on trace_begin and trace_end, then trace from trace_begin.\n");
trace_begin();
rc4_init(s, rc4_key, sizeof(rc4_key) - 1);
rc4_crypt(s, encrypted_serial, expected, SECRET_LEN);
expected[SECRET_LEN] = 0;
ok = slow_equals(user, expected, SECRET_LEN);
wipe_bytes(expected, sizeof(expected));
wipe_bytes(s, 256);
trace_end();
puts(ok ? "correct" : "wrong");
return ok ? 0 : 1;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment