#include "paging64.h" #include #include #include #include #include #include #include "memory.h" #include "parse/elf.h" #include "x86/asm.h" #include "x86/paging64.h" typedef struct { page_t *lvl4; bool nx; } paging64_state_t; static paging64_state_t paging64; static u32 table_type(bool is_kernel) { if (is_kernel) { return E820_KERNEL; } return E820_PAGE_TABLE; } static bool has_nx(void) { cpuid_regs_t regs = { 0 }; cpuid(0x80000000, ®s); if (regs.eax < CPUID_EXTENDED_INFO) { return false; } cpuid(CPUID_EXTENDED_INFO, ®s); return (regs.edx & CPUID_EI_NX) != 0; } static page_t *walk_table(page_t *table, size_t index, bool is_kernel) { page_t *next_table; if (table[index] & PT_PRESENT) { next_table = (page_t *)(uintptr_t)page_get_paddr(&table[index]); } else { u32 type = table_type(is_kernel); next_table = (page_t *)mmap_alloc(PAGE_4KIB, type, PAGE_4KIB); memset(next_table, 0, PAGE_4KIB); page_set_paddr(&table[index], (u64)(uintptr_t)next_table); table[index] |= PT_PRESENT; table[index] |= PT_WRITE; } return next_table; } // huge pages are used only when the caller has already aligned the range void map_page_64(size_t size, u64 vaddr, u64 paddr, u64 flags, bool is_kernel) { size_t lvl4_index = GET_LVL4_INDEX(vaddr); size_t lvl3_index = GET_LVL3_INDEX(vaddr); page_t *lvl3 = walk_table(paging64.lvl4, lvl4_index, is_kernel); page_t *entry; if (size == PAGE_1GIB) { entry = &lvl3[lvl3_index]; paddr = ALIGN_DOWN(paddr, PAGE_1GIB); flags |= PT_HUGE; goto finalize; } size_t lvl2_index = GET_LVL2_INDEX(vaddr); page_t *lvl2 = walk_table(lvl3, lvl3_index, is_kernel); if (size == PAGE_2MIB) { entry = &lvl2[lvl2_index]; paddr = ALIGN_DOWN(paddr, PAGE_2MIB); flags |= PT_HUGE; goto finalize; } size_t lvl1_index = GET_LVL1_INDEX(vaddr); page_t *lvl1 = walk_table(lvl2, lvl2_index, is_kernel); entry = &lvl1[lvl1_index]; finalize: page_set_paddr(entry, paddr); flags |= PT_PRESENT; u64 pat_huge = 0; if (flags & PT_HUGE) { pat_huge = flags & PT_PAT_HUGE; } *entry |= (flags & FLAGS_MASK) | pat_huge; } void map_region_64(size_t size, u64 vaddr, u64 paddr, u64 flags, bool is_kernel) { size_t remaining = ALIGN(size, PAGE_4KIB); while (remaining) { size_t page_size = PAGE_4KIB; if (remaining >= PAGE_2MIB && (vaddr & (PAGE_2MIB - 1)) == 0 && (paddr & (PAGE_2MIB - 1)) == 0) { page_size = PAGE_2MIB; } map_page_64(page_size, vaddr, paddr, flags, is_kernel); vaddr += page_size; paddr += page_size; remaining -= page_size; } } void identity_map_64(u64 top_address, u64 offset, bool is_kernel) { for (u64 i = 0; i < top_address; i += PAGE_2MIB) { map_page_64(PAGE_2MIB, i + offset, i, PT_WRITE, is_kernel); } } void setup_paging_64(void) { paging64.lvl4 = (page_t *)mmap_alloc(PAGE_4KIB, E820_KERNEL, PAGE_4KIB); memset(paging64.lvl4, 0, PAGE_4KIB); write_cr3((u32)(uintptr_t)paging64.lvl4); paging64.nx = has_nx(); // NX is optional on older machines, so the bootloader enables it only if CPUID says so if (paging64.nx) { u64 efer = read_msr(EFER_MSR); write_msr(EFER_MSR, efer | EFER_NX); } // write protect keeps supervisor writes honest once readonly kernel pages exist u32 cr0 = read_cr0(); write_cr0(cr0 | CR0_WP); } void init_paging_64(void) { u32 cr4 = read_cr4(); write_cr4(cr4 | CR4_PAE); u64 efer = read_msr(EFER_MSR); write_msr(EFER_MSR, efer | EFER_LME); u32 cr0 = read_cr0(); write_cr0(cr0 | CR0_PG); } static page_t elf_flags(u32 elf_flags) { u64 flags = PT_PRESENT; if (elf_flags & PF_W) { flags |= PT_WRITE; } if (paging64.nx && !(elf_flags & PF_X)) { flags |= PT_NO_EXECUTE; } return flags; } struct elf_load_ctx64 { void *elf; }; static bool load_segment_64(const elf_segment_t *seg, void *ctx_ptr) { struct elf_load_ctx64 *load_ctx = ctx_ptr; u64 size = ALIGN(seg->mem_size, PAGE_4KIB); u64 flags = elf_flags(seg->flags); u64 pbase = (u64)(uintptr_t)mmap_alloc(size, E820_KERNEL, (size_t)seg->align); u64 vbase = seg->vaddr; map_region_64(size, vbase, pbase, flags, true); memcpy((void *)(uintptr_t)pbase, (u8 *)load_ctx->elf + seg->offset, (size_t)seg->file_size); size_t zero_len = (size_t)(seg->mem_size - seg->file_size); memset((u8 *)(uintptr_t)pbase + seg->file_size, 0, zero_len); return true; } u64 load_elf_sections_64(void *elf_file) { struct elf_load_ctx64 ctx = { .elf = elf_file, }; elf_info_t info = { 0 }; if (!elf_foreach_segment(elf_file, 0, load_segment_64, &ctx, &info)) { return 0; } if (!info.is_64) { return 0; } return info.entry; }