haiku.git

/*
 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
 * Distributed under the terms of the MIT License.
 *
 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
 * Distributed under the terms of the NewOS License.
 */


#include "paging/64bit/X86PagingMethod64Bit.h"

#include <stdlib.h>
#include <string.h>

#include <boot/kernel_args.h>
#include <util/AutoLock.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/VMAddressSpace.h>

#include "paging/64bit/X86PagingStructures64Bit.h"
#include "paging/64bit/X86VMTranslationMap64Bit.h"
#include "paging/x86_physical_page_mapper.h"
#include "paging/x86_physical_page_mapper_mapped.h"


//#define TRACE_X86_PAGING_METHOD_64BIT
#ifdef TRACE_X86_PAGING_METHOD_64BIT
#	define TRACE(x...) dprintf(x)
#else
#	define TRACE(x...) ;
#endif


bool X86PagingMethod64Bit::la57 = false;


// #pragma mark - X86PagingMethod64Bit


X86PagingMethod64Bit::X86PagingMethod64Bit(bool la57)
	:
	fKernelPhysicalPMLTop(0),
	fKernelVirtualPMLTop(NULL),
	fPhysicalPageMapper(NULL),
	fKernelPhysicalPageMapper(NULL)
{
	X86PagingMethod64Bit::la57 = la57;
}


X86PagingMethod64Bit::~X86PagingMethod64Bit()
{
}


status_t
X86PagingMethod64Bit::Init(kernel_args* args,
	VMPhysicalPageMapper** _physicalPageMapper)
{
	fKernelPhysicalPMLTop = args->arch_args.phys_pgdir;
	fKernelVirtualPMLTop = (uint64*)(addr_t)args->arch_args.vir_pgdir;

	// if available enable NX-bit (No eXecute)
	if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD))
		call_all_cpus_sync(&_EnableExecutionDisable, NULL);

	// Create the physical page mapper.
	mapped_physical_page_ops_init(args, fPhysicalPageMapper,
		fKernelPhysicalPageMapper);

	// Ensure that the user half of the address space is clear. This removes
	// the temporary identity mapping made by the boot loader.
	if (la57) {
		uint64* virtualPML4 = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			*fKernelVirtualPMLTop & X86_64_PML5E_ADDRESS_MASK);
		memset(virtualPML4, 0, sizeof(uint64) * 256);
	}
	memset(fKernelVirtualPMLTop, 0, sizeof(uint64) * 256);
	arch_cpu_global_TLB_invalidate();

	*_physicalPageMapper = fPhysicalPageMapper;
	return B_ERROR;
}


status_t
X86PagingMethod64Bit::InitPostArea(kernel_args* args)
{
	// Create an area covering the physical map area.
	void* address = (void*)KERNEL_PMAP_BASE;
	area_id area = vm_create_null_area(VMAddressSpace::KernelID(),
		"physical map area", &address, B_EXACT_ADDRESS,
		KERNEL_PMAP_SIZE, 0);
	if (area < B_OK)
		return area;

	// Create an area to represent the kernel PMLTop.
	area = create_area("kernel pmltop", (void**)&fKernelVirtualPMLTop,
		B_EXACT_ADDRESS, B_PAGE_SIZE, B_ALREADY_WIRED,
		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
	if (area < B_OK)
		return area;

	return B_OK;
}


status_t
X86PagingMethod64Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
{
	X86VMTranslationMap64Bit* map = new(std::nothrow) X86VMTranslationMap64Bit(la57);
	if (map == NULL)
		return B_NO_MEMORY;

	status_t error = map->Init(kernel);
	if (error != B_OK) {
		delete map;
		return error;
	}

	*_map = map;
	return B_OK;
}


status_t
X86PagingMethod64Bit::MapEarly(kernel_args* args, addr_t virtualAddress,
	phys_addr_t physicalAddress, uint8 attributes)
{
	TRACE("X86PagingMethod64Bit::MapEarly(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR
		", %#" B_PRIx8 ")\n", virtualAddress, physicalAddress, attributes);

	uint64* virtualPML4 = fKernelVirtualPMLTop;
	if (la57) {
		// Get the PML4. We should be mapping on an existing PML4 at this stage.
		uint64* pml5e = &fKernelVirtualPMLTop[VADDR_TO_PML5E(virtualAddress)];
		ASSERT((*pml5e & X86_64_PML5E_PRESENT) != 0);
		virtualPML4 = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			*pml5e & X86_64_PML5E_ADDRESS_MASK);
	}

	// Get the PDPT. We should be mapping on an existing PDPT at this stage.
	uint64* pml4e = &virtualPML4[VADDR_TO_PML4E(virtualAddress)];
	ASSERT((*pml4e & X86_64_PML4E_PRESENT) != 0);
	uint64* virtualPDPT = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
		*pml4e & X86_64_PML4E_ADDRESS_MASK);

	// Get the page directory.
	uint64* pdpte = &virtualPDPT[VADDR_TO_PDPTE(virtualAddress)];
	uint64* virtualPageDir;
	if ((*pdpte & X86_64_PDPTE_PRESENT) == 0) {
		phys_addr_t physicalPageDir = vm_allocate_early_physical_page(args) * B_PAGE_SIZE;

		TRACE("X86PagingMethod64Bit::MapEarly(): creating page directory for va"
			" %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress,
			physicalPageDir);

		SetTableEntry(pdpte, (physicalPageDir & X86_64_PDPTE_ADDRESS_MASK)
			| X86_64_PDPTE_PRESENT
			| X86_64_PDPTE_WRITABLE
			| X86_64_PDPTE_USER);

		// Map it and zero it.
		virtualPageDir = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			physicalPageDir);
		memset(virtualPageDir, 0, B_PAGE_SIZE);
	} else {
		virtualPageDir = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			*pdpte & X86_64_PDPTE_ADDRESS_MASK);
	}

	// Get the page table.
	uint64* pde = &virtualPageDir[VADDR_TO_PDE(virtualAddress)];
	uint64* virtualPageTable;
	if ((*pde & X86_64_PDE_PRESENT) == 0) {
		phys_addr_t physicalPageTable = vm_allocate_early_physical_page(args) * B_PAGE_SIZE;

		TRACE("X86PagingMethod64Bit::MapEarly(): creating page table for va"
			" %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress,
			physicalPageTable);

		SetTableEntry(pde, (physicalPageTable & X86_64_PDE_ADDRESS_MASK)
			| X86_64_PDE_PRESENT
			| X86_64_PDE_WRITABLE
			| X86_64_PDE_USER);

		// Map it and zero it.
		virtualPageTable = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			physicalPageTable);
		memset(virtualPageTable, 0, B_PAGE_SIZE);
	} else {
		virtualPageTable = (uint64*)fKernelPhysicalPageMapper->GetPageTableAt(
			*pde & X86_64_PDE_ADDRESS_MASK);
	}

	// The page table entry must not already be mapped.
	uint64* pte = &virtualPageTable[VADDR_TO_PTE(virtualAddress)];
	ASSERT_PRINT(
		(*pte & X86_64_PTE_PRESENT) == 0,
		"virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64,
		virtualAddress, *pte);

	// Fill in the table entry.
	PutPageTableEntryInTable(pte, physicalAddress, attributes, 0,
		IS_KERNEL_ADDRESS(virtualAddress));

	return B_OK;
}


bool
X86PagingMethod64Bit::IsKernelPageAccessible(addr_t virtualAddress,
	uint32 protection)
{
	return true;
}


/*!	Traverses down the paging structure hierarchy to find the page directory
	for a virtual address, allocating new tables if required.
*/
/*static*/ uint64*
X86PagingMethod64Bit::PageDirectoryForAddress(uint64* virtualPMLTop,
	addr_t virtualAddress, bool isKernel, bool allocateTables,
	vm_page_reservation* reservation,
	TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
	uint64* virtualPML4 = virtualPMLTop;
	if (la57) {
		// Get the PDPT.
		uint64* pml5e = &virtualPMLTop[VADDR_TO_PML5E(virtualAddress)];
		if ((*pml5e & X86_64_PML5E_PRESENT) == 0) {
			if (!allocateTables)
				return NULL;

			// Allocate a new PDPT.
			vm_page* page = vm_page_allocate_page(reservation,
				PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);

			DEBUG_PAGE_ACCESS_END(page);

			phys_addr_t physicalPDPT
				= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;

			TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating PML4T"
				" for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n",
				virtualAddress, physicalPDPT);

			SetTableEntry(pml5e, (physicalPDPT & X86_64_PML5E_ADDRESS_MASK)
				| X86_64_PML5E_PRESENT
				| X86_64_PML5E_WRITABLE
				| X86_64_PML5E_USER);

			mapCount++;
		}

		virtualPML4 = (uint64*)pageMapper->GetPageTableAt(
			*pml5e & X86_64_PML5E_ADDRESS_MASK);
	}

	// Get the PDPT.
	uint64* pml4e = &virtualPML4[VADDR_TO_PML4E(virtualAddress)];
	if ((*pml4e & X86_64_PML4E_PRESENT) == 0) {
		if (!allocateTables)
			return NULL;

		// Allocate a new PDPT.
		vm_page* page = vm_page_allocate_page(reservation,
			PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);

		DEBUG_PAGE_ACCESS_END(page);

		phys_addr_t physicalPDPT
			= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;

		TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating PDPT "
			"for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress,
			physicalPDPT);

		SetTableEntry(pml4e, (physicalPDPT & X86_64_PML4E_ADDRESS_MASK)
			| X86_64_PML4E_PRESENT
			| X86_64_PML4E_WRITABLE
			| X86_64_PML4E_USER);

		mapCount++;
	}

	uint64* virtualPDPT = (uint64*)pageMapper->GetPageTableAt(
		*pml4e & X86_64_PML4E_ADDRESS_MASK);

	// Get the page directory.
	uint64* pdpte = &virtualPDPT[VADDR_TO_PDPTE(virtualAddress)];
	if ((*pdpte & X86_64_PDPTE_PRESENT) == 0) {
		if (!allocateTables)
			return NULL;

		// Allocate a new page directory.
		vm_page* page = vm_page_allocate_page(reservation,
			PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);

		DEBUG_PAGE_ACCESS_END(page);

		phys_addr_t physicalPageDir
			= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;

		TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating page "
			"directory for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n",
			virtualAddress, physicalPageDir);

		SetTableEntry(pdpte, (physicalPageDir & X86_64_PDPTE_ADDRESS_MASK)
			| X86_64_PDPTE_PRESENT
			| X86_64_PDPTE_WRITABLE
			| X86_64_PDPTE_USER);

		mapCount++;
	}

	return (uint64*)pageMapper->GetPageTableAt(
		*pdpte & X86_64_PDPTE_ADDRESS_MASK);
}


/*static*/ uint64*
X86PagingMethod64Bit::PageDirectoryEntryForAddress(uint64* virtualPMLTop,
	addr_t virtualAddress, bool isKernel, bool allocateTables,
	vm_page_reservation* reservation,
	TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
	uint64* virtualPageDirectory = PageDirectoryForAddress(virtualPMLTop,
		virtualAddress, isKernel, allocateTables, reservation, pageMapper,
		mapCount);
	if (virtualPageDirectory == NULL)
		return NULL;

	return &virtualPageDirectory[VADDR_TO_PDE(virtualAddress)];
}


/*!	Traverses down the paging structure hierarchy to find the page table for a
	virtual address, allocating new tables if required.
*/
/*static*/ uint64*
X86PagingMethod64Bit::PageTableForAddress(uint64* virtualPMLTop,
	addr_t virtualAddress, bool isKernel, bool allocateTables,
	vm_page_reservation* reservation,
	TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
	TRACE("X86PagingMethod64Bit::PageTableForAddress(%#" B_PRIxADDR ", "
		"%d)\n", virtualAddress, allocateTables);

	uint64* pde = PageDirectoryEntryForAddress(virtualPMLTop, virtualAddress,
		isKernel, allocateTables, reservation, pageMapper, mapCount);
	if (pde == NULL)
		return NULL;

	if ((*pde & X86_64_PDE_PRESENT) == 0) {
		if (!allocateTables)
			return NULL;

		// Allocate a new page table.
		vm_page* page = vm_page_allocate_page(reservation,
			PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);

		DEBUG_PAGE_ACCESS_END(page);

		phys_addr_t physicalPageTable
			= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;

		TRACE("X86PagingMethod64Bit::PageTableForAddress(): creating page "
			"table for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n",
			virtualAddress, physicalPageTable);

		SetTableEntry(pde, (physicalPageTable & X86_64_PDE_ADDRESS_MASK)
			| X86_64_PDE_PRESENT
			| X86_64_PDE_WRITABLE
			| X86_64_PDE_USER);

		mapCount++;
	}

	// No proper large page support at the moment, but they are used for the
	// physical map area. Ensure that nothing tries to treat that as normal
	// address space.
	ASSERT(!(*pde & X86_64_PDE_LARGE_PAGE));

	return (uint64*)pageMapper->GetPageTableAt(*pde & X86_64_PDE_ADDRESS_MASK);
}


/*static*/ uint64*
X86PagingMethod64Bit::PageTableEntryForAddress(uint64* virtualPMLTop,
	addr_t virtualAddress, bool isKernel, bool allocateTables,
	vm_page_reservation* reservation,
	TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
	uint64* virtualPageTable = PageTableForAddress(virtualPMLTop, virtualAddress,
		isKernel, allocateTables, reservation, pageMapper, mapCount);
	if (virtualPageTable == NULL)
		return NULL;

	return &virtualPageTable[VADDR_TO_PTE(virtualAddress)];
}


/*static*/ void
X86PagingMethod64Bit::PutPageTableEntryInTable(uint64* entry,
	phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
	bool globalPage)
{
	uint64 page = (physicalAddress & X86_64_PTE_ADDRESS_MASK)
		| X86_64_PTE_PRESENT | (globalPage ? X86_64_PTE_GLOBAL : 0)
		| MemoryTypeToPageTableEntryFlags(memoryType);

	// if the page is user accessible, it's automatically
	// accessible in kernel space, too (but with the same
	// protection)
	if ((attributes & B_USER_PROTECTION) != 0) {
		page |= X86_64_PTE_USER;
		if ((attributes & B_WRITE_AREA) != 0)
			page |= X86_64_PTE_WRITABLE;
		if ((attributes & B_EXECUTE_AREA) == 0
			&& x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
			page |= X86_64_PTE_NOT_EXECUTABLE;
		}
	} else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
		page |= X86_64_PTE_WRITABLE;

	// put it in the page table
	SetTableEntry(entry, page);
}


/*static*/ void
X86PagingMethod64Bit::_EnableExecutionDisable(void* dummy, int cpu)
{
	x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER)
		| IA32_MSR_EFER_NX);
}