/* $Id: memobj-r0drv-solaris.c 40966 2012-04-17 16:43:28Z vboxsync $ */ /** @file * IPRT - Ring-0 Memory Objects, Solaris. */ /* * Copyright (C) 2006-2007 Oracle Corporation * * This file is part of VirtualBox Open Source Edition (OSE), as * available from http://www.virtualbox.org. This file is free software; * you can redistribute it and/or modify it under the terms of the GNU * General Public License (GPL) as published by the Free Software * Foundation, in version 2 as it comes in the "COPYING" file of the * VirtualBox OSE distribution. VirtualBox OSE is distributed in the * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. * * The contents of this file may alternatively be used under the terms * of the Common Development and Distribution License Version 1.0 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the * VirtualBox OSE distribution, in which case the provisions of the * CDDL are applicable instead of those of the GPL. * * You may elect to license modified versions of this file under the * terms and conditions of either the GPL or the CDDL or both. */ /******************************************************************************* * Header Files * *******************************************************************************/ #include "../the-solaris-kernel.h" #include "internal/iprt.h" #include #include #include #include #include #include #include #include #include "internal/memobj.h" #include "memobj-r0drv-solaris.h" #define SOL_IS_KRNL_ADDR(vx) ((uintptr_t)(vx) >= kernelbase) static vnode_t s_PageVnode; /******************************************************************************* * Structures and Typedefs * *******************************************************************************/ /** * The Solaris version of the memory object structure. */ typedef struct RTR0MEMOBJSOL { /** The core structure. */ RTR0MEMOBJINTERNAL Core; /** Pointer to kernel memory cookie. */ ddi_umem_cookie_t Cookie; /** Shadow locked pages. */ void *pvHandle; /** Access during locking. */ int fAccess; /** Set if large pages are involved in an RTR0MEMOBJTYPE_PHYS * allocation. */ bool fLargePage; } RTR0MEMOBJSOL, *PRTR0MEMOBJSOL; /** * Returns the physical address for a virtual address. * * @param pv The virtual address. * * @returns The physical address corresponding to @a pv. */ static uint64_t rtR0MemObjSolVirtToPhys(void *pv) { struct hat *pHat = NULL; pfn_t PageFrameNum = 0; uintptr_t uVirtAddr = (uintptr_t)pv; if (SOL_IS_KRNL_ADDR(pv)) pHat = kas.a_hat; else { proc_t *pProcess = (proc_t *)RTR0ProcHandleSelf(); AssertRelease(pProcess); pHat = pProcess->p_as->a_hat; } PageFrameNum = hat_getpfnum(pHat, (caddr_t)(uVirtAddr & PAGEMASK)); AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolVirtToPhys failed. pv=%p\n", pv)); return (((uint64_t)PageFrameNum << PAGESHIFT) | (uVirtAddr & PAGEOFFSET)); } /** * Returns the physical address of a page from an array of pages. * * @param ppPages The array of pages. * @param iPage Index of the page in the array to get the physical * address. * * @returns Physical address of specific page within the list of pages specified * in @a ppPages. */ static inline uint64_t rtR0MemObjSolPageToPhys(page_t **ppPages, size_t iPage) { pfn_t PageFrameNum = page_pptonum(ppPages[iPage]); AssertReleaseMsg(PageFrameNum != PFN_INVALID, ("rtR0MemObjSolPageToPhys failed. ppPages=%p iPage=%u\n", ppPages, iPage)); return (uint64_t)PageFrameNum << PAGESHIFT; } /** * Retreives a free page from the kernel freelist. * * @param virtAddr The virtual address to which this page maybe mapped in * the future. * @param cbPage The size of the page. * * @returns Pointer to the allocated page, NULL on failure. */ static page_t *rtR0MemObjSolPageFromFreelist(caddr_t virtAddr, size_t cbPage) { seg_t KernelSeg; KernelSeg.s_as = &kas; page_t *pPage = page_get_freelist(&s_PageVnode, 0 /* offset */, &KernelSeg, virtAddr, cbPage, 0 /* flags */, NULL /* NUMA group */); if ( !pPage && g_frtSolUseKflt) { pPage = page_get_freelist(&s_PageVnode, 0 /* offset */, &KernelSeg, virtAddr, cbPage, 0x200 /* PG_KFLT */, NULL /* NUMA group */); } return pPage; } /** * Retrieves a free page from the kernel cachelist. * * @param virtAddr The virtual address to which this page maybe mapped in * the future. * @param cbPage The size of the page. * * @return Pointer to the allocated page, NULL on failure. */ static page_t *rtR0MemObjSolPageFromCachelist(caddr_t virtAddr, size_t cbPage) { seg_t KernelSeg; KernelSeg.s_as = &kas; page_t *pPage = page_get_cachelist(&s_PageVnode, 0 /* offset */, &KernelSeg, virtAddr, 0 /* flags */, NULL /* NUMA group */); if ( !pPage && g_frtSolUseKflt) { pPage = page_get_cachelist(&s_PageVnode, 0 /* offset */, &KernelSeg, virtAddr, 0x200 /* PG_KFLT */, NULL /* NUMA group */); } /* * Remove association with the vnode for pages from the cachelist. */ if (!PP_ISAGED(pPage)) page_hashout(pPage, NULL /* mutex */); return pPage; } /** * Allocates physical non-contiguous memory. * * @param uPhysHi The upper physical address limit (inclusive). * @param puPhys Where to store the physical address of first page. Optional, * can be NULL. * @param cb The size of the allocation. * * @return Array of allocated pages, NULL on failure. */ static page_t **rtR0MemObjSolPagesAlloc(uint64_t uPhysHi, uint64_t *puPhys, size_t cb) { /** @todo We need to satisfy the upper physical address constraint */ /* * The page freelist and cachelist both hold pages that are not mapped into any address space. * The cachelist is not really free pages but when memory is exhausted they'll be moved to the * free lists, it's the total of the free+cache list that we see on the 'free' column in vmstat. * * Reserve available memory for pages and create the pages. */ pgcnt_t cPages = (cb + PAGESIZE - 1) >> PAGESHIFT; int rc = page_resv(cPages, KM_NOSLEEP); if (rc) { rc = page_create_wait(cPages, 0 /* flags */); if (rc) { size_t cbPages = cPages * sizeof(page_t *); page_t **ppPages = kmem_zalloc(cbPages, KM_SLEEP); if (RT_LIKELY(ppPages)) { /* * Get pages from kseg, the 'virtAddr' here is only for colouring but unfortunately * we don't yet have the 'virtAddr' to which this memory may be mapped. */ caddr_t virtAddr = NULL; for (size_t i = 0; i < cPages; i++, virtAddr += PAGESIZE) { /* * Get a page from the freelist or cachelist. */ page_t *pPage = rtR0MemObjSolPageFromFreelist(virtAddr, PAGESIZE); if (!pPage) pPage = rtR0MemObjSolPageFromCachelist(virtAddr, PAGESIZE); if (RT_UNLIKELY(!pPage)) { /* * No more pages found, release was grabbed so far. */ page_create_putback(cPages - i); while (--i >= 0) page_free(ppPages[i], 0 /* don't need page, move to tail of pagelist */); kmem_free(ppPages, cbPages); page_unresv(cPages); return NULL; } PP_CLRFREE(pPage); /* Page is no longer free */ PP_CLRAGED(pPage); /* Page is not hashed in */ ppPages[i] = pPage; } /* * We now have the pages locked exclusively, before they are mapped in * we must downgrade the lock. */ if (puPhys) *puPhys = (uint64_t)page_pptonum(ppPages[0]) << PAGESHIFT; return ppPages; } page_create_putback(cPages); } page_unresv(cPages); } return NULL; } /** * Prepares pages allocated by rtR0MemObjSolPagesAlloc for mapping. * * @param ppPages Pointer to the page list. * @param cb Size of the allocation. * @param auPhys Where to store the physical address of the premapped * pages. * @param cPages The number of pages (entries) in @a auPhys. * * @returns IPRT status code. */ static int rtR0MemObjSolPagesPreMap(page_t **ppPages, size_t cb, uint64_t auPhys[], size_t cPages) { AssertPtrReturn(ppPages, VERR_INVALID_PARAMETER); AssertPtrReturn(auPhys, VERR_INVALID_PARAMETER); for (size_t iPage = 0; iPage < cPages; iPage++) { /* * Prepare pages for mapping into kernel/user-space. Downgrade the * exclusive page lock to a shared lock if necessary. */ if (page_tryupgrade(ppPages[iPage]) == 1) page_downgrade(ppPages[iPage]); auPhys[iPage] = rtR0MemObjSolPageToPhys(ppPages, iPage); } return VINF_SUCCESS; } /** * Frees pages allocated by rtR0MemObjSolPagesAlloc. * * @param ppPages Pointer to the page list. * @param cbPages Size of the allocation. */ static void rtR0MemObjSolPagesFree(page_t **ppPages, size_t cb) { size_t cPages = (cb + PAGESIZE - 1) >> PAGESHIFT; size_t cbPages = cPages * sizeof(page_t *); for (size_t iPage = 0; iPage < cPages; iPage++) { /* * We need to exclusive lock the pages before freeing them. */ int rc = page_tryupgrade(ppPages[iPage]); if (!rc) { page_unlock(ppPages[iPage]); while (!page_lock(ppPages[iPage], SE_EXCL, NULL /* mutex */, P_RECLAIM)) { /* nothing */; } } page_free(ppPages[iPage], 0 /* don't need page, move to tail of pagelist */); } kmem_free(ppPages, cbPages); page_unresv(cPages); } /** * Allocates a large page to cover the required allocation size. * * @param puPhys Where to store the physical address of the allocated * page. Optional, can be NULL. * @param cb Size of the allocation. * * @returns Pointer to the allocated large page, NULL on failure. */ static page_t *rtR0MemObjSolLargePageAlloc(uint64_t *puPhys, size_t cb) { /* * Reserve available memory and create the sub-pages. */ const pgcnt_t cPages = cb >> PAGESHIFT; int rc = page_resv(cPages, KM_NOSLEEP); if (rc) { rc = page_create_wait(cPages, 0 /* flags */); if (rc) { /* * Get a page off the free list. We set virtAddr to 0 since we don't know where * the memory is going to be mapped. */ seg_t KernelSeg; caddr_t virtAddr = NULL; KernelSeg.s_as = &kas; page_t *pRootPage = rtR0MemObjSolPageFromFreelist(virtAddr, cb); if (pRootPage) { AssertMsg(!(page_pptonum(pRootPage) & (cPages - 1)), ("%p:%lx cPages=%lx\n", pRootPage, page_pptonum(pRootPage), cPages)); /* * Mark all the sub-pages as non-free and not-hashed-in. * It is paramount that we destroy the list (before freeing it). */ page_t *pPageList = pRootPage; for (size_t iPage = 0; iPage < cPages; iPage++) { page_t *pPage = pPageList; AssertPtr(pPage); AssertMsg(page_pptonum(pPage) == iPage + page_pptonum(pRootPage), ("%p:%lx %lx+%lx\n", pPage, page_pptonum(pPage), iPage, page_pptonum(pRootPage))); page_sub(&pPageList, pPage); /* * Ensure page is now be free and the page size-code must match that of the root page. */ AssertMsg(PP_ISFREE(pPage), ("%p\n", pPage)); AssertMsg(pPage->p_szc == pRootPage->p_szc, ("%p - %d expected %d \n", pPage, pPage->p_szc, pRootPage->p_szc)); PP_CLRFREE(pPage); /* Page no longer free */ PP_CLRAGED(pPage); /* Page no longer hashed-in */ } uint64_t uPhys = (uint64_t)page_pptonum(pRootPage) << PAGESHIFT; AssertMsg(!(uPhys & (cb - 1)), ("%llx %zx\n", uPhys, cb)); if (puPhys) *puPhys = uPhys; return pRootPage; } page_create_putback(cPages); } page_unresv(cPages); } return NULL; } /** * Prepares the large page allocated by rtR0MemObjSolLargePageAlloc to be mapped. * * @param pRootPage Pointer to the root page. * @param cb Size of the allocation. * * @returns IPRT status code. */ static int rtR0MemObjSolLargePagePreMap(page_t *pRootPage, size_t cb) { const pgcnt_t cPages = cb >> PAGESHIFT; Assert(page_get_pagecnt(pRootPage->p_szc) == cPages); AssertMsg(!(page_pptonum(pRootPage) & (cPages - 1)), ("%p:%lx npages=%lx\n", pRootPage, page_pptonum(pRootPage), cPages)); /* * We need to downgrade the sub-pages from exclusive to shared locking * because otherweise we cannot . */ for (pgcnt_t iPage = 0; iPage < cPages; iPage++) { page_t *pPage = page_nextn(pRootPage, iPage); AssertMsg(page_pptonum(pPage) == iPage + page_pptonum(pRootPage), ("%p:%lx %lx+%lx\n", pPage, page_pptonum(pPage), iPage, page_pptonum(pRootPage))); AssertMsg(!PP_ISFREE(pPage), ("%p\n", pPage)); if (page_tryupgrade(pPage) == 1) page_downgrade(pPage); AssertMsg(!PP_ISFREE(pPage), ("%p\n", pPage)); } return VINF_SUCCESS; } /** * Frees the page allocated by rtR0MemObjSolLargePageAlloc. * * @param pRootPage Pointer to the root page. * @param cb Allocated size. */ static void rtR0MemObjSolLargePageFree(page_t *pRootPage, size_t cb) { pgcnt_t cPages = cb >> PAGESHIFT; Assert(page_get_pagecnt(pRootPage->p_szc) == cPages); AssertMsg(!(page_pptonum(pRootPage) & (cPages - 1)), ("%p:%lx cPages=%lx\n", pRootPage, page_pptonum(pRootPage), cPages)); /* * We need to exclusively lock the sub-pages before freeing the large one. */ for (pgcnt_t iPage = 0; iPage < cPages; iPage++) { page_t *pPage = page_nextn(pRootPage, iPage); AssertMsg(page_pptonum(pPage) == iPage + page_pptonum(pRootPage), ("%p:%lx %lx+%lx\n", pPage, page_pptonum(pPage), iPage, page_pptonum(pRootPage))); AssertMsg(!PP_ISFREE(pPage), ("%p\n", pPage)); int rc = page_tryupgrade(pPage); if (!rc) { page_unlock(pPage); while (!page_lock(pPage, SE_EXCL, NULL /* mutex */, P_RECLAIM)) { /* nothing */; } } } /* * Free the large page and unreserve the memory. */ page_free_pages(pRootPage); page_unresv(cPages); } /** * Unmaps kernel/user-space mapped memory. * * @param pv Pointer to the mapped memory block. * @param cb Size of the memory block. */ static void rtR0MemObjSolUnmap(void *pv, size_t cb) { if (SOL_IS_KRNL_ADDR(pv)) { hat_unload(kas.a_hat, pv, cb, HAT_UNLOAD | HAT_UNLOAD_UNLOCK); vmem_free(heap_arena, pv, cb); } else { struct as *pAddrSpace = ((proc_t *)RTR0ProcHandleSelf())->p_as; AssertPtr(pAddrSpace); as_rangelock(pAddrSpace); as_unmap(pAddrSpace, pv, cb); as_rangeunlock(pAddrSpace); } } /** * Lock down memory mappings for a virtual address. * * @param pv Pointer to the memory to lock down. * @param cb Size of the memory block. * @param fAccess Page access rights (S_READ, S_WRITE, S_EXEC) * * @returns IPRT status code. */ static int rtR0MemObjSolLock(void *pv, size_t cb, int fPageAccess) { /* * Kernel memory mappings on x86/amd64 are always locked, only handle user-space memory. */ if (!SOL_IS_KRNL_ADDR(pv)) { proc_t *pProc = (proc_t *)RTR0ProcHandleSelf(); AssertPtr(pProc); faultcode_t rc = as_fault(pProc->p_as->a_hat, pProc->p_as, (caddr_t)pv, cb, F_SOFTLOCK, fPageAccess); if (rc) { LogRel(("rtR0MemObjSolLock failed for pv=%pv cb=%lx fPageAccess=%d rc=%d\n", pv, cb, fPageAccess, rc)); return VERR_LOCK_FAILED; } } return VINF_SUCCESS; } /** * Unlock memory mappings for a virtual address. * * @param pv Pointer to the locked memory. * @param cb Size of the memory block. * @param fPageAccess Page access rights (S_READ, S_WRITE, S_EXEC). */ static void rtR0MemObjSolUnlock(void *pv, size_t cb, int fPageAccess) { if (!SOL_IS_KRNL_ADDR(pv)) { proc_t *pProcess = (proc_t *)RTR0ProcHandleSelf(); AssertPtr(pProcess); as_fault(pProcess->p_as->a_hat, pProcess->p_as, (caddr_t)pv, cb, F_SOFTUNLOCK, fPageAccess); } } /** * Maps a list of physical pages into user address space. * * @param pVirtAddr Where to store the virtual address of the mapping. * @param fPageAccess Page access rights (PROT_READ, PROT_WRITE, * PROT_EXEC) * @param paPhysAddrs Array of physical addresses to pages. * @param cb Size of memory being mapped. * * @returns IPRT status code. */ static int rtR0MemObjSolUserMap(caddr_t *pVirtAddr, unsigned fPageAccess, uint64_t *paPhysAddrs, size_t cb) { struct as *pAddrSpace = ((proc_t *)RTR0ProcHandleSelf())->p_as; int rc = VERR_INTERNAL_ERROR; SEGVBOX_CRARGS Args; Args.paPhysAddrs = paPhysAddrs; Args.fPageAccess = fPageAccess; as_rangelock(pAddrSpace); map_addr(pVirtAddr, cb, 0 /* offset */, 0 /* vacalign */, MAP_SHARED); if (*pVirtAddr != NULL) rc = as_map(pAddrSpace, *pVirtAddr, cb, rtR0SegVBoxSolCreate, &Args); else rc = ENOMEM; as_rangeunlock(pAddrSpace); return RTErrConvertFromErrno(rc); } DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem) { PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem; switch (pMemSolaris->Core.enmType) { case RTR0MEMOBJTYPE_LOW: rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb); break; case RTR0MEMOBJTYPE_PHYS: if (pMemSolaris->Core.u.Phys.fAllocated) { if (pMemSolaris->fLargePage) rtR0MemObjSolLargePageFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb); else rtR0SolMemFree(pMemSolaris->Core.pv, pMemSolaris->Core.cb); } break; case RTR0MEMOBJTYPE_PHYS_NC: rtR0MemObjSolPagesFree(pMemSolaris->pvHandle, pMemSolaris->Core.cb); break; case RTR0MEMOBJTYPE_PAGE: ddi_umem_free(pMemSolaris->Cookie); break; case RTR0MEMOBJTYPE_LOCK: rtR0MemObjSolUnlock(pMemSolaris->Core.pv, pMemSolaris->Core.cb, pMemSolaris->fAccess); break; case RTR0MEMOBJTYPE_MAPPING: rtR0MemObjSolUnmap(pMemSolaris->Core.pv, pMemSolaris->Core.cb); break; case RTR0MEMOBJTYPE_RES_VIRT: { if (pMemSolaris->Core.u.ResVirt.R0Process == NIL_RTR0PROCESS) vmem_xfree(heap_arena, pMemSolaris->Core.pv, pMemSolaris->Core.cb); else AssertFailed(); break; } case RTR0MEMOBJTYPE_CONT: /* we don't use this type here. */ default: AssertMsgFailed(("enmType=%d\n", pMemSolaris->Core.enmType)); return VERR_INTERNAL_ERROR; } return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable) { /* Create the object. */ PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PAGE, NULL, cb); if (RT_UNLIKELY(!pMemSolaris)) return VERR_NO_MEMORY; void *pvMem = ddi_umem_alloc(cb, DDI_UMEM_SLEEP, &pMemSolaris->Cookie); if (RT_UNLIKELY(!pvMem)) { rtR0MemObjDelete(&pMemSolaris->Core); return VERR_NO_PAGE_MEMORY; } pMemSolaris->Core.pv = pvMem; pMemSolaris->pvHandle = NULL; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable) { NOREF(fExecutable); /* Create the object */ PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOW, NULL, cb); if (!pMemSolaris) return VERR_NO_MEMORY; /* Allocate physically low page-aligned memory. */ uint64_t uPhysHi = _4G - 1; void *pvMem = rtR0SolMemAlloc(uPhysHi, NULL /* puPhys */, cb, PAGESIZE, false /* fContig */); if (RT_UNLIKELY(!pvMem)) { rtR0MemObjDelete(&pMemSolaris->Core); return VERR_NO_LOW_MEMORY; } pMemSolaris->Core.pv = pvMem; pMemSolaris->pvHandle = NULL; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable) { NOREF(fExecutable); return rtR0MemObjNativeAllocPhys(ppMem, cb, _4G - 1, PAGE_SIZE /* alignment */); } DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest) { #if HC_ARCH_BITS == 64 PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS_NC, NULL, cb); if (RT_UNLIKELY(!pMemSolaris)) return VERR_NO_MEMORY; uint64_t PhysAddr = UINT64_MAX; void *pvPages = rtR0MemObjSolPagesAlloc((uint64_t)PhysHighest, &PhysAddr, cb); if (!pvPages) { LogRel(("rtR0MemObjNativeAllocPhysNC: rtR0MemObjSolPagesAlloc failed for cb=%u.\n", cb)); rtR0MemObjDelete(&pMemSolaris->Core); return VERR_NO_MEMORY; } pMemSolaris->Core.pv = NULL; pMemSolaris->pvHandle = pvPages; Assert(PhysAddr != UINT64_MAX); Assert(!(PhysAddr & PAGE_OFFSET_MASK)); *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; #else /* 32 bit: */ return VERR_NOT_SUPPORTED; /* see the RTR0MemObjAllocPhysNC specs */ #endif } DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment) { AssertMsgReturn(PhysHighest >= 16 *_1M, ("PhysHigest=%RHp\n", PhysHighest), VERR_NOT_SUPPORTED); PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb); if (RT_UNLIKELY(!pMemSolaris)) return VERR_NO_MEMORY; /* * Allocating one large page gets special treatment. */ static uint32_t s_cbLargePage = UINT32_MAX; if (s_cbLargePage == UINT32_MAX) { #if 0 /* currently not entirely stable, so disabled. */ if (page_num_pagesizes() > 1) ASMAtomicWriteU32(&s_cbLargePage, page_get_pagesize(1)); else #endif ASMAtomicWriteU32(&s_cbLargePage, 0); } uint64_t PhysAddr; if ( cb == s_cbLargePage && cb == uAlignment && PhysHighest == NIL_RTHCPHYS) { /* * Allocate one large page. */ cmn_err(CE_NOTE, "calling rtR0MemObjSolLargePageAlloc\n"); void *pvPages = rtR0MemObjSolLargePageAlloc(&PhysAddr, cb); if (RT_LIKELY(pvPages)) { AssertMsg(!(PhysAddr & (cb - 1)), ("%RHp\n", PhysAddr)); pMemSolaris->Core.pv = NULL; pMemSolaris->Core.u.Phys.PhysBase = PhysAddr; pMemSolaris->Core.u.Phys.fAllocated = true; pMemSolaris->pvHandle = pvPages; pMemSolaris->fLargePage = true; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } } else { /* * Allocate physically contiguous memory aligned as specified. */ cmn_err(CE_NOTE, "rtR0MemObjNativeAllocPhys->rtR0SolMemAlloc\n"); AssertCompile(NIL_RTHCPHYS == UINT64_MAX); PhysAddr = PhysHighest; void *pvMem = rtR0SolMemAlloc(PhysHighest, &PhysAddr, cb, uAlignment, true /* fContig */); if (RT_LIKELY(pvMem)) { Assert(!(PhysAddr & PAGE_OFFSET_MASK)); Assert(PhysAddr < PhysHighest); Assert(PhysAddr + cb <= PhysHighest); pMemSolaris->Core.pv = pvMem; pMemSolaris->Core.u.Phys.PhysBase = PhysAddr; pMemSolaris->Core.u.Phys.fAllocated = true; pMemSolaris->pvHandle = NULL; pMemSolaris->fLargePage = false; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } } rtR0MemObjDelete(&pMemSolaris->Core); return VERR_NO_CONT_MEMORY; } DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy) { AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED); /* Create the object. */ PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_PHYS, NULL, cb); if (!pMemSolaris) return VERR_NO_MEMORY; /* There is no allocation here, it needs to be mapped somewhere first. */ pMemSolaris->Core.u.Phys.fAllocated = false; pMemSolaris->Core.u.Phys.PhysBase = Phys; pMemSolaris->Core.u.Phys.uCachePolicy = uCachePolicy; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process) { AssertReturn(R0Process == RTR0ProcHandleSelf(), VERR_INVALID_PARAMETER); NOREF(fAccess); /* Create the locking object */ PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb); if (!pMemSolaris) return VERR_NO_MEMORY; /* Lock down user pages. */ int fPageAccess = S_READ; if (fAccess & RTMEM_PROT_WRITE) fPageAccess = S_WRITE; if (fAccess & RTMEM_PROT_EXEC) fPageAccess = S_EXEC; int rc = rtR0MemObjSolLock((void *)R3Ptr, cb, fPageAccess); if (RT_FAILURE(rc)) { LogRel(("rtR0MemObjNativeLockUser: rtR0MemObjSolLock failed rc=%d\n", rc)); rtR0MemObjDelete(&pMemSolaris->Core); return rc; } /* Fill in the object attributes and return successfully. */ pMemSolaris->Core.u.Lock.R0Process = R0Process; pMemSolaris->pvHandle = NULL; pMemSolaris->fAccess = fPageAccess; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess) { NOREF(fAccess); PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_LOCK, pv, cb); if (!pMemSolaris) return VERR_NO_MEMORY; /* Lock down kernel pages. */ int fPageAccess = S_READ; if (fAccess & RTMEM_PROT_WRITE) fPageAccess = S_WRITE; if (fAccess & RTMEM_PROT_EXEC) fPageAccess = S_EXEC; int rc = rtR0MemObjSolLock(pv, cb, fPageAccess); if (RT_FAILURE(rc)) { LogRel(("rtR0MemObjNativeLockKernel: rtR0MemObjSolLock failed rc=%d\n", rc)); rtR0MemObjDelete(&pMemSolaris->Core); return rc; } /* Fill in the object attributes and return successfully. */ pMemSolaris->Core.u.Lock.R0Process = NIL_RTR0PROCESS; pMemSolaris->pvHandle = NULL; pMemSolaris->fAccess = fPageAccess; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment) { PRTR0MEMOBJSOL pMemSolaris; /* * Use xalloc. */ void *pv = vmem_xalloc(heap_arena, cb, uAlignment, 0 /* phase */, 0 /* nocross */, NULL /* minaddr */, NULL /* maxaddr */, VM_SLEEP); if (RT_UNLIKELY(!pv)) return VERR_NO_MEMORY; /* Create the object. */ pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_RES_VIRT, pv, cb); if (!pMemSolaris) { LogRel(("rtR0MemObjNativeReserveKernel failed to alloc memory object.\n")); vmem_xfree(heap_arena, pv, cb); return VERR_NO_MEMORY; } pMemSolaris->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS; *ppMem = &pMemSolaris->Core; return VINF_SUCCESS; } DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process) { return VERR_NOT_SUPPORTED; } DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment, unsigned fProt, size_t offSub, size_t cbSub) { /** @todo rtR0MemObjNativeMapKernel / Solaris - Should be fairly simple alloc kernel memory and memload it. */ return VERR_NOT_SUPPORTED; } DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, PRTR0MEMOBJINTERNAL pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process) { /* * Fend off things we cannot do. */ AssertMsgReturn(R3PtrFixed == (RTR3PTR)-1, ("%p\n", R3PtrFixed), VERR_NOT_SUPPORTED); AssertMsgReturn(R0Process == RTR0ProcHandleSelf(), ("%p != %p\n", R0Process, RTR0ProcHandleSelf()), VERR_NOT_SUPPORTED); if (uAlignment != PAGE_SIZE) return VERR_NOT_SUPPORTED; /* * Get parameters from the source object. */ PRTR0MEMOBJSOL pMemToMapSolaris = (PRTR0MEMOBJSOL)pMemToMap; void *pv = pMemToMapSolaris->Core.pv; size_t cb = pMemToMapSolaris->Core.cb; size_t cPages = cb >> PAGE_SHIFT; /* * Create the mapping object */ PRTR0MEMOBJSOL pMemSolaris; pMemSolaris = (PRTR0MEMOBJSOL)rtR0MemObjNew(sizeof(*pMemSolaris), RTR0MEMOBJTYPE_MAPPING, pv, cb); if (RT_UNLIKELY(!pMemSolaris)) return VERR_NO_MEMORY; int rc = VINF_SUCCESS; uint64_t *paPhysAddrs = kmem_zalloc(sizeof(uint64_t) * cPages, KM_SLEEP); if (RT_LIKELY(paPhysAddrs)) { /* * Prepare the pages according to type. */ if (pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS_NC) rc = rtR0MemObjSolPagesPreMap(pMemToMapSolaris->pvHandle, cb, paPhysAddrs, cPages); else if ( pMemToMapSolaris->Core.enmType == RTR0MEMOBJTYPE_PHYS && pMemToMapSolaris->fLargePage) { RTHCPHYS Phys = pMemToMapSolaris->Core.u.Phys.PhysBase; for (pgcnt_t iPage = 0; iPage < cPages; iPage++, Phys += PAGE_SIZE) paPhysAddrs[iPage] = Phys; rc = rtR0MemObjSolLargePagePreMap(pMemToMapSolaris->pvHandle, cb); } else { /* * Have kernel mapping, just translate virtual to physical. */ AssertPtr(pv); rc = VINF_SUCCESS; for (size_t iPage = 0; iPage < cPages; iPage++) { paPhysAddrs[iPage] = rtR0MemObjSolVirtToPhys(pv); if (RT_UNLIKELY(paPhysAddrs[iPage] == -(uint64_t)1)) { LogRel(("rtR0MemObjNativeMapUser: no page to map.\n")); rc = VERR_MAP_FAILED; break; } pv = (void *)((uintptr_t)pv + PAGE_SIZE); } } if (RT_SUCCESS(rc)) { unsigned fPageAccess = PROT_READ; if (fProt & RTMEM_PROT_WRITE) fPageAccess |= PROT_WRITE; if (fProt & RTMEM_PROT_EXEC) fPageAccess |= PROT_EXEC; /* * Perform the actual mapping. */ caddr_t UserAddr = NULL; rc = rtR0MemObjSolUserMap(&UserAddr, fPageAccess, paPhysAddrs, cb); if (RT_SUCCESS(rc)) { pMemSolaris->Core.u.Mapping.R0Process = R0Process; pMemSolaris->Core.pv = UserAddr; *ppMem = &pMemSolaris->Core; kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages); return VINF_SUCCESS; } LogRel(("rtR0MemObjNativeMapUser: rtR0MemObjSolUserMap failed rc=%d.\n", rc)); } rc = VERR_MAP_FAILED; kmem_free(paPhysAddrs, sizeof(uint64_t) * cPages); } else rc = VERR_NO_MEMORY; rtR0MemObjDelete(&pMemSolaris->Core); return rc; } DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt) { NOREF(pMem); NOREF(offSub); NOREF(cbSub); NOREF(fProt); return VERR_NOT_SUPPORTED; } DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage) { PRTR0MEMOBJSOL pMemSolaris = (PRTR0MEMOBJSOL)pMem; switch (pMemSolaris->Core.enmType) { case RTR0MEMOBJTYPE_PHYS_NC: if (pMemSolaris->Core.u.Phys.fAllocated) { uint8_t *pb = (uint8_t *)pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT); return rtR0MemObjSolVirtToPhys(pb); } return rtR0MemObjSolPageToPhys(pMemSolaris->pvHandle, iPage); case RTR0MEMOBJTYPE_PAGE: case RTR0MEMOBJTYPE_LOW: case RTR0MEMOBJTYPE_LOCK: { uint8_t *pb = (uint8_t *)pMemSolaris->Core.pv + ((size_t)iPage << PAGE_SHIFT); return rtR0MemObjSolVirtToPhys(pb); } /* * Although mapping can be handled by rtR0MemObjSolVirtToPhys(offset) like the above case, * request it from the parent so that we have a clear distinction between CONT/PHYS_NC. */ case RTR0MEMOBJTYPE_MAPPING: return rtR0MemObjNativeGetPagePhysAddr(pMemSolaris->Core.uRel.Child.pParent, iPage); case RTR0MEMOBJTYPE_CONT: case RTR0MEMOBJTYPE_PHYS: AssertFailed(); /* handled by the caller */ case RTR0MEMOBJTYPE_RES_VIRT: default: return NIL_RTHCPHYS; } }