---  amd64/amd64/pmap.c.orig
+++  amd64/amd64/pmap.c
@@ -77,7 +77,7 @@
  */
 
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.587 2007/05/31 22:52:10 attilio Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.581 2007/03/10 20:26:43 alc Exp $");
 
 /*
  *	Manages physical address maps.
@@ -111,6 +111,7 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/ktr.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
@@ -134,6 +135,7 @@
 #include <vm/vm_extern.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_pager.h>
+#include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
 #include <machine/cpu.h>
@@ -166,6 +168,9 @@
 #define PV_STAT(x)	do { } while (0)
 #endif
 
+#define	pa_index(pa)	((pa) >> PDRSHIFT)
+#define	pa_to_pvh(pa)	(&pv_table[pa_index(pa)])
+
 struct pmap kernel_pmap_store;
 
 vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
@@ -189,6 +194,8 @@
  * Data for the pv entry allocation mechanism
  */
 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
+static int pv_npg;
+static struct md_page *pv_table;
 static int shpgperproc = PMAP_SHPGPERPROC;
 
 /*
@@ -205,11 +212,28 @@
 
 static void	free_pv_entry(pmap_t pmap, pv_entry_t pv);
 static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
+static void	pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m);
+static void	pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static void	pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
+static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
+		    vm_offset_t va);
 
+static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
+    vm_prot_t prot);
 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
+static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
+static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
+static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
+    vm_prot_t prot);
+static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva);
 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
-		vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
+		vm_offset_t sva, pd_entry_t ptepde);
+static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 		vm_offset_t va);
@@ -221,9 +245,8 @@
 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
 
 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
-static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
-                vm_page_t* free);
-static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
+static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 
 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
@@ -364,21 +387,6 @@
 }
 
 
-static __inline pt_entry_t *
-pmap_pte_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *ptepde)
-{
-	pd_entry_t *pde;
-
-	pde = pmap_pde(pmap, va);
-	if (pde == NULL || (*pde & PG_V) == 0)
-		return NULL;
-	*ptepde = *pde;
-	if ((*pde & PG_PS) != 0)	/* compat with i386 pmap_pte() */
-		return ((pt_entry_t *)pde);
-	return (pmap_pde_to_pte(pde, va));
-}
-
-
 PMAP_INLINE pt_entry_t *
 vtopte(vm_offset_t va)
 {
@@ -513,6 +521,7 @@
 	 */
 	PMAP_LOCK_INIT(kernel_pmap);
 	kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
+	kernel_pmap->pm_root = NULL;
 	kernel_pmap->pm_active = -1;	/* don't allow deactivation */
 	TAILQ_INIT(&kernel_pmap->pm_pvchunk);
 	nkpt = NKPT;
@@ -613,8 +622,28 @@
 void
 pmap_init(void)
 {
+	pd_entry_t *pd;
+	vm_page_t mpte;
+	vm_size_t s;
+	int i;
 
 	/*
+	 * Initialize the vm page array entries for the kernel pmap's
+	 * page table pages.
+	 */ 
+	pd = pmap_pde(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
+	for (i = 0; i < nkpt; i++) {
+		if ((pd[i] & (PG_PS | PG_V)) == (PG_PS | PG_V))
+			continue;
+		mpte = PHYS_TO_VM_PAGE(pd[i] & PG_FRAME);
+		KASSERT(mpte >= vm_page_array &&
+		    mpte < &vm_page_array[vm_page_array_size],
+		    ("pmap_init: page table page is out of range"));
+		mpte->pindex = pmap_pde_pindex(VM_MIN_KERNEL_ADDRESS) + i;
+		mpte->phys_addr = pd[i] & PG_FRAME;
+	}
+
+	/*
 	 * Initialize the address space (zone) for the pv entries.  Set a
 	 * high water mark so that the system can recover from excessive
 	 * numbers of pv entries.
@@ -623,6 +652,23 @@
 	pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
 	TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
 	pv_entry_high_water = 9 * (pv_entry_max / 10);
+
+	/*
+	 * Calculate the size of the pv head table for superpages.
+	 */
+	for (i = 0; phys_avail[i + 1]; i += 2);
+	pv_npg = round_2mpage(phys_avail[(i - 2) + 1]) / NBPDR;
+
+	/*
+	 * Allocate memory for the pv head table for superpages.
+	 */
+	s = (vm_size_t)(pv_npg * sizeof(struct md_page));
+	s = round_page(s);
+	pv_table = (struct md_page *)kmem_alloc(kernel_map, s);
+	for (i = 0; i < pv_npg; i++) {
+		TAILQ_INIT(&pv_table[i].pv_list);
+		pv_table[i].pv_list_count = 0;
+	}
 }
 
 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
@@ -1066,16 +1112,73 @@
 /***************************************************
  * Page table page management routines.....
  ***************************************************/
-static PMAP_INLINE void
-pmap_free_zero_pages(vm_page_t free)
+
+/*
+ *
+ */
+static void
+pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
+{
+	vm_page_t root;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	root = pmap->pm_root;
+	if (root == NULL) {
+		mpte->left = NULL;
+		mpte->right = NULL;
+	} else {
+		root = vm_page_splay(mpte->pindex, root);
+		if (mpte->pindex < root->pindex) {
+			mpte->left = root->left;
+			mpte->right = root;
+			root->left = NULL;
+		} else if (mpte->pindex == root->pindex)
+			panic("pmap_insert_pt_page: pindex already inserted");
+		else {
+			mpte->right = root->right;
+			mpte->left = root;
+			root->right = NULL;
+		}
+	}
+	pmap->pm_root = mpte;
+}
+
+/*
+ *
+ */
+static vm_page_t
+pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+{
+	vm_page_t mpte;
+	vm_pindex_t pindex = pmap_pde_pindex(va);
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) {
+		mpte = vm_page_splay(pindex, mpte);
+		if ((pmap->pm_root = mpte)->pindex != pindex)
+			mpte = NULL;
+	}
+	return (mpte);
+}
+
+/*
+ *
+ */
+static void
+pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
 {
-	vm_page_t m;
+	vm_page_t root;
 
-	while (free != NULL) {
-		m = free;
-		free = m->right;
-		vm_page_free_zero(m);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if (mpte != pmap->pm_root)
+		vm_page_splay(mpte->pindex, pmap->pm_root);
+	if (mpte->left == NULL)
+		root = mpte->right;
+	else {
+		root = vm_page_splay(mpte->pindex, mpte->left);
+		root->right = mpte->right;
 	}
+	pmap->pm_root = root;
 }
 
 /*
@@ -1083,19 +1186,18 @@
  * drops to zero, then it decrements the wire count.
  */
 static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
+pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
 {
 
 	--m->wire_count;
 	if (m->wire_count == 0)
-		return _pmap_unwire_pte_hold(pmap, va, m, free);
+		return _pmap_unwire_pte_hold(pmap, va, m);
 	else
 		return 0;
 }
 
 static int 
-_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, 
-    vm_page_t *free)
+_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
 {
 	vm_offset_t pteva;
 
@@ -1127,14 +1229,14 @@
 		vm_page_t pdpg;
 
 		pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
-		pmap_unwire_pte_hold(pmap, va, pdpg, free);
+		pmap_unwire_pte_hold(pmap, va, pdpg);
 	}
 	if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 		/* We just released a PD, unhold the matching PDP */
 		vm_page_t pdppg;
 
 		pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
-		pmap_unwire_pte_hold(pmap, va, pdppg, free);
+		pmap_unwire_pte_hold(pmap, va, pdppg);
 	}
 
 	/*
@@ -1143,13 +1245,7 @@
 	 */
 	pmap_invalidate_page(pmap, pteva);
 
-	/* 
-	 * Put page on a list so that it is released after
-	 * *ALL* TLB shootdown is done
-	 */
-	m->right = *free;
-	*free = m;
-	
+	vm_page_free_zero(m);
 	atomic_subtract_int(&cnt.v_wire_count, 1);
 	return 1;
 }
@@ -1159,7 +1255,7 @@
  * conditionally free the page, and manage the hold/wire counts.
  */
 static int
-pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
+pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde)
 {
 	vm_page_t mpte;
 
@@ -1167,7 +1263,7 @@
 		return 0;
 	KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
-	return pmap_unwire_pte_hold(pmap, va, mpte, free);
+	return pmap_unwire_pte_hold(pmap, va, mpte);
 }
 
 void
@@ -1176,6 +1272,7 @@
 
 	PMAP_LOCK_INIT(pmap);
 	pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
+	pmap->pm_root = NULL;
 	pmap->pm_active = 0;
 	TAILQ_INIT(&pmap->pm_pvchunk);
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1212,6 +1309,7 @@
 	/* install self-referential address mapping entry(s) */
 	pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 
+	pmap->pm_root = NULL;
 	pmap->pm_active = 0;
 	TAILQ_INIT(&pmap->pm_pvchunk);
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1385,7 +1483,7 @@
 {
 	vm_pindex_t ptepindex;
 	pd_entry_t *pd;
-	vm_page_t m, free;
+	vm_page_t m;
 
 	KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 	    (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
@@ -1406,13 +1504,15 @@
 	 * normal 4K page.
 	 */
 	if (pd != 0 && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
-		*pd = 0;
-		pd = 0;
-		pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-		free = NULL;
-		pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free);
-		pmap_invalidate_all(kernel_pmap);
-		pmap_free_zero_pages(free);
+		if (!pmap_demote_pde(pmap, pd, va)) {
+			pmap_invalidate_page(pmap, va);
+
+			/*
+			 * Invalidation of the 2MB page mapping may have caused
+			 * the deallocation of the underlying PD page.
+			 */
+			pd = NULL;
+		}
 	}
 
 	/*
@@ -1452,6 +1552,8 @@
 	KASSERT(pmap->pm_stats.resident_count == 0,
 	    ("pmap_release: pmap resident count %ld != 0",
 	    pmap->pm_stats.resident_count));
+	KASSERT(pmap->pm_root == NULL,
+	    ("pmap_release: pmap has reserved page table page(s)"));
 
 	m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 
@@ -1539,7 +1641,7 @@
 		/*
 		 * This index is bogus, but out of the way
 		 */
-		nkpg = vm_page_alloc(NULL, nkpt,
+		nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end),
 		    VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 		if (!nkpg)
 			panic("pmap_growkernel: no memory to grow kernel");
@@ -1621,16 +1723,21 @@
  * drastic measures to free some pages so we can allocate
  * another pv entry chunk.  This is normally called to
  * unmap inactive pages, and if necessary, active pages.
+ *
+ * We do not, however, unmap 2mpages because subsequent accesses will
+ * allocate per-page pv entries until repromotion occurs, thereby
+ * exacerbating the shortage of free pv entries.
  */
 static void
 pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 {
-	pd_entry_t ptepde;
+	struct md_page *pvh;
+	pd_entry_t *pde;
 	pmap_t pmap;
 	pt_entry_t *pte, tpte;
 	pv_entry_t next_pv, pv;
 	vm_offset_t va;
-	vm_page_t m, free;
+	vm_page_t m;
 
 	TAILQ_FOREACH(m, &vpq->pl, pageq) {
 		if (m->hold_count || m->busy)
@@ -1644,7 +1751,10 @@
 			else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
 				continue;
 			pmap->pm_stats.resident_count--;
-			pte = pmap_pte_pde(pmap, va, &ptepde);
+			pde = pmap_pde(pmap, va);
+			KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found"
+			    " a 2mpage in page %p's pv list", m));
+			pte = pmap_pde_to_pte(pde, va);
 			tpte = pte_load_clear(pte);
 			KASSERT((tpte & PG_W) == 0,
 			    ("pmap_collect: wired pte %#lx", tpte));
@@ -1656,14 +1766,15 @@
 				    va, tpte));
 				vm_page_dirty(m);
 			}
-			free = NULL;
-			pmap_unuse_pt(pmap, va, ptepde, &free);
 			pmap_invalidate_page(pmap, va);
-			pmap_free_zero_pages(free);
 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-			if (TAILQ_EMPTY(&m->md.pv_list))
-				vm_page_flag_clear(m, PG_WRITEABLE);
+			if (TAILQ_EMPTY(&m->md.pv_list)) {
+				pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+				if (TAILQ_EMPTY(&pvh->pv_list))
+					vm_page_flag_clear(m, PG_WRITEABLE);
+			}
 			m->md.pv_list_count--;
+			pmap_unuse_pt(pmap, va, *pde);
 			free_pv_entry(pmap, pv);
 			if (pmap != locked_pmap)
 				PMAP_UNLOCK(pmap);
@@ -1717,7 +1828,6 @@
 {
 	static const struct timeval printinterval = { 60, 0 };
 	static struct timeval lastprint;
-	static vm_pindex_t colour;
 	int bit, field, page_req;
 	pv_entry_t pv;
 	struct pv_chunk *pc;
@@ -1752,7 +1862,7 @@
 	}
 	/* No free items, allocate another chunk */
 	page_req = try ? VM_ALLOC_NORMAL : VM_ALLOC_SYSTEM; 
-	m = vm_page_alloc(NULL, colour, page_req | VM_ALLOC_NOOBJ);
+	m = vm_page_alloc(NULL, 0, page_req | VM_ALLOC_NOOBJ);
 	if (m == NULL) {
 		if (try) {
 			pv_entry_count--;
@@ -1770,12 +1880,11 @@
 			    "vm.pmap.pv_entry_max\n");
 		PV_STAT(pmap_collect_inactive++);
 		pmap_collect(pmap, &vm_page_queues[PQ_INACTIVE]);
-		m = vm_page_alloc(NULL, colour,
-		    VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ);
+		m = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ);
 		if (m == NULL) {
 			PV_STAT(pmap_collect_active++);
 			pmap_collect(pmap, &vm_page_queues[PQ_ACTIVE]);
-			m = vm_page_alloc(NULL, colour,
+			m = vm_page_alloc(NULL, 0,
 			    VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ);
 			if (m == NULL)
 				panic("get_pv_entry: increase vm.pmap.shpgperproc");
@@ -1783,7 +1892,6 @@
 	}
 	PV_STAT(pc_chunk_count++);
 	PV_STAT(pc_chunk_allocs++);
-	colour++;
 	dump_add_page(m->phys_addr);
 	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 	pc->pc_pmap = pmap;
@@ -1796,25 +1904,115 @@
 	return (pv);
 }
 
-static void
-pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+static __inline pv_entry_t
+pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
-		if (pmap == PV_PMAP(pv) && va == pv->pv_va) 
+	TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+		if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
+			TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
+			pvh->pv_list_count--;
 			break;
+		}
 	}
-	KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
-	TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-	m->md.pv_list_count--;
-	if (TAILQ_EMPTY(&m->md.pv_list))
-		vm_page_flag_clear(m, PG_WRITEABLE);
+	return (pv);
+}
+
+static void
+pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+	vm_offset_t va_last;
+	vm_page_t m;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	KASSERT((pa & PDRMASK) == 0,
+	    ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
+
+	/*
+	 * Transfer the 2mpage's pv entry for this mapping to the first
+	 * page's pv list.
+	 */
+	pvh = pa_to_pvh(pa);
+	va = trunc_2mpage(va);
+	pv = pmap_pvh_remove(pvh, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
+	m = PHYS_TO_VM_PAGE(pa);
+	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+	m->md.pv_list_count++;
+	/* Instantiate the remaining NPTEPG - 1 pv entries. */
+	va_last = va + NBPDR - PAGE_SIZE;
+	do {
+		m++;
+		KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+		    ("pmap_pv_demote_pde: page %p is not managed", m));
+		va += PAGE_SIZE;
+		pmap_insert_entry(pmap, va, m);
+	} while (va < va_last);
+}
+
+static void
+pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+	vm_offset_t va_last;
+	vm_page_t m;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	KASSERT((pa & PDRMASK) == 0,
+	    ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
+
+	/*
+	 * Transfer the first page's pv entry for this mapping to the
+	 * 2mpage's pv list.  Aside from avoiding the cost of a call
+	 * to get_pv_entry(), a transfer avoids the possibility that
+	 * get_pv_entry() calls pmap_collect() and that pmap_collect()
+	 * removes one of the mappings that is being promoted.
+	 */
+	m = PHYS_TO_VM_PAGE(pa);
+	va = trunc_2mpage(va);
+	pv = pmap_pvh_remove(&m->md, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
+	pvh = pa_to_pvh(pa);
+	TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+	pvh->pv_list_count++;
+	/* Free the remaining NPTEPG - 1 pv entries. */
+	va_last = va + NBPDR - PAGE_SIZE;
+	do {
+		m++;
+		va += PAGE_SIZE;
+		pmap_pvh_free(&m->md, pmap, va);
+	} while (va < va_last);
+}
+
+static void
+pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
+{
+	pv_entry_t pv;
+
+	pv = pmap_pvh_remove(pvh, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 	free_pv_entry(pmap, pv);
 }
 
+static void
+pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+{
+	struct md_page *pvh;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pmap_pvh_free(&m->md, pmap, va);
+	if (TAILQ_EMPTY(&m->md.pv_list)) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		if (TAILQ_EMPTY(&pvh->pv_list))
+			vm_page_flag_clear(m, PG_WRITEABLE);
+	}
+}
+
 /*
  * Create a pv entry for page at pa for
  * (pmap, va).
@@ -1824,7 +2022,6 @@
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	pv = get_pv_entry(pmap, FALSE);
 	pv->pv_va = va;
@@ -1840,7 +2037,6 @@
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	if (pv_entry_count < pv_entry_high_water && 
 	    (pv = get_pv_entry(pmap, TRUE)) != NULL) {
@@ -1853,11 +2049,92 @@
 }
 
 /*
+ * Create the pv entries for each of the pages within a superpage.
+ */
+static boolean_t
+pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if (pv_entry_count < pv_entry_high_water && 
+	    (pv = get_pv_entry(pmap, TRUE)) != NULL) {
+		pv->pv_va = va;
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+		pvh->pv_list_count++;
+		return (TRUE);
+	} else
+		return (FALSE);
+}
+
+/*
+ * pmap_remove_pde: do the things to unmap a superpage in a process
+ */
+static int
+pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva)
+{
+	struct md_page *pvh;
+	pd_entry_t oldpde;
+	vm_offset_t eva, va;
+	vm_page_t m, mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	KASSERT((sva & PDRMASK) == 0,
+	    ("pmap_remove_pde: sva is not 2mpage aligned"));
+	oldpde = pte_load_clear(pdq);
+	if (oldpde & PG_W)
+		pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
+
+	/*
+	 * Machines that don't support invlpg, also don't support
+	 * PG_G.
+	 */
+	if (oldpde & PG_G)
+		pmap_invalidate_page(kernel_pmap, sva);
+	pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
+	if (oldpde & PG_MANAGED) {
+		pvh = pa_to_pvh(oldpde & PG_FRAME);
+		pmap_pvh_free(pvh, pmap, sva);
+		eva = sva + NBPDR;
+		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_FRAME);
+		    va < eva; va += PAGE_SIZE, m++) {
+			if (oldpde & PG_M) {
+				KASSERT((oldpde & PG_RW) != 0,
+	("pmap_remove_pde: modified 2mpage not writable: va: %#lx, pde: %#lx",
+				    va, oldpde));
+				vm_page_dirty(m);
+			}
+			if (oldpde & PG_A)
+				vm_page_flag_set(m, PG_REFERENCED);
+			if (TAILQ_EMPTY(&m->md.pv_list) &&
+			    TAILQ_EMPTY(&pvh->pv_list))
+				vm_page_flag_clear(m, PG_WRITEABLE);
+		}
+	}
+	if (pmap == kernel_pmap) {
+		if (!pmap_demote_pde(pmap, pdq, sva))
+			panic("pmap_remove_pde: failed demotion");
+	} else {
+		mpte = pmap_lookup_pt_page(pmap, sva);
+		if (mpte != NULL) {
+			pmap_remove_pt_page(pmap, mpte);
+			KASSERT(mpte->wire_count == NPTEPG,
+			    ("pmap_remove_pde: pte page wire count error"));
+			mpte->wire_count = 0;
+			vm_page_free(mpte);
+			atomic_subtract_int(&cnt.v_wire_count, 1);
+		}
+	}
+	return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva)));
+}
+
+/*
  * pmap_remove_pte: do the things to unmap a page in a process
  */
 static int
-pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
-    pd_entry_t ptepde, vm_page_t *free)
+pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde)
 {
 	pt_entry_t oldpte;
 	vm_page_t m;
@@ -1885,7 +2162,7 @@
 			vm_page_flag_set(m, PG_REFERENCED);
 		pmap_remove_entry(pmap, m, va);
 	}
-	return (pmap_unuse_pt(pmap, va, ptepde, free));
+	return (pmap_unuse_pt(pmap, va, ptepde));
 }
 
 /*
@@ -1895,7 +2172,6 @@
 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
 {
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((*pde & PG_V) == 0)
@@ -1903,9 +2179,8 @@
 	pte = pmap_pde_to_pte(pde, va);
 	if ((*pte & PG_V) == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va, *pde, &free);
+	pmap_remove_pte(pmap, pte, va, *pde);
 	pmap_invalidate_page(pmap, va);
-	pmap_free_zero_pages(free);
 }
 
 /*
@@ -1922,7 +2197,6 @@
 	pdp_entry_t *pdpe;
 	pd_entry_t ptpaddr, *pde;
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 	int anyvalid;
 
 	/*
@@ -1984,11 +2258,24 @@
 		 * Check for large page.
 		 */
 		if ((ptpaddr & PG_PS) != 0) {
-			*pde = 0;
-			pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-			pmap_unuse_pt(pmap, sva, *pdpe, &free);
-			anyvalid = 1;
-			continue;
+			/*
+			 * Are we removing the entire large page?  If not,
+			 * demote the mapping and fall through.
+			 */
+			if (sva + NBPDR == va_next && eva >= va_next) {
+				/*
+				 * The TLB entry for a PG_G mapping is
+				 * invalidated by pmap_remove_pde().
+				 */
+				if ((ptpaddr & PG_G) == 0)
+					anyvalid = 1;
+				pmap_remove_pde(pmap, pde, sva);
+				continue;
+			} else if (!pmap_demote_pde(pmap, pde, sva)) {
+				anyvalid = 1;
+				continue;
+			} else
+				ptpaddr = *pde;
 		}
 
 		/*
@@ -2010,16 +2297,14 @@
 			 */
 			if ((*pte & PG_G) == 0)
 				anyvalid = 1;
-			if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free))
+			if (pmap_remove_pte(pmap, pte, sva, ptpaddr))
 				break;
 		}
 	}
 out:
-	if (anyvalid) {
+	vm_page_unlock_queues();
+	if (anyvalid)
 		pmap_invalidate_all(pmap);
-		pmap_free_zero_pages(free);
-	}
-	vm_page_unlock_queues();	
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2039,11 +2324,12 @@
 void
 pmap_remove_all(vm_page_t m)
 {
+	struct md_page *pvh;
+	pmap_t pmap;
 	pv_entry_t pv;
-	pmap_t pmap;
 	pt_entry_t *pte, tpte;
-	pd_entry_t ptepde;
-	vm_page_t free;
+	pd_entry_t *pde;
+	vm_offset_t va;
 
 #if defined(PMAP_DIAGNOSTIC)
 	/*
@@ -2055,11 +2341,24 @@
 	}
 #endif
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		if (!pmap_demote_pde(pmap, pde, va))
+			pmap_invalidate_page(pmap, va);
+		PMAP_UNLOCK(pmap);
+	}
 	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
 		pmap->pm_stats.resident_count--;
-		pte = pmap_pte_pde(pmap, pv->pv_va, &ptepde);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
+		    " a 2mpage in page %p's pv list", m));
+		pte = pmap_pde_to_pte(pde, pv->pv_va);
 		tpte = pte_load_clear(pte);
 		if (tpte & PG_W)
 			pmap->pm_stats.wired_count--;
@@ -2075,12 +2374,10 @@
 			    pv->pv_va, tpte));
 			vm_page_dirty(m);
 		}
-		free = NULL;
-		pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 		pmap_invalidate_page(pmap, pv->pv_va);
-		pmap_free_zero_pages(free);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 		m->md.pv_list_count--;
+		pmap_unuse_pt(pmap, pv->pv_va, *pde);
 		free_pv_entry(pmap, pv);
 		PMAP_UNLOCK(pmap);
 	}
@@ -2088,6 +2385,50 @@
 }
 
 /*
+ * pmap_protect_pde: do the things to protect a 2mpage in a process
+ */
+static boolean_t
+pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
+{
+	pd_entry_t newpde, oldpde;
+	vm_offset_t eva, va;
+	vm_page_t m;
+	boolean_t anychanged;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	KASSERT((sva & PDRMASK) == 0,
+	    ("pmap_protect_pde: sva is not 2mpage aligned"));
+	anychanged = FALSE;
+retry:
+	oldpde = newpde = *pde;
+	if (oldpde & PG_MANAGED) {
+		eva = sva + NBPDR;
+		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_FRAME);
+		    va < eva; va += PAGE_SIZE, m++) {
+			if (oldpde & PG_A) {
+				vm_page_flag_set(m, PG_REFERENCED);
+				newpde &= ~PG_A;
+			}
+			if ((oldpde & PG_M) != 0)
+				vm_page_dirty(m);
+		}
+	}
+	if ((prot & VM_PROT_WRITE) == 0)
+		newpde &= ~(PG_RW | PG_M);
+	if ((prot & VM_PROT_EXECUTE) == 0)
+		newpde |= pg_nx;
+	if (newpde != oldpde) {
+		if (!atomic_cmpset_long(pde, oldpde, newpde))
+			goto retry;
+		if (oldpde & PG_G)
+			pmap_invalidate_page(pmap, sva);
+		else
+			anychanged = TRUE;
+	}
+	return (anychanged);
+}
+
+/*
  *	Set the physical protection on the
  *	specified range of this map as requested.
  */
@@ -2143,12 +2484,22 @@
 		 * Check for large page.
 		 */
 		if ((ptpaddr & PG_PS) != 0) {
-			if ((prot & VM_PROT_WRITE) == 0)
-				*pde &= ~(PG_M|PG_RW);
-			if ((prot & VM_PROT_EXECUTE) == 0)
-				*pde |= pg_nx;
-			anychanged = 1;
-			continue;
+			/*
+			 * Are we protecting the entire large page?  If not,
+			 * demote the mapping and fall through.
+			 */
+			if (sva + NBPDR == va_next && eva >= va_next) {
+				/*
+				 * The TLB entry for a PG_G mapping is
+				 * invalidated by pmap_protect_pde().
+				 */
+				if (pmap_protect_pde(pmap, pde, sva, prot))
+					anychanged = 1;
+				continue;
+			} else if (!pmap_demote_pde(pmap, pde, sva)) {
+				anychanged = 1;
+				continue;
+			}
 		}
 
 		if (va_next > eva)
@@ -2193,9 +2544,9 @@
 			}
 		}
 	}
+	vm_page_unlock_queues();
 	if (anychanged)
 		pmap_invalidate_all(pmap);
-	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2212,8 +2563,8 @@
  *	insert this page into the given map NOW.
  */
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-	   boolean_t wired)
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, vm_page_t m,
+    vm_prot_t prot, boolean_t wired)
 {
 	vm_paddr_t pa;
 	pd_entry_t *pde;
@@ -2363,9 +2714,12 @@
 	 * to update the pte.
 	 */
 	if ((origpte & ~(PG_M|PG_A)) != newpte) {
+		newpte |= PG_A;
+		if (fault_type & VM_PROT_WRITE)
+			newpte |= PG_M;
 		if (origpte & PG_V) {
 			invlva = FALSE;
-			origpte = pte_load_store(pte, newpte | PG_A);
+			origpte = pte_load_store(pte, newpte);
 			if (origpte & PG_A) {
 				if (origpte & PG_MANAGED)
 					vm_page_flag_set(om, PG_REFERENCED);
@@ -2385,8 +2739,23 @@
 			if (invlva)
 				pmap_invalidate_page(pmap, va);
 		} else
-			pte_store(pte, newpte | PG_A);
+			pte_store(pte, newpte);
+	}
+
+	/*
+	 * Promotion condition:
+	 * 1) Page has to be part of a fully populated reservation
+	 * 2) Virtual adress corresponding to the reservation has to
+	 *    be superpage aligned
+	 */
+	if (((mpte != NULL && mpte->wire_count == NPTEPG) ||
+	    m->object == kernel_object || m->object == kmem_object) &&
+	    m->reserv != NULL &&
+	    m->reserv->refcnt == NBPDR / PAGE_SIZE) {
+		KASSERT(m->object->flags & OBJ_SUPERPAGES, ("pmap_enter: xxx"));
+		pmap_promote_pde(pmap, pde, va);
 	}
+
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
@@ -2407,6 +2776,7 @@
 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
     vm_page_t m_start, vm_prot_t prot)
 {
+	vm_offset_t va;
 	vm_page_t m, mpte;
 	vm_pindex_t diff, psize;
 
@@ -2416,8 +2786,15 @@
 	m = m_start;
 	PMAP_LOCK(pmap);
 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
-		mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
-		    prot, mpte);
+		va = start + ptoa(diff);
+		if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
+		    m->reserv != NULL && m->reserv->first_page == m &&
+		    m->reserv->refcnt == SP_MAGN(m->reserv->level) &&
+		    pmap_enter_pde(pmap, va, m, prot))
+			m = &m[NBPDR / PAGE_SIZE - 1];
+		else
+			mpte = pmap_enter_quick_locked(pmap, va, m, prot,
+			    mpte);
 		m = TAILQ_NEXT(m, listq);
 	}
  	PMAP_UNLOCK(pmap);
@@ -2445,7 +2822,6 @@
 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
     vm_prot_t prot, vm_page_t mpte)
 {
-	vm_page_t free;
 	pt_entry_t *pte;
 	vm_paddr_t pa;
 
@@ -2481,7 +2857,7 @@
 			 */
 			if (ptepa && (*ptepa & PG_V) != 0) {
 				if (*ptepa & PG_PS)
-					panic("pmap_enter_quick: unexpected mapping into 2MB page");
+					return (NULL);
 				mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 				mpte->wire_count++;
 			} else {
@@ -2504,7 +2880,7 @@
 	pte = vtopte(va);
 	if (*pte) {
 		if (mpte != NULL) {
-			mpte->wire_count--;
+			pmap_unwire_pte_hold(pmap, va, mpte);
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2516,11 +2892,7 @@
 	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 	    !pmap_try_insert_pv_entry(pmap, va, m)) {
 		if (mpte != NULL) {
-			free = NULL;
-			if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
-				pmap_invalidate_page(pmap, va);
-				pmap_free_zero_pages(free);
-			}
+			pmap_unwire_pte_hold(pmap, va, mpte);
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2662,14 +3034,35 @@
 void
 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 {
+	pd_entry_t *pde;
 	pt_entry_t *pte;
+	boolean_t are_queues_locked;
+
+	are_queues_locked = FALSE;
 
 	/*
 	 * Wiring is not a hardware characteristic so there is no need to
 	 * invalidate TLB.
 	 */
+retry:
 	PMAP_LOCK(pmap);
-	pte = pmap_pte(pmap, va);
+	pde = pmap_pde(pmap, va);
+	if ((*pde & PG_PS) != 0) {
+		if (!wired != ((*pde & PG_W) == 0)) {
+			if (!are_queues_locked) {
+				are_queues_locked = TRUE;
+				if (!mtx_trylock(&vm_page_queue_mtx)) {
+					PMAP_UNLOCK(pmap);
+					vm_page_lock_queues();
+					goto retry;
+				}
+			}
+			if (!pmap_demote_pde(pmap, pde, va))
+				panic("pmap_change_wiring: demotion failed");
+		} else
+			goto out;
+	}
+	pte = pmap_pde_to_pte(pde, va);
 	if (wired && (*pte & PG_W) == 0) {
 		pmap->pm_stats.wired_count++;
 		atomic_set_long(pte, PG_W);
@@ -2677,6 +3070,9 @@
 		pmap->pm_stats.wired_count--;
 		atomic_clear_long(pte, PG_W);
 	}
+out:
+	if (are_queues_locked)
+		vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2694,7 +3090,6 @@
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 	  vm_offset_t src_addr)
 {
-	vm_page_t   free;
 	vm_offset_t addr;
 	vm_offset_t end_addr = src_addr + len;
 	vm_offset_t va_next;
@@ -2749,12 +3144,14 @@
 			pde = (pd_entry_t *)
 			    PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
 			pde = &pde[pmap_pde_index(addr)];
-			if (*pde == 0) {
+			if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
+			    pmap_pv_insert_pde(dst_pmap, addr,
+			    PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME)))) {
 				*pde = srcptepaddr & ~PG_W;
 				dst_pmap->pm_stats.resident_count +=
 				    NBPDR / PAGE_SIZE;
 			} else
-				dstmpde->wire_count--;
+				pmap_unwire_pte_hold(dst_pmap, addr, dstmpde);
 			continue;
 		}
 
@@ -2791,15 +3188,9 @@
 					*dst_pte = ptetemp & ~(PG_W | PG_M |
 					    PG_A);
 					dst_pmap->pm_stats.resident_count++;
-	 			} else {
-					free = NULL;
-					if (pmap_unwire_pte_hold(dst_pmap,
-					    addr, dstmpte, &free)) {
-					    	pmap_invalidate_page(dst_pmap,
-					 	    addr);
-				    	    	pmap_free_zero_pages(free);
-					}
-				}
+	 			} else
+					pmap_unwire_pte_hold(dst_pmap, addr,
+					    dstmpte);
 				if (dstmpte->wire_count >= srcmpte->wire_count)
 					break;
 			}
@@ -2880,6 +3271,7 @@
 boolean_t
 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 {
+	struct md_page *pvh;
 	pv_entry_t pv;
 	int loops = 0;
 
@@ -2895,10 +3287,37 @@
 		if (loops >= 16)
 			break;
 	}
+	if (loops < 16) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+			if (PV_PMAP(pv) == pmap)
+				return (TRUE);
+			loops++;
+			if (loops >= 16)
+				break;
+		}
+	}
 	return (FALSE);
 }
 
 /*
+ * Returns TRUE if the given page is mapped individually or as part of
+ * a 2mpage.  Otherwise, returns FALSE.
+ */
+boolean_t
+pmap_page_is_mapped(vm_page_t m)
+{
+	struct md_page *pvh;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if (TAILQ_EMPTY(&m->md.pv_list)) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		return (!TAILQ_EMPTY(&pvh->pv_list));
+	} else
+		return (TRUE);
+}
+
+/*
  * Remove all pages from specified address space
  * this aids process exit speeds.  Also, this code
  * is special cased for current process only, but
@@ -2909,9 +3328,11 @@
 void
 pmap_remove_pages(pmap_t pmap)
 {
+	pd_entry_t *pde;
 	pt_entry_t *pte, tpte;
-	vm_page_t m, free = NULL;
+	vm_page_t m, mpte, mt;
 	pv_entry_t pv;
+	struct md_page *pvh;
 	struct pv_chunk *pc, *npc;
 	int field, idx;
 	int64_t bit;
@@ -2935,8 +3356,14 @@
 				pv = &pc->pc_pventry[idx];
 				inuse &= ~bitmask;
 
-				pte = vtopte(pv->pv_va);
-				tpte = *pte;
+				pde = vtopde(pv->pv_va);
+				tpte = *pde;
+				if ((tpte & PG_PS) != 0)
+					pte = pde;
+				else {
+					pte = vtopte(pv->pv_va);
+					tpte = *pte & ~PG_PTE_PAT;
+				}
 
 				if (tpte == 0) {
 					printf(
@@ -2963,27 +3390,59 @@
 					("pmap_remove_pages: bad tpte %#jx",
 					(uintmax_t)tpte));
 
-				pmap->pm_stats.resident_count--;
-
 				pte_clear(pte);
 
 				/*
 				 * Update the vm_page_t clean/reference bits.
 				 */
-				if (tpte & PG_M)
-					vm_page_dirty(m);
+				if (tpte & PG_M) {
+					KASSERT((tpte & PG_RW) != 0,
+	("pmap_remove_pages: modified page not writable: va: %#lx, pte: %#lx",
+					    pv->pv_va, tpte));
+					if ((tpte & PG_PS) != 0) {
+						for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
+							vm_page_dirty(mt);
+					} else
+						vm_page_dirty(m);
+				}
 
 				/* Mark free */
 				PV_STAT(pv_entry_frees++);
 				PV_STAT(pv_entry_spare++);
 				pv_entry_count--;
 				pc->pc_map[field] |= bitmask;
-				m->md.pv_list_count--;
-				TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-				if (TAILQ_EMPTY(&m->md.pv_list))
-					vm_page_flag_clear(m, PG_WRITEABLE);
-				pmap_unuse_pt(pmap, pv->pv_va,
-				    *vtopde(pv->pv_va), &free);
+				if ((tpte & PG_PS) != 0) {
+					pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
+					pvh = pa_to_pvh(tpte & PG_FRAME);
+					pvh->pv_list_count--;
+					TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
+					if (TAILQ_EMPTY(&pvh->pv_list)) {
+						for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
+							if (TAILQ_EMPTY(&mt->md.pv_list))
+								vm_page_flag_clear(mt, PG_WRITEABLE);
+					}
+					mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
+					if (mpte != NULL) {
+						pmap_remove_pt_page(pmap, mpte);
+						KASSERT(mpte->wire_count == NPTEPG,
+						    ("pmap_remove_pages: pte page wire count error"));
+						mpte->wire_count = 0;
+						vm_page_free(mpte);
+						atomic_subtract_int(&cnt.v_wire_count, 1);
+					}
+					pmap_unuse_pt(pmap, pv->pv_va,
+					    *pmap_pdpe(pmap, pv->pv_va));
+				} else {
+					pmap->pm_stats.resident_count--;
+					m->md.pv_list_count--;
+					TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+					if (TAILQ_EMPTY(&m->md.pv_list)) {
+						pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+						if (TAILQ_EMPTY(&pvh->pv_list))
+							vm_page_flag_clear(m, PG_WRITEABLE);
+					}
+					pmap_unuse_pt(pmap, pv->pv_va, *pde);
+				}
 			}
 		}
 		if (allfree) {
@@ -2996,9 +3455,8 @@
 			vm_page_free(m);
 		}
 	}
+	vm_page_unlock_queues();
 	pmap_invalidate_all(pmap);
-	pmap_free_zero_pages(free);
-	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -3011,17 +3469,30 @@
 boolean_t
 pmap_is_modified(vm_page_t m)
 {
+
+	if (m->flags & PG_FICTITIOUS)
+		return (FALSE);
+	if (pmap_is_modified_pvh(&m->md))
+		return (TRUE);
+	return (pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
+}
+
+/*
+ * Returns TRUE if any of the given mappings were used to modify
+ * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
+ * mappings are supported.
+ */
+static boolean_t
+pmap_is_modified_pvh(struct md_page *pvh)
+{
 	pv_entry_t pv;
 	pt_entry_t *pte;
 	pmap_t pmap;
 	boolean_t rv;
 
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	rv = FALSE;
-	if (m->flags & PG_FICTITIOUS)
-		return (rv);
-
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+	TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
 		pte = pmap_pte(pmap, pv->pv_va);
@@ -3049,7 +3520,7 @@
 	rv = FALSE;
 	PMAP_LOCK(pmap);
 	pde = pmap_pde(pmap, addr);
-	if (pde != NULL && (*pde & PG_V)) {
+	if (pde != NULL && (*pde & (PG_PS | PG_V)) == PG_V) {
 		pte = vtopte(addr);
 		rv = (*pte & PG_V) == 0;
 	}
@@ -3063,18 +3534,34 @@
 void
 pmap_remove_write(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
 	pmap_t pmap;
+	pv_entry_t next_pv, pv;
+	pd_entry_t *pde;
 	pt_entry_t oldpte, *pte;
+	vm_offset_t va;
 
 	if ((m->flags & PG_FICTITIOUS) != 0 ||
 	    (m->flags & PG_WRITEABLE) == 0)
 		return;
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		if ((*pde & PG_RW) != 0 && !pmap_demote_pde(pmap, pde, va))
+			pmap_invalidate_page(pmap, va);
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
-		pte = pmap_pte(pmap, pv->pv_va);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
+		    " a 2mpage in page %p's pv list", m));
+		pte = pmap_pde_to_pte(pde, pv->pv_va);
 retry:
 		oldpte = *pte;
 		if (oldpte & PG_RW) {
@@ -3105,14 +3592,46 @@
 int
 pmap_ts_referenced(vm_page_t m)
 {
+	struct md_page *pvh;
+	pmap_t pmap;
 	pv_entry_t pv, pvf, pvn;
-	pmap_t pmap;
+	pd_entry_t oldpde, *pde;
 	pt_entry_t *pte;
+	vm_offset_t va;
 	int rtval = 0;
 
 	if (m->flags & PG_FICTITIOUS)
 		return (rtval);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_A) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				if ((oldpde & PG_W) == 0) {
+					/*
+					 * Remove the mapping to a single page
+					 * so that a subsequent access may
+					 * repromote.
+					 */
+					va += VM_PAGE_TO_PHYS(m) - (oldpde &
+					    PG_FRAME);
+					pmap_remove_page(pmap, va, pde);
+					rtval++;
+					if (rtval > 4) {
+						PMAP_UNLOCK(pmap);
+						return (rtval);
+					}
+				}
+			} else
+				pmap_invalidate_page(pmap, va);
+		}
+		PMAP_UNLOCK(pmap);
+	}
 	if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 		pvf = pv;
 		do {
@@ -3121,7 +3640,10 @@
 			TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 			pmap = PV_PMAP(pv);
 			PMAP_LOCK(pmap);
-			pte = pmap_pte(pmap, pv->pv_va);
+			pde = pmap_pde(pmap, pv->pv_va);
+			KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
+			    " found a 2mpage in page %p's pv list", m));
+			pte = pmap_pde_to_pte(pde, pv->pv_va);
 			if ((*pte & PG_A) != 0) {
 				atomic_clear_long(pte, PG_A);
 				pmap_invalidate_page(pmap, pv->pv_va);
@@ -3141,17 +3663,58 @@
 void
 pmap_clear_modify(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
 	pmap_t pmap;
-	pt_entry_t *pte;
+	pv_entry_t next_pv, pv;
+	pd_entry_t oldpde, *pde;
+	pt_entry_t oldpte, *pte;
+	vm_offset_t va;
 
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		return;
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_RW) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				if ((oldpde & PG_W) == 0) {
+					/*
+					 * Write protect the mapping to a
+					 * single page so that a subsequent
+					 * write access may repromote.
+					 */
+					va += VM_PAGE_TO_PHYS(m) - (oldpde &
+					    PG_FRAME);
+					pte = pmap_pde_to_pte(pde, va);
+					oldpte = *pte;
+					if ((oldpte & PG_V) != 0) {
+						while (!atomic_cmpset_long(pte,
+						    oldpte,
+						    oldpte & ~(PG_M | PG_RW)))
+							oldpte = *pte;
+						vm_page_dirty(m);
+						pmap_invalidate_page(pmap, va);
+					}
+				}
+			} else
+				pmap_invalidate_page(pmap, va);
+		} else
+			KASSERT((oldpde & PG_M) == 0,
+			    ("pmap_clear_modify: modified page not writable"));
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
-		pte = pmap_pte(pmap, pv->pv_va);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
+		    " a 2mpage in page %p's pv list", m));
+		pte = pmap_pde_to_pte(pde, pv->pv_va);
 		if (*pte & PG_M) {
 			atomic_clear_long(pte, PG_M);
 			pmap_invalidate_page(pmap, pv->pv_va);
@@ -3168,17 +3731,43 @@
 void
 pmap_clear_reference(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
 	pmap_t pmap;
+	pv_entry_t next_pv, pv;
+	pd_entry_t oldpde, *pde;
 	pt_entry_t *pte;
+	vm_offset_t va;
 
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		return;
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_A) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				/*
+				 * Remove the mapping to a single page so
+				 * that a subsequent access may repromote.
+				 */
+				va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_FRAME);
+				pmap_remove_page(pmap, va, pde);
+			} else
+				pmap_invalidate_page(pmap, va);
+		}
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
-		pte = pmap_pte(pmap, pv->pv_va);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_reference: found"
+		    " a 2mpage in page %p's pv list", m));
+		pte = pmap_pde_to_pte(pde, pv->pv_va);
 		if (*pte & PG_A) {
 			atomic_clear_long(pte, PG_A);
 			pmap_invalidate_page(pmap, pv->pv_va);
@@ -3306,7 +3895,7 @@
 	pd_entry_t *pde;
 	pt_entry_t *pte;
 
-	base = trunc_page(va);
+	base = va & PG_FRAME;
 	offset = va & PAGE_MASK;
 	size = roundup(offset + size, PAGE_SIZE);
 
@@ -3369,24 +3958,35 @@
 int
 pmap_mincore(pmap_t pmap, vm_offset_t addr)
 {
-	pt_entry_t *ptep, pte;
+	pd_entry_t *pdep;
+	pt_entry_t pte;
+	vm_paddr_t pa;
 	vm_page_t m;
 	int val = 0;
 	
 	PMAP_LOCK(pmap);
-	ptep = pmap_pte(pmap, addr);
-	pte = (ptep != NULL) ? *ptep : 0;
+	pdep = pmap_pde(pmap, addr);
+	if (pdep != NULL && (*pdep & PG_V)) {
+		if (*pdep & PG_PS) {
+			KASSERT((*pdep & PG_FRAME & PDRMASK) == 0,
+			    ("pmap_mincore: bad pde"));
+			pte = *pdep;
+			pa = (*pdep & PG_FRAME) | (addr & PDRMASK);
+		} else {
+			pte = *pmap_pde_to_pte(pdep, addr);
+			pa = pte & PG_FRAME;
+		}
+	} else {
+		pte = 0;
+		pa = 0;
+	}
 	PMAP_UNLOCK(pmap);
 
 	if (pte != 0) {
-		vm_paddr_t pa;
-
 		val = MINCORE_INCORE;
 		if ((pte & PG_MANAGED) == 0)
 			return val;
 
-		pa = pte & PG_FRAME;
-
 		m = PHYS_TO_VM_PAGE(pa);
 
 		/*
@@ -3459,3 +4059,238 @@
 	addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
 	return addr;
 }
+
+#define COMPATIBLE_PTE_MASK (PG_NX | PG_MANAGED | PG_W | PG_G | PG_PTE_PAT | PG_M | PG_A | PG_NC_PCD | PG_NC_PWT | PG_U | PG_RW | PG_V)
+#define COMPATIBLE_PTE(a,b) (((a) & COMPATIBLE_PTE_MASK) == ((b) & COMPATIBLE_PTE_MASK))
+
+SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, "");
+
+static u_long pmap_pde_promotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
+    &pmap_pde_promotions, 0, "pde promotions");
+
+static void
+pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
+{
+	pd_entry_t newpde;
+	pt_entry_t *firstpte, oldpte, *pte;
+	vm_paddr_t pa;
+	vm_page_t mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
+	KASSERT((*firstpte & PG_V) != 0,
+	    ("pmap_promote_pde: firstpte is missing PG_V"));
+	if ((*firstpte & PG_A) == 0)
+		return;
+	pa = *firstpte & PG_FRAME & ~PDRMASK;
+	newpde = *firstpte;
+	if ((newpde & (PG_M | PG_RW)) == PG_RW)
+		newpde &= ~PG_RW;
+
+	/* 
+	 * Check all the ptes before promotion
+	 */
+	for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
+retry:
+		oldpte = *pte;
+		if ((oldpte & PG_FRAME) != pa) {
+			CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
+			    " in pmap %p", va, pmap);
+			return;
+		}
+		if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
+			/*
+			 * When PG_M is already clear, PG_RW can be cleared
+			 * without a TLB invalidation.
+			 */
+			if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
+				goto retry;
+			oldpte = *pte;
+		}
+		if (!COMPATIBLE_PTE(oldpte, newpde)) {
+			CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
+			    " in pmap %p", va, pmap);
+			return;
+		}
+		pa += PAGE_SIZE;
+	}
+
+	/*
+	 * Save the page table page in its current state until the PDE
+	 * mapping the superpage is demoted by pmap_demote_pde() or
+	 * destroyed by pmap_remove_pde(). 
+	 */
+	mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
+	KASSERT(mpte >= vm_page_array &&
+	    mpte < &vm_page_array[vm_page_array_size],
+	    ("pmap_promote_pde: page table page is out of range"));
+	KASSERT(mpte->pindex == pmap_pde_pindex(va),
+	    ("pmap_promote_pde: page table page's pindex is wrong"));
+	pmap_insert_pt_page(pmap, mpte);
+
+	/*
+	 * Promote the pv entries.
+	 */
+	if ((newpde & PG_MANAGED) != 0)
+		pmap_pv_promote_pde(pmap, va, newpde & PG_FRAME);
+
+	/*
+	 * Propagate the PAT index to its proper position.
+	 */
+	if ((newpde & PG_PTE_PAT) != 0)
+		newpde ^= PG_PDE_PAT | PG_PTE_PAT;
+
+	/*
+	 * Map the superpage.
+	 */
+	pde_store(pde, PG_PS | newpde);
+
+	pmap_pde_promotions++;
+	CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
+	    " in pmap %p", va, pmap);
+}
+
+static u_long pmap_pde_demotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
+    &pmap_pde_demotions, 0, "pde demotions");
+
+static boolean_t
+pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
+{
+	pd_entry_t newpde, oldpde;
+	pt_entry_t *firstpte, newpte, *pte;
+	vm_paddr_t mptepa;
+	vm_page_t mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	mpte = pmap_lookup_pt_page(pmap, va);
+	if (mpte != NULL)
+		pmap_remove_pt_page(pmap, mpte);
+	else {
+		KASSERT((*pde & PG_W) == 0,
+		    ("pmap_demote_pde: page table page for a wired mapping"
+		    " is missing"));
+		pmap_remove_pde(pmap, pde, trunc_2mpage(va));
+		CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
+		    " in pmap %p", va, pmap);
+		return (FALSE);
+	}
+	mptepa = VM_PAGE_TO_PHYS(mpte);
+	firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
+	oldpde = *pde;
+	newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
+	KASSERT((oldpde & (PG_A | PG_V)) == (PG_A | PG_V),
+	    ("pmap_demote_pde: oldpde is missing PG_A and/or PG_V"));
+	KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
+	    ("pmap_demote_pde: oldpde is missing PG_M"));
+	KASSERT((oldpde & PG_PS) != 0,
+	    ("pmap_demote_pde: oldpde is missing PG_PS"));
+	newpte = oldpde & ~PG_PS;
+	if ((newpte & PG_PDE_PAT) != 0)
+		newpte ^= PG_PDE_PAT | PG_PTE_PAT;
+
+	/*
+	 * If the mapping has changed attributes, update the page table
+	 * entries.
+	 */
+	KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
+	    ("pmap_demote_pde: firstpte and newpte map different physical"
+	    " addresses"));
+	if (!COMPATIBLE_PTE(*firstpte, newpte))
+		for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
+			*pte = newpte;
+			newpte += PAGE_SIZE;
+		}
+
+	/*
+	 * Demote the mapping.  This pmap is locked.  The old PDE has
+	 * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
+	 * set.  Thus, there is no danger of a race with another
+	 * processor changing the setting of PG_A and/or PG_M between
+	 * the read above and the store below. 
+	 */
+	pde_store(pde, newpde);	
+
+	/*
+	 * Invalidate a stale mapping of the page table page.
+	 */
+	pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
+
+	/*
+	 * Demote the pv entry.  This depends on the earlier demotion
+	 * of the mapping.  Specifically, the (re)creation of a per-
+	 * page pv entry might trigger the execution of pmap_collect(),
+	 * which might reclaim a newly (re)created per-page pv entry
+	 * and destroy the associated mapping.  In order to destroy
+	 * the mapping, the PDE must have already changed from mapping
+	 * the 2mpage to referencing the page table page.
+	 */
+	if ((oldpde & PG_MANAGED) != 0)
+		pmap_pv_demote_pde(pmap, va, oldpde & PG_FRAME);
+
+	pmap_pde_demotions++;
+	CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx"
+	    " in pmap %p", va, pmap);
+	return (TRUE);
+}
+
+static u_long pmap_pde_mappings;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
+    &pmap_pde_mappings, 0, "pde mappings");
+
+static boolean_t
+pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+	pd_entry_t *pde, newpde;
+	vm_page_t mpde;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if ((mpde = pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
+		CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
+		    " in pmap %p", va, pmap);
+		return (FALSE);
+	}
+	pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpde));
+	pde = &pde[pmap_pde_index(va)];
+	if ((*pde & PG_V) != 0) {
+		pmap_unwire_pte_hold(pmap, va, mpde);
+		CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
+		    " in pmap %p", va, pmap);
+		return (FALSE);
+	}
+	newpde = VM_PAGE_TO_PHYS(m) | PG_PS | PG_V;
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
+		newpde |= PG_MANAGED;
+
+		/*
+		 * Create a PV entry for each of the managed pages.
+		 */
+		if (!pmap_pv_insert_pde(pmap, va, m)) {
+			pmap_unwire_pte_hold(pmap, va, mpde);
+			CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
+			    " in pmap %p", va, pmap);
+			return (FALSE);
+		}
+	}
+	if ((prot & VM_PROT_EXECUTE) == 0)
+		newpde |= pg_nx;
+	if (va < VM_MAXUSER_ADDRESS)
+		newpde |= PG_U;
+
+	/*
+	 * Increment counters.
+	 */
+	pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
+
+	/*
+	 * Map the superpage.
+	 */
+	pde_store(pde, newpde);
+
+	pmap_pde_mappings++;
+	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
+	    " in pmap %p", va, pmap);
+	return (TRUE);
+}
---  amd64/include/param.h.orig
+++  amd64/include/param.h
@@ -115,6 +115,27 @@
 #define	NBPML4		(1ul<<PML4SHIFT)/* bytes/page map lev4 table */
 #define	PML4MASK	(NBPML4-1)
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#ifndef	SP_LEVELS
+#define	SP_LEVELS	1
+#endif
+
+/*
+ * SP_SMALL_SHIFT is LOG2(NBPDR / PAGE_SIZE).
+ */
+#ifndef	SP_SMALL_SHIFT
+#define	SP_SMALL_SHIFT	9
+#endif
+
+/*
+ * If there is only one superpage size, then SP_FACTOR_SHIFT is zero.
+ */
+#ifndef	SP_FACTOR_SHIFT
+#define	SP_FACTOR_SHIFT	0
+#endif
+
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef	KSTACK_PAGES
---  amd64/include/pmap.h.orig
+++  amd64/include/pmap.h
@@ -242,6 +242,7 @@
 	u_int			pm_active;	/* active on cpus */
 	/* spare u_int here due to padding */
 	struct pmap_statistics	pm_stats;	/* pmap statistics */
+	vm_page_t		pm_root;	/* spare page table pages */
 };
 
 typedef struct pmap	*pmap_t;
@@ -302,7 +303,6 @@
 extern vm_offset_t virtual_avail;
 extern vm_offset_t virtual_end;
 
-#define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.pv_list))
 #define	pmap_unmapbios(va, sz)	pmap_unmapdev((va), (sz))
 
 void	pmap_bootstrap(vm_paddr_t *);
@@ -316,6 +316,7 @@
 void	*pmap_mapbios(vm_paddr_t, vm_size_t);
 void	*pmap_mapdev(vm_paddr_t, vm_size_t);
 void	*pmap_mapdev_attr(vm_paddr_t, vm_size_t, int);
+boolean_t pmap_page_is_mapped(vm_page_t m);
 void	pmap_unmapdev(vm_offset_t, vm_size_t);
 void	pmap_invalidate_page(pmap_t, vm_offset_t);
 void	pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
---  amd64/include/vmparam.h.orig
+++  amd64/include/vmparam.h
@@ -106,9 +106,10 @@
  * the pool from which physical pages for page tables and small UMA
  * objects are allocated.
  */
-#define	VM_NFREEPOOL		2
+#define	VM_NFREEPOOL		3
 #define	VM_FREEPOOL_DEFAULT	0
 #define	VM_FREEPOOL_DIRECT	1
+#define	VM_FREEPOOL_RESERV	2
 
 /*
  * Create two free page lists: VM_FREELIST_DEFAULT is for physical
@@ -116,11 +117,17 @@
  * accessible by ISA DMA and VM_FREELIST_ISADMA is for physical pages
  * that are below that address.
  */
-#define	VM_NFREELIST		2
+#define	VM_NFREELIST		3
 #define	VM_FREELIST_DEFAULT	0
 #define	VM_FREELIST_ISADMA	1
+#define	VM_FREELIST_HIGHMEM	2
 
 /*
+ * VM_HIGHMEM_ADDRESS is the physical address at which HIGHMEM starts.
+ */
+#define	VM_HIGHMEM_ADDRESS	4294967296UL
+
+/*
  * An allocation size of 16MB is supported in order to optimize the
  * use of the direct map by UMA.  Specifically, a cache line contains
  * at most 8 PDEs, collectively mapping 16MB of physical memory.  By
@@ -185,4 +192,8 @@
 #define	VM_INITIAL_PAGEIN	16
 #endif
 
+#ifndef	VM_IMAGE_BASE
+#define	VM_IMAGE_BASE		0x400000
+#endif
+
 #endif /* _MACHINE_VMPARAM_H_ */
---  arm/arm/pmap.c.orig
+++  arm/arm/pmap.c
@@ -167,6 +167,7 @@
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_reserv.h>
 #include <vm/vm_extern.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
@@ -197,8 +198,12 @@
 static void pmap_free_pv_entry (pv_entry_t);
 static pv_entry_t pmap_get_pv_entry(void);
 
+static int		pmap_fixup_l1_entry(pmap_t pm, vm_offset_t va);
+static void 		pmap_set_section(pmap_t pm, vm_offset_t va, vm_page_t m);
+static void 		pmap_remove_section(pmap_t pm, vm_offset_t va, vm_page_t m);
 static void		pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t,
     vm_prot_t, boolean_t, int);
+static vm_paddr_t	pmap_extract_locked(pmap_t, vm_offset_t);
 static void		pmap_vac_me_harder(struct vm_page *, pmap_t,
     vm_offset_t);
 static void		pmap_vac_me_kpmap(struct vm_page *, pmap_t, 
@@ -211,10 +216,22 @@
 static int		pmap_clearbit(struct vm_page *, u_int);
 
 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t);
-static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t);
+static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t, int flags);
 static void		pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
 static vm_offset_t	kernel_pt_lookup(vm_paddr_t);
 
+static void		pmap_promote_small(pmap_t, struct l2_bucket *, 
+    vm_offset_t);
+static void		pmap_promote_large(pmap_t, vm_offset_t va);
+static void		pmap_demote(pmap_t, vm_offset_t);
+static void		pmap_demote_large(pmap_t, vm_offset_t);
+
+static void 		pmap_nuke_pv(vm_page_t, pmap_t, struct pv_entry *);
+
+static void		pmap_enter_pv(vm_page_t, struct pv_entry *, pmap_t,
+    vm_offset_t, u_int);
+static struct pv_entry *pmap_remove_pv(vm_page_t, pmap_t, vm_offset_t);
+static u_int	pmap_modify_pv(vm_page_t, pmap_t, vm_offset_t, u_int, u_int);
 static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1");
 
 vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
@@ -280,6 +297,9 @@
 
 char *_tmppt;
 
+#ifdef ARM_USE_SMALL_ALLOC
+extern struct mtx smallalloc_mtx;
+#endif
 /*
  * Metadata for L1 translation tables.
  */
@@ -306,8 +326,13 @@
 
 	/* KVA of this L1 page table */
 	pd_entry_t *l1_kva;
+	struct mtx l1_mtx;
 };
 
+#define L1_LOCK(l1)		mtx_lock(&(l1)->l1_mtx)
+#define L1_UNLOCK(l1)		mtx_unlock(&(l1)->l1_mtx)
+#define PMAP_LOCK_L1(pmap)	L1_LOCK((pmap)->pm_l1)
+#define PMAP_UNLOCK_L1(pmap)	L1_UNLOCK((pmap)->pm_l1)
 /*
  * Convert a virtual address into its L1 table index. That is, the
  * index used to locate the L2 descriptor table pointer in an L1 table.
@@ -446,6 +471,8 @@
 		panic("pmap_init_l1: can't get PA of L1 at %p", l1pt);
 	SLIST_INSERT_HEAD(&l1_list, l1, l1_link);
 	TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
+	if (l1pt != pmap_kernel()->pm_l1->l1_kva)
+		mtx_init(&l1->l1_mtx, "L1 mtx", NULL, MTX_DEF | MTX_RECURSE);
 }
 
 static vm_offset_t
@@ -897,7 +924,7 @@
  * the bucket/page in the meantime.
  */
 static struct l2_bucket *
-pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va)
+pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va, int flags)
 {
 	struct l2_dtable *l2;
 	struct l2_bucket *l2b;
@@ -1070,6 +1097,8 @@
 	ptep = l2b->l2b_kva;
 	l2b->l2b_kva = NULL;
 
+	PMAP_LOCK_L1(pm);
+	pmap_fixup_l1_entry(pm, l1idx << L1_S_SHIFT);
 	pl1pd = &pm->pm_l1->l1_kva[l1idx];
 
 	/*
@@ -1081,6 +1110,7 @@
 		*pl1pd = 0;
 		PTE_SYNC(pl1pd);
 	}
+	PMAP_UNLOCK_L1(pm);
 
 	/*
 	 * Release the L2 descriptor table back to the pool cache.
@@ -1117,9 +1147,7 @@
 #ifndef PMAP_INCLUDE_PTE_SYNC
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, pte;
-#ifdef ARM_USE_SMALL_ALLOC
 	pd_entry_t *pde;
-#endif
 	vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK;
 
 	/*
@@ -1130,28 +1158,38 @@
 	 * page tables, we simply fix up the cache-mode here if it's not
 	 * correct.
 	 */
+	pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
 #ifdef ARM_USE_SMALL_ALLOC
-	pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
-	if (!l1pte_section_p(*pde)) {
+	if (l1pte_section_p(*pde) && va >= alloc_firstaddr)
+		return (0);
 #endif
+	if (l1pte_section_p(*pde) && (*pde & pte_l1_s_cache_mask) !=
+	    pte_l1_s_cache_mode_pt) {
+		pmap_demote_large(pmap_kernel(), va);
+		cpu_tlb_flushID_SE(va & L1_S_ADDR_MASK);
+	} else if (l1pte_section_p(*pde))
+		return (0);
+	
+	l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+	ptep = &l2b->l2b_kva[l2pte_index(va)];
+	if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L && 
+	    (*ptep & L2_L_CACHE_MASK) != pte_l2_l_cache_mode_pt) {
+		pmap_demote(pmap_kernel(), va);
 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
-		ptep = &l2b->l2b_kva[l2pte_index(va)];
-		pte = *ptep;
+	}
+	pte = *ptep;
 		
-		if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
-			/*
-			 * Page tables must have the cache-mode set to 
-			 * Write-Thru.
-			 */
-			*ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
-			PTE_SYNC(ptep);
-			cpu_tlb_flushD_SE(va);
-			cpu_cpwait();
-		}
-#ifdef ARM_USE_SMALL_ALLOC
+	if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
+		/*
+		 * Page tables must have the cache-mode set to 
+		 * Write-Thru.
+		 */
+		*ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
+		PTE_SYNC(ptep);
+		cpu_tlb_flushD_SE(va);
+		cpu_cpwait();
 	}
 #endif
-#endif
 	memset(mem, 0, L2_TABLE_SIZE_REAL);
 	PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
 	return (0);
@@ -1418,21 +1456,25 @@
 	struct pv_entry *pv, *npv;
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, pte;
+	vm_page_t m;
 	u_int entries = 0;
 	u_int writable = 0;
 	u_int cacheable_entries = 0;
 	u_int kern_cacheable = 0;
 	u_int other_writable = 0;
+	int had_large = 0, had_section = 0;
 
 	/*
 	 * Count mappings and writable mappings in this pmap.
 	 * Include kernel mappings as part of our own.
 	 * Keep a pointer to the first one.
 	 */
+count:
 	npv = TAILQ_FIRST(&pg->md.pv_list);
 	TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) {
 		/* Count mappings in the same pmap */
-		if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) {
+		if (!(pv->pv_flags & (PVF_LARGE | PVF_SECTION)) &&
+		    (pm == pv->pv_pmap || kpmap == pv->pv_pmap)) {
 			if (entries++ == 0)
 				npv = pv;
 
@@ -1450,6 +1492,37 @@
 		if (pv->pv_flags & PVF_WRITE)
 			other_writable = 1;
 	}
+	/* Check if this page is part of a superpage. */
+	m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L1_S_FRAME);
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		if (pv->pv_flags & PVF_SECTION) {
+			if (pv->pv_pmap == pm || pv->pv_pmap == kpmap) {
+				if ((pv->pv_flags & PVF_NC) == 0) {
+					had_section = 1;
+					cacheable_entries++;
+					if (kpmap == pv->pv_pmap)
+						kern_cacheable++;
+					if (pv->pv_flags & PVF_WRITE)
+						++writable;
+				}
+			}
+		}
+	}
+	m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L2_L_FRAME);
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		if (pv->pv_flags & PVF_LARGE) {
+			if (pv->pv_pmap == pm || pv->pv_pmap == kpmap) {
+				if ((pv->pv_flags & PVF_NC) == 0) {
+					had_large = 1;
+					cacheable_entries++;
+					if (kpmap == pv->pv_pmap)
+						kern_cacheable++;
+					if (pv->pv_flags & PVF_WRITE)
+						++writable;
+				}
+			}
+		}
+	}
 
 	/*
 	 * Enable or disable caching as necessary.
@@ -1461,6 +1534,62 @@
 	    (entries > 0 && pm == kpmap && other_writable)) {
 		if (cacheable_entries == 0)
 			return;
+		/* Not optimal, but this won't happen very often. */
+		/* Demote any cached superpages. */
+		if (had_section) {
+			m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L1_S_FRAME);
+redo_tailq:
+			TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+				if (pv->pv_flags & PVF_SECTION) {
+					if (pv->pv_pmap == pm ||
+					    pv->pv_pmap == kpmap) {
+						if ((pv->pv_flags & PVF_NC)
+						    == 0) {
+							pmap_t pmap =
+							    pv->pv_pmap;
+							if (pmap != pm)
+								PMAP_LOCK(pmap);
+							PMAP_LOCK_L1(pmap);
+							pmap_fixup_l1_entry(pmap,
+							    pv->pv_va);
+							pmap_demote(pmap, 
+							    pv->pv_va);
+							PMAP_UNLOCK_L1(pmap);
+							if (pmap != pm)
+								PMAP_UNLOCK(pmap);
+							goto redo_tailq;
+						}
+					}
+				}
+			}
+		}
+
+		if (had_large) {
+			m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L2_L_FRAME);
+redo_tailq_large:
+			TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+				if (pv->pv_flags & PVF_LARGE) {
+					
+					if (pv->pv_pmap == pm ||
+					    pv->pv_pmap == kpmap) {
+						if ((pv->pv_flags & PVF_NC)
+						    == 0) {
+							pmap_t pmap =
+							    pv->pv_pmap;
+							if (pmap != pm)
+								PMAP_LOCK(pmap);
+							pmap_demote(pmap, 
+							    pv->pv_va);
+							if (pmap != pm)
+								PMAP_UNLOCK(pmap);
+							goto redo_tailq_large;
+						}
+					}
+				}
+			}
+		}
+		if (had_large || had_section)
+			goto count;
 
 		for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) {
 			if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) ||
@@ -1468,8 +1597,18 @@
 				continue;
 
 			pv->pv_flags |= PVF_NC;
+			PMAP_LOCK_L1(pm);
+			pmap_fixup_l1_entry(pm, va);
 
+			if (l1pte_section_p(
+			    pm->pm_l1->l1_kva[L1_IDX(pv->pv_va)])) {
+				pmap_demote(pv->pv_pmap, pv->pv_va);
+				pmap_tlb_flushID_SE(pv->pv_pmap,
+				    pv->pv_va & L1_S_ADDR_MASK);
+			}
+			PMAP_UNLOCK_L1(pm);
 			l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+			
 			ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
 			pte = *ptep & ~L2_S_CACHE_MASK;
 
@@ -1524,11 +1663,459 @@
 
 			*ptep = pte;
 			PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
+			if (pv->pv_flags & PVF_DEMOTED) {
+				pmap_t pmap = pv->pv_pmap;
+				vm_offset_t va = pv->pv_va;
+				PMAP_LOCK_L1(pmap);
+				pmap_promote_small(pmap, l2b, va);
+				pmap_promote_large(pmap, va);
+				PMAP_UNLOCK_L1(pmap);
+			}
 		}
 	}
 }
 
 /*
+ *
+ * pmap_find_pv: Find a pv entry
+ *
+ * => caller should hold lock on vm_page
+ */
+static PMAP_INLINE struct pv_entry *
+pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va)
+{
+	struct pv_entry *pv;
+	vm_page_t m;
+	
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list)
+	    if (pm == pv->pv_pmap && va == pv->pv_va)
+		    break;
+	/* Search a superpage PV entry. */
+	if (pv == NULL) {
+		m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L2_L_FRAME);
+		if (m != pg) {
+			TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
+			    if (pm == pv->pv_pmap && (va & L2_L_FRAME) == 
+				pv->pv_va)
+				    break;
+		}
+	}
+	if (pv == NULL) {
+		m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L1_S_FRAME);
+		if (m != pg) {
+			TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
+			    if (pm == pv->pv_pmap && (va & L1_S_FRAME) ==
+				pv->pv_va)
+				    break;
+		}
+	}
+	return (pv);
+}
+
+/* XXX: misleading, promote_small actually promotes to large pages. */
+static void
+pmap_promote_small(pmap_t pmap, struct l2_bucket *l2b, vm_offset_t va)
+{
+	int i;
+	pt_entry_t *ptep;
+	struct pv_entry *pv;
+	vm_page_t m;
+	vm_paddr_t pa0;
+	vm_offset_t va0;
+	int cachemode;
+	int oldwire = -1;
+	
+	ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)];
+	pa0 = *ptep & L2_L_FRAME;
+	va0 = va & L2_L_FRAME;
+	if ((*ptep & L2_S_FRAME) != pa0)
+		return;
+	/* First check it is eligible for promotion. */
+	for (i = 1; i < 0x10; i++) {
+		int wire1, wire2;
+		if (ptep[i] != (ptep[i - 1] + PAGE_SIZE)) {
+			/* There's no way we can do it. */
+			return;
+		}
+		pv = pmap_find_pv(PHYS_TO_VM_PAGE(ptep[i] & L2_S_FRAME), pmap,
+		    va0 + i * PAGE_SIZE);
+		if (!pv)
+			wire1 = -2;
+		else
+			wire1 = (pv->pv_flags & PVF_WIRED);
+		if (oldwire != -1 && wire1 != oldwire)
+			return;
+		pv = pmap_find_pv(PHYS_TO_VM_PAGE(ptep[i - 1] & L2_S_FRAME), 
+		    pmap, va0 + (i - 1) * PAGE_SIZE);
+		if (!pv)
+			wire2 = -2;
+		else
+			wire2 = (pv->pv_flags & PVF_WIRED);
+		if (wire1 != wire2)
+			return;
+		if (oldwire == -1)
+			oldwire = wire1;
+		
+	}
+#ifdef SP_DEBUG
+	printf("promoting small %x\n", va);
+#endif
+	if (*ptep & L2_S_PROT_U)
+		pa0 |= L2_L_PROT_U;
+	if (*ptep & L2_S_PROT_W)
+		pa0 |= L2_L_PROT_W;
+	/* Let's do it. */
+	if ((*ptep & pte_l2_s_cache_mask) == pte_l2_s_cache_mode)
+		cachemode = pte_l2_l_cache_mode;
+	else if ((*ptep & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt)
+		cachemode = pte_l2_l_cache_mode_pt;
+	else
+		cachemode = 0;
+	for (i = 0; i < 0x10; i++) {
+		ptep[i] = pa0 | L2_L_PROTO | cachemode;
+		m = PHYS_TO_VM_PAGE(pa0 + i * PAGE_SIZE);
+		pv = pmap_find_pv(m, pmap, va0 + i * PAGE_SIZE);
+		if (pv) {
+			if (i == 0) {
+				pv->pv_flags |= PVF_LARGE;
+				pv->pv_flags &= ~PVF_DEMOTED;
+			} else {
+				pmap_nuke_pv(m, pmap, pv);
+				pmap_free_pv_entry(pv);
+			}
+		}
+	}
+}
+
+static void
+pmap_promote_large(pmap_t pmap, vm_offset_t va)
+{
+	pd_entry_t *pd;
+	pt_entry_t *pt;
+	vm_offset_t va0;
+	vm_paddr_t pa;
+	struct l2_dtable *l2;
+	struct l1_ttable *l1;
+	struct pv_entry *pv;
+	uint16_t l1idx;
+	int i;
+	int cachemode, cachemode2;
+	int oldwire = -1;
+	
+	va0 = va & L1_S_ADDR_MASK;
+	l1idx = L1_IDX(va0);
+	pd = &pmap->pm_l1->l1_kva[l1idx];
+	KASSERT(pd != NULL, ("NO pd_entry in pmap_promote_large"));
+	l2 = pmap->pm_l2[L2_IDX(l1idx)];
+	KASSERT(l2 != NULL, ("No l2 bucket in pmap_promote_large"));
+	pt = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva;
+	KASSERT(pt != NULL, ("No pt in pmap_promote_large"));
+	if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
+		pa = *pt & L2_L_FRAME;
+	else
+		pa = *pt & L2_S_FRAME;
+	if (pa != (*pt & L1_S_FRAME))
+		return;
+	for (i = 1; i < 0x100; i++) {
+		vm_paddr_t pa, pa2;
+		int wire1, wire2;
+		int flags1 = 0, flags2 = 0;
+
+		if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L) {
+			pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE;
+			if ((pt[i] & pte_l2_l_cache_mask) == 
+			    pte_l2_l_cache_mode)
+				cachemode = pte_l1_s_cache_mode;
+			else if ((pt[i] & pte_l2_l_cache_mask) ==
+			    pte_l2_l_cache_mode_pt)
+				cachemode = pte_l1_s_cache_mode_pt;
+			else
+				cachemode = 0;
+			if (pt[i] & L2_L_PROT_U)
+				flags1 |= L2_S_PROT_U;
+			if (pt[i] & L2_L_PROT_W)
+				flags1 |= L2_S_PROT_W;
+		} else {
+			pa = pt[i] & L2_S_FRAME;
+			if ((pt[i] & pte_l2_s_cache_mask) == 
+			    pte_l2_s_cache_mode)
+				cachemode = pte_l1_s_cache_mode;
+			else if ((pt[i] & pte_l2_s_cache_mask) ==
+			    pte_l2_s_cache_mode_pt)
+				cachemode = pte_l1_s_cache_mode_pt;
+			else
+				cachemode = 0;
+			flags1 = pt[i] & (L2_S_PROT_U|L2_S_PROT_W);
+		}
+		if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L) {
+			pa2 = (pt[i - 1] & L2_L_FRAME) +
+			    ((i - 1) & 0xf) * PAGE_SIZE;
+			if ((pt[i - 1] & pte_l2_l_cache_mask) == 
+			    pte_l2_l_cache_mode)
+				cachemode2 = pte_l1_s_cache_mode;
+			else if ((pt[i - 1] & pte_l2_l_cache_mask) ==
+			    pte_l2_l_cache_mode_pt)
+				cachemode2 = pte_l1_s_cache_mode_pt;
+			else
+				cachemode2 = 0;
+		if (pt[i - 1] & L2_L_PROT_U)
+			flags2 |= L2_S_PROT_U;
+		if (pt[i - 1] & L2_L_PROT_W)
+			flags2 |= L2_S_PROT_W;
+		} else {
+			pa2 = pt[i - 1] & L2_S_FRAME;
+			if ((pt[i - 1] & pte_l2_s_cache_mask) == 
+			    pte_l2_s_cache_mode)
+				cachemode2 = pte_l1_s_cache_mode;
+			else if ((pt[i - 1] & pte_l2_s_cache_mask) ==
+				pte_l2_s_cache_mode_pt)
+				cachemode2 = pte_l1_s_cache_mode_pt;
+			else
+				cachemode2 = 0;
+			flags2 = pt[i - 1] & (L2_S_PROT_U|L2_S_PROT_W);
+		}
+		if (pa != pa2 + PAGE_SIZE)
+			/* Nothing much we can do. */
+			return;
+		if (flags1 != flags2)
+			return;
+		if (cachemode != cachemode2)
+			return;
+		pv = pmap_find_pv(PHYS_TO_VM_PAGE(pa), pmap,
+		    va0 + i * PAGE_SIZE);
+		if (!pv)
+			wire1 = -2;
+		else
+			wire1 = (pv->pv_flags & PVF_WIRED);
+		if (oldwire != -1 && wire1 != oldwire)
+			return;
+		pv = pmap_find_pv(PHYS_TO_VM_PAGE(pa2), pmap,
+		    va0 + (i - 1) * PAGE_SIZE);
+		if (!pv)
+			wire2 = -2;
+		else
+			wire2 = (pv->pv_flags & PVF_WIRED);
+		if (wire1 != wire2)
+			return;
+		if (oldwire == -1)
+			oldwire = wire1;
+	}
+#ifdef SP_DEBUG
+	printf("promoting large %x\n", va);
+#endif
+	/* Now free the unused pts. */
+	if (*pt & L2_S_PROT_U)
+		pa |= L1_S_PROT_U;
+	if (*pt & L2_S_PROT_W)
+		pa |= L1_S_PROT_W;
+	*pd = L1_S_PROTO | pa | cachemode | L1_S_DOM(pmap->pm_domain);
+	bzero(pt, 0x100 * sizeof(*pt));
+	pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100);
+	if (pmap == kernel_pmap) {
+		SLIST_FOREACH(l1, &l1_list, l1_link) {
+			l1->l1_kva[l1idx] = *pd;
+			PTE_SYNC(&l1->l1_kva[l1idx]);
+		}
+	}
+	for (i = 1; i < 0x100; i++) {
+		pv = pmap_remove_pv(PHYS_TO_VM_PAGE((pa & L1_S_ADDR_MASK) +
+		    i * PAGE_SIZE), pmap, va0 + i * PAGE_SIZE);
+		if (pv)
+			/* 
+			 * It can happen that we don't find any pv, as it may
+			 * certainly was a large page before.
+			 */
+			pmap_free_pv_entry(pv);
+	}
+	if (oldwire != -2)
+		pmap_set_section(pmap, va0, 
+		    PHYS_TO_VM_PAGE(pa & L1_S_ADDR_MASK));
+}
+
+/* 
+ * Special case of pmap_demote(), where we know we want to demote a section
+ * mapping into large pages, except for the concerned page.
+ */
+
+static void
+pmap_demote_large(pmap_t pmap, vm_offset_t va)
+{
+	pd_entry_t *pd;
+	pt_entry_t *pt;
+	struct l2_bucket *l2b;
+	struct l1_ttable *l1;
+	struct pv_entry *pv;
+	vm_offset_t va0;
+	uint16_t l1idx;
+	vm_paddr_t pa;
+	int i, pv_flags;
+	int mode_l = 0, mode_s = 0;
+
+#ifdef SP_DEBUG
+	printf("demoting section mapping at %x\n", va);
+#endif
+	l1idx = L1_IDX(va);
+	pd = &pmap->pm_l1->l1_kva[l1idx];
+	va0 = va & L1_S_ADDR_MASK;
+	if (pmap == pmap_kernel())
+		l2b = pmap_get_l2_bucket(pmap, va0);
+	else
+		l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT);
+	pa = *pd & L1_S_ADDR_MASK;
+	if (*pd & L1_S_PROT_U) {
+		mode_l |= L2_L_PROT_U;
+		mode_s |= L2_S_PROT_U;
+	}
+	if (*pd & L1_S_PROT_W) {
+		mode_l |= L2_L_PROT_W;
+		mode_s |= L2_S_PROT_W;
+	}
+	if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode) {
+		mode_l |= pte_l2_l_cache_mode;
+		mode_s |= pte_l2_s_cache_mode;
+	} else {
+		mode_l |= pte_l2_l_cache_mode_pt;
+		mode_s |= pte_l2_s_cache_mode_pt;
+	}
+	pt = &l2b->l2b_kva[l2pte_index(va0)];
+	*pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
+	l2b->l2b_occupancy += 0x100;
+	if (pmap == kernel_pmap) {
+		SLIST_FOREACH(l1, &l1_list, l1_link) {
+			l1->l1_kva[l1idx] = *pd;
+			PTE_SYNC(&l1->l1_kva[l1idx]);
+		}
+	}
+	pmap_remove_section(pmap, va0, PHYS_TO_VM_PAGE(pa & L1_S_ADDR_MASK));
+	pv = pmap_find_pv(PHYS_TO_VM_PAGE(pa), pmap, va0);
+	if (pv)
+		pv_flags = pv->pv_flags | PVF_DEMOTED;
+	else
+		pv_flags = 0;
+	for (i = 0; i < 0x100; i++, pa += PAGE_SIZE) {
+		if (va0 + i * PAGE_SIZE != va) {
+			if (pv_flags != 0) {
+				if (i != 0 && (i & 0xf) == 0)
+					pv = pmap_get_pv_entry();
+				if ((i & 0xf) == 0)
+					pmap_enter_pv(PHYS_TO_VM_PAGE(pa), pv,
+					    pmap, va0 + i * PAGE_SIZE,
+					    pv_flags | PVF_LARGE);
+			}
+			pt[i] = (pa & L2_L_FRAME) | mode_l;
+		} else {
+			if (pv_flags != 0) {
+				if (i != 0)
+					pv = pmap_get_pv_entry();
+				pmap_enter_pv(PHYS_TO_VM_PAGE(pa), pv,
+				    pmap, va0 + i * PAGE_SIZE, pv_flags);
+			}
+			if (pv_flags != 0) {
+				if (i != 0 && (i & 0xf) == 0)
+					pv = pmap_get_pv_entry();
+				if (i == 0)
+					pmap_modify_pv(PHYS_TO_VM_PAGE(pa),
+					    pmap, va0, PVF_LARGE | PVF_SECTION,
+					    PVF_LARGE);
+				else if ((i & 0xf) == 0)
+					pmap_enter_pv(PHYS_TO_VM_PAGE(pa), pv,
+					    pmap, va0 + i * PAGE_SIZE,
+					    pv_flags | PVF_LARGE);
+			}
+			pt[i] = pa | mode_s;
+		}
+	}
+}
+
+static void
+pmap_demote(pmap_t pmap, vm_offset_t va)
+{
+	pd_entry_t *pd;
+	pt_entry_t *pt;
+	struct l2_bucket *l2b;
+	struct l1_ttable *l1;
+	struct pv_entry *pv;
+	vm_offset_t va0;
+	uint16_t l1idx;
+	uint16_t demote_size;
+	vm_paddr_t pa;
+	int i;
+	int cachemode;
+	int pv_flags;
+
+#ifdef SP_DEBUG
+	printf("demoting %x\n", va);
+#endif
+	l1idx = L1_IDX(va);
+	pd = &pmap->pm_l1->l1_kva[l1idx];
+	if (l1pte_section_p(*pd)) {
+		va0 = va & L1_S_ADDR_MASK;
+		if (pmap == pmap_kernel())
+			l2b = pmap_get_l2_bucket(pmap, va0);
+		else
+			l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT);
+		demote_size = 0x100;
+		pa = *pd & L1_S_ADDR_MASK;
+		if (*pd & L1_S_PROT_U)
+			pa |= L2_S_PROT_U;
+		if (*pd & L1_S_PROT_W)
+			pa |= L2_S_PROT_W;
+		if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode)
+			cachemode = pte_l2_s_cache_mode;
+		else
+			cachemode = pte_l2_s_cache_mode_pt;
+		pt = &l2b->l2b_kva[l2pte_index(va0)];
+		*pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
+		l2b->l2b_occupancy += 0x100;
+		if (pmap == kernel_pmap) {
+			SLIST_FOREACH(l1, &l1_list, l1_link) {
+				l1->l1_kva[l1idx] = *pd;
+				PTE_SYNC(&l1->l1_kva[l1idx]);
+			}
+		}
+		pmap_remove_section(pmap, va0, 
+		    PHYS_TO_VM_PAGE(pa & L1_S_ADDR_MASK));
+	} else {
+		va0 = va & L2_L_FRAME;
+		l2b = pmap_get_l2_bucket(pmap, va0);
+		demote_size = 0x10;
+		pt = &l2b->l2b_kva[l2pte_index(va0)];
+		if ((*pt & L2_TYPE_MASK) != L2_TYPE_L)
+			return;
+		pa = *pt & L2_L_FRAME;
+		if (*pt & L2_L_PROT_U)
+			pa |= L2_S_PROT_U;
+		if (*pt & L2_L_PROT_W)
+			pa |= L2_S_PROT_W;
+		if ((*pt & pte_l2_l_cache_mask) == pte_l2_l_cache_mode)
+			cachemode = pte_l2_s_cache_mode;
+		else
+			cachemode = pte_l2_s_cache_mode_pt;
+	}
+	pv = pmap_find_pv(PHYS_TO_VM_PAGE(pa), pmap, va0);
+	pa |= L2_S_PROTO;
+	if (pv)
+		pv_flags = (pv->pv_flags &~ PVF_LARGE) | PVF_DEMOTED;
+	else
+		pv_flags = 0;
+	for (i = 0; i < demote_size; i++, pa += PAGE_SIZE) {
+		if (pv_flags != 0) {
+			if (i != 0) {
+				pv = pmap_get_pv_entry();
+				pmap_enter_pv(PHYS_TO_VM_PAGE(pa), pv, pmap, 
+				    va0 + i * PAGE_SIZE, pv_flags);
+			} else
+				pmap_modify_pv(PHYS_TO_VM_PAGE(pa), pmap,
+				    va0, PVF_LARGE | PVF_SECTION, 0);
+		}
+		pt[i] = (pa) | cachemode;
+		PTE_SYNC(pt[i]);
+	}
+}
+
+
+/*
  * Modify pte bits for all ptes corresponding to the given physical address.
  * We use `maskbits' rather than `clearbits' because we're always passing
  * constants and the latter would require an extra inversion at run-time.
@@ -1538,6 +2125,7 @@
 {
 	struct l2_bucket *l2b;
 	struct pv_entry *pv;
+	vm_page_t m;
 	pt_entry_t *ptep, npte, opte;
 	pmap_t pm;
 	vm_offset_t va;
@@ -1551,6 +2139,37 @@
 	 */
 	pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF));
 
+	/* Check if we're part of a superpage. */
+	m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L1_S_FRAME);
+redo_tailq_section:
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		if (pv->pv_flags & PVF_SECTION) {
+			if ((pv->pv_flags & (PVF_REF | PVF_WRITE)) &
+			    (maskbits & (PVF_REF | PVF_WRITE))) {
+				pm = pv->pv_pmap;
+				va = pv->pv_va;
+				PMAP_LOCK_L1(pm);
+				pmap_fixup_l1_entry(pm, va);
+				pmap_demote(pm, va);
+				PMAP_UNLOCK_L1(pm);
+				goto redo_tailq_section;
+			}
+		}
+	}
+	m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(pg) & L2_L_FRAME);
+redo_tailq_large:
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+		if (pv->pv_flags & PVF_LARGE) {
+			if ((pv->pv_flags & (PVF_REF | PVF_WRITE)) &
+			    (maskbits & (PVF_REF | PVF_WRITE))) {
+				va = pv->pv_va;
+				pm = pv->pv_pmap;
+				pmap_demote(pm, va);
+				goto redo_tailq_large;
+			}
+		}
+	}
+
 	if (TAILQ_EMPTY(&pg->md.pv_list)) {
 		return (0);
 	}
@@ -1566,9 +2185,16 @@
 
 		PMAP_LOCK(pm);
 
+		PMAP_LOCK_L1(pm);
+		pmap_fixup_l1_entry(pm, va);
+
+		PMAP_UNLOCK_L1(pm);
 		l2b = pmap_get_l2_bucket(pm, va);
-
 		ptep = &l2b->l2b_kva[l2pte_index(va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+			pmap_demote(pm, va);
+			pv->pv_flags |= PVF_DEMOTED;
+		}
 		npte = opte = *ptep;
 
 		if (maskbits & (PVF_WRITE|PVF_MOD)) {
@@ -1681,6 +2307,12 @@
 			if (PV_BEEN_REFD(oflags))
 				pmap_tlb_flushD_SE(pm, pv->pv_va);
 		}
+		if (pv->pv_flags & PVF_DEMOTED) {
+			PMAP_LOCK_L1(pm);
+			pmap_promote_small(pm, l2b, va);
+			pmap_promote_large(pm, va);
+			PMAP_UNLOCK_L1(pm);
+		}
 
 		PMAP_UNLOCK(pm);
 
@@ -1719,9 +2351,16 @@
 	pve->pv_pmap = pm;
 	pve->pv_va = va;
 	pve->pv_flags = flags;
+	pve->pv_page = pg;
 
 	TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list);
-	TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist);
+	if (pm->pm_firstpv)
+		TAILQ_INSERT_AFTER(&pm->pm_pvlist, pm->pm_firstpv,
+		    pve, pv_plist);
+	else {
+		TAILQ_INSERT_TAIL(&pm->pm_pvlist, pve, pv_plist);
+		pm->pm_firstpv = pve;
+	}
 	pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD);
 	if (pm == pmap_kernel()) {
 		if (flags & PVF_WRITE)
@@ -1739,22 +2378,56 @@
 	vm_page_flag_set(pg, PG_REFERENCED);
 }
 
-/*
- *
- * pmap_find_pv: Find a pv entry
- *
- * => caller should hold lock on vm_page
- */
 static PMAP_INLINE struct pv_entry *
-pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va)
+pmap_find_superpage(pmap_t pm, vm_offset_t va)
+{
+	struct pv_entry *pv;
+
+	TAILQ_FOREACH(pv, &pm->pm_pvlist, pv_plist) {
+		if (!(pv->pv_flags & PVF_SECTION))
+			break;
+		if (pv->pv_va == va)
+			return (pv);
+	}
+	return (NULL);
+}
+
+/* Make it so all the section mappings are first in the pmap PV list. */
+
+static void 
+pmap_set_section(pmap_t pm, vm_offset_t va, vm_page_t m)
+{
+	struct pv_entry *pv;
+
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
+		if (pv->pv_va == va && pv->pv_pmap == pm)
+			break;
+	KASSERT(pv != NULL, ("No pv for new section"));
+	pv->pv_flags |= PVF_SECTION;
+	TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist);
+	TAILQ_INSERT_HEAD(&pm->pm_pvlist, pv, pv_plist);
+}
+
+static void
+pmap_remove_section(pmap_t pm, vm_offset_t va, vm_page_t m)
 {
 	struct pv_entry *pv;
 
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-	TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list)
-	    if (pm == pv->pv_pmap && va == pv->pv_va)
-		    break;
-	return (pv);
+	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
+		if (pv->pv_va == va && pv->pv_pmap == pm)
+			break;
+	/* We can ending up having no PV for unmanaged or fictitious pages. */
+	if (!pv)
+		return;
+	pv->pv_flags &= ~PVF_SECTION;
+	TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist);
+	if (pm->pm_firstpv)
+		TAILQ_INSERT_AFTER(&pm->pm_pvlist, pm->pm_firstpv,
+		    pv, pv_plist);
+	else {
+		TAILQ_INSERT_TAIL(&pm->pm_pvlist, pv, pv_plist);
+		pm->pm_firstpv = pv;
+	}
 }
 
 /*
@@ -1792,11 +2465,15 @@
 static void
 pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve)
 {
+	struct pv_entry *tmppve;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	PMAP_ASSERT_LOCKED(pm);
 	TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list);
+	tmppve = TAILQ_NEXT(pve, pv_plist);
 	TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist);
+	if (pm->pm_firstpv == pve)
+		pm->pm_firstpv = tmppve;
 	if (pve->pv_flags & PVF_WIRED)
 		--pm->pm_stats.wired_count;
 	pg->md.pv_list_count--;
@@ -1843,6 +2520,22 @@
 
 	return(pve);				/* return removed pve */
 }
+
+static int
+pmap_clearbit_pv_range(pmap_t pm, vm_offset_t sva, vm_offset_t eva, int bits)
+{
+	struct pv_entry *pve;
+	int ret = 0;
+
+	TAILQ_FOREACH(pve, &pm->pm_pvlist, pv_plist) {
+		if (pve->pv_va >= sva && pve->pv_va < eva) {
+			ret |= pve->pv_flags;
+			pve->pv_flags &= ~bits;
+		}
+	}
+	return (ret);
+}
+
 /*
  *
  * pmap_modify_pv: Update pv flags
@@ -1985,16 +2678,38 @@
 {
 	struct l2_dtable *l2;
 	struct l2_bucket *l2b;
-	pd_entry_t *pl1pd, l1pd;
+	pd_entry_t *pl1pd;
 	pt_entry_t *ptep, pte;
 	vm_paddr_t pa;
 	u_int l1idx;
+	struct pv_entry *pv;
+	struct vm_page *pg;
 	int rv = 0;
 
 	l1idx = L1_IDX(va);
 	vm_page_lock_queues();
 	PMAP_LOCK(pm);
+	PMAP_LOCK_L1(pm);
+	rv = pmap_fixup_l1_entry(pm, va);
+	if (rv == 1) {
+		PMAP_UNLOCK_L1(pm);
+		goto out;
+	}
+	pl1pd = &pm->pm_l1->l1_kva[l1idx];
+	if (l1pte_section_p(*pl1pd)) {
+		if (user && !(*pl1pd & L1_S_PROT_U))
+			goto out;
+		pg = PHYS_TO_VM_PAGE((*pl1pd & L1_S_FRAME) + 
+		    (va & L1_S_OFFSET));
+		pv = pmap_find_pv(pg, pm, va);
+		if (!pv || (ftype == VM_PROT_WRITE &&
+		    !(pv->pv_flags & PVF_WRITE)))
+			goto out;
+		rv = 1;
+		goto out;
+	}
 
+	PMAP_UNLOCK_L1(pm);
 	/*
 	 * If there is no l2_dtable for this address, then the process
 	 * has no business accessing it.
@@ -2024,31 +2739,30 @@
 	/*
 	 * Catch a userland access to the vector page mapped at 0x0
 	 */
-	if (user && (pte & L2_S_PROT_U) == 0)
+	if (user && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+	    (pte & L2_L_PROT_U) == 0) || (pte & L2_S_PROT_U) == 0))
 		goto out;
 	if (va == vector_page)
 		goto out;
 
-	pa = l2pte_pa(pte);
+	pa = (pte & L2_TYPE_MASK) == L2_TYPE_L ? 
+	    ((pte & L2_L_FRAME) | (va & L2_L_OFFSET)) : l2pte_pa(pte);
 
-	if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) {
+	if ((ftype & VM_PROT_WRITE) && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+	    (pte & L2_L_PROT_W) == 0) || (pte & L2_S_PROT_W) == 0)) {
 		/*
 		 * This looks like a good candidate for "page modified"
 		 * emulation...
 		 */
-		struct pv_entry *pv;
-		struct vm_page *pg;
 
 		/* Extract the physical address of the page */
-		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) {
+		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
 			goto out;
-		}
 		/* Get the current flags for this page. */
 
 		pv = pmap_find_pv(pg, pm, va);
-		if (pv == NULL) {
+		if (pv == NULL)
 			goto out;
-		}
 
 		/*
 		 * Do the flags say this page is writable? If not then it
@@ -2057,9 +2771,8 @@
 		 * PTE. Now we know a write has occurred we can correct this
 		 * and also set the modified bit
 		 */
-		if ((pv->pv_flags & PVF_WRITE) == 0) {
+		if ((pv->pv_flags & PVF_WRITE) == 0)
 			goto out;
-		}
 
 		pg->md.pvh_attrs |= PVF_REF | PVF_MOD;
 		vm_page_dirty(pg);
@@ -2072,8 +2785,14 @@
 		 * changing. We've already set the cacheable bits based on
 		 * the assumption that we can write to this page.
 		 */
-		*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W;
+		*ptep = pte | L2_S_PROTO | L2_S_PROT_W;
 		PTE_SYNC(ptep);
+		if (pv->pv_flags & PVF_DEMOTED) {
+			PMAP_LOCK_L1(pm);
+			pmap_promote_small(pm, l2b, va);
+			pmap_promote_large(pm, va);
+			PMAP_UNLOCK_L1(pm);
+		}
 		rv = 1;
 	} else
 	if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) {
@@ -2096,24 +2815,17 @@
 		pg->md.pvh_attrs |= PVF_REF;
 		pv->pv_flags |= PVF_REF;
 
-
 		*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO;
+		if (pv->pv_flags & PVF_DEMOTED) {
+			PMAP_LOCK_L1(pm);
+			pmap_promote_small(pm, l2b, va);
+			pmap_promote_large(pm, va);
+			PMAP_UNLOCK_L1(pm);
+		}
 		PTE_SYNC(ptep);
 		rv = 1;
 	}
 
-	/*
-	 * We know there is a valid mapping here, so simply
-	 * fix up the L1 if necessary.
-	 */
-	pl1pd = &pm->pm_l1->l1_kva[l1idx];
-	l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO;
-	if (*pl1pd != l1pd) {
-		*pl1pd = l1pd;
-		PTE_SYNC(pl1pd);
-		rv = 1;
-	}
-
 #ifdef CPU_SA110
 	/*
 	 * There are bugs in the rev K SA110.  This is a check for one
@@ -2241,6 +2953,57 @@
 #endif
 }
 
+/* Fixup a L1 Section entry if it belongs to an another process. */
+static int
+pmap_fixup_l1_entry(pmap_t pm, vm_offset_t va)
+{
+	struct pv_entry *pv;
+	pd_entry_t *pl1pd;
+	u_int l1idx = L1_IDX(va);
+	int ret = 0;
+	
+	pl1pd = &pm->pm_l1->l1_kva[l1idx];
+	if (l1pte_section_p(*pl1pd) && (*pl1pd & L1_S_DOM_MASK) 
+	    == L1_S_DOM(pm->pm_domain))
+		return (0);
+	else if (l1pte_page_p(*pl1pd) && (*pl1pd & L1_C_DOM_MASK)
+	    == L1_C_DOM(pm->pm_domain))
+		return (0);
+	pv = pmap_find_superpage(pm, va & L1_S_ADDR_MASK);
+	if (pv) {
+		*pl1pd = VM_PAGE_TO_PHYS(pv->pv_page) |
+		    L1_S_PROTO | L1_S_PROT_U |
+		    pte_l1_s_cache_mode | L1_S_DOM(pm->pm_domain);
+		if (pv->pv_flags & PVF_WRITE)
+			*pl1pd |= L1_S_PROT_W;
+		PTE_SYNC(pl1pd);
+		pmap_tlb_flushID_SE(pm, va & L1_S_ADDR_MASK);
+		ret = 1;
+	} else {
+		struct l2_dtable *l2;
+		struct l2_bucket *l2b;
+		u_int l1idx;
+
+		
+		l1idx = L1_IDX(va);
+		l2 = pm->pm_l2[L2_IDX(l1idx)];
+		if (l2) {
+			l2b = &l2->l2_bucket[L2_BUCKET(l1idx)];
+			if (l2b && l2b->l2b_kva) {
+				*pl1pd = l2b->l2b_phys | 
+				    L1_C_DOM(pm->pm_domain) | L1_C_PROTO;
+				pmap_tlb_flushID_SE(pm, va);
+				ret = 1;
+			}
+		} 
+		PTE_SYNC(pl1pd);
+	}
+	if (l1pte_section_p(*pl1pd) && (*pl1pd & L1_S_DOM_MASK)
+		    != L1_S_DOM(pm->pm_domain))
+		*pl1pd = 0;
+	return (ret);
+}
+
 /*
  * This is used to stuff certain critical values into the PCB where they
  * can be accessed quickly from cpu_switch() et al.
@@ -2389,9 +3152,6 @@
  *	(physical) address starting relative to 0]
  */
 #define PMAP_STATIC_L2_SIZE 16
-#ifdef ARM_USE_SMALL_ALLOC
-extern struct mtx smallalloc_mtx;
-#endif
 
 void
 pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt)
@@ -2416,6 +3176,7 @@
 	kernel_pmap = &kernel_pmap_store;
 	kernel_pmap->pm_l1 = l1;
 	kernel_l1pa = l1pt->pv_pa;
+	mtx_init(&l1->l1_mtx, "L1 mtx", NULL, MTX_DEF | MTX_RECURSE);
 	
 	/*
 	 * Scan the L1 translation table created by initarm() and create
@@ -2568,9 +3329,6 @@
 {
 	struct pcb *pcb;
 	
-	pmap_idcache_wbinv_all(pmap);
-	pmap_tlb_flushID(pmap);
-	cpu_cpwait();
 	if (vector_page < KERNBASE) {
 		struct pcb *curpcb = PCPU_GET(curpcb);
 		pcb = thread0.td_pcb;
@@ -2776,22 +3534,82 @@
 {
 	struct pv_entry *pv, *npv;
 	struct l2_bucket *l2b = NULL;
+	pt_entry_t *pt;
+	pd_entry_t *pd;
 	vm_page_t m;
-	pt_entry_t *pt;
-	
+	int i;
+
 	vm_page_lock_queues();
 	PMAP_LOCK(pmap);
 	for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
-		if (pv->pv_flags & PVF_WIRED) {
+		if (pv->pv_flags & PVF_WIRED) {                  
 			/* The page is wired, cannot remove it now. */
 			npv = TAILQ_NEXT(pv, pv_plist);
 			continue;
 		}
+			    
+		PMAP_LOCK_L1(pmap);
+		pmap_fixup_l1_entry(pmap, pv->pv_va);
+		pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+		if (l1pte_section_p(*pd)) {
+			struct l1_ttable *l1;
+
+			for (i = 0; i < 0x100; i++) {
+				m = PHYS_TO_VM_PAGE(
+				    (*pd & L1_S_ADDR_MASK) +
+				    i * PAGE_SIZE);
+				if (TAILQ_EMPTY(&m->md.pv_list))
+					vm_page_flag_clear(m,
+					    PG_WRITEABLE);
+			}
+			*pd = 0;
+			npv = TAILQ_NEXT(pv, pv_plist);
+			pmap_nuke_pv(m, pmap, pv);
+			if (TAILQ_EMPTY(&m->md.pv_list))
+				vm_page_flag_clear(m, PG_WRITEABLE);
+			pmap_free_pv_entry(pv);
+
+			if (pmap == pmap_kernel()) {
+				SLIST_FOREACH(l1, &l1_list,
+				    l1_link) {
+					l1->l1_kva[
+					    L1_IDX(pv->pv_va &
+						L1_S_ADDR_MASK)] = 0;
+					PTE_SYNC(
+					    &l1->l1_kva[L1_IDX(
+					    pv->pv_va &
+					    L1_S_ADDR_MASK)]);
+				}
+			}
+			PMAP_UNLOCK_L1(pmap);
+			continue;
+		}
+		PMAP_UNLOCK_L1(pmap);
 		pmap->pm_stats.resident_count--;
 		l2b = pmap_get_l2_bucket(pmap, pv->pv_va);
 		KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages"));
 		pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
-		m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK);
+		if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) {
+			pt = &l2b->l2b_kva[l2pte_index(pv->pv_va &
+			    L2_L_FRAME)];
+			for (int i = 0; i < 0x10; i++) {
+				m = PHYS_TO_VM_PAGE((pt[i] &
+				    L2_L_FRAME) + i * PAGE_SIZE);
+				if (TAILQ_EMPTY(&m->md.pv_list))
+					vm_page_flag_clear(m,
+					    PG_WRITEABLE);
+				pt[i] = 0;
+			}
+			pmap_free_l2_bucket(pmap, l2b, 0x10);
+			npv = TAILQ_NEXT(pv, pv_plist);
+			pmap_nuke_pv(m, pmap, pv);
+			if (TAILQ_EMPTY(&m->md.pv_list))
+				vm_page_flag_clear(m, PG_WRITEABLE);
+			pmap_free_pv_entry(pv);
+
+			continue;
+		}
+		m = pv->pv_page;
 #ifdef ARM_USE_SMALL_ALLOC
 		KASSERT((vm_offset_t)m >= alloc_firstaddr, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt));
 #else
@@ -3052,7 +3870,6 @@
 	}
 }
 
-
 /*
  * pmap_object_init_pt preloads the ptes for a given object
  * into the specified pmap.  This eliminates the blast of soft
@@ -3083,7 +3900,8 @@
 
 	if (!pmap_get_pde_pte(pmap, addr, &pde, &pte))
 		return (FALSE);
-	KASSERT(pte != NULL, ("Valid mapping but no pte ?"));
+	if (!pte)
+		return (FALSE);
 	if (*pte == 0)
 		return (TRUE);
 	return (FALSE);
@@ -3116,14 +3934,17 @@
 		return (FALSE);
 
 	l1idx = L1_IDX(va);
+	PMAP_LOCK_L1(pm);
+	pmap_fixup_l1_entry(pm, va);
 	*pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx];
 	l1pd = *pl1pd;
 
 	if (l1pte_section_p(l1pd)) {
+		PMAP_UNLOCK_L1(pm);
 		*ptp = NULL;
 		return (TRUE);
 	}
-
+	PMAP_UNLOCK_L1(pm);
 	if (pm->pm_l2 == NULL)
 		return (FALSE);
 
@@ -3154,6 +3975,7 @@
 pmap_remove_all(vm_page_t m)
 {
 	pv_entry_t pv;
+	pd_entry_t *pd;
 	pt_entry_t *ptep, pte;
 	struct l2_bucket *l2b;
 	boolean_t flush = FALSE;
@@ -3174,14 +3996,33 @@
 		return;
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	curpm = vmspace_pmap(curproc->p_vmspace);
+redo:
 	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 		if (flush == FALSE && (pv->pv_pmap == curpm ||
 		    pv->pv_pmap == pmap_kernel()))
 			flush = TRUE;
 		PMAP_LOCK(pv->pv_pmap);
+		PMAP_LOCK_L1(pv->pv_pmap);
+		pmap_fixup_l1_entry(pv->pv_pmap, pv->pv_va);
+		pd = &pv->pv_pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+		/* Demotion mess up with the PV list. */
+		if (l1pte_section_p(*pd)) {
+			pmap_t pm = pv->pv_pmap;
+			pmap_demote(pv->pv_pmap, pv->pv_va);
+			PMAP_UNLOCK_L1(pm);
+			PMAP_UNLOCK(pm);
+			goto redo;
+		}
+		PMAP_UNLOCK_L1(pv->pv_pmap);
 		l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+		
 		KASSERT(l2b != NULL, ("No l2 bucket"));
 		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+			pmap_demote(pv->pv_pmap, pv->pv_va);
+			PMAP_UNLOCK(pv->pv_pmap);
+			goto redo;
+		}
 		pte = *ptep;
 		*ptep = 0;
 		PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
@@ -3217,6 +4058,9 @@
 	vm_offset_t next_bucket;
 	u_int flags;
 	int flush;
+	int demoted = 0;
+	int i;
+	pd_entry_t *pd;
 
 	if ((prot & VM_PROT_READ) == 0) {
 		pmap_remove(pm, sva, eva);
@@ -3244,10 +4088,53 @@
 	flags = 0;
 
 	while (sva < eva) {
+		demoted = 0;
 		next_bucket = L2_NEXT_BUCKET(sva);
 		if (next_bucket > eva)
 			next_bucket = eva;
+		PMAP_LOCK_L1(pm);
+		pmap_fixup_l1_entry(pm, sva);
+
+		if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(sva)])) {
+			if ((sva & L1_S_OFFSET) == 0 &&
+			    sva + L1_S_SIZE < eva) {
+				struct l1_ttable *l1;
 
+				/* Change the whole 1MB superpage. */
+				pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
+				if (*pd & L1_S_PROT_W) {
+					*pd &= ~L1_S_PROT_W;
+					PTE_SYNC(*pd);
+					flush++;
+					flags |= pmap_clearbit_pv_range(pm, sva,
+					    sva + L1_S_SIZE, PVF_WRITE);
+					for (i = 0; i < 0x100; i++) {
+						vm_page_t m =
+						    PHYS_TO_VM_PAGE(
+							(*pd & L1_S_FRAME) +
+							i * PAGE_SIZE);
+						pmap_vac_me_harder(m, pm,
+						    sva + i * PAGE_SIZE);
+						vm_page_dirty(m);
+					}
+				}
+				if (pm == pmap_kernel()) {
+					SLIST_FOREACH(l1, &l1_list, l1_link) {
+						l1->l1_kva[L1_IDX(sva)] =
+						    *pd;
+						PTE_SYNC(&l1->l1_kva[L1_IDX(sva)]);
+					}
+				}
+				PMAP_UNLOCK_L1(pm);
+				sva += L1_S_SIZE;
+				continue;
+			}
+			pmap_demote(pm, sva);
+			flush = 1;
+			l2b = pmap_get_l2_bucket(pm, sva);
+			demoted = 1;
+		}
+		PMAP_UNLOCK_L1(pm);
 		l2b = pmap_get_l2_bucket(pm, sva);
 		if (l2b == NULL) {
 			sva = next_bucket;
@@ -3257,6 +4144,37 @@
 		ptep = &l2b->l2b_kva[l2pte_index(sva)];
 
 		while (sva < next_bucket) {
+			demoted = 0;
+			if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+				if ((sva & L2_L_OFFSET) == 0 &&
+				    sva + L2_L_SIZE < eva) {
+					if (*ptep & L2_L_PROT_W) {
+						flags |= pmap_clearbit_pv_range(
+						    pm, sva, 
+						    sva + L2_L_SIZE, PVF_WRITE);
+						for (i = 0; i < 0x10; i++) {
+							vm_page_t m;
+
+							m = PHYS_TO_VM_PAGE(
+							    (ptep[i] & 
+							    L2_L_FRAME) + 
+							    i * PAGE_SIZE);
+							ptep[i] &= ~L2_L_PROT_W;
+							PTE_SYNC(&ptep[i]);
+							pmap_vac_me_harder(m,
+							    pm, sva + 
+							    i * PAGE_SIZE);
+							vm_page_dirty(m);
+						}
+						flush++;
+					}
+					sva += L2_L_SIZE;
+					continue;
+				} else {
+					pmap_demote(pm, sva);
+					demoted = 1;
+				}
+			}
 			if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) {
 				struct vm_page *pg;
 				u_int f;
@@ -3285,6 +4203,12 @@
 					pmap_tlb_flushD_SE(pm, sva);
 			}
 
+			if (demoted) {
+				PMAP_LOCK_L1(pm);
+				pmap_promote_small(pm, l2b, sva);
+				pmap_promote_large(pm, sva);
+				PMAP_UNLOCK_L1(pm);
+			}
 			sva += PAGE_SIZE;
 			ptep++;
 		}
@@ -3303,7 +4227,6 @@
  	PMAP_UNLOCK(pm);
 }
 
-
 /*
  *	Insert the given physical page (p) at
  *	the specified virtual address (v) in the
@@ -3318,8 +4241,8 @@
  */
 
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-    boolean_t wired)
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, vm_page_t m, 
+    vm_prot_t prot, boolean_t wired)
 {
 
 	vm_page_lock_queues();
@@ -3337,15 +4260,56 @@
     boolean_t wired, int flags)
 {
 	struct l2_bucket *l2b = NULL;
-	struct vm_page *opg;
+	struct vm_page *opg = NULL;
 	struct pv_entry *pve = NULL;
 	pt_entry_t *ptep, npte, opte;
+	pd_entry_t l1pd;
 	u_int nflags;
 	u_int oflags;
 	vm_paddr_t pa;
 
 	PMAP_ASSERT_LOCKED(pmap);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_L1(pmap);
+	pmap_fixup_l1_entry(pmap, va);
+	l1pd = pmap->pm_l1->l1_kva[L1_IDX(va)];
+	if (l1pte_section_p(l1pd) &&
+	    ((l1pd & L1_S_FRAME) != (VM_PAGE_TO_PHYS(m) & L1_S_FRAME) ||
+	     (((VM_PAGE_TO_PHYS(m) & L2_S_FRAME) & L2_S_OFFSET) !=
+	      (va & L2_S_OFFSET)) ||
+	     ((l1pd & L1_S_PROT_U) != 0) != (pmap != pmap_kernel() && va !=
+		vector_page) || ((l1pd & L1_S_PROT_W) != 0) !=
+	     ((prot & VM_PROT_WRITE) != 0))) {
+		/* There's already a superpage at this address, demote. */
+		pmap_demote(pmap, va);
+		pmap_tlb_flushID_SE(pmap, va & L1_S_ADDR_MASK);
+	} else if (l1pte_section_p(l1pd)) {
+		opg = m;
+		l2b = NULL;
+	}
+	PMAP_UNLOCK_L1(pmap);
+	
+	if (opg == NULL) {
+		if (pmap == pmap_kernel()) {
+			l2b = pmap_get_l2_bucket(pmap, va);
+			if (l2b == NULL)
+				l2b = pmap_grow_l2_bucket(pmap, va);
+		} else {
+do_l2b_alloc:
+			l2b = pmap_alloc_l2_bucket(pmap, va, M_NOWAIT);
+			if (l2b == NULL) {
+				if (flags & M_WAITOK) {
+					PMAP_UNLOCK(pmap);
+					vm_page_unlock_queues();
+					VM_WAIT;
+					vm_page_lock_queues();
+					PMAP_LOCK(pmap);
+					goto do_l2b_alloc;
+				}
+				return;
+			}
+		}
+	}
 	if (va == vector_page) {
 		pa = systempage.pv_pa;
 		m = NULL;
@@ -3361,43 +4325,39 @@
 	PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, "
 	    "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired));
 	    
-	if (pmap == pmap_kernel()) {
-		l2b = pmap_get_l2_bucket(pmap, va);
-		if (l2b == NULL)
-			l2b = pmap_grow_l2_bucket(pmap, va);
+	if (l2b != NULL) {
+		ptep = &l2b->l2b_kva[l2pte_index(va)];
+		opte = *ptep;
+		npte = pa; 
+		oflags = 0;
 	} else {
-do_l2b_alloc:
-		l2b = pmap_alloc_l2_bucket(pmap, va);
-		if (l2b == NULL) {
-			if (flags & M_WAITOK) {
-				PMAP_UNLOCK(pmap);
-				vm_page_unlock_queues();
-				VM_WAIT;
-				vm_page_lock_queues();
-				PMAP_LOCK(pmap);
-				goto do_l2b_alloc;
+		npte = oflags = opte = 0;
+		ptep = NULL;
+	}
+	if (opte) {
+		if ((opte & L2_TYPE_MASK) == L2_TYPE_L) {
+			if (((opte & L2_L_FRAME) | (va & L2_L_OFFSET)) ==
+			    VM_PAGE_TO_PHYS(m) &&
+			    (((opte & L2_L_PROT_W) != 0) ==
+			    (prot & VM_PROT_WRITE) != 0) &&
+			    (((opte & L2_L_PROT_U) == 0) ==
+			     (pmap == pmap_kernel() || va == vector_page)))
+				opg = m;
+			else {
+				pmap_demote(pmap, va);
+				opte = *ptep;
 			}
-			return;
 		}
-	}
-
-	ptep = &l2b->l2b_kva[l2pte_index(va)];
-		    
-	opte = *ptep;
-	npte = pa;
-	oflags = 0;
-	if (opte) {
 		/*
 		 * There is already a mapping at this address.
 		 * If the physical address is different, lookup the
 		 * vm_page.
 		 */
-		if (l2pte_pa(opte) != pa)
+		if (!opg && l2pte_pa(opte) != pa)
 			opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
-		else
+		else if (!opg)
 			opg = m;
-	} else
-		opg = NULL;
+	}
 
 	if ((prot & (VM_PROT_ALL)) ||
 	    (!m || m->md.pvh_attrs & PVF_REF)) {
@@ -3516,17 +4476,19 @@
 	/*
 	 * Keep the stats up to date
 	 */
-	if (opte == 0) {
+	if (opte == 0 && l2b) {
 		l2b->l2b_occupancy++;
 		pmap->pm_stats.resident_count++;
 	} 
 
+	PMAP_LOCK_L1(pmap);
+	pmap_fixup_l1_entry(pmap, va);
 
 	/*
 	 * If this is just a wiring change, the two PTEs will be
 	 * identical, so there's no need to update the page table.
 	 */
-	if (npte != opte) {
+	if (l2b && npte != opte && ((opte & L2_TYPE_MASK) != L2_TYPE_L)) {
 		boolean_t is_cached = pmap_is_current(pmap);
 
 		*ptep = npte;
@@ -3564,7 +4526,18 @@
 
 		if (m)
 			pmap_vac_me_harder(m, pmap, va);
+		if (m && ((va < VM_MAXUSER_ADDRESS ||
+		    m->object == kernel_object || 
+		    (m->object == kmem_object)) && m->reserv)) {
+			if (m->reserv->refcnt > 0 &&
+			    m->reserv->refcnt != SP_LARGE &&
+			    (m->reserv->refcnt % SP_SMALL) == 0)
+				pmap_promote_small(pmap, l2b, va);
+			else if (m->reserv->refcnt == SP_LARGE)
+				pmap_promote_large(pmap, va);
+		}
 	}
+	PMAP_UNLOCK_L1(pmap);
 }
 
 /*
@@ -3626,17 +4599,11 @@
 void
 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 {
-	struct l2_bucket *l2b;
-	pt_entry_t *ptep, pte;
 	vm_page_t pg;
 
 	vm_page_lock_queues();
  	PMAP_LOCK(pmap);
-	l2b = pmap_get_l2_bucket(pmap, va);
-	KASSERT(l2b, ("No l2b bucket in pmap_change_wiring"));
-	ptep = &l2b->l2b_kva[l2pte_index(va)];
-	pte = *ptep;
-	pg = PHYS_TO_VM_PAGE(l2pte_pa(pte));
+	pg = PHYS_TO_VM_PAGE(pmap_extract_locked(pmap, va));
 	if (pg) 
 		pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired);
 	vm_page_unlock_queues();
@@ -3664,8 +4631,8 @@
  *		Extract the physical page address associated
  *		with the given map/virtual_address pair.
  */
-vm_paddr_t
-pmap_extract(pmap_t pm, vm_offset_t va)
+static vm_paddr_t
+pmap_extract_locked(pmap_t pm, vm_offset_t va)
 {
 	struct l2_dtable *l2;
 	pd_entry_t l1pd;
@@ -3674,15 +4641,13 @@
 	u_int l1idx;
 	l1idx = L1_IDX(va);
 
-	PMAP_LOCK(pm);
+	PMAP_LOCK_L1(pm);
+	pmap_fixup_l1_entry(pm, va);
 	l1pd = pm->pm_l1->l1_kva[l1idx];
-	if (l1pte_section_p(l1pd)) {
-		/*
-		 * These should only happen for pmap_kernel()
-		 */
-		KASSERT(pm == pmap_kernel(), ("huh"));
+	PMAP_UNLOCK_L1(pm);
+	if (l1pte_section_p(l1pd))
 		pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
-	} else {
+	else {
 		/*
 		 * Note that we can't rely on the validity of the L1
 		 * descriptor as an indication that a mapping exists.
@@ -3691,18 +4656,14 @@
 		l2 = pm->pm_l2[L2_IDX(l1idx)];
 
 		if (l2 == NULL ||
-		    (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) {
-			PMAP_UNLOCK(pm);
+		    (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL)
 			return (0);
-		}
 
 		ptep = &ptep[l2pte_index(va)];
 		pte = *ptep;
 
-		if (pte == 0) {
-			PMAP_UNLOCK(pm);
+		if (pte == 0)
 			return (0);
-		}
 
 		switch (pte & L2_TYPE_MASK) {
 		case L2_TYPE_L:
@@ -3714,11 +4675,19 @@
 			break;
 		}
 	}
+	return (pa);
+}
 
+vm_paddr_t
+pmap_extract(pmap_t pm, vm_offset_t va)
+{
+	vm_paddr_t ret;
+
+	PMAP_LOCK(pm);
+	ret = pmap_extract_locked(pm, va);
 	PMAP_UNLOCK(pm);
-	return (pa);
+	return (ret);
 }
-
 /*
  * Atomically extract and hold the physical page with the given
  * pmap and virtual address pair if that mapping permits the given
@@ -3738,19 +4707,19 @@
 
 	vm_page_lock_queues();
  	PMAP_LOCK(pmap);
+	PMAP_LOCK_L1(pmap);
+	pmap_fixup_l1_entry(pmap, va);
 	l1pd = pmap->pm_l1->l1_kva[l1idx];
 	if (l1pte_section_p(l1pd)) {
-		/*
-		 * These should only happen for pmap_kernel()
-		 */
-		KASSERT(pmap == pmap_kernel(), ("huh"));
 		pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
 		if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
 			m = PHYS_TO_VM_PAGE(pa);
 			vm_page_hold(m);
 		}
+		PMAP_UNLOCK_L1(pmap);
 			
 	} else {
+		PMAP_UNLOCK_L1(pmap);
 		/*
 		 * Note that we can't rely on the validity of the L1
 		 * descriptor as an indication that a mapping exists.
@@ -3814,8 +4783,8 @@
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 	pmap->pm_stats.resident_count = 1;
 	if (vector_page < KERNBASE) {
-		pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa),
-		    VM_PROT_READ, 1);
+		pmap_enter(pmap, vector_page, 0, 
+		    PHYS_TO_VM_PAGE(systempage.pv_pa), VM_PROT_READ, 1);
 	} 
 }
 
@@ -3866,6 +4835,8 @@
 	vm_offset_t next_bucket;
 	pt_entry_t *ptep;
 	u_int cleanlist_idx, total, cnt;
+	vm_page_t m;
+	struct pv_entry *pve;
 	struct {
 		vm_offset_t va;
 		pt_entry_t *pte;
@@ -3894,12 +4865,48 @@
 		if (next_bucket > eva)
 			next_bucket = eva;
 
+		PMAP_LOCK_L1(pm);
+		pmap_fixup_l1_entry(pm, sva);
 		l2b = pmap_get_l2_bucket(pm, sva);
 		if (l2b == NULL) {
-			sva = next_bucket;
-			continue;
+			pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
+			if (l1pte_section_p(*pd)) {
+				struct l1_ttable *l1;
+				/* We can just remove the superpage. */
+				flushall = 1;
+				if ((sva == (sva & L1_S_ADDR_MASK)) &&
+				    (sva + L1_S_SIZE < eva)) {
+					m = PHYS_TO_VM_PAGE(*pd & L1_S_FRAME);
+					pve = pmap_find_pv(m, pm, sva);
+					if (pve) {
+						pmap_nuke_pv(m, pm, pve);
+						pmap_free_pv_entry(pve);
+					}
+					*pd = 0;
+					sva += L1_S_SIZE;
+					if (pm == pmap_kernel()) {
+						SLIST_FOREACH(l1, &l1_list,
+						    l1_link) {
+							l1->l1_kva[L1_IDX(sva)] =
+							    0;
+							PTE_SYNC(&l1->l1_kva[L1_IDX(sva)]);
+						}
+					}
+					PMAP_UNLOCK_L1(pm);
+					continue;
+				} else {
+					pmap_demote(pm, sva);
+					PMAP_UNLOCK_L1(pm);
+					l2b = pmap_get_l2_bucket(pm, sva);
+				}
+			} else {
+				PMAP_UNLOCK_L1(pm);
+				sva = next_bucket;
+				continue;
+			}
 		}
 
+		PMAP_UNLOCK_L1(pm);
 		ptep = &l2b->l2b_kva[l2pte_index(sva)];
 		mappings = 0;
 
@@ -3908,6 +4915,28 @@
 			pt_entry_t pte;
 			vm_paddr_t pa;
 
+			if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+			/* 64KB superpage. */
+				if ((sva & L2_L_FRAME) == sva &&
+				    sva + L2_L_SIZE < eva) {
+					m = PHYS_TO_VM_PAGE(*ptep &
+					    L2_L_FRAME);
+					
+					pve = pmap_find_pv(m, pm, sva);
+					if (pve) {
+						pmap_nuke_pv(m, pm, pve);
+						pmap_free_pv_entry(pve);
+					}
+					for (int i = 0; i < 0x10; i++)
+						ptep[i] = 0;
+					sva += L2_L_SIZE;
+					ptep += 0x10;
+					flushall = 1;
+					mappings += 0x10;
+					continue;
+				}
+				pmap_demote(pm, sva);
+			}
 			pte = *ptep;
 
 			if (pte == 0) {
@@ -3929,15 +4958,13 @@
 			 * we could cluster a lot of these and do a
 			 * number of sequential pages in one go.
 			 */
-			if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
-				struct pv_entry *pve;
+			pg = PHYS_TO_VM_PAGE(pa);
 
-				pve = pmap_remove_pv(pg, pm, sva);
-				if (pve) {
-					is_exec = PV_BEEN_EXECD(pve->pv_flags);
-					is_refd = PV_BEEN_REFD(pve->pv_flags);
-					pmap_free_pv_entry(pve);
-				}
+			pve = pmap_remove_pv(pg, pm, sva);
+			if (pve) {
+				is_exec = PV_BEEN_EXECD(pve->pv_flags);
+				is_refd = PV_BEEN_REFD(pve->pv_flags);
+				pmap_free_pv_entry(pve);
 			}
 
 			if (!l2pte_valid(pte)) {
@@ -3958,8 +4985,7 @@
 			} else
 			if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) {
 				/* Nuke everything if needed. */
-				pmap_idcache_wbinv_all(pm);
-				pmap_tlb_flushID(pm);
+				flushall = 1;
 
 				/*
 				 * Roll back the previous PTE list,
@@ -3972,14 +4998,13 @@
 				*ptep = 0;
 				PTE_SYNC(ptep);
 				cleanlist_idx++;
-				flushall = 1;
 			} else {
 				*ptep = 0;
 				PTE_SYNC(ptep);
-					if (is_exec)
-						pmap_tlb_flushID_SE(pm, sva);
-					else
-					if (is_refd)
+				if (!flushall && is_exec)
+					pmap_tlb_flushID_SE(pm, sva);
+				else
+					if (!flushall && is_refd)
 						pmap_tlb_flushD_SE(pm, sva);
 			}
 
@@ -3996,15 +5021,17 @@
 			for (cnt = 0; cnt < cleanlist_idx; cnt++) {
 				vm_offset_t clva =
 				    cleanlist[cnt].va & ~1;
-				if (cleanlist[cnt].va & 1) {
-					pmap_idcache_wbinv_range(pm,
-					    clva, PAGE_SIZE);
-					pmap_tlb_flushID_SE(pm, clva);
-				} else {
-					pmap_dcache_wb_range(pm,
-					    clva, PAGE_SIZE, TRUE,
-					    FALSE);
-					pmap_tlb_flushD_SE(pm, clva);
+				if (!flushall) {
+					if (cleanlist[cnt].va & 1) {
+						pmap_idcache_wbinv_range(pm,
+						    clva, PAGE_SIZE);
+						pmap_tlb_flushID_SE(pm, clva);
+					} else {
+						pmap_dcache_wb_range(pm,
+						    clva, PAGE_SIZE, TRUE,
+						    FALSE);
+						pmap_tlb_flushD_SE(pm, clva);
+					}
 				}
 				*cleanlist[cnt].pte = 0;
 				PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte);
@@ -4018,7 +5045,6 @@
 				 * easier to flush the whole cache.
 				 */
 				cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
-				pmap_idcache_wbinv_all(pm);
 				flushall = 1;
 			}
 		}
@@ -4026,9 +5052,11 @@
 		pmap_free_l2_bucket(pm, l2b, mappings);
 	}
 
+	if (flushall) {
+		pmap_idcache_wbinv_all(pm);
+		cpu_tlb_flushID();
+	}
 	vm_page_unlock_queues();
-	if (flushall)
-		cpu_tlb_flushID();
  	PMAP_UNLOCK(pm);
 }
 
@@ -4146,7 +5174,11 @@
 
 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 
+		if (!l2b)
+			panic("fuck2");
 		sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+			panic("bordel");
 
 		while (va < next_bucket) {
 			pte = *ptep;
@@ -4872,4 +5904,3 @@
 
 	return (NULL);
 }
-
---  arm/include/param.h.orig
+++  arm/include/param.h
@@ -90,6 +90,21 @@
 #define NBPDR		(1 << PDR_SHIFT)
 #define NPDEPG          (1 << (32 - PDR_SHIFT))
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#define	SP_LEVELS	2
+
+/*
+ * SP_SMALL_SHIFT is LOG2("The Smallest Superpage Size" / PAGE_SIZE).
+ */
+#define	SP_SMALL_SHIFT	4
+
+/*
+ * XXX
+ */
+#define	SP_FACTOR_SHIFT	4
+
 #ifndef KSTACK_PAGES
 #define KSTACK_PAGES    2
 #endif /* !KSTACK_PAGES */
---  arm/include/pmap.h.orig
+++  arm/include/pmap.h
@@ -136,7 +136,6 @@
  */
 #define	L2_LOG2		((32 - L1_S_SHIFT) - L2_BUCKET_LOG2)
 #define	L2_SIZE		(1 << L2_LOG2)
-
 struct	pmap {
 	struct mtx		pm_mtx;
 	u_int8_t		pm_domain;
@@ -146,6 +145,7 @@
 	int			pm_count;	/* reference count */
 	int			pm_active;	/* active on cpus */
 	struct pmap_statistics	pm_stats;	/* pmap statictics */
+	struct pv_entry		*pm_firstpv;	/* First pv not a section map */
 	TAILQ_HEAD(,pv_entry)	pm_pvlist;	/* list of mappings in pmap */
 };
 
@@ -178,6 +178,7 @@
 	TAILQ_ENTRY(pv_entry)   pv_list;
 	TAILQ_ENTRY(pv_entry)	pv_plist;
 	int		pv_flags;	/* flags (wired, etc...) */
+	vm_page_t	pv_page;	/* ASsociated bage. */
 } *pv_entry_t;
 
 #define PV_ENTRY_NULL   ((pv_entry_t) 0)
@@ -266,6 +267,7 @@
 #define	L1_S_CACHE_MASK_generic	(L1_S_B|L1_S_C)
 #define	L1_S_CACHE_MASK_xscale	(L1_S_B|L1_S_C|L1_S_XSCALE_TEX(TEX_XSCALE_X))
 
+
 #define	L2_L_PROT_U		(L2_AP(AP_U))
 #define	L2_L_PROT_W		(L2_AP(AP_W))
 #define	L2_L_PROT_MASK		(L2_L_PROT_U|L2_L_PROT_W)
@@ -494,6 +496,9 @@
 #define	PVF_EXEC	0x10		/* mapping is executable */
 #define	PVF_UNC		0x20		/* mapping is 'user' non-cacheable */
 #define	PVF_KNC		0x40		/* mapping is 'kernel' non-cacheable */
+#define PVF_DEMOTED	0x80		/* Page was demoted. */
+#define PVF_SECTION	0x100		/* Page is a section mapping. */
+#define PVF_LARGE	0x200		/* Page is a large page. */
 #define	PVF_NC		(PVF_UNC|PVF_KNC)
 
 void vector_page_setprot(int);
@@ -544,6 +549,7 @@
 void arm_unmap_nocache(void *, vm_size_t);
 
 extern vm_paddr_t dump_avail[];
+
 #endif	/* _KERNEL */
 
 #endif	/* !LOCORE */
---  arm/include/vmparam.h.orig
+++  arm/include/vmparam.h
@@ -67,6 +67,7 @@
 #define	VM_NFREEPOOL		2
 #define	VM_FREEPOOL_DEFAULT	0
 #define	VM_FREEPOOL_DIRECT	1
+#define	VM_FREEPOOL_RESERV	0
 
 /*
  * we support 2 free lists:
@@ -131,4 +132,8 @@
 #ifdef ARM_USE_SMALL_ALLOC
 #define UMA_MD_SMALL_ALLOC
 #endif /* ARM_USE_SMALL_ALLOC */
+
+#ifndef VM_IMAGE_BASE
+#define VM_IMAGE_BASE	0x8000
+#endif /* VM_IMAGE_BASE */
 #endif	/* _MACHINE_VMPARAM_H_ */
---  conf/files.orig
+++  conf/files
@@ -2079,6 +2079,7 @@
 vm/vm_pageq.c			standard
 vm/vm_pager.c			standard
 vm/vm_phys.c			standard
+vm/vm_reserv.c			standard
 vm/vm_unix.c			standard
 vm/vm_zeroidle.c		standard
 vm/vnode_pager.c		standard
---  conf/options.orig
+++  conf/options
@@ -549,6 +549,9 @@
 VFS_BIO_DEBUG		opt_global.h
 
 # These are VM related options
+SP_LEVELS		opt_vm.h
+SP_SMALL_SHIFT		opt_vm.h
+SP_FACTOR_SHIFT		opt_vm.h
 VM_KMEM_SIZE		opt_vm.h
 VM_KMEM_SIZE_SCALE	opt_vm.h
 VM_KMEM_SIZE_MAX	opt_vm.h
---  i386/i386/pmap.c.orig
+++  i386/i386/pmap.c
@@ -75,7 +75,7 @@
  */
 
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/i386/i386/pmap.c,v 1.592 2007/05/31 22:52:12 attilio Exp $");
+__FBSDID("$FreeBSD: src/sys/i386/i386/pmap.c,v 1.587 2007/04/13 16:07:29 alc Exp $");
 
 /*
  *	Manages physical address maps.
@@ -112,6 +112,7 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/ktr.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
@@ -135,6 +136,7 @@
 #include <vm/vm_extern.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_pager.h>
+#include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
 #include <machine/cpu.h>
@@ -175,6 +177,9 @@
 #define PV_STAT(x)	do { } while (0)
 #endif
 
+#define	pa_index(pa)	((pa) >> PDRSHIFT)
+#define	pa_to_pvh(pa)	(&pv_table[pa_index(pa)])
+
 /*
  * Get PDEs and PTEs for user/kernel address space
  */
@@ -214,6 +219,8 @@
  * Data for the pv entry allocation mechanism
  */
 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
+static int pv_npg;
+static struct md_page *pv_table;
 static int shpgperproc = PMAP_SHPGPERPROC;
 
 struct pv_chunk *pv_chunkbase;		/* KVA block for pv_chunks */
@@ -266,11 +273,27 @@
 
 static void	free_pv_entry(pmap_t pmap, pv_entry_t pv);
 static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
+static void	pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m);
+static void	pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static void	pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
+static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
+		    vm_offset_t va);
 
+static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
+    vm_prot_t prot);
 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
-static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
-    vm_page_t *free);
+static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
+static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
+static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
+    vm_prot_t prot);
+static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva);
+static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva);
+static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 					vm_offset_t va);
@@ -281,10 +304,10 @@
 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
 
 static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags);
-static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free);
+static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m);
 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
 static void pmap_pte_release(pt_entry_t *pte);
-static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *);
+static int pmap_unuse_pt(pmap_t, vm_offset_t);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 #ifdef PAE
 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
@@ -350,6 +373,7 @@
 #ifdef PAE
 	kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
 #endif
+	kernel_pmap->pm_root = NULL;
 	kernel_pmap->pm_active = -1;	/* don't allow deactivation */
 	TAILQ_INIT(&kernel_pmap->pm_pvchunk);
 	LIST_INIT(&allpmaps);
@@ -599,8 +623,24 @@
 void
 pmap_init(void)
 {
+	vm_page_t mpte;
+	vm_size_t s;
+	int i;
 
 	/*
+	 * Initialize the vm page array entries for the kernel pmap's
+	 * page table pages.
+	 */ 
+	for (i = 0; i < nkpt; i++) {
+		mpte = PHYS_TO_VM_PAGE(PTD[i + KPTDI] & PG_FRAME);
+		KASSERT(mpte >= vm_page_array &&
+		    mpte < &vm_page_array[vm_page_array_size],
+		    ("pmap_init: page table page is out of range"));
+		mpte->pindex = i + KPTDI;
+		mpte->phys_addr = PTD[i + KPTDI] & PG_FRAME;
+	}
+
+	/*
 	 * Initialize the address space (zone) for the pv entries.  Set a
 	 * high water mark so that the system can recover from excessive
 	 * numbers of pv entries.
@@ -611,6 +651,23 @@
 	pv_entry_max = roundup(pv_entry_max, _NPCPV);
 	pv_entry_high_water = 9 * (pv_entry_max / 10);
 
+	/*
+	 * Calculate the size of the pv head table for superpages.
+	 */
+	for (i = 0; phys_avail[i + 1]; i += 2);
+	pv_npg = round_4mpage(phys_avail[(i - 2) + 1]) / NBPDR;
+
+	/*
+	 * Allocate memory for the pv head table for superpages.
+	 */
+	s = (vm_size_t)(pv_npg * sizeof(struct md_page));
+	s = round_page(s);
+	pv_table = (struct md_page *)kmem_alloc(kernel_map, s);
+	for (i = 0; i < pv_npg; i++) {
+		TAILQ_INIT(&pv_table[i].pv_list);
+		pv_table[i].pv_list_count = 0;
+	}
+
 	pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
 	pv_chunkbase = (struct pv_chunk *)kmem_alloc_nofault(kernel_map,
 	    PAGE_SIZE * pv_maxchunks);
@@ -1130,16 +1187,73 @@
 /***************************************************
  * Page table page management routines.....
  ***************************************************/
-static PMAP_INLINE void
-pmap_free_zero_pages(vm_page_t free)
+
+/*
+ *
+ */
+static void
+pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
+{
+	vm_page_t root;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	root = pmap->pm_root;
+	if (root == NULL) {
+		mpte->left = NULL;
+		mpte->right = NULL;
+	} else {
+		root = vm_page_splay(mpte->pindex, root);
+		if (mpte->pindex < root->pindex) {
+			mpte->left = root->left;
+			mpte->right = root;
+			root->left = NULL;
+		} else if (mpte->pindex == root->pindex)
+			panic("pmap_insert_pt_page: pindex already inserted");
+		else {
+			mpte->right = root->right;
+			mpte->left = root;
+			root->right = NULL;
+		}
+	}
+	pmap->pm_root = mpte;
+}
+
+/*
+ *
+ */
+static vm_page_t
+pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+{
+	vm_page_t mpte;
+	vm_pindex_t pindex = va >> PDRSHIFT;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) {
+		mpte = vm_page_splay(pindex, mpte);
+		if ((pmap->pm_root = mpte)->pindex != pindex)
+			mpte = NULL;
+	}
+	return (mpte);
+}
+
+/*
+ *
+ */
+static void
+pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
 {
-	vm_page_t m;
+	vm_page_t root;
 
-	while (free != NULL) {
-		m = free;
-		free = m->right;
-		vm_page_free_zero(m);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	if (mpte != pmap->pm_root)
+		vm_page_splay(mpte->pindex, pmap->pm_root);
+	if (mpte->left == NULL)
+		root = mpte->right;
+	else {
+		root = vm_page_splay(mpte->pindex, mpte->left);
+		root->right = mpte->right;
 	}
+	pmap->pm_root = root;
 }
 
 /*
@@ -1147,18 +1261,18 @@
  * drops to zero, then it decrements the wire count.
  */
 static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free)
+pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
 {
 
 	--m->wire_count;
 	if (m->wire_count == 0)
-		return _pmap_unwire_pte_hold(pmap, m, free);
+		return _pmap_unwire_pte_hold(pmap, m);
 	else
 		return 0;
 }
 
 static int 
-_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free)
+_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
 {
 	vm_offset_t pteva;
 
@@ -1168,8 +1282,6 @@
 	pmap->pm_pdir[m->pindex] = 0;
 	--pmap->pm_stats.resident_count;
 
-	atomic_subtract_int(&cnt.v_wire_count, 1);
-
 	/*
 	 * Do an invltlb to make the invalidated mapping
 	 * take effect immediately.
@@ -1177,13 +1289,8 @@
 	pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex);
 	pmap_invalidate_page(pmap, pteva);
 
-	/* 
-	 * Put page on a list so that it is released after
-	 * *ALL* TLB shootdown is done
-	 */
-	m->right = *free;
-	*free = m;
-
+	vm_page_free_zero(m);
+	atomic_subtract_int(&cnt.v_wire_count, 1);
 	return 1;
 }
 
@@ -1192,7 +1299,7 @@
  * conditionally free the page, and manage the hold/wire counts.
  */
 static int
-pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t *free)
+pmap_unuse_pt(pmap_t pmap, vm_offset_t va)
 {
 	pd_entry_t ptepde;
 	vm_page_t mpte;
@@ -1201,7 +1308,7 @@
 		return 0;
 	ptepde = *pmap_pde(pmap, va);
 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
-	return pmap_unwire_pte_hold(pmap, mpte, free);
+	return pmap_unwire_pte_hold(pmap, mpte);
 }
 
 void
@@ -1213,6 +1320,7 @@
 #ifdef PAE
 	pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
 #endif
+	pmap->pm_root = NULL;
 	pmap->pm_active = 0;
 	PCPU_SET(curpmap, pmap);
 	TAILQ_INIT(&pmap->pm_pvchunk);
@@ -1251,7 +1359,10 @@
 		KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
 		    ("pmap_pinit: pdpt above 4g"));
 #endif
+		pmap->pm_root = NULL;
 	}
+	KASSERT(pmap->pm_root == NULL,
+	    ("pmap_pinit: pmap has reserved page table page(s)"));
 
 	/*
 	 * allocate the page directory page(s)
@@ -1374,10 +1485,9 @@
 	 * normal 4K page.
 	 */
 	if (ptepa & PG_PS) {
-		pmap->pm_pdir[ptepindex] = 0;
-		ptepa = 0;
-		pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-		pmap_invalidate_all(kernel_pmap);
+		if (!pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va))
+			pmap_invalidate_page(pmap, va);
+		ptepa = pmap->pm_pdir[ptepindex];
 	}
 
 	/*
@@ -1511,6 +1621,8 @@
 	KASSERT(pmap->pm_stats.resident_count == 0,
 	    ("pmap_release: pmap resident count %ld != 0",
 	    pmap->pm_stats.resident_count));
+	KASSERT(pmap->pm_root == NULL,
+	    ("pmap_release: pmap has reserved page table page(s)"));
 
 	pmap_lazyfix(pmap);
 	mtx_lock_spin(&allpmaps_lock);
@@ -1603,7 +1715,7 @@
 		/*
 		 * This index is bogus, but out of the way
 		 */
-		nkpg = vm_page_alloc(NULL, nkpt,
+		nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT,
 		    VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 		if (!nkpg)
 			panic("pmap_growkernel: no memory to grow kernel");
@@ -1698,11 +1810,13 @@
 static void
 pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 {
+	struct md_page *pvh;
+	pd_entry_t *pde;
 	pmap_t pmap;
 	pt_entry_t *pte, tpte;
 	pv_entry_t next_pv, pv;
 	vm_offset_t va;
-	vm_page_t m, free;
+	vm_page_t m;
 
 	sched_pin();
 	TAILQ_FOREACH(m, &vpq->pl, pageq) {
@@ -1717,6 +1831,9 @@
 			else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
 				continue;
 			pmap->pm_stats.resident_count--;
+			pde = pmap_pde(pmap, va);
+			KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found"
+			    " a 4mpage in page %p's pv list", m));
 			pte = pmap_pte_quick(pmap, va);
 			tpte = pte_load_clear(pte);
 			KASSERT((tpte & PG_W) == 0,
@@ -1729,14 +1846,15 @@
 				    va, (uintmax_t)tpte));
 				vm_page_dirty(m);
 			}
-			free = NULL;
-			pmap_unuse_pt(pmap, va, &free);
 			pmap_invalidate_page(pmap, va);
-			pmap_free_zero_pages(free);
 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-			if (TAILQ_EMPTY(&m->md.pv_list))
-				vm_page_flag_clear(m, PG_WRITEABLE);
+			if (TAILQ_EMPTY(&m->md.pv_list)) {
+				pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+				if (TAILQ_EMPTY(&pvh->pv_list))
+					vm_page_flag_clear(m, PG_WRITEABLE);
+			}
 			m->md.pv_list_count--;
+			pmap_unuse_pt(pmap, va);
 			free_pv_entry(pmap, pv);
 			if (pmap != locked_pmap)
 				PMAP_UNLOCK(pmap);
@@ -1891,25 +2009,115 @@
 	return (pv);
 }
 
-static void
-pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+static __inline pv_entry_t
+pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
-		if (pmap == PV_PMAP(pv) && va == pv->pv_va)
+	TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+		if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
+			TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
+			pvh->pv_list_count--;
 			break;
+		}
 	}
-	KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
-	TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-	m->md.pv_list_count--;
-	if (TAILQ_EMPTY(&m->md.pv_list))
-		vm_page_flag_clear(m, PG_WRITEABLE);
+	return (pv);
+}
+
+static void
+pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+	vm_offset_t va_last;
+	vm_page_t m;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	KASSERT((pa & PDRMASK) == 0,
+	    ("pmap_pv_demote_pde: pa is not 4mpage aligned"));
+
+	/*
+	 * Transfer the 4mpage's pv entry for this mapping to the first
+	 * page's pv list.
+	 */
+	pvh = pa_to_pvh(pa);
+	va = trunc_4mpage(va);
+	pv = pmap_pvh_remove(pvh, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
+	m = PHYS_TO_VM_PAGE(pa);
+	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+	m->md.pv_list_count++;
+	/* Instantiate the remaining NPTEPG - 1 pv entries. */
+	va_last = va + NBPDR - PAGE_SIZE;
+	do {
+		m++;
+		KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+		    ("pmap_pv_demote_pde: page %p is not managed", m));
+		va += PAGE_SIZE;
+		pmap_insert_entry(pmap, va, m);
+	} while (va < va_last);
+}
+
+static void
+pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+	vm_offset_t va_last;
+	vm_page_t m;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	KASSERT((pa & PDRMASK) == 0,
+	    ("pmap_pv_promote_pde: pa is not 4mpage aligned"));
+
+	/*
+	 * Transfer the first page's pv entry for this mapping to the
+	 * 4mpage's pv list.  Aside from avoiding the cost of a call
+	 * to get_pv_entry(), a transfer avoids the possibility that
+	 * get_pv_entry() calls pmap_collect() and that pmap_collect()
+	 * removes one of the mappings that is being promoted.
+	 */
+	m = PHYS_TO_VM_PAGE(pa);
+	va = trunc_4mpage(va);
+	pv = pmap_pvh_remove(&m->md, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
+	pvh = pa_to_pvh(pa);
+	TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+	pvh->pv_list_count++;
+	/* Free the remaining NPTEPG - 1 pv entries. */
+	va_last = va + NBPDR - PAGE_SIZE;
+	do {
+		m++;
+		va += PAGE_SIZE;
+		pmap_pvh_free(&m->md, pmap, va);
+	} while (va < va_last);
+}
+
+static void
+pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
+{
+	pv_entry_t pv;
+
+	pv = pmap_pvh_remove(pvh, pmap, va);
+	KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 	free_pv_entry(pmap, pv);
 }
 
+static void
+pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+{
+	struct md_page *pvh;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pmap_pvh_free(&m->md, pmap, va);
+	if (TAILQ_EMPTY(&m->md.pv_list)) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		if (TAILQ_EMPTY(&pvh->pv_list))
+			vm_page_flag_clear(m, PG_WRITEABLE);
+	}
+}
+
 /*
  * Create a pv entry for page at pa for
  * (pmap, va).
@@ -1919,7 +2127,6 @@
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	pv = get_pv_entry(pmap, FALSE);
 	pv->pv_va = va;
@@ -1935,7 +2142,6 @@
 {
 	pv_entry_t pv;
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	if (pv_entry_count < pv_entry_high_water && 
 	    (pv = get_pv_entry(pmap, TRUE)) != NULL) {
@@ -1948,10 +2154,91 @@
 }
 
 /*
+ * Create the pv entries for each of the pages within a superpage.
+ */
+static boolean_t
+pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m)
+{
+	struct md_page *pvh;
+	pv_entry_t pv;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if (pv_entry_count < pv_entry_high_water && 
+	    (pv = get_pv_entry(pmap, TRUE)) != NULL) {
+		pv->pv_va = va;
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+		pvh->pv_list_count++;
+		return (TRUE);
+	} else
+		return (FALSE);
+}
+
+/*
+ * pmap_remove_pde: do the things to unmap a superpage in a process
+ */
+static void
+pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva)
+{
+	struct md_page *pvh;
+	pd_entry_t oldpde;
+	vm_offset_t eva, va;
+	vm_page_t m, mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	KASSERT((sva & PDRMASK) == 0,
+	    ("pmap_remove_pde: sva is not 4mpage aligned"));
+	oldpde = pte_load_clear(pdq);
+	if (oldpde & PG_W)
+		pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
+
+	/*
+	 * Machines that don't support invlpg, also don't support
+	 * PG_G.
+	 */
+	if (oldpde & PG_G)
+		pmap_invalidate_page(kernel_pmap, sva);
+	pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
+	if (oldpde & PG_MANAGED) {
+		pvh = pa_to_pvh(oldpde & PG_FRAME);
+		pmap_pvh_free(pvh, pmap, sva);
+		eva = sva + NBPDR;
+		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_FRAME);
+		    va < eva; va += PAGE_SIZE, m++) {
+			if (oldpde & PG_M) {
+				KASSERT((oldpde & PG_RW) != 0,
+	("pmap_remove_pde: modified 4mpage not writable: va: %#x, pde: %#jx",
+				    va, (uintmax_t)oldpde));
+				vm_page_dirty(m);
+			}
+			if (oldpde & PG_A)
+				vm_page_flag_set(m, PG_REFERENCED);
+			if (TAILQ_EMPTY(&m->md.pv_list) &&
+			    TAILQ_EMPTY(&pvh->pv_list))
+				vm_page_flag_clear(m, PG_WRITEABLE);
+		}
+	}
+	if (pmap == kernel_pmap) {
+		if (!pmap_demote_pde(pmap, pdq, sva))
+			panic("pmap_remove_pde: failed demotion");
+	} else {
+		mpte = pmap_lookup_pt_page(pmap, sva);
+		if (mpte != NULL) {
+			pmap_remove_pt_page(pmap, mpte);
+			KASSERT(mpte->wire_count == NPTEPG,
+			    ("pmap_remove_pde: pte page wire count error"));
+			mpte->wire_count = 0;
+			vm_page_free(mpte);
+			atomic_subtract_int(&cnt.v_wire_count, 1);
+		}
+	}
+}
+
+/*
  * pmap_remove_pte: do the things to unmap a page in a process
  */
 static int
-pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free)
+pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
 {
 	pt_entry_t oldpte;
 	vm_page_t m;
@@ -1980,7 +2267,7 @@
 			vm_page_flag_set(m, PG_REFERENCED);
 		pmap_remove_entry(pmap, m, va);
 	}
-	return (pmap_unuse_pt(pmap, va, free));
+	return (pmap_unuse_pt(pmap, va));
 }
 
 /*
@@ -1990,16 +2277,14 @@
 pmap_remove_page(pmap_t pmap, vm_offset_t va)
 {
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va, &free);
+	pmap_remove_pte(pmap, pte, va);
 	pmap_invalidate_page(pmap, va);
-	pmap_free_zero_pages(free);
 }
 
 /*
@@ -2014,7 +2299,6 @@
 	vm_offset_t pdnxt;
 	pd_entry_t ptpaddr;
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 	int anyvalid;
 
 	/*
@@ -2064,10 +2348,25 @@
 		 * Check for large page.
 		 */
 		if ((ptpaddr & PG_PS) != 0) {
-			pmap->pm_pdir[pdirindex] = 0;
-			pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-			anyvalid = 1;
-			continue;
+			/*
+			 * Are we removing the entire large page?  If not,
+			 * demote the mapping and fall through.
+			 */
+			if (sva + NBPDR == pdnxt && eva >= pdnxt) {
+				/*
+				 * The TLB entry for a PG_G mapping is
+				 * invalidated by pmap_remove_pde().
+				 */
+				if ((ptpaddr & PG_G) == 0)
+					anyvalid = 1;
+				pmap_remove_pde(pmap,
+				    &pmap->pm_pdir[pdirindex], sva);
+				continue;
+			} else if (!pmap_demote_pde(pmap,
+			    &pmap->pm_pdir[pdirindex], sva)) {
+				anyvalid = 1;
+				continue;
+			}
 		}
 
 		/*
@@ -2089,17 +2388,15 @@
 			 */
 			if ((*pte & PG_G) == 0)
 				anyvalid = 1;
-			if (pmap_remove_pte(pmap, pte, sva, &free))
+			if (pmap_remove_pte(pmap, pte, sva))
 				break;
 		}
 	}
 out:
 	sched_unpin();
-	if (anyvalid) {
+	vm_page_unlock_queues();
+	if (anyvalid)
 		pmap_invalidate_all(pmap);
-		pmap_free_zero_pages(free);
-	}
-	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2119,10 +2416,12 @@
 void
 pmap_remove_all(vm_page_t m)
 {
+	struct md_page *pvh;
+	pmap_t pmap;
 	pv_entry_t pv;
-	pmap_t pmap;
 	pt_entry_t *pte, tpte;
-	vm_page_t free;
+	pd_entry_t *pde;
+	vm_offset_t va;
 
 #if defined(PMAP_DIAGNOSTIC)
 	/*
@@ -2135,10 +2434,23 @@
 #endif
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	sched_pin();
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		if (!pmap_demote_pde(pmap, pde, va))
+			pmap_invalidate_page(pmap, va);
+		PMAP_UNLOCK(pmap);
+	}
 	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
 		pmap->pm_stats.resident_count--;
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
+		    " a 4mpage in page %p's pv list", m));
 		pte = pmap_pte_quick(pmap, pv->pv_va);
 		tpte = pte_load_clear(pte);
 		if (tpte & PG_W)
@@ -2155,12 +2467,10 @@
 			    pv->pv_va, (uintmax_t)tpte));
 			vm_page_dirty(m);
 		}
-		free = NULL;
-		pmap_unuse_pt(pmap, pv->pv_va, &free);
 		pmap_invalidate_page(pmap, pv->pv_va);
-		pmap_free_zero_pages(free);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 		m->md.pv_list_count--;
+		pmap_unuse_pt(pmap, pv->pv_va);
 		free_pv_entry(pmap, pv);
 		PMAP_UNLOCK(pmap);
 	}
@@ -2169,6 +2479,52 @@
 }
 
 /*
+ * pmap_protect_pde: do the things to protect a 4mpage in a process
+ */
+static boolean_t
+pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
+{
+	pd_entry_t newpde, oldpde;
+	vm_offset_t eva, va;
+	vm_page_t m;
+	boolean_t anychanged;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	KASSERT((sva & PDRMASK) == 0,
+	    ("pmap_protect_pde: sva is not 4mpage aligned"));
+	anychanged = FALSE;
+retry:
+	oldpde = newpde = *pde;
+	if (oldpde & PG_MANAGED) {
+		eva = sva + NBPDR;
+		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_FRAME);
+		    va < eva; va += PAGE_SIZE, m++) {
+			if (oldpde & PG_A) {
+				vm_page_flag_set(m, PG_REFERENCED);
+				newpde &= ~PG_A;
+			}
+			if ((oldpde & PG_M) != 0)
+				vm_page_dirty(m);
+		}
+	}
+	if ((prot & VM_PROT_WRITE) == 0)
+		newpde &= ~(PG_RW | PG_M);
+#ifdef PAE
+	if ((prot & VM_PROT_EXECUTE) == 0)
+		newpde |= pg_nx;
+#endif
+	if (newpde != oldpde) {
+		if (!pde_cmpset(pde, oldpde, newpde))
+			goto retry;
+		if (oldpde & PG_G)
+			pmap_invalidate_page(pmap, sva);
+		else
+			anychanged = TRUE;
+	}
+	return (anychanged);
+}
+
+/*
  *	Set the physical protection on the
  *	specified range of this map as requested.
  */
@@ -2219,14 +2575,24 @@
 		 * Check for large page.
 		 */
 		if ((ptpaddr & PG_PS) != 0) {
-			if ((prot & VM_PROT_WRITE) == 0)
-				pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
-#ifdef PAE
-			if ((prot & VM_PROT_EXECUTE) == 0)
-				pmap->pm_pdir[pdirindex] |= pg_nx;
-#endif
-			anychanged = 1;
-			continue;
+			/*
+			 * Are we protecting the entire large page?  If not,
+			 * demote the mapping and fall through.
+			 */
+			if (sva + NBPDR == pdnxt && eva >= pdnxt) {
+				/*
+				 * The TLB entry for a PG_G mapping is
+				 * invalidated by pmap_protect_pde().
+				 */
+				if (pmap_protect_pde(pmap,
+				    &pmap->pm_pdir[pdirindex], sva, prot))
+					anychanged = 1;
+				continue;
+			} else if (!pmap_demote_pde(pmap,
+			    &pmap->pm_pdir[pdirindex], sva)) {
+				anychanged = 1;
+				continue;
+			}
 		}
 
 		if (pdnxt > eva)
@@ -2283,9 +2649,9 @@
 		}
 	}
 	sched_unpin();
+	vm_page_unlock_queues();
 	if (anychanged)
 		pmap_invalidate_all(pmap);
-	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2302,8 +2668,8 @@
  *	insert this page into the given map NOW.
  */
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-	   boolean_t wired)
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, vm_page_t m,
+    vm_prot_t prot, boolean_t wired)
 {
 	vm_paddr_t pa;
 	pd_entry_t *pde;
@@ -2455,9 +2821,12 @@
 	 * to update the pte.
 	 */
 	if ((origpte & ~(PG_M|PG_A)) != newpte) {
+		newpte |= PG_A;
+		if (fault_type & VM_PROT_WRITE)
+			newpte |= PG_M;
 		if (origpte & PG_V) {
 			invlva = FALSE;
-			origpte = pte_load_store(pte, newpte | PG_A);
+			origpte = pte_load_store(pte, newpte);
 			if (origpte & PG_A) {
 				if (origpte & PG_MANAGED)
 					vm_page_flag_set(om, PG_REFERENCED);
@@ -2481,7 +2850,21 @@
 			if (invlva)
 				pmap_invalidate_page(pmap, va);
 		} else
-			pte_store(pte, newpte | PG_A);
+			pte_store(pte, newpte);
+	}
+
+	/*
+	 * Promotion condition:
+	 * 1) Page has to be part of a fully populated reservation
+	 * 2) Virtual adress corresponding to the reservation has to
+	 *    be superpage aligned
+	 */
+	if (((mpte != NULL && mpte->wire_count == NPTEPG) ||
+	    m->object == kernel_object || m->object == kmem_object) &&
+	    m->reserv != NULL &&
+	    m->reserv->refcnt == NBPDR / PAGE_SIZE) {
+		KASSERT(m->object->flags & OBJ_SUPERPAGES, ("pmap_enter: xxx"));
+		pmap_promote_pde(pmap, pde, va);
 	}
 	sched_unpin();
 	vm_page_unlock_queues();
@@ -2504,6 +2887,7 @@
 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
     vm_page_t m_start, vm_prot_t prot)
 {
+	vm_offset_t va;
 	vm_page_t m, mpte;
 	vm_pindex_t diff, psize;
 
@@ -2513,8 +2897,15 @@
 	m = m_start;
 	PMAP_LOCK(pmap);
 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
-		mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
-		    prot, mpte);
+		va = start + ptoa(diff);
+		if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
+		    m->reserv != NULL && m->reserv->first_page == m &&
+		    m->reserv->refcnt == SP_MAGN(m->reserv->level) &&
+		    pmap_enter_pde(pmap, va, m, prot))
+			m = &m[NBPDR / PAGE_SIZE - 1];
+		else
+			mpte = pmap_enter_quick_locked(pmap, va, m, prot,
+			    mpte);
 		m = TAILQ_NEXT(m, listq);
 	}
  	PMAP_UNLOCK(pmap);
@@ -2544,7 +2935,6 @@
 {
 	pt_entry_t *pte;
 	vm_paddr_t pa;
-	vm_page_t free;
 
 	KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 	    (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
@@ -2578,7 +2968,7 @@
 			 */
 			if (ptepa) {
 				if (ptepa & PG_PS)
-					panic("pmap_enter_quick: unexpected mapping into 4MB page");
+					return (NULL);
 				mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 				mpte->wire_count++;
 			} else {
@@ -2601,7 +2991,7 @@
 	pte = vtopte(va);
 	if (*pte) {
 		if (mpte != NULL) {
-			mpte->wire_count--;
+			pmap_unwire_pte_hold(pmap, mpte);
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2613,12 +3003,7 @@
 	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 	    !pmap_try_insert_pv_entry(pmap, va, m)) {
 		if (mpte != NULL) {
-			free = NULL;
-			if (pmap_unwire_pte_hold(pmap, mpte, &free)) {
-				pmap_invalidate_page(pmap, va);
-				pmap_free_zero_pages(free);
-			}
-			
+			pmap_unwire_pte_hold(pmap, mpte);
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2742,9 +3127,29 @@
 void
 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 {
+	pd_entry_t *pde;
 	pt_entry_t *pte;
+	boolean_t are_queues_locked;
 
+	are_queues_locked = FALSE;
+retry:
 	PMAP_LOCK(pmap);
+	pde = pmap_pde(pmap, va);
+	if ((*pde & PG_PS) != 0) {
+		if (!wired != ((*pde & PG_W) == 0)) {
+			if (!are_queues_locked) {
+				are_queues_locked = TRUE;
+				if (!mtx_trylock(&vm_page_queue_mtx)) {
+					PMAP_UNLOCK(pmap);
+					vm_page_lock_queues();
+					goto retry;
+				}
+			}
+			if (!pmap_demote_pde(pmap, pde, va))
+				panic("pmap_change_wiring: demotion failed");
+		} else
+			goto out;
+	}
 	pte = pmap_pte(pmap, va);
 
 	if (wired && !pmap_pte_w(pte))
@@ -2758,6 +3163,9 @@
 	 */
 	pmap_pte_set_w(pte, wired);
 	pmap_pte_release(pte);
+out:
+	if (are_queues_locked)
+		vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2775,7 +3183,6 @@
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 	  vm_offset_t src_addr)
 {
-	vm_page_t   free;
 	vm_offset_t addr;
 	vm_offset_t end_addr = src_addr + len;
 	vm_offset_t pdnxt;
@@ -2812,7 +3219,10 @@
 			continue;
 			
 		if (srcptepaddr & PG_PS) {
-			if (dst_pmap->pm_pdir[ptepindex] == 0) {
+			if (dst_pmap->pm_pdir[ptepindex] == 0 &&
+			    ((srcptepaddr & PG_MANAGED) == 0 ||
+			    pmap_pv_insert_pde(dst_pmap, addr,
+			    PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME)))) {
 				dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
 				    ~PG_W;
 				dst_pmap->pm_stats.resident_count +=
@@ -2852,15 +3262,8 @@
 					*dst_pte = ptetemp & ~(PG_W | PG_M |
 					    PG_A);
 					dst_pmap->pm_stats.resident_count++;
-	 			} else {
-					free = NULL;
-					if (pmap_unwire_pte_hold( dst_pmap,
-					    dstmpte, &free)) {
-						pmap_invalidate_page(dst_pmap,
-						    addr);
-						pmap_free_zero_pages(free);
-					}
-				}
+	 			} else
+					pmap_unwire_pte_hold(dst_pmap, dstmpte);
 				if (dstmpte->wire_count >= srcmpte->wire_count)
 					break;
 			}
@@ -2998,6 +3401,7 @@
 boolean_t
 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 {
+	struct md_page *pvh;
 	pv_entry_t pv;
 	int loops = 0;
 
@@ -3013,10 +3417,37 @@
 		if (loops >= 16)
 			break;
 	}
+	if (loops < 16) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+			if (PV_PMAP(pv) == pmap)
+				return (TRUE);
+			loops++;
+			if (loops >= 16)
+				break;
+		}
+	}
 	return (FALSE);
 }
 
 /*
+ * Returns TRUE if the given page is mapped individually or as part of
+ * a 4mpage.  Otherwise, returns FALSE.
+ */
+boolean_t
+pmap_page_is_mapped(vm_page_t m)
+{
+	struct md_page *pvh;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if (TAILQ_EMPTY(&m->md.pv_list)) {
+		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+		return (!TAILQ_EMPTY(&pvh->pv_list));
+	} else
+		return (TRUE);
+}
+
+/*
  * Remove all pages from specified address space
  * this aids process exit speeds.  Also, this code
  * is special cased for current process only, but
@@ -3028,8 +3459,9 @@
 pmap_remove_pages(pmap_t pmap)
 {
 	pt_entry_t *pte, tpte;
-	vm_page_t m, free = NULL;
+	vm_page_t m, mpte, mt;
 	pv_entry_t pv;
+	struct md_page *pvh;
 	struct pv_chunk *pc, *npc;
 	int field, idx;
 	int32_t bit;
@@ -3054,8 +3486,12 @@
 				pv = &pc->pc_pventry[idx];
 				inuse &= ~bitmask;
 
-				pte = vtopte(pv->pv_va);
+				pte = pmap_pde(pmap, pv->pv_va);
 				tpte = *pte;
+				if ((tpte & PG_PS) == 0) {
+					pte = vtopte(pv->pv_va);
+					tpte = *pte & ~PG_PTE_PAT;
+				}
 
 				if (tpte == 0) {
 					printf(
@@ -3082,27 +3518,57 @@
 					("pmap_remove_pages: bad tpte %#jx",
 					(uintmax_t)tpte));
 
-				pmap->pm_stats.resident_count--;
-
 				pte_clear(pte);
 
 				/*
 				 * Update the vm_page_t clean/reference bits.
 				 */
-				if (tpte & PG_M)
-					vm_page_dirty(m);
+				if (tpte & PG_M) {
+					KASSERT((tpte & PG_RW) != 0,
+	("pmap_remove_pages: modified page not writable: va: %#x, pte: %#jx",
+					    pv->pv_va, (uintmax_t)tpte));
+					if ((tpte & PG_PS) != 0) {
+						for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
+							vm_page_dirty(mt);
+					} else
+						vm_page_dirty(m);
+				}
 
 				/* Mark free */
 				PV_STAT(pv_entry_frees++);
 				PV_STAT(pv_entry_spare++);
 				pv_entry_count--;
 				pc->pc_map[field] |= bitmask;
-				m->md.pv_list_count--;
-				TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
-				if (TAILQ_EMPTY(&m->md.pv_list))
-					vm_page_flag_clear(m, PG_WRITEABLE);
-
-				pmap_unuse_pt(pmap, pv->pv_va, &free);
+				if ((tpte & PG_PS) != 0) {
+					pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
+					pvh = pa_to_pvh(tpte & PG_FRAME);
+					pvh->pv_list_count--;
+					TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
+					if (TAILQ_EMPTY(&pvh->pv_list)) {
+						for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
+							if (TAILQ_EMPTY(&mt->md.pv_list))
+								vm_page_flag_clear(mt, PG_WRITEABLE);
+					}
+					mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
+					if (mpte != NULL) {
+						pmap_remove_pt_page(pmap, mpte);
+						KASSERT(mpte->wire_count == NPTEPG,
+						    ("pmap_remove_pages: pte page wire count error"));
+						mpte->wire_count = 0;
+						vm_page_free(mpte);
+						atomic_subtract_int(&cnt.v_wire_count, 1);
+					}
+				} else {
+					pmap->pm_stats.resident_count--;
+					m->md.pv_list_count--;
+					TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+					if (TAILQ_EMPTY(&m->md.pv_list)) {
+						pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+						if (TAILQ_EMPTY(&pvh->pv_list))
+							vm_page_flag_clear(m, PG_WRITEABLE);
+					}
+					pmap_unuse_pt(pmap, pv->pv_va);
+				}
 			}
 		}
 		if (allfree) {
@@ -3118,9 +3584,8 @@
 		}
 	}
 	sched_unpin();
+	vm_page_unlock_queues();
 	pmap_invalidate_all(pmap);
-	pmap_free_zero_pages(free);
-	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -3133,18 +3598,31 @@
 boolean_t
 pmap_is_modified(vm_page_t m)
 {
+
+	if (m->flags & PG_FICTITIOUS)
+		return (FALSE);
+	if (pmap_is_modified_pvh(&m->md))
+		return (TRUE);
+	return (pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
+}
+
+/*
+ * Returns TRUE if any of the given mappings were used to modify
+ * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
+ * mappings are supported.
+ */
+static boolean_t
+pmap_is_modified_pvh(struct md_page *pvh)
+{
 	pv_entry_t pv;
 	pt_entry_t *pte;
 	pmap_t pmap;
 	boolean_t rv;
 
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	rv = FALSE;
-	if (m->flags & PG_FICTITIOUS)
-		return (rv);
-
 	sched_pin();
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+	TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
 		pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3166,12 +3644,14 @@
 boolean_t
 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 {
+	pd_entry_t *pde;
 	pt_entry_t *pte;
 	boolean_t rv;
 
 	rv = FALSE;
 	PMAP_LOCK(pmap);
-	if (*pmap_pde(pmap, addr)) {
+	pde = pmap_pde(pmap, addr);
+	if (*pde != 0 && (*pde & PG_PS) == 0) {
 		pte = vtopte(addr);
 		rv = *pte == 0;
 	}
@@ -3185,18 +3665,34 @@
 void
 pmap_remove_write(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
+	pv_entry_t next_pv, pv;
 	pmap_t pmap;
+	pd_entry_t *pde;
 	pt_entry_t oldpte, *pte;
+	vm_offset_t va;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	if ((m->flags & PG_FICTITIOUS) != 0 ||
 	    (m->flags & PG_WRITEABLE) == 0)
 		return;
 	sched_pin();
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		if ((*pde & PG_RW) != 0 && !pmap_demote_pde(pmap, pde, va))
+			pmap_invalidate_page(pmap, va);
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
+		    " a 4mpage in page %p's pv list", m));
 		pte = pmap_pte_quick(pmap, pv->pv_va);
 retry:
 		oldpte = *pte;
@@ -3234,15 +3730,47 @@
 int
 pmap_ts_referenced(vm_page_t m)
 {
+	struct md_page *pvh;
 	pv_entry_t pv, pvf, pvn;
 	pmap_t pmap;
+	pd_entry_t oldpde, *pde;
 	pt_entry_t *pte;
+	vm_offset_t va;
 	int rtval = 0;
 
 	if (m->flags & PG_FICTITIOUS)
 		return (rtval);
 	sched_pin();
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_A) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				if ((oldpde & PG_W) == 0) {
+					/*
+					 * Remove the mapping to a single page
+					 * so that a subsequent access may
+					 * repromote.
+					 */
+					va += VM_PAGE_TO_PHYS(m) - (oldpde &
+					    PG_FRAME);
+					pmap_remove_page(pmap, va);
+					rtval++;
+					if (rtval > 4) {
+						PMAP_UNLOCK(pmap);
+						return (rtval);
+					}
+				}
+			} else
+				pmap_invalidate_page(pmap, va);
+		}
+		PMAP_UNLOCK(pmap);
+	}
 	if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 		pvf = pv;
 		do {
@@ -3251,6 +3779,9 @@
 			TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 			pmap = PV_PMAP(pv);
 			PMAP_LOCK(pmap);
+			pde = pmap_pde(pmap, pv->pv_va);
+			KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
+			    " found a 4mpage in page %p's pv list", m));
 			pte = pmap_pte_quick(pmap, pv->pv_va);
 			if ((*pte & PG_A) != 0) {
 				atomic_clear_int((u_int *)pte, PG_A);
@@ -3272,17 +3803,63 @@
 void
 pmap_clear_modify(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
+	pv_entry_t next_pv, pv;
 	pmap_t pmap;
-	pt_entry_t *pte;
+	pd_entry_t oldpde, *pde;
+	pt_entry_t oldpte, *pte;
+	vm_offset_t va;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		return;
 	sched_pin();
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_RW) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				if ((oldpde & PG_W) == 0) {
+					/*
+					 * Write protect the mapping to a
+					 * single page so that a subsequent
+					 * write access may repromote.
+					 */
+					va += VM_PAGE_TO_PHYS(m) - (oldpde &
+					    PG_FRAME);
+					pte = pmap_pte_quick(pmap, va);
+					oldpte = *pte;
+					if ((oldpte & PG_V) != 0) {
+						/*
+						 * Regardless of whether a pte is 32 or 64 bits
+						 * in size, PG_RW and PG_M are among the least
+						 * significant 32 bits.
+						 */
+						while (!atomic_cmpset_int((u_int *)pte,
+						    oldpte,
+						    oldpte & ~(PG_M | PG_RW)))
+							oldpte = *pte;
+						vm_page_dirty(m);
+						pmap_invalidate_page(pmap, va);
+					}
+				}
+			} else
+				pmap_invalidate_page(pmap, va);
+		} else
+			KASSERT((oldpde & PG_M) == 0,
+			    ("pmap_clear_modify: modified page not writable"));
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
+		    " a 4mpage in page %p's pv list", m));
 		pte = pmap_pte_quick(pmap, pv->pv_va);
 		if ((*pte & PG_M) != 0) {
 			/*
@@ -3306,17 +3883,43 @@
 void
 pmap_clear_reference(vm_page_t m)
 {
-	pv_entry_t pv;
+	struct md_page *pvh;
+	pv_entry_t next_pv, pv;
 	pmap_t pmap;
+	pd_entry_t oldpde, *pde;
 	pt_entry_t *pte;
+	vm_offset_t va;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		return;
 	sched_pin();
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
+		va = pv->pv_va;
+		pmap = PV_PMAP(pv);
+		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, va);
+		oldpde = *pde;
+		if ((oldpde & PG_A) != 0) {
+			if (pmap_demote_pde(pmap, pde, va)) {
+				/*
+				 * Remove the mapping to a single page so
+				 * that a subsequent access may repromote.
+				 */
+				va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_FRAME);
+				pmap_remove_page(pmap, va);
+			} else
+				pmap_invalidate_page(pmap, va);
+		}
+		PMAP_UNLOCK(pmap);
+	}
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 		pmap = PV_PMAP(pv);
 		PMAP_LOCK(pmap);
+		pde = pmap_pde(pmap, pv->pv_va);
+		KASSERT((*pde & PG_PS) == 0, ("pmap_clear_reference: found"
+		    " a 4mpage in page %p's pv list", m));
 		pte = pmap_pte_quick(pmap, pv->pv_va);
 		if ((*pte & PG_A) != 0) {
 			/*
@@ -3466,25 +4069,35 @@
 int
 pmap_mincore(pmap_t pmap, vm_offset_t addr)
 {
+	pd_entry_t *pdep;
 	pt_entry_t *ptep, pte;
+	vm_paddr_t pa;
 	vm_page_t m;
 	int val = 0;
 	
 	PMAP_LOCK(pmap);
-	ptep = pmap_pte(pmap, addr);
-	pte = (ptep != NULL) ? *ptep : 0;
-	pmap_pte_release(ptep);
+	pdep = pmap_pde(pmap, addr);
+	if (*pdep != 0) {
+		if (*pdep & PG_PS) {
+			pte = *pdep;
+			pa = (*pdep & ~PDRMASK) | (addr & PDRMASK);
+		} else {
+			ptep = pmap_pte(pmap, addr);
+			pte = *ptep;
+			pmap_pte_release(ptep);
+			pa = pte & PG_FRAME;
+		}
+	} else {
+		pte = 0;
+		pa = 0;
+	}
 	PMAP_UNLOCK(pmap);
 
 	if (pte != 0) {
-		vm_paddr_t pa;
-
 		val = MINCORE_INCORE;
 		if ((pte & PG_MANAGED) == 0)
 			return val;
 
-		pa = pte & PG_FRAME;
-
 		m = PHYS_TO_VM_PAGE(pa);
 
 		/*
@@ -3564,6 +4177,301 @@
 	return addr;
 }
 
+#define COMPATIBLE_PTE_MASK (PG_MANAGED | PG_W | PG_G | PG_PTE_PAT | PG_M | PG_A | PG_NC_PCD | PG_NC_PWT | PG_U | PG_RW | PG_V)
+#define COMPATIBLE_PTE(a,b) (((a) & COMPATIBLE_PTE_MASK) == ((b) & COMPATIBLE_PTE_MASK))
+
+SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, "");
+
+static u_long pmap_pde_promotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
+    &pmap_pde_promotions, 0, "pde promotions");
+
+/*
+ * Managed (PG_MANAGED) mappings within the kernel address space are
+ * not promoted.  The reason is that kernel PDEs are replicated in
+ * each pmap but pmap_clear_ptes() and pmap_ts_referenced() only read
+ * the PDE from the kernel pmap.
+ */
+static void
+pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
+{
+	pd_entry_t newpde;
+	pmap_t allpmaps_entry;
+	pt_entry_t *firstpte, oldpte, *pte;
+	vm_paddr_t pa;
+	vm_page_t mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	firstpte = vtopte(trunc_4mpage(va));
+	KASSERT((*firstpte & PG_V) != 0,
+	    ("pmap_promote_pde: firstpte is missing PG_V"));
+	if ((*firstpte & PG_A) == 0)
+		return;
+	if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap)
+		return;
+	pa = *firstpte & PG_FRAME & ~PDRMASK;
+	newpde = *firstpte;
+	if ((newpde & (PG_M | PG_RW)) == PG_RW)
+		newpde &= ~PG_RW;
+
+	/* 
+	 * Check all the ptes before promotion
+	 */
+	for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
+retry:
+		oldpte = *pte;
+		if ((oldpte & PG_FRAME) != pa) {
+			CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
+			    " in pmap %p", va, pmap);
+			return;
+		}
+		if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
+			/*
+			 * When PG_M is already clear, PG_RW can be cleared
+			 * without a TLB invalidation.
+			 */
+			if (!atomic_cmpset_int((u_int *)pte, oldpte,
+			    oldpte & ~PG_RW))
+				goto retry;
+			oldpte = *pte;
+		}
+		if (!COMPATIBLE_PTE(oldpte, newpde)) {
+			CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
+			    " in pmap %p", va, pmap);
+			return;
+		}
+		pa += PAGE_SIZE;
+	}
+
+	/*
+	 * Save the page table page in its current state until the PDE
+	 * mapping the superpage is demoted by pmap_demote_pde() or
+	 * destroyed by pmap_remove_pde(). 
+	 */
+	mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
+	KASSERT(mpte >= vm_page_array &&
+	    mpte < &vm_page_array[vm_page_array_size],
+	    ("pmap_promote_pde: page table page is out of range"));
+	KASSERT(mpte->pindex == va >> PDRSHIFT,
+	    ("pmap_promote_pde: page table page's pindex is wrong"));
+	pmap_insert_pt_page(pmap, mpte);
+
+	/*
+	 * Promote the pv entries.
+	 */
+	if ((newpde & PG_MANAGED) != 0)
+		pmap_pv_promote_pde(pmap, va, newpde & PG_FRAME);
+
+	/*
+	 * Propagate the PAT index to its proper position.
+	 */
+	if ((newpde & PG_PTE_PAT) != 0)
+		newpde ^= PG_PDE_PAT | PG_PTE_PAT;
+
+	/*
+	 * Map the superpage.
+	 */
+	if (pmap == kernel_pmap) {
+		mtx_lock_spin(&allpmaps_lock);
+		LIST_FOREACH(allpmaps_entry, &allpmaps, pm_list) {
+			pde = pmap_pde(allpmaps_entry, va);
+			pde_store(pde, PG_PS | newpde);
+		}
+		mtx_unlock_spin(&allpmaps_lock);
+	} else
+		pde_store(pde, PG_PS | newpde);
+
+	pmap_pde_promotions++;
+	CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
+	    " in pmap %p", va, pmap);
+}
+
+static u_long pmap_pde_demotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
+    &pmap_pde_demotions, 0, "pde demotions");
+
+static boolean_t
+pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
+{
+	pd_entry_t newpde, oldpde;
+	pmap_t allpmaps_entry;
+	pt_entry_t *firstpte, newpte, *pte;
+	vm_paddr_t mptepa;
+	vm_page_t mpte;
+
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	mpte = pmap_lookup_pt_page(pmap, va);
+	if (mpte != NULL)
+		pmap_remove_pt_page(pmap, mpte);
+	else {
+		KASSERT((*pde & PG_W) == 0,
+		    ("pmap_demote_pde: page table page for a wired mapping"
+		    " is missing"));
+		pmap_remove_pde(pmap, pde, trunc_4mpage(va));
+		CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
+		    " in pmap %p", va, pmap);
+		return (FALSE);
+	}
+	mptepa = VM_PAGE_TO_PHYS(mpte);
+
+	/*
+	 * Temporarily map the page table page (mpte) into the kernel's
+	 * address space at either PADDR1 or PADDR2.
+	 */
+	if (curthread->td_pinned > 0 && mtx_owned(&vm_page_queue_mtx)) {
+		if ((*PMAP1 & PG_FRAME) != mptepa) {
+			*PMAP1 = mptepa | PG_RW | PG_V | PG_A | PG_M;
+#ifdef SMP
+			PMAP1cpu = PCPU_GET(cpuid);
+#endif
+			invlcaddr(PADDR1);
+			PMAP1changed++;
+		} else
+#ifdef SMP
+		if (PMAP1cpu != PCPU_GET(cpuid)) {
+			PMAP1cpu = PCPU_GET(cpuid);
+			invlcaddr(PADDR1);
+			PMAP1changedcpu++;
+		} else
+#endif
+			PMAP1unchanged++;
+		firstpte = PADDR1;
+	} else {
+		mtx_lock(&PMAP2mutex);
+		if ((*PMAP2 & PG_FRAME) != mptepa) {
+			*PMAP2 = mptepa | PG_RW | PG_V | PG_A | PG_M;
+			pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
+		}
+		firstpte = PADDR2;
+	}
+	oldpde = *pde;
+	newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
+	KASSERT((oldpde & (PG_A | PG_V)) == (PG_A | PG_V),
+	    ("pmap_demote_pde: oldpde is missing PG_A and/or PG_V"));
+	KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
+	    ("pmap_demote_pde: oldpde is missing PG_M"));
+	KASSERT((oldpde & PG_PS) != 0,
+	    ("pmap_demote_pde: oldpde is missing PG_PS"));
+	newpte = oldpde & ~PG_PS;
+	if ((newpte & PG_PDE_PAT) != 0)
+		newpte ^= PG_PDE_PAT | PG_PTE_PAT;
+
+	/*
+	 * If the mapping has changed attributes, update the page table
+	 * entries.
+	 */ 
+	KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
+	    ("pmap_demote_pde: firstpte and newpte map different physical"
+	    " addresses"));
+	if (!COMPATIBLE_PTE(*firstpte, newpte))
+		for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
+			*pte = newpte;	
+			newpte += PAGE_SIZE;
+		}
+	
+	/*
+	 * Demote the mapping.  This pmap is locked.  The old PDE has
+	 * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
+	 * set.  Thus, there is no danger of a race with another
+	 * processor changing the setting of PG_A and/or PG_M between
+	 * the read above and the store below. 
+	 */
+	if (pmap == kernel_pmap) {
+		/*
+		 * A harmless race exists between this loop and the bcopy()
+		 * in pmap_pinit() that initializes the kernel segment of
+		 * the new page table.  Specifically, that bcopy() may copy
+		 * the new PDE from the PTD, which is first in allpmaps, to
+		 * the new page table before this loop updates that new
+		 * page table.
+		 */
+		mtx_lock_spin(&allpmaps_lock);
+		LIST_FOREACH(allpmaps_entry, &allpmaps, pm_list) {
+			pde = pmap_pde(allpmaps_entry, va);
+			KASSERT(*pde == newpde || COMPATIBLE_PTE(*pde,
+			    oldpde),
+			    ("pmap_demote_pde: pde was %#jx, expected %#jx",
+			    (uintmax_t)*pde, (uintmax_t)oldpde));
+			pde_store(pde, newpde);
+		}
+		mtx_unlock_spin(&allpmaps_lock);
+	} else
+		pde_store(pde, newpde);	
+	if (firstpte == PADDR2)
+		mtx_unlock(&PMAP2mutex);
+
+	/*
+	 * Invalidate the recursive mapping of the page table page.
+	 */
+	pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
+
+	/*
+	 * Demote the pv entry.  This depends on the earlier demotion
+	 * of the mapping.  Specifically, the (re)creation of a per-
+	 * page pv entry might trigger the execution of pmap_collect(),
+	 * which might reclaim a newly (re)created per-page pv entry
+	 * and destroy the associated mapping.  In order to destroy
+	 * the mapping, the PDE must have already changed from mapping
+	 * the 2mpage to referencing the page table page.
+	 */
+	if ((oldpde & PG_MANAGED) != 0)
+		pmap_pv_demote_pde(pmap, va, oldpde & PG_FRAME);
+
+	pmap_pde_demotions++;
+	CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#x"
+	    " in pmap %p", va, pmap);
+	return (TRUE);
+}
+
+static u_long pmap_pde_mappings;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
+    &pmap_pde_mappings, 0, "pde mappings");
+
+static boolean_t
+pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+	pd_entry_t *pde, newpde;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	pde = pmap_pde(pmap, va);
+	if (*pde != 0) {
+		CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
+		    " in pmap %p", va, pmap);
+		return (FALSE);
+	}
+	newpde = VM_PAGE_TO_PHYS(m) | PG_PS | PG_V;
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
+		newpde |= PG_MANAGED;
+
+		/*
+		 * Create a PV entry for each of the managed pages.
+		 */
+		if (!pmap_pv_insert_pde(pmap, va, m)) {
+			CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
+			    " in pmap %p", va, pmap);
+			return (FALSE);
+		}
+	}
+	if (va < VM_MAXUSER_ADDRESS)
+		newpde |= PG_U;
+
+	/*
+	 * Increment counters.
+	 */
+	pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
+
+	/*
+	 * Map the superpage.
+	 */
+	pde_store(pde, newpde);
+
+	pmap_pde_mappings++;
+	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
+	    " in pmap %p", va, pmap);
+	return (TRUE);
+}
+
 
 #if defined(PMAP_DEBUG)
 pmap_pid_dump(int pid)
---  i386/include/param.h.orig
+++  i386/include/param.h
@@ -93,6 +93,33 @@
 #define NBPDR		(1<<PDRSHIFT)	/* bytes/page dir */
 #define PDRMASK		(NBPDR-1)
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#ifndef	SP_LEVELS
+#define	SP_LEVELS	1
+#endif
+
+/*
+ * SP_SMALL_SHIFT is LOG2(NBPDR / PAGE_SIZE).
+ */
+#ifdef PAE
+#ifndef	SP_SMALL_SHIFT
+#define	SP_SMALL_SHIFT	9
+#endif
+#else
+#ifndef	SP_SMALL_SHIFT
+#define	SP_SMALL_SHIFT	10
+#endif
+#endif
+
+/*
+ * If there is only one superpage size, then SP_FACTOR_SHIFT is zero.
+ */
+#ifndef	SP_FACTOR_SHIFT
+#define	SP_FACTOR_SHIFT	0
+#endif
+
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef KSTACK_PAGES
---  i386/include/pmap.h.orig
+++  i386/include/pmap.h
@@ -222,6 +222,9 @@
 
 #ifdef PAE
 
+#define	pde_cmpset(pdep, old, new) \
+				atomic_cmpset_64((pdep), (old), (new))
+
 static __inline pt_entry_t
 pte_load(pt_entry_t *ptep)
 {
@@ -278,6 +281,9 @@
 
 #else /* PAE */
 
+#define	pde_cmpset(pdep, old, new) \
+				atomic_cmpset_int((pdep), (old), (new))
+
 static __inline pt_entry_t
 pte_load(pt_entry_t *ptep)
 {
@@ -340,6 +346,7 @@
 	pdpt_entry_t		*pm_pdpt;	/* KVA of page director pointer
 						   table */
 #endif
+	vm_page_t		pm_root;	/* spare page table pages */
 };
 
 typedef struct pmap	*pmap_t;
@@ -403,7 +410,6 @@
 extern vm_offset_t virtual_avail;
 extern vm_offset_t virtual_end;
 
-#define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.pv_list))
 #define	pmap_unmapbios(va, sz)	pmap_unmapdev((va), (sz))
 
 void	pmap_bootstrap(vm_paddr_t);
@@ -416,6 +422,7 @@
 void	*pmap_mapbios(vm_paddr_t, vm_size_t);
 void	*pmap_mapdev(vm_paddr_t, vm_size_t);
 void	*pmap_mapdev_attr(vm_paddr_t, vm_size_t, int);
+boolean_t pmap_page_is_mapped(vm_page_t m);
 void	pmap_unmapdev(vm_offset_t, vm_size_t);
 pt_entry_t *pmap_pte(pmap_t, vm_offset_t) __pure2;
 void	pmap_set_pg(void);
---  i386/include/vmparam.h.orig
+++  i386/include/vmparam.h
@@ -129,6 +129,49 @@
 #endif
 
 /*
+ * Allow for one more than the number of phys_avail[] entries because
+ * the phys_avail[] entry that spans the largest physical address that
+ * is accessible by ISA DMA is split into two physseg entries.
+ */
+#define	VM_PHYSSEG_MAX		17
+
+/*
+ * Create XXX free page pools: XXX
+ */
+#define	VM_NFREEPOOL		1
+#define	VM_FREEPOOL_DEFAULT	0
+#define	VM_FREEPOOL_DIRECT	0
+#define	VM_FREEPOOL_RESERV	0
+
+/*
+ * Create two free page lists: VM_FREELIST_DEFAULT is for physical
+ * pages that are above the largest physical address that is
+ * accessible by ISA DMA and VM_FREELIST_ISADMA is for physical pages
+ * that are below that address.
+ */
+#ifdef PAE
+#define	VM_NFREELIST		3
+#else
+#define	VM_NFREELIST		2
+#endif
+#define	VM_FREELIST_DEFAULT	0
+#define	VM_FREELIST_ISADMA	1
+#ifdef PAE
+#define	VM_FREELIST_HIGHMEM	2
+#endif
+
+/*
+ * VM_HIGHMEM_ADDRESS is the physical address at which HIGHMEM starts.
+ */
+#define	VM_HIGHMEM_ADDRESS	4294967296ULL
+
+#ifdef PAE
+#define	VM_NFREEORDER		10
+#else
+#define	VM_NFREEORDER		11
+#endif
+
+/*
  * Virtual addresses of things.  Derived from the page directory and
  * page table indexes from pmap.h for precision.
  * Because of the page that is both a PD and PT, it looks a little
@@ -177,4 +220,8 @@
 #define	VM_INITIAL_PAGEIN	16
 #endif
 
+#ifndef	VM_IMAGE_BASE
+#define	VM_IMAGE_BASE		0x8048000
+#endif
+
 #endif /* _MACHINE_VMPARAM_H_ */
---  ia64/ia64/pmap.c.orig
+++  ia64/ia64/pmap.c
@@ -50,6 +50,7 @@
 
 #include <sys/param.h>
 #include <sys/kernel.h>
+#include <sys/ktr.h>
 #include <sys/lock.h>
 #include <sys/mman.h>
 #include <sys/mutex.h>
@@ -60,9 +61,11 @@
 
 #include <vm/vm.h>
 #include <vm/vm_page.h>
+#include <vm/vm_phys.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
 #include <machine/md_var.h>
@@ -249,27 +252,12 @@
 vm_offset_t
 pmap_steal_memory(vm_size_t size)
 {
-	vm_size_t bank_size;
-	vm_offset_t pa, va;
+	vm_offset_t va;
+	vm_paddr_t pa;
 
 	size = round_page(size);
 
-	bank_size = phys_avail[1] - phys_avail[0];
-	while (size > bank_size) {
-		int i;
-		for (i = 0; phys_avail[i+2]; i+= 2) {
-			phys_avail[i] = phys_avail[i+2];
-			phys_avail[i+1] = phys_avail[i+3];
-		}
-		phys_avail[i] = 0;
-		phys_avail[i+1] = 0;
-		if (!phys_avail[0])
-			panic("pmap_steal_memory: out of memory");
-		bank_size = phys_avail[1] - phys_avail[0];
-	}
-
-	pa = phys_avail[0];
-	phys_avail[0] += size;
+	pa = vm_phys_bootstrap_alloc(size, 0);
 
 	va = IA64_PHYS_TO_RR7(pa);
 	bzero((caddr_t) va, size);
@@ -1177,7 +1165,7 @@
  */
 static void
 pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa,
-    boolean_t wired, boolean_t managed)
+    int page_shift, boolean_t wired, boolean_t managed)
 {
 
 	pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED;
@@ -1186,7 +1174,7 @@
 	pte->pte |= (wired) ? PTE_WIRED : 0;
 	pte->pte |= pa & PTE_PPN_MASK;
 
-	pte->itir = PAGE_SHIFT << 2;
+	pte->itir = page_shift << 2;
 
 	pte->tag = ia64_ttag(va);
 }
@@ -1284,7 +1272,8 @@
 		else
 			pmap_enter_vhpt(pte, va);
 		pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
-		pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), FALSE, FALSE);
+		pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m[i]), PAGE_SHIFT, FALSE,
+		    FALSE);
 		va += PAGE_SIZE;
 	}
 }
@@ -1325,7 +1314,7 @@
 	else
 		pmap_enter_vhpt(pte, va);
 	pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
-	pmap_set_pte(pte, va, pa, FALSE, FALSE);
+	pmap_set_pte(pte, va, pa, PAGE_SHIFT, FALSE, FALSE);
 }
 
 /*
@@ -1552,8 +1541,8 @@
  *	insert this page into the given map NOW.
  */
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-    boolean_t wired)
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, vm_page_t m,
+    vm_prot_t prot, boolean_t wired)
 {
 	pmap_t oldpmap;
 	vm_offset_t pa;
@@ -1654,7 +1643,7 @@
 	 * adds the pte to the VHPT if necessary.
 	 */
 	pmap_pte_prot(pmap, pte, prot);
-	pmap_set_pte(pte, va, pa, wired, managed);
+	pmap_set_pte(pte, va, pa, PAGE_SHIFT, wired, managed);
 
 	if ((prot & VM_PROT_WRITE) != 0)
 		vm_page_flag_set(m, PG_WRITEABLE);
@@ -1663,6 +1652,55 @@
 	PMAP_UNLOCK(pmap);
 }
 
+#define	NBPDR	SP_SIZE(0)
+#define	PDRMASK	(NBPDR - 1)
+
+SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+
+SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, "");
+
+static u_long pmap_pde_mappings;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
+    &pmap_pde_mappings, 0, "pde mappings");
+
+static boolean_t
+pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+{
+	struct ia64_lpte *pte;
+	vm_page_t m_end;
+	boolean_t managed;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	m_end = &m[SP_MAGN(m->reserv->level)];
+	while (m < m_end) {
+		if ((pte = pmap_find_pte(va)) == NULL)
+			return (FALSE);
+		if (!pmap_present(pte)) {
+			if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
+				if (!pmap_try_insert_pv_entry(pmap, va, m)) {
+					pmap_free_pte(pte, va);
+					return (FALSE);
+				}
+				managed = TRUE;
+			} else
+				managed = FALSE;
+			pmap->pm_stats.resident_count++;
+			pmap_enter_vhpt(pte, va);
+			pmap_pte_prot(pmap, pte,
+			    prot & (VM_PROT_READ | VM_PROT_EXECUTE));
+			pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), PAGE_SHIFT +
+			    SP_SMALL_SHIFT, FALSE, managed);
+		}
+		va += PAGE_SIZE;
+		m++;
+	}
+	pmap_pde_mappings++;
+	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
+	    " in pmap %p", va, pmap);
+	return (TRUE);
+}
+
 /*
  * Maps a sequence of resident pages belonging to the same object.
  * The sequence begins with the given page m_start.  This page is
@@ -1680,6 +1718,7 @@
     vm_page_t m_start, vm_prot_t prot)
 {
 	pmap_t oldpmap;
+	vm_offset_t va;
 	vm_page_t m;
 	vm_pindex_t diff, psize;
 
@@ -1689,6 +1728,13 @@
 	PMAP_LOCK(pmap);
 	oldpmap = pmap_switch(pmap);
 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
+		va = start + ptoa(diff);
+		if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
+		    m->reserv != NULL && m->reserv->first_page == m &&
+		    m->reserv->refcnt == SP_MAGN(m->reserv->level) &&
+		    pmap_enter_pde(pmap, va, m, prot))
+			m = &m[SP_MAGN(m->reserv->level) - 1];
+		else
 		pmap_enter_quick_locked(pmap, start + ptoa(diff), m, prot);
 		m = TAILQ_NEXT(m, listq);
 	}
@@ -1751,7 +1797,8 @@
 		pmap_enter_vhpt(pte, va);
 		pmap_pte_prot(pmap, pte,
 		    prot & (VM_PROT_READ | VM_PROT_EXECUTE));
-		pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), FALSE, managed);
+		pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), PAGE_SHIFT, FALSE,
+		    managed);
 	}
 }
 
---  ia64/include/param.h.orig
+++  ia64/include/param.h
@@ -107,6 +107,27 @@
 #define PAGE_MASK	(PAGE_SIZE-1)
 #define NPTEPG		(PAGE_SIZE/(sizeof (pt_entry_t)))
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#ifndef	SP_LEVELS
+#define	SP_LEVELS	1
+#endif
+
+/*
+ * SP_SMALL_SHIFT is LOG2(NBPDR / PAGE_SIZE).
+ */
+#ifndef	SP_SMALL_SHIFT
+#define	SP_SMALL_SHIFT	9
+#endif
+
+/*
+ * If there is only one superpage size, then SP_FACTOR_SHIFT is zero.
+ */
+#ifndef	SP_FACTOR_SHIFT
+#define	SP_FACTOR_SHIFT	0
+#endif
+
 #ifndef	KSTACK_PAGES
 #define	KSTACK_PAGES	4		/* pages of kernel stack */
 #endif
---  ia64/include/vmparam.h.orig
+++  ia64/include/vmparam.h
@@ -130,14 +130,21 @@
 #define	VM_NFREEPOOL		2
 #define	VM_FREEPOOL_DEFAULT	0
 #define	VM_FREEPOOL_DIRECT	1
+#define	VM_FREEPOOL_RESERV	0
 
 /*
  * Create one free page list.
  */
-#define	VM_NFREELIST		1
+#define	VM_NFREELIST		2
 #define	VM_FREELIST_DEFAULT	0
+#define	VM_FREELIST_HIGHMEM	1
 
 /*
+ * VM_HIGHMEM_ADDRESS is the physical address at which HIGHMEM starts.
+ */
+#define	VM_HIGHMEM_ADDRESS	4294967296UL
+
+/*
  * An allocation size of 256MB is supported in order to optimize the
  * use of the identity mappings in region 7 by UMA.
  */
@@ -199,4 +206,8 @@
 #define	VM_INITIAL_PAGEIN	16
 #endif
 
+#ifndef	VM_IMAGE_BASE
+#define	VM_IMAGE_BASE		0x2000000000000000
+#endif
+
 #endif	/* !_MACHINE_VMPARAM_H */
---  kern/kern_exec.c.orig
+++  kern/kern_exec.c
@@ -818,6 +818,10 @@
 	if (object == NULL)
 		return (EACCES);
 	VM_OBJECT_LOCK(object);
+	if ((object->flags & OBJ_SUPERPAGES) == 0) {
+		object->flags |= OBJ_SUPERPAGES;
+		object->pg_color = (u_short)atop(VM_IMAGE_BASE);
+	}
 	ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
 	if ((ma[0]->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) {
 		initial_pagein = VM_INITIAL_PAGEIN;
---  powerpc/powerpc/mmu_if.m.orig
+++  powerpc/powerpc/mmu_if.m
@@ -206,6 +206,7 @@
 	mmu_t		_mmu;
 	pmap_t		_pmap;
 	vm_offset_t	_va;
+	vm_prot_t	_fault_type;
 	vm_page_t	_p;
 	vm_prot_t	_prot;
 	boolean_t	_wired;
---  powerpc/powerpc/mmu_oea.c.orig
+++  powerpc/powerpc/mmu_oea.c
@@ -311,7 +311,8 @@
 void moea_clear_modify(mmu_t, vm_page_t);
 void moea_clear_reference(mmu_t, vm_page_t);
 void moea_copy_page(mmu_t, vm_page_t, vm_page_t);
-void moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t);
+void moea_enter(mmu_t, pmap_t, vm_offset_t, vm_prot_t, vm_page_t, vm_prot_t,
+    boolean_t);
 void moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
     vm_prot_t);
 void moea_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
@@ -1056,8 +1057,8 @@
  * will be wired down.
  */
 void
-moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-	   boolean_t wired)
+moea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t fault_type,
+    vm_page_t m, vm_prot_t prot, boolean_t wired)
 {
 
 	vm_page_lock_queues();
---  powerpc/powerpc/pmap_dispatch.c.orig
+++  powerpc/powerpc/pmap_dispatch.c
@@ -107,10 +107,10 @@
 }
 
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t p, vm_prot_t prot,
-    boolean_t wired)
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, vm_page_t p,
+    vm_prot_t prot, boolean_t wired)
 {
-	MMU_ENTER(mmu_obj, pmap, va, p, prot, wired);
+	MMU_ENTER(mmu_obj, pmap, va, fault_type, p, prot, wired);
 }
 
 void
---  rpc/rpcclnt.c.orig
+++  rpc/rpcclnt.c
@@ -1,4 +1,4 @@
-/* $FreeBSD: src/sys/rpc/rpcclnt.c,v 1.19 2007/06/16 05:42:26 mjacob Exp $ */
+/* $FreeBSD: src/sys/rpc/rpcclnt.c,v 1.18 2007/03/25 21:44:24 rwatson Exp $ */
 /* $Id: rpcclnt.c,v 1.9 2003/11/05 14:59:03 rees Exp $ */
 
 /*-
@@ -1180,18 +1180,6 @@
 
 	m = rpcclnt_buildheader(rpc, procnum, mrest, mrest_len, &xid, &mheadend,
 	    cred);
-	/*
-	 * This can happen if the auth_type is neither UNIX or NULL
-	 */
-	if (m == NULL) {
-#ifdef __OpenBSD__
-		pool_put(&rpctask_pool, task);
-#else
-		FREE(task, M_RPC);
-#endif
-		error = EPROTONOSUPPORT;
-		goto rpcmout;
-	}
 
 	/*
 	 * For stream protocols, insert a Sun RPC Record Mark.
@@ -1879,7 +1867,6 @@
 	*tl++ = txdr_unsigned(procid);
 
 	if ((error = rpcauth_buildheader(rc->rc_auth, cred, &mb, &bpos))) {
-		m_freem(mreq);
 		RPCDEBUG("rpcauth_buildheader failed %d", error);
 		return NULL;
 	}
---  sparc64/include/param.h.orig
+++  sparc64/include/param.h
@@ -105,6 +105,21 @@
 #define PAGE_SIZE_MAX	PAGE_SIZE_4M
 #define PAGE_MASK_MAX	PAGE_MASK_4M
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#define	SP_LEVELS	3
+
+/*
+ * SP_SMALL_SHIFT is LOG2("The Smallest Superpage Size" / PAGE_SIZE).
+ */
+#define	SP_SMALL_SHIFT	3
+
+/*
+ * SP_FACTOR_SHIFT is ...
+ */
+#define	SP_FACTOR_SHIFT	3
+
 #ifndef KSTACK_PAGES
 #define KSTACK_PAGES		4	/* pages of kernel stack (with pcb) */
 #endif
---  sparc64/include/vmparam.h.orig
+++  sparc64/include/vmparam.h
@@ -96,9 +96,10 @@
  * the pool from which physical pages for small UMA objects are
  * allocated.
  */
-#define	VM_NFREEPOOL		2
+#define	VM_NFREEPOOL		3
 #define	VM_FREEPOOL_DEFAULT	0
 #define	VM_FREEPOOL_DIRECT	1
+#define	VM_FREEPOOL_RESERV	2
 
 /*
  * Create two free page lists: VM_FREELIST_DEFAULT is for physical
@@ -215,6 +216,13 @@
 #endif
 
 /*
+ * The default virtual address for the beginning of the text segment
+ */
+#ifndef	VM_IMAGE_BASE
+#define	VM_IMAGE_BASE		(0x0000000100000000UL)
+#endif
+
+/*
  * Initial pagein size of beginning of executable file.
  */
 #ifndef	VM_INITIAL_PAGEIN
---  sparc64/sparc64/pmap.c.orig
+++  sparc64/sparc64/pmap.c
@@ -1262,8 +1262,8 @@
  * will be wired down.
  */
 void
-pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-	   boolean_t wired)
+pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t fault_type, vm_page_t m,
+    vm_prot_t prot, boolean_t wired)
 {
 
 	vm_page_lock_queues();
---  sun4v/include/param.h.orig
+++  sun4v/include/param.h
@@ -102,6 +102,21 @@
 #define PAGE_SIZE_MAX	PAGE_SIZE_4M
 #define PAGE_MASK_MAX	PAGE_MASK_4M
 
+/*
+ * SP_LEVELS is the number of superpage sizes.
+ */
+#define	SP_LEVELS	3
+
+/*
+ * SP_SMALL_SHIFT is LOG2("The Smallest Superpage Size" / PAGE_SIZE).
+ */
+#define	SP_SMALL_SHIFT	3
+
+/*
+ * SP_FACTOR_SHIFT is ...
+ */
+#define	SP_FACTOR_SHIFT	3
+
 #ifndef KSTACK_PAGES
 #define KSTACK_PAGES		4	/* pages of kernel stack (with pcb) */
 #endif
---  sun4v/include/vmparam.h.orig
+++  sun4v/include/vmparam.h
@@ -96,9 +96,10 @@
  * the pool from which physical pages for small UMA objects are
  * allocated.
  */
-#define	VM_NFREEPOOL		2
+#define	VM_NFREEPOOL		3
 #define	VM_FREEPOOL_DEFAULT	0
 #define	VM_FREEPOOL_DIRECT	1
+#define	VM_FREEPOOL_RESERV	2
 
 /*
  * Create two free page lists: VM_FREELIST_DEFAULT is for physical
@@ -215,6 +216,13 @@
 #endif
 
 /*
+ * The default virtual address for the beginning of the text segment
+ */
+#ifndef	VM_IMAGE_BASE
+#define	VM_IMAGE_BASE		(0x0000000100000000UL)
+#endif
+
+/*
  * Initial pagein size of beginning of executable file.
  */
 #ifndef	VM_INITIAL_PAGEIN
---  sun4v/sun4v/pmap.c.orig
+++  sun4v/sun4v/pmap.c
@@ -57,7 +57,7 @@
 #include <vm/vm_object.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_pageout.h>
-#include <vm/vm_pager.h>
+#include <vm/vm_phys.h>
 #include <vm/vm_phys.h>
 #include <vm/uma.h>
 
@@ -1039,7 +1039,8 @@
  * will be wired down.
  */
 void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t fault_type, 
+	   vm_page_t m, vm_prot_t prot,
 	   boolean_t wired)
 {
 	vm_paddr_t pa, opa;
@@ -1124,6 +1125,8 @@
 	
 	invlva = FALSE;
 	if ((otte_data & ~(VTD_W|VTD_REF)) != tte_data) {
+		if (fault_type & VM_PROT_WRITE)
+			tte_data |= VTD_W; 
 		if (otte_data & VTD_V) {
 			if (otte_data & VTD_REF) {
 				if (otte_data & VTD_MANAGED) 
@@ -1141,7 +1144,8 @@
 				pmap_invalidate_page(pmap, va, TRUE);
 		}
 	} 
-
+	tte_hash_insert(pmap->pm_hash, va, tte_data|TTE_MINFLAGS|VTD_REF);
+	tsb_set_tte(&pmap->pm_tsb, va, tte_data|TTE_MINFLAGS|VTD_REF, pmap->pm_context);
 
 	tte_hash_insert(pmap->pm_hash, va, tte_data|TTE_MINFLAGS|VTD_REF);
 	/*
---  vm/pmap.h.orig
+++  vm/pmap.h
@@ -95,8 +95,8 @@
 void		 pmap_clear_reference(vm_page_t m);
 void		 pmap_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
 void		 pmap_copy_page(vm_page_t, vm_page_t);
-void		 pmap_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,
-		    boolean_t);
+void		 pmap_enter(pmap_t, vm_offset_t, vm_prot_t, vm_page_t,
+		    vm_prot_t, boolean_t);
 void	 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
 		    vm_prot_t prot);
 void		 pmap_enter_object(pmap_t pmap, vm_offset_t start,
---  vm/vm.h.orig
+++  vm/vm.h
@@ -114,6 +114,9 @@
 typedef struct vm_page *vm_page_t;
 #endif				/* _KERNEL */
 
+struct vm_reserv;
+typedef struct vm_reserv *vm_reserv_t;
+
 /*
  * Information passed from the machine-independant VM initialization code
  * for use by machine-dependant code (mainly for MMU support)
---  vm/vm_fault.c.orig
+++  vm/vm_fault.c
@@ -117,6 +117,17 @@
 #define VM_FAULT_READ_BEHIND 7
 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
 
+SYSCTL_NODE(_debug, OID_AUTO, prefetch, CTLFLAG_RD, 0, "");
+
+static int aligned = 1;
+SYSCTL_INT(_debug_prefetch, OID_AUTO, aligned, CTLFLAG_RW, &aligned, 0, "");
+
+static int calls;
+SYSCTL_INT(_debug_prefetch, OID_AUTO, calls, CTLFLAG_RD, &calls, 0, "");
+
+static int pages;
+SYSCTL_INT(_debug_prefetch, OID_AUTO, pages, CTLFLAG_RD, &pages, 0, "");
+
 struct faultstate {
 	vm_page_t m;
 	vm_object_t object;
@@ -209,10 +220,10 @@
 {
 	vm_prot_t prot;
 	int is_first_object_locked, result;
-	boolean_t growstack, wired;
+	boolean_t are_queues_locked, growstack, wired;
 	int map_generation;
 	vm_object_t next_object;
-	vm_page_t marray[VM_FAULT_READ];
+	vm_page_t marray[VM_FAULT_READ], mt;
 	int hardfault;
 	int faultcount;
 	struct faultstate fs;
@@ -442,6 +453,10 @@
 			 */
 			fs.m = NULL;
 			if (!vm_page_count_severe()) {
+				if ((fs.object->flags & OBJ_SUPERPAGES) == 0) {
+					fs.object->flags |= OBJ_SUPERPAGES;
+					fs.object->pg_color = atop(vaddr) - fs.pindex;
+				}
 				fs.m = vm_page_alloc(fs.object, fs.pindex,
 				    (fs.vp || fs.object->backing_object)? VM_ALLOC_NORMAL: VM_ALLOC_ZERO);
 			}
@@ -471,6 +486,17 @@
 			if (behavior == MAP_ENTRY_BEHAV_RANDOM) {
 				ahead = 0;
 				behind = 0;
+			} else if (aligned) {
+				/*
+				 * Compute an aligned window of VM_FAULT_READ
+				 * pages that are ahead and/or behind the
+				 * faulting virtual address.
+				 */
+				ahead = atop(MIN(roundup2(vaddr + PAGE_SIZE,
+				    VM_FAULT_READ * PAGE_SIZE), fs.entry->end) -
+				    (vaddr + PAGE_SIZE));
+				behind = atop(vaddr - MAX(fs.entry->start,
+				    vaddr & ~(VM_FAULT_READ * PAGE_SIZE - 1)));
 			} else {
 				behind = (vaddr - fs.entry->start) >> PAGE_SHIFT;
 				if (behind > VM_FAULT_READ_BEHIND)
@@ -480,6 +506,8 @@
 				if (ahead > VM_FAULT_READ_AHEAD)
 					ahead = VM_FAULT_READ_AHEAD;
 			}
+			KASSERT(behind + ahead < VM_FAULT_READ,
+			    ("vm_fault: reading too many pages"));
 			is_first_object_locked = FALSE;
 			if ((behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
 			     (behavior != MAP_ENTRY_BEHAV_RANDOM &&
@@ -495,20 +523,26 @@
 				else
 					firstpindex = fs.first_pindex - 2 * VM_FAULT_READ;
 
-				vm_page_lock_queues();
+				mt = fs.first_object == fs.object ? fs.m :
+				    fs.first_m;
+				KASSERT(mt != NULL, ("vm_fault: missing mt"));
+				are_queues_locked = FALSE;
+
 				/*
 				 * note: partially valid pages cannot be 
 				 * included in the lookahead - NFS piecemeal
 				 * writes will barf on it badly.
 				 */
 				for (tmppindex = fs.first_pindex - 1;
-					tmppindex >= firstpindex;
-					--tmppindex) {
-					vm_page_t mt;
-
-					mt = vm_page_lookup(fs.first_object, tmppindex);
-					if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL))
+				    tmppindex >= firstpindex && (mt =
+				    TAILQ_PREV(mt, pglist, listq)) != NULL &&
+				    mt->pindex == tmppindex; --tmppindex) {
+					if (mt->valid != VM_PAGE_BITS_ALL)
 						break;
+					if (!are_queues_locked) {
+						are_queues_locked = TRUE;
+						vm_page_lock_queues();
+					}
 					if (mt->busy ||
 					    (mt->oflags & VPO_BUSY) ||
 					    (mt->flags & (PG_FICTITIOUS | PG_UNMANAGED)) ||
@@ -522,9 +556,12 @@
 						vm_page_cache(mt);
 					}
 				}
-				vm_page_unlock_queues();
-				ahead += behind;
-				behind = 0;
+				if (are_queues_locked)
+					vm_page_unlock_queues();
+				if (!aligned) {
+					ahead += behind;
+					behind = 0;
+				}
 			}
 			if (is_first_object_locked)
 				VM_OBJECT_UNLOCK(fs.first_object);
@@ -545,8 +582,10 @@
 			 * XXX vm_fault_additional_pages() can block
 			 * without releasing the map lock.
 			 */
+			calls++;
 			faultcount = vm_fault_additional_pages(
 			    fs.m, behind, ahead, marray, &reqpage);
+			pages += faultcount;
 
 			/*
 			 * update lastr imperfectly (we do not know how much
@@ -891,7 +930,7 @@
 	 * back on the active queue until later so that the pageout daemon
 	 * won't find it (yet).
 	 */
-	pmap_enter(fs.map->pmap, vaddr, fs.m, prot, wired);
+	pmap_enter(fs.map->pmap, vaddr, fault_type, fs.m, prot, wired);
 	if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) {
 		vm_fault_prefault(fs.map->pmap, vaddr, fs.entry);
 	}
@@ -987,7 +1026,6 @@
 			break;
 		}
 		if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
-			(m->busy == 0) &&
 		    (m->flags & PG_FICTITIOUS) == 0) {
 
 			vm_page_lock_queues();
@@ -1121,6 +1159,8 @@
 	 */
 	dst_object = vm_object_allocate(OBJT_DEFAULT,
 	    OFF_TO_IDX(dst_entry->end - dst_entry->start));
+	dst_object->flags |= OBJ_SUPERPAGES;
+	dst_object->pg_color = atop(dst_entry->start);
 
 	VM_OBJECT_LOCK(dst_object);
 	dst_entry->object.vm_object = dst_object;
@@ -1178,9 +1218,10 @@
 		VM_OBJECT_UNLOCK(dst_object);
 
 		/*
-		 * Enter it in the pmap...
+		 * Enter it in the pmap as if a read fault occurred.
 		 */
-		pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
+		pmap_enter(dst_map->pmap, vaddr, prot & ~VM_PROT_WRITE, dst_m,
+		    prot, FALSE);
 
 		/*
 		 * Mark it no longer busy, and put it on the active list.
---  vm/vm_kern.c.orig
+++  vm/vm_kern.c
@@ -406,7 +406,7 @@
 		/*
 		 * Because this is kernel_pmap, this call will not block.
 		 */
-		pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL, 1);
+		pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL, 1);
 		vm_page_wakeup(m);
 	}
 	VM_OBJECT_UNLOCK(kmem_object);
---  vm/vm_object.c.orig
+++  vm/vm_object.c
@@ -90,6 +90,7 @@
 #include <vm/swap_pager.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_extern.h>
+#include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
 #define EASY_SCAN_FACTOR       8
@@ -237,10 +238,14 @@
 	VM_OBJECT_LOCK_INIT(&kernel_object_store, "kernel object");
 	_vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
 	    kernel_object);
+	kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
+	vm_object_set_flag(kernel_object, OBJ_SUPERPAGES);
 
 	VM_OBJECT_LOCK_INIT(&kmem_object_store, "kmem object");
 	_vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
 	    kmem_object);
+	kmem_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
+	vm_object_set_flag(kmem_object, OBJ_SUPERPAGES);
 
 	/*
 	 * The lock portion of struct vm_object must be type stable due
@@ -643,7 +648,7 @@
 			vm_page_free(p);
 			cnt.v_pfree++;
 		} else {
-			vm_page_remove(p);
+			vm_page_remove(p, TRUE);
 		}
 	}
 	vm_page_unlock_queues();
@@ -1244,7 +1249,10 @@
 		LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
 		source->shadow_count++;
 		source->generation++;
-		result->flags |= source->flags & OBJ_NEEDGIANT;
+		result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
+		    ((1 << (VM_NFREEORDER - 1)) - 1);
+		result->flags |= source->flags &
+		    (OBJ_NEEDGIANT | OBJ_SUPERPAGES);
 		VM_OBJECT_UNLOCK(source);
 	}
 
@@ -1458,6 +1466,7 @@
 		 */
 		if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
 			vm_page_t pp;
+			vm_reserv_t reserv;
 
 			if (op & OBSC_COLLAPSE_NOWAIT) {
 				if ((p->oflags & VPO_BUSY) ||
@@ -1519,7 +1528,7 @@
 				if (p->wire_count == 0)
 					vm_page_free(p);
 				else
-					vm_page_remove(p);
+					vm_page_remove(p, TRUE);
 				vm_page_unlock_queues();
 				p = next;
 				continue;
@@ -1543,13 +1552,22 @@
 				if (p->wire_count == 0)
 					vm_page_free(p);
 				else
-					vm_page_remove(p);
+					vm_page_remove(p, TRUE);
 				vm_page_unlock_queues();
 				p = next;
 				continue;
 			}
 
 			/*
+			 * Rename the reservation.
+			 */
+			if ((reserv = vm_reserv_get(p)) != NULL &&
+			    reserv->object != object) {
+				reserv->object = object;
+				reserv->first_pindex -= backing_offset_index;
+			}
+
+			/*
 			 * Page does not exist in parent, rename the
 			 * page from the backing object to the main object. 
 			 *
---  vm/vm_object.h.orig
+++  vm/vm_object.h
@@ -142,6 +142,7 @@
 #define OBJ_PIPWNT	0x0040		/* paging in progress wanted */
 #define OBJ_MIGHTBEDIRTY 0x0100		/* object might be dirty */
 #define OBJ_CLEANING	0x0200
+#define	OBJ_SUPERPAGES	0x1000
 #define	OBJ_ONEMAPPING	0x2000		/* One USE (a single, non-forked) mapping flag */
 #define	OBJ_DISCONNECTWNT 0x4000	/* disconnect from vnode wanted */
 #define	OBJ_NEEDGIANT	0x8000		/* object requires Giant */
---  vm/vm_page.c.orig
+++  vm/vm_page.c
@@ -99,6 +99,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: src/sys/vm/vm_page.c,v 1.348 2007/06/16 04:57:05 alc Exp $");
 
+#include "opt_vm.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
@@ -118,6 +120,7 @@
 #include <vm/vm_pageout.h>
 #include <vm/vm_pager.h>
 #include <vm/vm_phys.h>
+#include <vm/vm_reserv.h>
 #include <vm/vm_extern.h>
 #include <vm/uma.h>
 #include <vm/uma_int.h>
@@ -134,7 +137,6 @@
 
 vm_page_t vm_page_array = 0;
 int vm_page_array_size = 0;
-long first_page = 0;
 int vm_page_zero_count = 0;
 
 static int boot_pages = UMA_BOOT_PAGES;
@@ -201,7 +203,7 @@
 vm_page_startup(vm_offset_t vaddr)
 {
 	vm_offset_t mapped;
-	vm_size_t npages;
+	vm_size_t nreservations;
 	vm_paddr_t page_range;
 	vm_paddr_t new_end;
 	int i;
@@ -213,7 +215,7 @@
 	/* the biggest memory array is the second group of pages */
 	vm_paddr_t end;
 	vm_paddr_t biggestsize;
-	vm_paddr_t low_water, high_water;
+	vm_paddr_t high_water;
 	int biggestone;
 
 	vm_paddr_t total;
@@ -229,7 +231,6 @@
 		phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
 	}
 
-	low_water = phys_avail[0];
 	high_water = phys_avail[1];
 
 	for (i = 0; phys_avail[i + 1]; i += 2) {
@@ -239,8 +240,6 @@
 			biggestone = i;
 			biggestsize = size;
 		}
-		if (phys_avail[i] < low_water)
-			low_water = phys_avail[i];
 		if (phys_avail[i + 1] > high_water)
 			high_water = phys_avail[i + 1];
 		++nblocks;
@@ -273,7 +272,21 @@
 	    VM_PROT_READ | VM_PROT_WRITE);
 	bzero((void *)mapped, end - new_end);
 	uma_startup((void *)mapped, boot_pages);
+	end = new_end;
 
+	/*
+	 * Initialize the reservation structures.
+	 */
+	nreservations = 0;
+	for (i = 0; i < SP_LEVELS; i++)
+		nreservations += total >> (SP_ORDER(i) + PAGE_SHIFT);
+	new_end = end - (nreservations * sizeof(struct vm_reserv));
+	new_end = trunc_page(new_end);
+	mapped = pmap_map(&vaddr, new_end, end, VM_PROT_READ | VM_PROT_WRITE);
+	bzero((void *)mapped, end - new_end);
+	vm_reserv_init((void *)mapped, nreservations);
+	end = new_end;
+
 #if defined(__amd64__) || defined(__i386__)
 	/*
 	 * Allocate a bitmap to indicate that a random physical page
@@ -288,29 +301,27 @@
 	 */
 	page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE;
 	vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
-	new_end -= vm_page_dump_size;
+	new_end = end - vm_page_dump_size;
+	new_end = trunc_page(new_end);
 	vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
-	    new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
+	    end, VM_PROT_READ | VM_PROT_WRITE);
 	bzero((void *)vm_page_dump, vm_page_dump_size);
+	end = new_end;
 #endif
 	/*
 	 * Compute the number of pages of memory that will be available for
 	 * use (taking into account the overhead of a page structure per
 	 * page).
 	 */
-	first_page = low_water / PAGE_SIZE;
 #ifdef VM_PHYSSEG_SPARSE
 	page_range = 0;
 	for (i = 0; phys_avail[i + 1] != 0; i += 2)
 		page_range += atop(phys_avail[i + 1] - phys_avail[i]);
 #elif defined(VM_PHYSSEG_DENSE)
-	page_range = high_water / PAGE_SIZE - first_page;
+	page_range = high_water / PAGE_SIZE;
 #else
 #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
 #endif
-	npages = (total - (page_range * sizeof(struct vm_page)) -
-	    (end - new_end)) / PAGE_SIZE;
-	end = new_end;
 
 	/*
 	 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
@@ -345,14 +356,9 @@
 	vm_page_array_size = page_range;
 
 	/*
-	 * This assertion tests the hypothesis that npages and total are
-	 * redundant.  XXX
+	 * Initialize the physical memory allocator.
 	 */
-	page_range = 0;
-	for (i = 0; phys_avail[i + 1] != 0; i += 2)
-		page_range += atop(phys_avail[i + 1] - phys_avail[i]);
-	KASSERT(page_range == npages,
-	    ("vm_page_startup: inconsistent page counts"));
+	vm_phys_init();
 
 	/*
 	 * Initialize the physical memory allocator.
@@ -697,10 +703,11 @@
  *	This routine may not block.
  */
 void
-vm_page_remove(vm_page_t m)
+vm_page_remove(vm_page_t m, boolean_t release)
 {
 	vm_object_t object;
 	vm_page_t root;
+	vm_reserv_t reserv;
 
 	if ((object = m->object) == NULL)
 		return;
@@ -712,6 +719,12 @@
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 
 	/*
+	 * Release the superpage reservation if specified.
+	 */
+	if ((reserv = vm_reserv_get(m)) != NULL && __predict_false(release))
+		vm_reserv_release(reserv, -1, m);
+
+	/*
 	 * Now remove from the object's list of backed pages.
 	 */
 	if (m != object->root)
@@ -787,8 +800,10 @@
 void
 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
 {
+	vm_reserv_t reserv;
 
-	vm_page_remove(m);
+	vm_page_remove(m, (reserv = vm_reserv_get(m)) != NULL &&
+	    reserv->object != new_object);
 	vm_page_insert(m, new_object, new_pindex);
 	if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
 		vm_page_deactivate(m);
@@ -856,6 +871,7 @@
 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 {
 	vm_page_t m = NULL;
+	vm_reserv_t reserv;
 	int flags, page_req;
 
 	page_req = req & VM_ALLOC_CLASS_MASK;
@@ -867,7 +883,9 @@
 		KASSERT(object != NULL,
 		    ("vm_page_alloc: NULL object."));
 		VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	}
+		reserv = vm_reserv_acquire(object, pindex, req);
+	} else
+		reserv = NULL;
 
 	/*
 	 * The pager is allowed to eat deeper into the free page list.
@@ -878,7 +896,14 @@
 
 loop:
 	mtx_lock(&vm_page_queue_free_mtx);
-	if (cnt.v_free_count > cnt.v_free_reserved ||
+	if (reserv != NULL) {
+		KASSERT(!reserv->broken,
+		    ("vm_page_alloc: reservation %p is broken", reserv));
+		m = &reserv->first_page[SP_POS(object, pindex, reserv->level)];
+		KASSERT(m->queue == PQ_NONE,
+		    ("vm_page_alloc: page %p has unexpected queue %d",
+		    m, m->queue));
+	} else if (cnt.v_free_count > cnt.v_free_reserved ||
 	    (page_req == VM_ALLOC_SYSTEM && 
 	     cnt.v_cache_count == 0 && 
 	     cnt.v_free_count > cnt.v_interrupt_free_min) ||
@@ -902,6 +927,14 @@
 			    ("vm_page_alloc: cache queue is missing %d pages",
 			    cnt.v_cache_count));
 			vm_page_unlock_queues();
+
+			/*
+			 * Retry the allocation if the unused pages
+			 * from an incompletely utilized reservation
+			 * are reclaimed.
+			 */ 
+			if (vm_reserv_reclaim(-1))
+				goto loop;
 			atomic_add_int(&vm_pageout_deficit, 1);
 			pagedaemon_wakeup();
 
@@ -966,11 +999,15 @@
 	m->busy = 0;
 	m->valid = 0;
 	KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
+	KASSERT(m->reserv == NULL,
+	    ("vm_page_alloc: free page %p has a reservation", m));
 	mtx_unlock(&vm_page_queue_free_mtx);
 
-	if ((req & VM_ALLOC_NOOBJ) == 0)
+	if ((req & VM_ALLOC_NOOBJ) == 0) {
 		vm_page_insert(m, object, pindex);
-	else
+		if (reserv != NULL)
+			vm_reserv_populate(m, reserv);
+	} else
 		m->pindex = pindex;
 
 	/*
@@ -1109,6 +1146,7 @@
 void
 vm_page_free_toq(vm_page_t m)
 {
+	vm_reserv_t reserv;
 
 	if (VM_PAGE_GETQUEUE(m) != PQ_NONE)
 		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@@ -1134,7 +1172,7 @@
 	 * appropriate free queue.
 	 */
 	vm_pageq_remove_nowakeup(m);
-	vm_page_remove(m);
+	vm_page_remove(m, m->hold_count != 0);
 
 	/*
 	 * If fictitious remove object association and
@@ -1159,16 +1197,20 @@
 		vm_pageq_enqueue(PQ_HOLD, m);
 	} else {
 		m->flags |= PG_FREE;
-		mtx_lock(&vm_page_queue_free_mtx);
-		if ((m->flags & PG_ZERO) != 0) {
-			vm_phys_free_pages_locked(m, 0);
-			++vm_page_zero_count;
-		} else {
-			vm_phys_free_pages_locked(m, 0);
-			vm_page_zero_idle_wakeup();
+		if ((reserv = vm_reserv_get(m)) != NULL)
+			vm_reserv_depopulate(reserv, m);
+		else {
+			mtx_lock(&vm_page_queue_free_mtx);
+			if ((m->flags & PG_ZERO) != 0) {
+				vm_phys_free_pages_locked(m, 0);
+				++vm_page_zero_count;
+			} else {
+				vm_phys_free_pages_locked(m, 0);
+				vm_page_zero_idle_wakeup();
+			}
+			vm_page_free_wakeup();
+			mtx_unlock(&vm_page_queue_free_mtx);
 		}
-		vm_page_free_wakeup();
-		mtx_unlock(&vm_page_queue_free_mtx);
 	}
 }
 
@@ -1710,7 +1752,7 @@
 
  retry_alloc:
 	pmap_remove_all(m);
-	vm_page_remove(m);
+	vm_page_remove(m, TRUE);
 	mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
 	if (mnew == NULL) {
 		vm_page_insert(m, object, pindex);
---  vm/vm_page.h.orig
+++  vm/vm_page.h
@@ -106,6 +106,7 @@
 	struct vm_page *left;		/* splay tree link (O)		*/
 	struct vm_page *right;		/* splay tree link (O)		*/
 
+	vm_reserv_t reserv;		/* which superpage am I in (O) */
 	vm_object_t object;		/* which object am I in (O,P)*/
 	vm_pindex_t pindex;		/* offset into object (O,P) */
 	vm_paddr_t phys_addr;		/* physical address of page */
@@ -254,7 +255,8 @@
 
 extern vm_page_t vm_page_array;		/* First resident page in table */
 extern int vm_page_array_size;		/* number of vm_page_t's */
-extern long first_page;			/* first physical page number */
+
+#define	VM_PAGE_IS_FREE(m)	(((m)->flags & PG_FREE) != 0)
 
 #define	VM_PAGE_IS_FREE(m)	(((m)->flags & PG_FREE) != 0)
 
@@ -270,7 +272,7 @@
 #ifdef VM_PHYSSEG_SPARSE
 	return (vm_phys_paddr_to_vm_page(pa));
 #elif defined(VM_PHYSSEG_DENSE)
-	return (&vm_page_array[atop(pa) - first_page]);
+	return (&vm_page_array[atop(pa)]);
 #else
 #error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
 #endif
@@ -331,7 +333,7 @@
 void vm_page_deactivate (vm_page_t);
 void vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
 vm_page_t vm_page_lookup (vm_object_t, vm_pindex_t);
-void vm_page_remove (vm_page_t);
+void vm_page_remove(vm_page_t, boolean_t);
 void vm_page_rename (vm_page_t, vm_object_t, vm_pindex_t);
 vm_page_t vm_page_select_cache(void);
 void vm_page_sleep(vm_page_t m, const char *msg);
---  vm/vm_phys.h.orig
+++  vm/vm_phys.h
@@ -27,7 +27,7 @@
  * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: src/sys/vm/vm_phys.h,v 1.1 2007/06/10 00:49:16 alc Exp $
+ * $FreeBSD$
  */
 
 /*
--- /dev/null	2007-08-19 10:55:00.000000000 +0200
+++ vm/vm_reserv.c	2007-08-19 10:58:32.000000000 +0200
@@ -0,0 +1,788 @@
+/*-
+ * Copyright (c) 2002-2006 Rice University
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project by Alan L. Cox,
+ * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ *	Superpage reservation management module
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_vm.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/sbuf.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
+#include <vm/vm_reserv.h>
+
+/* 
+ * Returns the index of the reservation in which the page is located. This index refers
+ * to a reservation of size res->level - 1 (i.e res->order - SP_FACTOR_SHIFT)
+ */
+#define IDX(m,res) (((m) - (res)->first_page) >> \
+					((res)->order - SP_FACTOR_SHIFT))
+
+#define NO_RESQ		SP_LEVELS		/*reserv->rql==NO_RESQ -> reservation is in no reservation queue */
+
+static vm_reserv_t vm_reserv_alloc(void);
+static vm_reserv_t vm_reserv_create(int, vm_object_t);
+static void vm_reserv_create_children(vm_reserv_t, vm_page_t);
+static void vm_reserv_dequeue(vm_reserv_t);
+static void vm_reserv_free(vm_reserv_t);
+static void vm_reserv_move(vm_reserv_t, boolean_t);
+static int vm_reserv_policy(vm_object_t, vm_pindex_t, int);
+static void vm_reserv_update_maxavail(vm_reserv_t);
+
+/*
+ * Reservations in rres[level], with level=-1..SP_LEVELS-2, are those that, 
+ * if broken, will yield chunks of size at most SP_MAGN(level).
+ * Fully populated reservations don't go here
+ */
+static struct vm_reserv_queue rres_store[SP_LEVELS];
+static struct vm_reserv_queue *const rres = &rres_store[1];
+static struct mtx rres_mtx;
+
+/*
+ * The free reservation queue:
+ */
+static struct vm_reserv_queue vm_resq_free;
+static struct mtx vm_resq_free_mtx;
+static int vm_resq_free_count;
+
+SYSCTL_NODE(_vm, OID_AUTO, reserv, CTLFLAG_RD, 0, "Reservation Info");
+
+SYSCTL_INT(_vm_reserv, OID_AUTO, free, CTLFLAG_RD, &vm_resq_free_count, 0,
+    "Free Reservations");
+
+static int sysctl_vm_reserv_queues(SYSCTL_HANDLER_ARGS);
+
+SYSCTL_OID(_vm_reserv, OID_AUTO, queues, CTLTYPE_STRING | CTLFLAG_RD,
+    NULL, 0, sysctl_vm_reserv_queues, "A", "Reservation Queues");
+
+/*
+ *
+ */
+static int
+sysctl_vm_reserv_queues(SYSCTL_HANDLER_ARGS)
+{
+	vm_reserv_t res;
+	struct sbuf sbuf;
+	char *cbuf;
+	const int cbufsize = (SP_LEVELS + 1) * 81;
+	int counter, error, level;
+
+	cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO);
+	sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN);
+	sbuf_printf(&sbuf, "\nLEVEL     SIZE  NUMBER\n\n");
+	for (level = -1; level <= SP_LEVELS - 2; level++) {
+		counter = 0;
+		mtx_lock(&rres_mtx);
+		TAILQ_FOREACH(res, &rres[level], next)
+			counter++;
+		mtx_unlock(&rres_mtx);
+		sbuf_printf(&sbuf, "%5.5d: %6.6dK, %6.6d\n", level,
+		    SP_SIZE(level) / 1024, counter);
+	}
+	sbuf_finish(&sbuf);
+	error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
+	sbuf_delete(&sbuf);
+	free(cbuf, M_TEMP);
+	return (error);
+}
+
+/*
+ * Determine the size of the reservation to create for the given
+ * vm_object
+ * Returns the level of the reservation corresponding to the reservation size
+ */
+static int
+vm_reserv_policy(vm_object_t object, vm_pindex_t pindex, int level)
+{
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+
+	/* 
+	 * If vnode or backed by vnode, do not reserve beyond object's size.
+	 * We don't bother looking too far in the object chain.
+	 */
+	if (object->type == OBJT_VNODE || (object->backing_object != NULL &&
+	    object->backing_object->type == OBJT_VNODE))
+		while (level >= 0 && object->size < pindex -
+		    SP_POS(object, pindex, level) + SP_MAGN(level))
+			level--;
+	KASSERT(level >= -1, ("vm_reserv_policy: invalid level"));
+	return (level);
+}
+
+/*
+ * Update maxavail in the given superpage reservation and each of its
+ * ancestors.  As a side effect, free any unused descendants of these
+ * superpage reservations.
+ */
+static void
+vm_reserv_update_maxavail(vm_reserv_t res)
+{
+	vm_reserv_t child;
+	int i, newmax;
+	boolean_t nochild;
+
+	VM_OBJECT_LOCK_ASSERT(res->object, MA_OWNED);
+	res->maxavail = MAX(res->popfrom, (1 << res->order) - res->popto);
+	while ((res = res->parent) != NULL) {	
+		KASSERT(!res->broken,
+		    ("vm_reserv_update_maxavail: reservation %p is broken", res));
+		newmax = 0;
+		nochild = TRUE;
+		for (i = 0; i < SP_FACTOR; i++) {
+			child = res->child[i];
+			if (child != NULL && child->popfrom == child->popto) {
+				KASSERT(child->refcnt == 0,
+				    ("vm_reserv_update_maxavail: child->refcnt != 0"));
+				vm_reserv_free(child);
+				child = res->child[i] = NULL;
+			}
+			if (child == NULL)
+				newmax = 1 << (res->order - SP_FACTOR_SHIFT);
+			else {
+				newmax = MAX(newmax, child->maxavail);
+				nochild = FALSE;
+			}
+		}
+		if (nochild) {
+			/* must have been marked */
+			KASSERT(res->popfrom > res->popto,
+			    ("vm_reserv_update_maxavail: popfrom <= popto"));
+			res->popfrom = res->popto = 0;
+			res->maxavail = 1 << res->order;
+		} else {
+			if (newmax == res->maxavail)
+				break;
+			res->maxavail = newmax;
+		}
+	}
+}
+
+/*
+ * Acquire a reservation for the given object and offset. 
+ */
+vm_reserv_t
+vm_reserv_acquire(vm_object_t object, vm_pindex_t pindex, int req)
+{
+	int l, level;
+	vm_page_t m, p, s;
+	vm_pindex_t first, leftcap, rightcap;
+	vm_reserv_t res;
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+	if ((object->flags & OBJ_SUPERPAGES) == 0 || pindex >= object->size)
+		return (NULL);
+
+	/*
+	 * Look for an existing reservation.
+	 */
+	s = NULL;
+	p = object->root;
+	while (p != NULL) {
+		KASSERT(p->pindex != pindex,
+		    ("vm_reserv_acquire: pindex already allocated"));
+		res = vm_reserv_get(p);
+		if (res != NULL && vm_reserv_has_pindex(res, pindex))
+			return (res);
+		else if (p->pindex < pindex) {
+			if (s != NULL || (s = TAILQ_NEXT(p, listq)) == NULL)
+				break;
+			KASSERT(s->pindex != pindex,
+			    ("vm_reserv_acquire: pindex already allocated"));
+			res = vm_reserv_get(s);
+			if (res != NULL && vm_reserv_has_pindex(res, pindex))
+				return (res);
+			else if (pindex < s->pindex)
+				break;
+		} else if (s == NULL) {
+			s = p;
+			p = TAILQ_PREV(s, pglist, listq);
+			continue;
+		}
+		s = NULL;
+		p = object->root = vm_page_splay(pindex, object->root);
+	}
+
+	/*
+	 * Determine the first index to the left that can be used.
+	 */
+	if (p == NULL)
+		leftcap = 0;
+	else if (p->reserv == NULL)
+		leftcap = p->pindex + 1;
+	else
+		leftcap = p->reserv->first_pindex + SP_MAGN(p->reserv->level);
+
+	/*
+	 * Determine the first index to the right that cannot be used.
+	 */
+	if (s == NULL)
+		rightcap = pindex + SP_LARGE;
+	else if (s->reserv == NULL)
+		rightcap = s->pindex;
+	else
+		rightcap = s->reserv->first_pindex;
+
+	/*
+	 * Determine the level of the largest superpage that fits
+	 * between leftcap and rightcap.
+	 */
+	for (l = 0; l < SP_LEVELS; l++) {
+		if (pindex < SP_POS(object, pindex, l))
+			break;
+		first = pindex - SP_POS(object, pindex, l);
+		if (first < leftcap || first + SP_MAGN(l) > rightcap)
+			break;
+	}
+	l--;
+	KASSERT(l >= -1 && l < SP_LEVELS,
+	    ("vm_reserv_acquire: invalid level"));
+	level = vm_reserv_policy(object, pindex, l);
+	for (; level > -1; level--) {
+retry:
+		mtx_lock(&vm_page_queue_free_mtx);
+		if (cnt.v_free_count > cnt.v_free_reserved + SP_MAGN(level)) {
+			m = vm_phys_alloc_pages_locked(VM_FREEPOOL_RESERV,
+			    SP_ORDER(level));
+			mtx_unlock(&vm_page_queue_free_mtx);
+			if (m != NULL) {
+				res = vm_reserv_create(level, object);
+				res->first_page = m;
+				res->first_pindex = pindex - SP_POS(object,
+				    pindex, level);
+				return (res);
+			}
+		} else
+			mtx_unlock(&vm_page_queue_free_mtx);
+		if (level < SP_LEVELS - 1 && vm_reserv_reclaim(level))
+			goto retry;
+	}
+	return (NULL);
+}
+
+/*
+ * Returns the superpage reservation containing the given page. 
+ */
+vm_reserv_t
+vm_reserv_lookup(vm_page_t m)
+{
+	vm_reserv_t res;
+
+	while ((res = m->reserv) != NULL && __predict_false(res->broken)) {
+		m->reserv = res->child[IDX(m, res)];
+		KASSERT(res->refcnt > 0,
+		    ("vm_reserv_lookup: reservation %p has"
+		    " an invalid refcnt", res));
+		res->refcnt--;
+		if (res->refcnt == 0)
+			vm_reserv_free(res);
+	}	
+	return (res);
+}
+
+static long vm_reserv_allocations;
+SYSCTL_LONG(_vm_reserv, OID_AUTO, allocations, CTLFLAG_RD,
+    &vm_reserv_allocations, 0, "");
+
+/*
+ * Allocates a superpage reservation.  Initializes all of its fields
+ * to zero.  Always returns a superpage reservation; never fails.
+ */
+static vm_reserv_t
+vm_reserv_alloc(void)
+{
+	vm_reserv_t res;
+
+	vm_reserv_allocations++;
+	mtx_lock(&vm_resq_free_mtx);
+	res = TAILQ_FIRST(&vm_resq_free);
+	KASSERT(res != NULL,
+	    ("vm_reserv_alloc: free reservation queue is empty"));
+	TAILQ_REMOVE(&vm_resq_free, res, next);
+	vm_resq_free_count--;
+	mtx_unlock(&vm_resq_free_mtx);
+	bzero(res, sizeof(*res));
+	return (res);
+}
+
+/*
+ * Allocates a superpage reservation.  Initializes it to the given
+ * object and level.  Always returns a superpage reservation; never
+ * fails.
+ */
+static vm_reserv_t
+vm_reserv_create(int level, vm_object_t object)
+{
+	vm_reserv_t sp;
+
+	KASSERT(level >= 0, ("vm_reserv_create: invalid level"));
+	sp = vm_reserv_alloc();
+	sp->level = level;
+	sp->order = SP_ORDER(level);
+	sp->rql = NO_RESQ;
+	sp->maxavail = 1 << SP_ORDER(level);
+	sp->object = object;
+	return (sp);
+}
+
+/*
+ * Instantiates the children of the given superpage reservation.
+ */ 
+static void
+vm_reserv_create_children(vm_reserv_t res, vm_page_t m /* NULL => noupdate */)
+{
+	vm_reserv_t ch;
+	int from, idx, magn, to;
+
+	VM_OBJECT_LOCK_ASSERT(res->object, MA_OWNED);
+	magn = 1 << (res->order - SP_FACTOR_SHIFT);
+	idx = res->popfrom >> (res->order - SP_FACTOR_SHIFT);
+	KASSERT(idx >= 0 && idx < SP_FACTOR,
+	    ("vm_reserv_create_children: idx invalid"));
+	for (from = res->popfrom - idx * magn, to = res->popto - idx * magn;
+	    from < to; from = 0, to -= magn, idx++) {
+		ch = res->child[idx];
+		if (ch == NULL) {
+			ch = res->child[idx] = vm_reserv_create(res->level - 1, res->object);
+			ch->parent = res;
+			ch->first_page = &res->first_page[idx * (1 << ch->order)];
+			ch->first_pindex = res->first_pindex + idx * (1 << ch->order);
+			KASSERT(from < magn, ("vm_reserv_create_children: xxx"));
+			ch->popfrom = from;
+			ch->popto = to > magn ? magn : to;
+			ch->refcnt = ch->popto - ch->popfrom;			
+			if (ch->order == SP_SMALL_SHIFT) {
+				/* adjust things */
+				ch->popto = ch->popto - ch->popfrom;
+				ch->popfrom = 0;
+			}
+		}
+
+		/* 
+		 * this is subtle: we update maxavail only if ch is
+		 * not in the path to m. If it is, we don't want to
+		 * update maxavail to ensure propper propagation up.
+		 */
+		if (m == NULL || m < ch->first_page ||
+		    m >= ch->first_page + (1 << ch->order))
+			ch->maxavail = (1 << ch->order) - (ch->popto - ch->popfrom);
+	}
+	/* invalid popfrom/popto as a mark that means "has children" */
+	res->popfrom = 1;
+	res->popto = 0;
+}
+
+/*
+ *
+ */
+static __inline void
+vm_reserv_dequeue(vm_reserv_t sp)
+{
+
+	if (sp->rql != NO_RESQ) {
+		KASSERT(sp->rql >= -1 && sp->rql < SP_LEVELS - 1,
+		    ("vm_reserv_dequeue: sp->rql invalid"));
+		mtx_lock(&rres_mtx);
+		TAILQ_REMOVE(&rres[sp->rql], sp, next);
+		mtx_unlock(&rres_mtx);
+		sp->rql = NO_RESQ;
+	}
+}
+
+/*
+ * Frees a superpage reservation.
+ */
+static void
+vm_reserv_free(vm_reserv_t res)
+{
+
+	mtx_lock(&vm_resq_free_mtx);
+	TAILQ_INSERT_HEAD(&vm_resq_free, res, next);
+	vm_resq_free_count++;
+	mtx_unlock(&vm_resq_free_mtx);
+}
+
+static vm_reserv_t
+follow_child(vm_reserv_t res, int idx)
+{
+	vm_reserv_t ch;
+
+	KASSERT(idx >= 0 && idx < SP_FACTOR, ("follow_child: idx invalid"));
+	if (res->child[idx] == NULL) {
+		ch = vm_reserv_create(res->level - 1, res->object);
+		res->child[idx] = ch;
+		ch->parent = res;
+		ch->first_page = &res->first_page[idx * (1 << ch->order)];
+		ch->first_pindex = res->first_pindex + idx * (1 << ch->order);
+	}
+	return (res->child[idx]);
+}
+
+/*
+ * The given page is used.
+ */
+void
+vm_reserv_populate(vm_page_t m, vm_reserv_t res)
+{
+	int pos;
+
+	VM_OBJECT_LOCK_ASSERT(res->object, MA_OWNED);
+	m->reserv = res;
+	for (;;) {
+		KASSERT(!res->broken,
+		    ("vm_reserv_populate: reservation %p is broken", res));
+		KASSERT(m >= res->first_page &&
+		    m < &res->first_page[1 << res->order],
+		    ("vm_reserv_populate: page %p is outside"
+		    " of reservation %p", m, res));
+		KASSERT(res->refcnt < 1 << res->order,
+		    ("vm_reserv_populate: reservation %p has"
+		    " an invalid refcnt", res));
+		res->refcnt++;
+		if (res->order == SP_SMALL_SHIFT) {
+			/* at this level we just count */
+			KASSERT(res->popto < SP_SMALL,
+			    ("vm_reserv_populate: reservation %p has"
+			    " an invalid popto", res));
+			res->popto++;
+			/* if popto == SP_SMALL we could colapse, but don't bother */
+			break;
+		} else if (res->popfrom == res->popto) {
+			/* first population */
+			pos = m - res->first_page;
+			res->popfrom = pos;
+			res->popto = pos + 1;
+			break;
+		} else if (res->popfrom < res->popto) {
+			pos = m - res->first_page;
+			if (pos == res->popfrom - 1) {
+				res->popfrom--;
+				break;
+			} else if (pos == res->popto) {
+				res->popto++;
+				break;
+			}
+			vm_reserv_create_children(res, m);
+		}
+		res = follow_child(res, IDX(m, res));
+		KASSERT(res != NULL,
+		    ("vm_reserv_populate: missing reservation"));
+	}
+	vm_reserv_update_maxavail(res);
+	vm_reserv_move(m->reserv, TRUE);
+}
+
+/*
+ * The given page is not used.
+ */
+void
+vm_reserv_depopulate(vm_reserv_t res, vm_page_t m)
+{
+	vm_reserv_t sp;
+	int pos;
+
+	VM_OBJECT_LOCK_ASSERT(res->object, MA_OWNED);
+	for (sp = res;;) {
+		KASSERT(!sp->broken,
+		    ("vm_reserv_depopulate: reservation %p is broken", sp));
+		KASSERT(sp->popfrom != sp->popto,
+		    ("vm_reserv_depopulate: reservation %p is empty", sp));
+		KASSERT(sp->refcnt > 0,
+		    ("vm_reserv_depopulate: reservation %p has"
+		    " an invalid refcnt", sp));
+		sp->refcnt--;
+		if (sp->order == SP_SMALL_SHIFT) {
+			/* at this level we just count */
+			KASSERT(sp->popto > 0,
+			    ("vm_reserv_depopulate: reservation %p has"
+			    " an invalid popto", sp));
+			sp->popto--;
+			break;
+		} else if (sp->popfrom < sp->popto) {
+			pos = m - sp->first_page;
+			if (sp->popfrom == pos) {
+				sp->popfrom++;
+				break;
+			} else if (sp->popto == pos + 1) {
+				sp->popto--;
+				break;
+			}
+			vm_reserv_create_children(sp, m);
+		}
+		sp = sp->child[IDX(m, sp)];
+		KASSERT(sp != NULL,
+		    ("vm_reserv_depopulate: missing reservation"));
+	}
+	if (sp->popfrom == sp->popto)
+		sp->popfrom = sp->popto = 0;
+	m->reserv = NULL;
+	vm_reserv_update_maxavail(sp);
+	vm_reserv_move(res, FALSE);
+}
+
+/*
+ * Initializes the superpage reservation system.
+ */
+void
+vm_reserv_init(vm_reserv_t reservation, int nreservations) 
+{
+	int i;
+
+	TAILQ_INIT(&vm_resq_free);
+	for (i = 0; i < nreservations; i++) {
+		TAILQ_INSERT_TAIL(&vm_resq_free, reservation, next);
+		reservation++;
+	}
+	vm_resq_free_count += nreservations;
+	mtx_init(&vm_resq_free_mtx, "free reservation queue", NULL, MTX_DEF);
+	for (i = -1; i < SP_LEVELS - 1; i++)
+		TAILQ_INIT(&rres[i]);
+	mtx_init(&rres_mtx, "reservation queues", NULL, MTX_DEF);
+}
+
+static int vm_reserv_reclamations;
+SYSCTL_INT(_vm_reserv, OID_AUTO, reclamations, CTLFLAG_RD,
+    &vm_reserv_reclamations, 0, "Cumulative number of reclaimed reservations");
+
+/*
+ * Finds a superpage reservation at or above the given level and
+ * releases it.
+ */
+boolean_t
+vm_reserv_reclaim(int level)
+{
+	vm_object_t object;
+	vm_reserv_t sp;
+	int l;
+	boolean_t was_trylocked;
+
+	KASSERT(level >= -1 && level < SP_LEVELS - 1,
+	    ("vm_reserv_reclaim: invalid level")); 	
+	mtx_lock(&rres_mtx);
+	for (l = level; l < SP_LEVELS - 1; l++) {
+		TAILQ_FOREACH(sp, &rres[l], next) {
+			KASSERT(sp->rql == l && sp->rql < sp->level,
+			    ("vm_reserv_reclaim: reservation %p has"
+			    " an invalid rql", sp));
+			object = sp->object;
+			was_trylocked = VM_OBJECT_TRYLOCK(object);
+			if (was_trylocked ||
+			    VM_OBJECT_LOCKED(object)) {
+				TAILQ_REMOVE(&rres[l], sp, next);
+				vm_reserv_reclamations++;
+				mtx_unlock(&rres_mtx);
+				sp->rql = NO_RESQ;
+				vm_reserv_release(sp, level, NULL);
+				if (was_trylocked)
+					VM_OBJECT_UNLOCK(object);
+				return (TRUE);
+			}
+		}
+	}
+	mtx_unlock(&rres_mtx);
+	return (FALSE);
+}
+
+static long vm_reserv_releases;
+SYSCTL_LONG(_vm_reserv, OID_AUTO, releases, CTLFLAG_RD,
+    &vm_reserv_releases, 0, "");
+
+/* 
+ * Frees a chunk of size SP_MAGN(lev) by breaking a reservation,
+ * or, if m!=NULL, break the part of the reservation where m lies.
+ */
+void
+vm_reserv_release(vm_reserv_t sp, int lev, vm_page_t m)
+{
+	int i, magn, recovered = 0;
+	vm_reserv_t ch, nxt, res;
+	vm_page_t p;
+
+	VM_OBJECT_LOCK_ASSERT(sp->object, MA_OWNED);
+	vm_reserv_releases++;
+	if (m) {
+		if (m->reserv->broken) vm_reserv_lookup(m);
+	}
+	KASSERT(sp->popfrom != sp->popto,("preempt: popfrom = popto"));
+	for (; sp != NULL; sp = nxt) {
+		if (sp->rql != NO_RESQ) {
+			/* remove from queue */
+			KASSERT(sp->rql < sp->level && sp->rql >= -1,
+			    ("vm_reserv_release: reservation %p has"
+			    " an invalid rql", sp));
+			mtx_lock(&rres_mtx);
+			TAILQ_REMOVE(&rres[sp->rql], sp, next);
+			mtx_unlock(&rres_mtx);
+			sp->rql = NO_RESQ;
+		}
+		KASSERT(!sp->broken,
+		    ("vm_reserv_release: reservation %p is broken", sp));
+		sp->broken = TRUE;
+		nxt = NULL;
+		if (sp->order == SP_SMALL_SHIFT) {
+			/*
+			 * The reservation sp may be freed before all of its
+			 * pages are freed.  Therefore, a local reference to
+			 * its pages is used within the loop.
+			 */
+			p = sp->first_page;
+			for (i = 0; i < SP_SMALL; i++) {
+				if (p->reserv != NULL) {
+					/* Deallocate the reservation. */
+					res = vm_reserv_lookup(p);
+					KASSERT(res == NULL,
+					    ("vm_reserv_release: deallocation"
+					    " of reservation %p failed", res));
+				} else {
+					vm_phys_free_pages(p, 0);
+					if (recovered < 1)
+						recovered = 1;
+				}
+				p++;
+			}
+		} else {
+			if (sp->popfrom < sp->popto)
+				vm_reserv_create_children(sp, NULL);
+			if (m != NULL) {
+				nxt = sp->child[IDX(m, sp)];
+				KASSERT(nxt != NULL,
+				    ("vm_reserv_release: missing reservation"));
+			}
+			/* put sub-sps back in queues, with some exceptions */
+			for (i = 0; i < SP_FACTOR; i++) {
+				ch = sp->child[i];
+				if (ch == NULL) {
+					magn = 1 << (sp->order - SP_FACTOR_SHIFT);
+					p = &sp->first_page[i * magn];
+					vm_phys_free_pages(p, sp->order - SP_FACTOR_SHIFT);
+					if (recovered < magn)
+						recovered = magn;
+				} else {
+					KASSERT(ch->maxavail < 1 << ch->order,
+					    ("preempt: ch invalid"));
+					ch->parent = NULL;
+					if (nxt == NULL &&
+					    ch->maxavail == sp->maxavail)
+						nxt = ch;
+					else if (ch->maxavail > 0) {
+						KASSERT(ch->popto != ch->popfrom,
+						    ("preempt: ch->popto = ch->popfrom"));
+						ch->rql = logf(ch->maxavail);
+						KASSERT(ch->rql < ch->level,
+						    ("preempt: ch->rql >= ch->level"));
+						mtx_lock(&rres_mtx);
+						TAILQ_INSERT_TAIL(&rres[ch->rql], ch, next);
+						mtx_unlock(&rres_mtx);
+					}
+				}
+			}
+		}
+	}
+	if (!m)
+		KASSERT(recovered >= (lev == -1 ? 1 : SP_MAGN(lev)),
+		    ("preempt: recovered < SP_MAGN(level)"));
+}
+
+/*
+ *
+ */
+static void
+vm_reserv_move(vm_reserv_t sp, boolean_t to_tail)
+{
+	int rql;
+
+	VM_OBJECT_LOCK_ASSERT(sp->object, MA_OWNED);
+	if (sp->maxavail == 0) {
+		/* Full reservation: Dequeue the superpage reservation. */
+		vm_reserv_dequeue(sp);
+	} else if (sp->refcnt == 0) {
+		KASSERT(sp->maxavail == 1 << sp->order,
+		    ("vm_reserv_move maxavail=%d", sp->maxavail));
+		/* Empty reservation: Free the superpage reservation. */
+		vm_reserv_dequeue(sp);
+		vm_phys_free_pages(sp->first_page, sp->order);
+		vm_reserv_free(sp);
+	} else {
+		KASSERT(sp->popto != sp->popfrom,
+		    ("vm_reserv_move: popto = popfrom"));
+
+		/*
+		 * Return if the superpage reservation is already at
+		 * the designated place.  Inaccuracy due to a race is
+		 * of small consequence because the only effect is on
+		 * the order of reclamation.
+		 */
+		rql = logf(sp->maxavail);
+		KASSERT(rql < sp->level,
+		    ("vm_reserv_move: rql >= level: rql: %d, level: %d",
+		    rql, sp->level));
+		if (rql == sp->rql) {
+			if (to_tail) {
+				if (TAILQ_NEXT(sp, next) == NULL)
+					return;
+			} else {
+				if (TAILQ_FIRST(&rres[rql]) == sp)
+					return;
+			}
+		}
+
+		/*
+		 * Move the superpage reservation to the head or the
+		 * tail of the appropriate reservation queue.
+		 */
+		mtx_lock(&rres_mtx);
+		if (sp->rql != NO_RESQ) {
+			KASSERT(sp->rql >= -1 && sp->rql < SP_LEVELS - 1,
+			    ("vm_reserv_move: sp->rql invalid"));
+			TAILQ_REMOVE(&rres[sp->rql], sp, next);
+		}
+		sp->rql = rql;
+		if (to_tail)
+			TAILQ_INSERT_TAIL(&rres[sp->rql], sp, next);
+		else
+			TAILQ_INSERT_HEAD(&rres[sp->rql], sp, next);
+		mtx_unlock(&rres_mtx);
+	}	
+}
--- /dev/null	2007-08-19 10:55:00.000000000 +0200
+++ vm/vm_reserv.h	2007-08-19 10:58:33.000000000 +0200
@@ -0,0 +1,138 @@
+/*-
+ * Copyright (c) 2002-2006 Rice University
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project by Alan L. Cox,
+ * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ *	Superpage reservation system definitions
+ */
+
+#ifndef	_VM_RESERV_H_
+#define	_VM_RESERV_H_
+
+#include <sys/param.h>
+#include <sys/queue.h>
+
+#include <vm/vm.h>
+
+/* SUPERPAGES:
+ * PAGE_SIZE = 2**PAGE_SHIFT
+ * SMALLEST SUPERPAGE SIZE (level 0) = 2**(PAGE_SHIFT + SMALL_SHIFT)
+ * NEXT SUPERPAGE SIZES (level l) = 2**(PAGE_SHIFT + SMALL_SHIFT + l*SP_FACTOR_SHIFT) 
+ */
+#define	SP_SMALL	(1 << SP_SMALL_SHIFT)
+#define SP_FACTOR	(1 << SP_FACTOR_SHIFT)
+
+#define	SP_MAGN(l)	(SP_SMALL << (SP_FACTOR_SHIFT * (l))) /* magn of level l>=0 */
+#define SP_ORDER(l)	((l) == -1 ? 0 : SP_SMALL_SHIFT + SP_FACTOR_SHIFT * (l))
+#define SP_SIZE(l)	(1 << (SP_ORDER(l) + PAGE_SHIFT))
+
+#define	SP_LARGE	SP_MAGN(SP_LEVELS - 1)
+
+/* position within a superpage of magnitud M */
+#define	SP_POS(object, pindex, level) \
+	    (((object)->pg_color + (pindex)) & (SP_MAGN(level) - 1))
+
+/*
+ * Compute the reservation queue level for a superpage reservation
+ * with maxavail unused pages. 
+ */
+static __inline int
+logf(int maxavail)
+{
+	int i, rql;
+
+	rql = -1;
+	for (i = SP_SMALL_SHIFT; 1 << i < maxavail + 1; i += SP_FACTOR_SHIFT)
+		rql++;
+	return (rql);
+}
+
+struct	vm_reserv {
+	vm_object_t object;	/* Object to which this reservation is tied */
+	vm_pindex_t	first_pindex;
+	vm_page_t	first_page;
+	TAILQ_ENTRY(vm_reserv) next; /* to link reserved superpages */
+
+	/* 
+	 * level is redundant (== (order-SMALL_SHIFT)/SP_FACTOR_SHIFT) and a legacy thing
+	 * With a little work can be eliminated
+	 */
+	signed char level;
+	signed char rql;				/* rres queue level */
+	unsigned char broken;				/* boolean: was this reserv broken? */
+	signed char order;				/* such that size (in base pages) = 1 << order */
+	short popfrom, popto;			/* popfrom > popto iif has children */
+	short refcnt;				/* refcnt == 0 iff popfrom == popto */
+	short maxavail;				/* maxavail == (1 << order) iif refcnt == 0 */
+
+	/* 
+	 * parent is only used to propagate maxavail up; can be replaced
+	 * by a stack of pointers local to a few functions in reserve.c
+	 */
+	vm_reserv_t parent;			
+	vm_reserv_t child[SP_FACTOR];
+};
+
+TAILQ_HEAD(vm_reserv_queue, vm_reserv);
+
+vm_reserv_t	vm_reserv_acquire(vm_object_t, vm_pindex_t, int);
+void		vm_reserv_depopulate(vm_reserv_t, vm_page_t);
+void		vm_reserv_init(vm_reserv_t, int);
+vm_reserv_t	vm_reserv_lookup(vm_page_t);
+void		vm_reserv_populate(vm_page_t, vm_reserv_t);
+boolean_t	vm_reserv_reclaim(int);
+void		vm_reserv_release(vm_reserv_t, int, vm_page_t);
+
+/*
+ * Returns the superpage reservation containing the given page. 
+ */
+static __inline vm_reserv_t
+vm_reserv_get(vm_page_t m)
+{
+	vm_reserv_t reserv;
+
+	reserv = m->reserv;
+	if (reserv != NULL && __predict_false(reserv->broken))
+		reserv = vm_reserv_lookup(m);
+	return (reserv);
+}
+
+/*
+ * Returns TRUE if the given superpage reservation contains the given pindex;
+ * otherwise, returns FALSE.
+ */
+static __inline boolean_t
+vm_reserv_has_pindex(vm_reserv_t reserv, vm_pindex_t pindex)
+{
+
+	return (((pindex - reserv->first_pindex) & ~(SP_MAGN(reserv->level) - 1)) == 0);
+}
+
+#endif	/* !_VM_RESERV_H_ */
