#ifndef lint
static	char sccsid[] = "@(#)vm_hat.c 1.1 94/10/31 SMI";
#endif

/*
 * Copyright (c) 1991 by Sun Microsystems, Inc.
 */

/*
 * VM - Hardware Address Translation management.
 *
 * This file implements the machine specific hardware translation
 * needed by the VM system.  The machine independent interface is
 * described in <vm/hat.h> while the machine dependent interface
 * and data structures are described in <machine/vm_hat.h>.  For
 * Sun computers, we actually share this same source for more than
 * one architecture (Sun-2, Sun-3, and Sun-4), since the static
 * segment and page map mmu's are very similar.	 For this reason,
 * this file is located in the sun directory and we use ifdef's
 * for the few cases where they differ enough to be noticed here.
 * The actual loading of the hardware registers is done at the mmu
 * layer which is different for each Sun architecture type.  In
 * reality we probably need a more general cross architecture
 * sharing structure so that we can more easily share certain code
 * (like this file) across SOME Sun architectures without giving
 * the idea that the file MUST be shared across all Sun architectures.
 *
 * The hat layer manages the address translation hardware as a cache
 * driven by calls from the higher levels in the VM system.  Nearly
 * all the details of how the hardware is managed shound not be visable
 * above this layer except for miscellaneous machine specific functions
 * (e.g. mapin/mapout) that work in conjunction with this code.	 Other
 * than a small number of machine specific places, the hat data
 * structures seen by the higher levels in the VM system are opaque
 * and are only operated on by the hat routines.  Each address space
 * contains a struct hat and a page contains an opaque pointer which
 * is used by the hat code to hold a list of active translations to
 * that page.
 *
 * XXX - At integration into 5.0, replace all #define'd symbols such as
 *	 NPMGRPSSW with the actual variable names (npmgrpssw in this case).
 */

#include <sys/param.h>
#include <sys/mman.h>
#include <sys/debug.h>
#include <sys/user.h>			/* for u_ru.ru_minflt */
#include <sys/trace.h>
#include <sys/vmmeter.h>		/* for flush_cnt */

#include <machine/pte.h>
#include <machine/cpu.h>
#include <machine/mmu.h>
#ifdef IOC
#include <machine/iocache.h>
#endif

#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/page.h>
#include <vm/mp.h>
#include <vm/vpage.h>
#include <vm/rm.h>
#include <vm/seg_u.h>
#include <vm/seg_vn.h>
#include <vm/faultcode.h>

#if !(defined(sun4) || defined(sun4c))
ERROR - This HAT works only for sun4 and sun4c
#endif !(defined(sun4) || defined(sun4c))

/*
 * XXX - Review all ASSERT's in this vm_hat.c. Change inexpensive ones to
 *	 test and panic code. Remove #include <sys/debug>.
 */
#define	ASSERTPMGMAPPED(pmg, msg) /* assertpmgmapped(pmg, msg) */
#define	TEST_PANIC(cond, panicmsg) {if (cond) panic(panicmsg); }

/*
 * Machine specific public data structures.
 */
struct as kas;
struct ctx *kctx;
struct pmgrp *pmgrp_invalid;
#ifdef MMU_3LEVEL
struct smgrp *smgrp_invalid;
#endif MMU_3LEVEL

/*
 * Semi-private vm_hat data structures.
 * Other machine specific routines need to access these.
 */
extern int	npmgrpssw;		/* now defined in param.c */
int		npmghash;
u_short		ctx_time;

struct	ctx	*ctxs, *ctxsNCTXS;	/* used by <machine/mmu.c> */
struct	pmgrp	*pmgrps, *pmgrpsNPMGRPS; /* used by <machine/mmu.c> */
struct	pment	*pments, *pmentsNPMENTS; /* used by <machine/machdep.c> */
struct	hwpmg	*hwpmgs, *hwpmgsNHWPMGS; /* used by <machine/mmu.c> */
struct	pte	*ptes, *ptesNPTES;	/* used by <machine/machdep.c> */
struct	pmgrp	**pmghash, **pmghashNPMGHASH; /* used by <machine/machdep.c> */
#ifdef MMU_3LEVEL
struct	smgrp *smgrps, *smgrpsNSMGRPS;
struct	sment *sments, *smentsNSMENTS;
#endif MMU_3LEVEL

#if	defined(SUNDBE) && defined(sun4)
#ifndef MMU_3LEVEL
ERROR - ISM will work only with 3 level MMU
#endif  MMU_3LEVEL
static int ispseudo_smgrp(/* smgrp */);
#endif  ISM

extern void vac_flushallctx();

/*
 * hat layer statistics.
 */

/*
 * Context, smeg, and pmeg statistics.
 */
struct vmhatstat {

	/* Context allocation statistics */
	u_int	vh_ctxfree;	  /* ctx allocations without a ctx steal */
	u_int	vh_ctxstealclean; /* ctx allocations requiring a ctx steal */
	u_int	vh_ctxstealflush; /* ctx allocations requiring a ctx steal */
	u_int	vh_ctxmappmgs;	  /* pmgs mapped at ctx allocation */

	/* SW pmg statistics */
	u_int	vh_pmgallocfree;  /* pmg allocation without a pmg steal */
	u_int	vh_pmgallocsteal; /* pmg allocations requiring a pmg steal */

	/* HW pmg map and load/unload statistics */
	u_int	vh_pmgmap;	  /* mappings of loaded pmg's */
	u_int	vh_pmgldfree;	/* alloc.s. of free HW pmg */
	u_int	vh_pmgldnoctx;	/* allocs. of HW pmg with no ctx */
	u_int	vh_pmgldcleanctx; /* allocs. of HW pmg with clean ctx */
	u_int	vh_pmgldflush;	/* allocs. of HW pmg needing VAC flush */
	u_int	vh_pmgldnomap;	/* allocs. of HW pmg taking unmapped pmg */

	/* hat_fault statitistics */
	u_int	vh_faultmap;	  /* hat_fault mapped HW pmg */
	u_int	vh_faultload;	  /* hat_fault loaded SW pmg in HW */
	u_int	vh_faultinhw;	  /* hat_fault failed to resolve the fault */
	u_int	vh_faultnopmg;	  /* hat_fault failed to resolve the fault */

	/* HW smg allocation statistics */
	u_int	vh_smgfree;
	u_int	vh_smgnoctx;
	u_int	vh_smgcleanctx;
	u_int	vh_smgflush;

	/* added later to the end not to break pmegstat */
	u_int	vh_pmgallochas;	/* has already a pmeg */

} vmhatstat;

/*
 * Page cacheability statistics.
 */
struct cache_stats {
	int cs_ionc;		/* non-cached IO translations */
	int cs_ioc;		/* cached IO translations */
	int cs_knc;		/* non-cached kernel translations */
	int cs_kc;		/* cached kernel translations */
	int cs_unc;		/* non-cached user translations */
	int cs_uc;		/* cached user translations */
	int cs_other;		/* other non-type 0 pages */
	int cs_iowantchg;	/* # of IO cached to NC changes skipped */
	int cs_kchange;		/* # of kernel cached to non-cached changes */
	int cs_uchange;		/* # of user cached to non-cached changes */
	int cs_unloadfix;	/* # of unload's that made pages cachable */
	int cs_unloadnofix;	/* # of	 " that didn't made pages cachable */
	int cs_skip;		/* XXX should be after cs_other */
} cache_stats;

/*
 * hat_pmgfind() look aside buffer hit/miss statistics.
 */
struct pmgfindstat {
	u_int	pf_hit;
	u_int	pf_miss;
	u_int	pf_notfound;
} pmgfindstat;


/*
 * Private vm_hat data structures
 */

enum ptesflag { PTESFLAG_SKIP, PTESFLAG_UNLOAD };

static	struct ctx *ctxhand;

static	struct pmgrp *pmgrphand;
static	struct pmgrp *pmgrpfree;
static	struct pmgrp *pmgrpmin;

static	struct hwpmg *hwpmghand;
static	struct hwpmg *hwpmgfree;
static	struct hwpmg *hwpmgmin;

#ifdef MMU_3LEVEL
static	struct smgrp *smgrphand;
static	struct smgrp *smgrpfree;
static	struct smgrp *smgrpmin;
#endif MMU_3LEVEL

static void		hat_xfree(/* as */);
static void		hat_pteunload(/* pmg, pme, addr, flags */);
static void		hat_ptesync(/* pp, pmg, pme, addr, flags */);

static void		hat_pmgfree(/* pmg */);
static void		hat_pmglink(/* pmg, as, addr */);
static void		hat_pmgload(/* pmg */);
static void		hat_pmgunload(/* pmg, ptesflag */);
static struct pmgrp *	hat_pmgalloc(/* seg, addr */);
static struct pmgrp *	hat_pmgfind(/* addr, as */);
static void		hat_pmgloadptes(/* a, ppte */);
static void		hat_pmgunloadptes(/* a, ppte */);
static void		hat_pmgswapptes(/* a, ppte1, ppte2 */);
static void		hat_pmgmap(/* pmg */);
static void		hat_clrcleanbit();
static void		hat_unmap_aspmgs(/* as */);

#ifdef MMU_3LEVEL
static	struct smgrp	*hat_getsmg(/* addr */);
static	void		hat_smgfree(/* smg */);
static	void		hat_smglink(/* smg, as, addr */);
static	void		hat_smgalloc(/* as, addr, pmg */);
#define	hat_pmgtosmg(pmg) \
	(&smgrps[((pmg)->pmg_sme - sments) >> NSMENTPERSMGRPSHIFT])
#endif MMU_3LEVEL

static int hatunmaplimit = 30;		/* % limit used by hat_unmap_aspmgs() */

/* local inline functions */
#define	hat_pmgbase(a) ((addr_t)((u_int)a & PMGRPMASK))
#define	hat_pmgisloaded(pmg) (pmg->pmg_num != PMGNUM_SW)
#define	hat_addrtopte(pmg, a) \
	((pmg)->pmg_pte + (((u_int)(a) & PMGRPOFFSET) >> PAGESHIFT))
#define	hat_pmetopte(pmg, pme) ((pmg)->pmg_pte + ((pme) - (pmg)->pmg_pme))

/**************** misc. macros and declarations below ************************/

/*
 * XXX - Function defitions should be in a header file.
 */
extern	u_int map_getsgmap();

extern char DVMA[];		/* addresses in kas above DVMA are for "IO" */

#ifdef VAC
/*
 * XXX - should be made more efficent by using pment
 * indexing instead of virtual address checking.
 */
#define	VAC_MASK(a) (((u_int)(a) & MMU_PAGEMASK) & (shm_alignment - 1))
#define	VAC_ALIGNED(a1, a2) ((VAC_MASK(a1) ^ VAC_MASK(a2)) == 0)
#endif

/*
 * These macros allow us to walk the translation list in a sane manner.
 * The list walking must be protected, and if the translation we land
 * on is interrupt replaceable, all work on that translation must also
 * be protected. This complexity is hidden in the macros. NOTE that these
 * macros assume that variables "pme", "pmg", "pp", and "s" are declared as
 * locals in the calling function. The sequence for use is as follows:
 *
 *	PP_LIST_OPEN
 *	{
 *		do operations;
 *		PP_LIST_NEXT(next item)
 *	}
 *	PP_LIST_CLOSE
 *
 * NOTE that you CANNOT put a semicolon after PP_LIST_OPEN or it will not
 * function.
 */
#define	PP_LIST_OPEN \
	s = splvm(); \
	pmg = (struct pmgrp *)NULL; \
	PP_LIST_NEXT((struct pment *)pp->p_mapping) \
	while (pme)

#define	PP_LIST_NEXT(x)	\
	(void) splx(s); \
	s = splvm(); \
	if (pmg) \
	    pmg->pmg_keepcnt--; \
	pme = x; \
	if (pme) { \
	    pmg = &pmgrps[(pme - pments) >> NPMENTPERPMGRPSHIFT]; \
	    pmg->pmg_keepcnt++; \
	}

#define	PP_LIST_CLOSE  (void) splx(s);

/*
 *
 * The next set of routines implements the machine
 * independent hat interface described in <vm/hat.h>
 *
 */

/*
 * Initialize the hardware address translation structures.
 * Called by startup.
 *
 * Initialize the SW page tables in the range [0..NPMGRPS) as loaded
 * and mapped. This is required for hat_pmgreserve to work. After startup
 * reserves kernel pmg's, it calls hat_pmginit. hat_pmginit will create
 * a list of free SW page tables and a list of free HW pmg's by skipping
 * pmg's reserved pmg's.
 */
void
hat_init()
{
	register struct ctx	*ctx;
	register struct pmgrp	*pmg;
	register struct pment	*pme;
	register struct pte	*pte;
	register int i;


	i = 0;
	for (ctx = ctxs; ctx < ctxsNCTXS; ctx++)
		ctx->c_num = i++;
	ctxhand = ctxs;

#ifdef MMU_3LEVEL
	if (mmu_3level) {
		register struct smgrp *smg;
		register struct sment *sme;

		i = 0;
		sme = sments;
		for (smg = smgrps; smg < smgrpsNSMGRPS; smg++) {
			smg->smg_num = i++;
			smg->smg_sme = sme;
			sme += NSMENTPERSMGRP;
		}
		smgrps[SMGRP_INVALID].smg_lock = 1;	/* lock invalid smgrp */
		smgrps[SMGRP_INVALID].smg_keepcnt++;
		smgrp_invalid = &smgrps[SMGRP_INVALID];
		smgrphand = smgrps;
	}
#endif MMU_3LEVEL

	i = 0;
	pme = pments;
	pte = ptes;

	/*
	 * First NPMGRS pmgs are loaded in HW. hat_pmgreserve() assumes this.
	 *
	 * XXX - Note that &pmgrps[NPMGRPS] cannot be changed to pmgrpsNPMGRPS
	 */
	for (pmg = pmgrps; pmg < &pmgrps[NPMGRPS]; i++, pmg++) {

		pmg->pmg_num = i;
		pmg->pmg_mapped = 1;
		hwpmgs[i].hwp_pmgrp = pmg;

		pmg->pmg_pme = pme;
		pmg->pmg_pte = pte;
		pme += NPMENTPERPMGRP;
		pte += NPMENTPERPMGRP;
	}

	/*
	 * The remaining SW pmgrp are not loaded in HW.
	 */
	for (; pmg < pmgrpsNPMGRPS; pmg++) {

		pmg->pmg_num = PMGNUM_SW;
		pmg->pmg_mapped = 0;

		pmg->pmg_pme = pme;
		pmg->pmg_pte = pte;
		pme += NPMENTPERPMGRP;
		pte += NPMENTPERPMGRP;
	}

	pmgrps[PMGRP_INVALID].pmg_lock = 1;	/* lock invalid pmgrp */
	pmgrps[PMGRP_INVALID].pmg_keepcnt++;
	pmgrp_invalid = &pmgrps[PMGRP_INVALID];

	/*
	 * For now, we just grab a context and keep it locked, as well
	 * as locking all of the kernel which was loaded into memory.
	 */
	kctx = &ctxs[KCONTEXT];
	kctx->c_lock = 1;
	kctx->c_as = &kas;
	kas.a_hat.hat_ctx = kctx;
}

/*
 * Free all the hat resources held by an address space.
 * Called from as_free when an address space is being
 * destroyed and when it is to be "swapped out".
 *
 * XXX - should we do anything about locked translations here?
 */
void
hat_free(as)
	register struct as *as;
{
	register struct ctx *ctx;

	if ((ctx = as->a_hat.hat_ctx) != NULL) {

		TEST_PANIC(ctx->c_lock, "hat_free - ctx is locked");

		/*
		 * Clean context now. This will prevent expensive segment
		 * and page flushing when freeing individual pmgs.
		 */
		mmu_setctx(ctx);
		if (!ctx->c_clean) {
			vac_ctxflush();
			ctx->c_clean = 1;
		}
		hat_xfree(as);

		as->a_hat.hat_ctx = NULL;
		ctx->c_as = NULL;
	} else {
		hat_xfree(as);
	}

	/*
	 * Switch to kernel context.
	 */
	mmu_setctx(kctx);
}

/*
 * Set up addr to map to page pp with protection prot.
 */
void
hat_memload(seg, addr, pp, prot, lock)
	struct seg *seg;
	addr_t addr;
	struct page *pp;
	u_int prot;
	int lock;
{
	struct pte pte;

	hat_mempte(pp, prot, &pte);
	hat_pteload(seg, addr, pp, pte, lock ? PTELD_LOCK : 0);
}

/*
 * Cons up a struct pte using the device's pf bits and protection
 * prot to load into the hardware for address addr; treat as minflt.
 */
void
hat_devload(seg, addr, pf, prot, lock)
	struct seg *seg;
	addr_t addr;
	int pf;
	u_int prot;
	int lock;
{
	struct page *pp, *page_numtouserpp();
	union {
		struct pte u_pte;
		int u_pf;
	} apte;
	int	s = -1;

	/*
	 * If the request is to load a page which is really something
	 * for which we have a struct page, then we must be sure we
	 * maintain cache consistency.	However, we don't want to
	 * maintain such consistency for processes just running through
	 * physical memory, so we only pass along the page struct if it's
	 * not in a transition state.  This is why we use page_numtouserpp
	 * instead of page_numtopp here.  This should fix some readers
	 * of /dev/mem, but also allow those using the window system lock
	 * structures to work right too.
	 */
	if ((pf & PGT_MASK) == PGT_OBMEM) {
		s = splvm();
		pp = page_numtouserpp((u_int)(pf & PG_PFNUM));
	} else
		pp = NULL;

	apte.u_pf = pf & PG_PFNUM;
	apte.u_pte.pg_v = 1;
	apte.u_pte.pg_prot = hat_vtop_prot(prot);
	hat_pteload(seg, addr, pp, apte.u_pte, lock ? PTELD_LOCK : 0);
	u.u_ru.ru_minflt++;
	if (s == -1)
		return;
	else
		(void) splx(s);
}

/*
 * Release one hardware address translation lock on the given address.
 * For the Sun MMU, this means decrementing the counter on the pmgrp.
 */
void
hat_unlock(seg, addr)
	struct seg *seg;
	addr_t addr;
{
	register struct pmgrp	*pmg;
	int			s;

	pmg = hat_pmgfind(addr, seg->s_as);

	TEST_PANIC(pmg == NULL || pmg->pmg_keepcnt < 2, "hat_unlock");

#ifdef VAC
	if (vac && pmg->pmg_keepcnt == 2) {
		register struct pment *pme = pmg->pmg_pme;
		register int tcnt;
		struct page *pp;

		/*
		 * Now check to see if we now can cache any non-cached pages.
		 * For now, we use the simple minded algorithm and just
		 * unload any locked of the locked translations if the
		 * corresponding page is currently marked as non-cachable.
		 * This situation doesn't happen all the much, so the
		 * efficency doesn't have to be all that great.
		 */
		for (tcnt = 0; tcnt < NPMENTPERPMGRP; tcnt++, pme++) {
			if (pme->pme_valid && (pp = pme->pme_page) != NULL &&
			    pp->p_nc) {
				hat_pteunload(pmg, pme, (addr_t)NULL,
				    HAT_RMSYNC);
				if (pp->p_nc) {
					/*
					 * We lost - unloading the mmu
					 * translation wasn't enough to
					 * make the page cacheable again.
					 */
					cache_stats.cs_unloadnofix++;
				} else {
					/*
					 * We won - unloading the mmu
					 * translation made the page
					 * cacheable again.
					 */
					cache_stats.cs_unloadfix++;
				}
			}
		}
	}
#endif VAC
	s = splvm();
	pmg->pmg_keepcnt -= 2;	/* once for hat_getpmg, once for unlock */
#ifdef MMU_3LEVEL
	/* Decr. once for unlock. */
	if (mmu_3level)
		hat_pmgtosmg(pmg)->smg_keepcnt--;
#endif MMU_3LEVEL
	(void) splx(s);
}

/*
 * Change the protections in the virtual address range
 * given to the specified virtual protection.  If
 * vprot == ~PROT_WRITE, then all the write permission
 * is taken away for the current translations, else if
 * vprot == ~PROT_USER, then all the user permissions
 * are takem away for the current translations, otherwise
 * vprot gives the new virtual protections to load up.
 */
void
hat_chgprot(seg, addr, len, vprot)
	struct seg	*seg;
	addr_t		addr;
	u_int		len;
	u_int		vprot;		/* virtual page protections */
{
	register addr_t		a, ea;
	register struct pmgrp	*pmg = NULL;
	register u_int		pprot;	/* physical page protections */
	register int		newprot;
	struct pte		pte;
	struct pte		*ppte;

	if (vprot != ~PROT_WRITE && vprot != ~PROT_USER)
		pprot = hat_vtop_prot(vprot);

	/*
	 * We must get a context for the AS because we will be
	 * synchronizing SW PTE by reads from MMU.
	 */
	hat_setup(seg->s_as);

	for (a = addr, ea = addr + len; a < ea; a += MMU_PAGESIZE) {
		if (pmg == NULL ||
		    ((u_int)a & (PMGRPSIZE - 1)) < MMU_PAGESIZE) {
			if (pmg) {
				register int s;
				s = splvm();
				pmg->pmg_keepcnt--;
				(void) splx(s);
			}
			pmg = hat_pmgfind(a, seg->s_as);
			if (pmg == NULL) {
				/*
				 * Bump up `a' to avoid checking all
				 * the pte's in the invalid pmgrp.
				 */
				a = (addr_t)((u_int)a & ~(PMGRPSIZE - 1)) +
				    PMGRPSIZE - MMU_PAGESIZE;
				continue;
			}
			/*
			 * Make sure that loaded pmg is also mapped.
			 */
			if (hat_pmgisloaded(pmg))
				hat_pmgmap(pmg);
		}

		ppte = hat_addrtopte(pmg, a);
		if (!pte_valid(ppte))
			continue;

		/*
		 * Synchronize PTE from MMU.
		 */
		if (hat_pmgisloaded(pmg)) {
			mmu_getpte(a, ppte);
		}

		pte = *ppte;

		if (vprot == ~PROT_WRITE) {
			switch (pte.pg_prot) {
			case KW: pprot = KR; newprot = 1; break;
			case UW: pprot = UR; newprot = 1; break;
			default: newprot = 0; break;
			}
		} else if (vprot == ~PROT_USER) {
#ifdef sun2
			/* XXX - need a better way to do this */
			if (pte.pg_prot & 07) {
				pprot = pte.pg_prot & ~07;
				newprot = 1;
			} else {
				newprot = 0;
			}
#else sun2
			switch (pte.pg_prot) {
			case UW: pprot = KW; newprot = 1; break;
			case UR: pprot = KR; newprot = 1; break;
			default: newprot = 0; break;
			}
#endif sun2
		} else if (pte.pg_prot != pprot) {
			newprot = 1;
		} else {
			newprot = 0;
		}
		if (newprot) {
			pte.pg_prot = pprot;

			*ppte = pte;	/* Set SW PTE */

			/*
			 * Synchronize HW PTE if this pmg is loaded.
			 * We assume that hat_setup has been done.
			 */
			if (hat_pmgisloaded(pmg)) {
#ifdef VAC
				if (vac && !pte.pg_nc && pte.pg_r)
					vac_pageflush(a);
#endif VAC
				mmu_setpte(a, pte);
			}
		}
	}
	if (pmg != NULL) {
		register int s;
		s = splvm();
		pmg->pmg_keepcnt--;
		(void) splx(s);
	}
}

/*
 * Associate all the mappings in the range [addr..addr+len) with
 * segment seg. Since we don't cache segments in this hat implementation,
 * this routine is a noop.
 */
/*ARGSUSED*/
void
hat_newseg(seg, addr, len, nseg)
	struct seg	*seg;
	addr_t		addr;
	u_int		len;
	struct seg	*nseg;
{
	return;
}

/*
 * Unload all the mappings in the range [addr..addr+len).
 */
void
hat_unload(seg, addr, len)
	struct seg	*seg;
	addr_t		addr;
	u_int		len;
{
	register addr_t		a;
	register struct pment	*pme;
	register struct pmgrp	*pmg = NULL;
	addr_t			ea;
	int			s;

	if (seg->s_as->a_hat.hat_pmgrps == NULL) {
		/*
		 * If there are no allocated pmgrps for this
		 * address space, we don't want to do anything.
		 */
		return;
	}

	/*
	 * hat_setup needed for HAT_VADDR optimization to work.
	 */
	hat_setup(seg->s_as);

	for (a = addr, ea = addr + len; a < ea; a += MMU_PAGESIZE) {
		if (pmg == NULL ||
		    ((u_int)a & (PMGRPSIZE - 1)) < MMU_PAGESIZE) {
			if (pmg != NULL) {
				s = splvm();
				pmg->pmg_keepcnt--;
				(void) splx(s);
			}
			pmg = hat_pmgfind(a, seg->s_as);
			if (pmg == NULL) {
				/*
				 * Bump up `a' to avoid checking all
				 * the pme's in the invalid pmgrp.
				 */
				a = (addr_t)((u_int)a & ~(PMGRPSIZE - 1)) +
				    PMGRPSIZE - MMU_PAGESIZE;
				continue;
			}
			pme = &pmg->pmg_pme[mmu_btop(a - pmg->pmg_base)];
		} else {
			pme++;
		}
		/*
		 * Throw out the mapping.
		 */
		if (pme->pme_nosync) {

			TEST_PANIC(pmg->pmg_keepcnt < 2,
			    "hat_unload - pmg not kept");

			/*
			 * Decrement keepcnt on pmg and smg to indicate
			 * 'unlock' on the address.
			 */
			s = splvm();
#ifdef MMU_3LEVEL
			if (mmu_3level) {
				hat_pmgtosmg(pmg)->smg_keepcnt--;
			}
#endif MMU_3LEVEL
			pmg->pmg_keepcnt--;
			(void) splx(s);

			hat_pteunload(pmg, pme, a, HAT_NOSYNC | HAT_VADDR);
		} else {
			hat_pteunload(pmg, pme, a, HAT_RMSYNC | HAT_VADDR);
		}
	}
	if (pmg) {
		s = splvm();
		pmg->pmg_keepcnt--;
		(void) splx(s);
	}
}

/*
 * Unload all the hardware translations that map page `pp'.
 */
void
hat_pageunload(pp)
	register struct page *pp;
{
	register struct pmgrp *pmg;
	register struct pment *pme;
	int s;

	PP_LIST_OPEN
	{
		hat_pteunload(pmg, pme, (addr_t)NULL, HAT_RMSYNC);
		PP_LIST_NEXT((struct pment *)pp->p_mapping)
	}
	PP_LIST_CLOSE
}

/*
 * Get all the hardware dependent attributes for a page struct
 */
void
hat_pagesync(pp)
	struct page *pp;
{
	register struct pment *pme;
	register struct pmgrp *pmg;
	int s;

	PP_LIST_OPEN
	{
		/*
		 * Get page dependent info from hardware for
		 * each translation, but don't unload them.
		 */
		hat_ptesync(pp, pmg, pme, (addr_t)NULL, HAT_RMSYNC);
		PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
	}
	PP_LIST_CLOSE
}

/*
 * Returns the page frame number for a given kernel virtual address.
 */
u_int
hat_getkpfnum(addr)
	addr_t	addr;
{
	struct	pte pte;

	mmu_getkpte(addr, &pte);
	return (MAKE_PFNUM(&pte));
}

/*
 * Resolve a page fault by loading a cached translation.
 *
 * hat_fault() is called from the fault handler in locore.s and pagefault.
 *
 */
hat_fault(addr)
	caddr_t		addr;
{
	struct pmgrp	*pmg;
	int		s;
	struct as	*as;

	/*
	 * We assume that addresses above KERNELBASE belong to kas.
	 */
	if (addr >= (addr_t)KERNELBASE) {
		as = &kas;
	} else {

		/*
		 * We punt if the address space of the running process
		 * hasn't been set up. pagefault will allocate the context
		 * and call hat_fault again if this was a real pagefault.
		 */
		if ((as = u.u_procp->p_as) != mmu_getctx()->c_as)
			return (FC_NOMAP);
	}

	if ((pmg = hat_pmgfind(addr, as)) != NULL) {

		if (pmg->pmg_mapped) {
			/*
			 * Pmg is mapped and loaded. The hat layer cannot
			 * resolve the fault.
			 */
			ASSERT(pmg->pmg_num != PMGNUM_SW);
			s = splvm();
			pmg->pmg_keepcnt--;
			vmhatstat.vh_faultinhw++;
			(void) splx(s);
			return (FC_NOMAP);
		} else if (hat_pmgisloaded(pmg)) {

			/*
			 * pmg is loaded but not mapped.
			 */
			s = splvm();

			/*
			 * The code below is inline hat_pmgmap.
			 */
#ifdef MMU_3LEVEL
			if (mmu_3level) {
				hat_smgalloc(pmg->pmg_as, pmg->pmg_base, pmg);
				hat_pmgtosmg(pmg)->smg_keepcnt--;
			}
#endif MMU_3LEVEL
			mmu_setpmg(pmg->pmg_base, pmg);
			pmg->pmg_mapped = 1;
			/* End of inline hat_pmgmap. */

			pmg->pmg_keepcnt--;
			vmhatstat.vh_faultmap++;
			(void) splx(s);
			return ((faultcode_t)0);
		} else {
			/*
			 * Pmg is not loaded.
			 */
			hat_pmgload(pmg);
			s = splvm();
			pmg->pmg_keepcnt--;
#ifdef MMU_3LEVEL
			if (mmu_3level)
				hat_pmgtosmg(pmg)->smg_keepcnt--;
#endif MMU_3LEVEL
			(void) splx(s);
			vmhatstat.vh_faultload++;
			return ((faultcode_t)0);
		}
	}

	/*
	 * HAT couldn't resolve the fault because there is no SW pmg.
	 */
	vmhatstat.vh_faultnopmg++;
	return (FC_NOMAP);
}


/*
 * End of machine independent interface routines.
 *
 * The next few routines implement some machine dependent functions
 * needed for the Sun MMU.  Note that each hat implementation can define
 * whatever additional interfaces that make sense for that machine.
 * These routines are defined in <machine/vm_hat.h>.
 *
 * Start machine specific interface routines.
 */


/*
 * This routine is called for kernel initialization
 * to cause a pmgrp to be reserved w/o any unloading,
 * links the pmgrp into the address space (if not already there),
 * and return with the pmgrp in question leaving its keepcnt incremented.
 */
void
hat_pmgreserve(seg, addr)
	struct seg *seg;
	addr_t addr;
{
	register struct as	*as = seg->s_as;
	register struct pmgrp	*pmg;
	u_int			pmgnum;
	int			s;

	pmgnum = map_getsgmap(addr);
	TEST_PANIC(pmgnum == PMGRP_INVALID, "hat_pmgreserve: invalid pmg");

	pmg = hwpmgs[pmgnum].hwp_pmgrp;
	ASSERT(pmg != NULL);
	ASSERT(pmg->pmg_num == pmgnum);
	s = splvm();
	pmg->pmg_keepcnt++;
	(void) splx(s);

	TEST_PANIC(as->a_hat.hat_ctx == NULL ||
	    (pmg->pmg_as != NULL && pmg->pmg_as != as),
	    "hat_pmgreserve");

	if (pmg != pmgrp_invalid && pmg->pmg_as == NULL) {
		hat_pmglink(pmg, as, addr);
	}

#ifdef MMU_3LEVEL
	if (mmu_3level)
		hat_smgreserve(seg, addr);
#endif MMU_3LEVEL
}

/*
 * Initialize all the unlocked pmgs to have invalid pme's
 * and add them to the free list.
 * This routine is called during startup after all the
 * kernel pmgs have been reserved.  This routine will
 * also set the pmgrpmin variable for use in hat_pmgalloc.
 */
void
hat_pmginit()
{
	register struct pmgrp	*pmg;
	register addr_t		addr;
	int			s;
	int			i;
	struct hwpmg		*hwpmg;

	s = splvm();

	/*
	 * Make HW free list. Skip locked and kept pmgrps.
	 * Here we assume that <0..NPMGRPS) were loaded in MMU by hat_init().
	 */
	for (pmg = pmgrps, hwpmg = hwpmgs;
	    hwpmg < hwpmgsNHWPMGS; pmg++, hwpmg++) {

		if (pmg->pmg_lock || pmg->pmg_keepcnt > 0) {
			pmg->pmg_mapped = 1;
			continue;
		}

		if (pmgrpmin == NULL) {
			pmgrpmin = pmgrphand = pmg;
			hwpmgmin = hwpmghand = hwpmg;
		}

		mmu_settpmg(SEGTEMP, pmg);

		for (addr = SEGTEMP; addr < SEGTEMP + PMGRPSIZE;
		    addr += PAGESIZE)
			mmu_setpte(addr, mmu_pteinvalid);

		pmg->pmg_num = PMGNUM_SW;

		hwpmg->hwp_next = hwpmgfree;
		hwpmg->hwp_pmgrp = NULL;
		hwpmgfree	= hwpmg;

	}
	mmu_pmginval(SEGTEMP);

	/*
	 * Make SW pmg free list.
	 */
	for (pmg = pmgrps; pmg < pmgrpsNPMGRPS; pmg++) {

		if (pmg->pmg_lock || pmg->pmg_keepcnt > 0)
			continue;

		for (i = 0; i < NPMENTPERPMGRP; i++) {
			pmg->pmg_pte[i] = mmu_pteinvalid;
		}
		pmg->pmg_num = PMGNUM_SW;
		pmg->pmg_mapped = 0;

		pmg->pmg_next = pmgrpfree;
		pmgrpfree = pmg;
	}

	(void) splx(s);
#ifdef MMU_3LEVEL
	if (mmu_3level)
		hat_smginit();

	/*
	 * Check keepcnt on smegs and pmegs to detect double mapped pmegs.
	 */
	hat_smgcheck_keepcntall();
#endif MMU_3LEVEL
}

/*
 * Set addr in segment seg to use pte to (possibly) map to page pp.
 * This is the common routine used for hat_memload and hat_devload
 * in addition to the machine dependent mapin implementation.
 */
void
hat_pteload(seg, addr, pp, pte, flags)
	struct seg	*seg;
	addr_t		addr;
	struct page	*pp;
	struct pte	pte;
	int		flags;
{
	register struct pment	*ppme;
	register struct pmgrp	*ppmg;
	int			s;
	struct pte		opte;

	TEST_PANIC(pp != NULL && pp->p_free, "hat_pteload free page");

	/*
	 * We need to setup context because we will (potentially) be reading
	 * HW PTE's.
	 */
	hat_setup(seg->s_as);
	ppmg = hat_pmgalloc(seg, addr);
	hat_pmgload(ppmg);

	ppme = &ppmg->pmg_pme[mmu_btop(addr - ppmg->pmg_base)];

	/*
	 * We must be sure that setting the pte and adding to the list
	 * of mappings is atomic.
	 */
	s = splvm();
#ifdef VAC
	/*
	 * If there's no page structure associated with this mapping,
	 * or the vac is turned off, or the page is non-cacheable,
	 * then force the mapping to be non-cached.
	 */
	if (pp == NULL || !vac || pp->p_nc)
		pte.pg_nc = 1;
#endif VAC
	if (ppme->pme_valid) {
		/*
		 * Reloading a translation - be sure to preserve the
		 * exiting ref and mod bits for this translation.
		 *
		 * XXX - should cache all the attributes of a loaded
		 * translation in the pme structure so that we can
		 * avoid reloading all together unless something
		 * is actually going to change.
		 */

		/*
		 * Synch SW PTE by reading MMU.
		 */
		opte = *hat_addrtopte(ppmg, addr);

		mmu_getpte(addr, &opte);
		pte.pg_r = opte.pg_r;
		pte.pg_m = opte.pg_m;

	} else {
		ppme->pme_valid = 1;
	}

#ifdef IOC
	if (ioc) {
		struct pte *p = &pte;

		if (flags & PTELD_IOCACHE) {
			ioc_pteset(p);
		}
		ioc_mbufset(p, addr);
	}
#endif IOC

	*hat_addrtopte(ppmg, addr) = pte; /* Set SW PTE */

	mmu_setpte(addr, pte);	/* load the SW pte in HW */

	/*
	 * Check to see if this pme needs to be added
	 * to the list of pme's mapping this page.
	 */
	if (pp != ppme->pme_page) {

		TEST_PANIC(ppme->pme_page != NULL, "hat_pteload");
		ASSERT(pp != NULL && ppme->pme_page == NULL);

		ppme->pme_page = pp;
		ppme->pme_next = PMENXT_INDEX(pp->p_mapping);
		pp->p_mapping = (caddr_t)ppme;
		(void) splx(s);

		seg->s_as->a_rss += 1;
#ifdef VAC
		/*
		 * If (vac) active, then check for conflicts.
		 * A conflict exists if the new and existent mappings
		 * do not match in their "shm_alignment" fields
		 * XXX and one of them is writable XXX.	 If conflicts
		 * exist, the extant mappings are flushed UNLESS
		 * one of them is locked.  If one of them is locked,
		 * then the mappings are flushed and converted to
		 * non-cacheable mappings [must be deconverted in
		 * hat_pteunload].
		 * XXX	need to store protections in pme
		 *	to employ writable optimization.
		 */
		if (vac && !pp->p_nc) {
			struct pmgrp *pmg;	/* temporary pmg */
			struct pment *pme;	/* temporary pme */
			struct pmgrp
			    *pmgpc = (struct pmgrp *)0,	/* user's text pmg */
			    *pmgsp = (struct pmgrp *)0;	/* user's stack pmg */
			int ccf;		/* cache conflict found flag */
			int first = 1;

			/*
			 * mappings to the user's current stack and
			 * text locations must be locked in memory,
			 * or we run the risk of getting into an
			 * infinite paging loop if the program tries
			 * to read the physical pages containing either
			 * via a mapping that is not cache aliased.
			 */
			s = splvm();
			if (!servicing_interrupt()) {
			    if (u.u_ar0 != (int *)0) {
				if (u.u_ar0[SP] != 0) {
				    pmgsp = mmu_getpmg ((addr_t)u.u_ar0[SP]);
				    if (pmgsp != (struct pmgrp *)0)
					pmgsp->pmg_keepcnt ++;
				}
				if (u.u_ar0[PC] != 0) {
				    pmgpc = mmu_getpmg ((addr_t)u.u_ar0[PC]);
				    if (pmgpc != (struct pmgrp *)0)
					pmgpc->pmg_keepcnt ++;
				}
			    }
			}
			(void)splx(s);

			ccf = 0;
			PP_LIST_OPEN
			{
				if (pme == ppme) {
					PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
					continue;
				}
				if (first && VAC_ALIGNED(addr, pmg->pmg_base +
				    mmu_ptob(pme - pmg->pmg_pme))) {
					PP_LIST_NEXT((struct pment *)NULL)
					continue;
				}
				first = 0;
				/*
				 * Compare keep to 1 because
				 * list walker keeps it too.
				 */
				if (pmg->pmg_lock || pmg->pmg_keepcnt > 1) {
					ccf = 1;
					PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
					continue;
				}
				hat_pteunload(pmg, pme, (addr_t)NULL,
				    HAT_RMSYNC);
				PP_LIST_NEXT((struct pment *)pp->p_mapping)
			}
			PP_LIST_CLOSE

			/*
			 * Release locked pmgrps.
			 */
			s = splvm();
			if (pmgsp != (struct pmgrp *)0)
				pmgsp->pmg_keepcnt --;
			if (pmgpc != (struct pmgrp *)0)
				pmgpc->pmg_keepcnt --;
			(void) splx(s);

			if (ccf) {
				pte.pg_nc = 1;
				pp->p_nc = 1;
				/*
				 * This time we don't exclude our translation,
				 * so it will get remarked noncacheable.
				 */
				PP_LIST_OPEN
				{
					hat_ptesync(pp, pmg, pme, (addr_t)NULL,
					    HAT_NCSYNC);
					PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
				}
				PP_LIST_CLOSE
			}
		}
#endif VAC
	} else {
		(void) splx(s);

		TEST_PANIC(pp != NULL &&
			((ppme->pme_intrep != ((flags & PTELD_INTREP) != 0)) ||
			(ppme->pme_nosync != ((flags & PTELD_NOSYNC) != 0))),
				"pteload - remap flags");
#ifdef VAC
		/*
		 * Reloading a pte for an already mapped page.	If
		 * the page is "real", then if we've got a VAC, then
		 * flush the cache to ensure no protection mismatches
		 * between cache and MMU.
		 * XXX - need to store protections in pme to avoid this.
		 *
		 * XXX - this could cause writeback errors if protection
		 *	 was changed from writeable to read-only.
		 */
		if ((pp != NULL) && vac && pte.pg_r &&
		    *(int *)&pte != *(int *)&opte)
			vac_pageflush(addr);
#endif VAC
	}
	ppme->pme_intrep = (flags & PTELD_INTREP) != 0;
	ppme->pme_nosync = (flags & PTELD_NOSYNC) != 0;

	/* keep some statistics on the cache-ability of the translation */
#if defined(VAC)
	if (pte.pg_type == OBMEM) {
		if (seg->s_as == &kas) {
			if (addr >= DVMA) {
				if (!pte.pg_nc)
					cache_stats.cs_ioc++;
				else
					cache_stats.cs_ionc++;
			} else {
				if (!pte.pg_nc)
					cache_stats.cs_kc++;
				else
					cache_stats.cs_knc++;
			}
		} else {
			if (!pte.pg_nc)
				cache_stats.cs_uc++;
			else
				cache_stats.cs_unc++;
		}
	} else {
		cache_stats.cs_other++;
	}
#else defined(VAC)
	if (pte.pg_type == OBMEM) {
		if (seg->s_as == &kas) {
			if (addr >= DVMA) {
				cache_stats.cs_ionc++;
			} else {
				cache_stats.cs_knc++;
			}
		} else {
			cache_stats.cs_unc++;
		}
	} else {
		cache_stats.cs_other++;
	}
#endif defined(VAC)

	if ((flags & PTELD_LOCK) == 0) {
		/*
		 * Decrement lock count. It was incr. by hat_pmgalloc().
		 */
		s = splvm();
		ppmg->pmg_keepcnt--;
#ifdef MMU_3LEVEL
		if (mmu_3level)
			hat_pmgtosmg(ppmg)->smg_keepcnt--;
#endif MMU_3LEVEL
		(void) splx(s);
	}
}

void
hat_mempte(pp, vprot, ppte)
	struct page *pp;
	u_int vprot;
	register struct pte *ppte;
{

	*ppte = mmu_pteinvalid;
	ppte->pg_prot = hat_vtop_prot(vprot);
	ppte->pg_v = 1;
	ppte->pg_type = OBMEM;
	ppte->pg_pfnum = page_pptonum(pp);
}

void
hat_getctx(as)
	struct as *as;
{
	register struct ctx	*ctx, *ttctx;
	register struct pmgrp	*pmg;
	register u_short	tt = 0;
	int			s;
#ifdef MMU_3LEVEL
	register struct smgrp *smg;
#endif MMU_3LEVEL

	if (as->a_hat.hat_ctx) {
		as->a_hat.hat_ctx->c_clean = 0;
		return;
	}


	/* find a free ctx or an old one */
	ttctx = NULL;
	for (ctx = ctxhand + 1; ctx != ctxhand; ctx++) {
		if (ctx == ctxsNCTXS) /* wrap around the ctx table */
			ctx = ctxs;
		if (ctx->c_lock != 0)	/* can't touch */
			continue;
		/*
		 * Take ctx with no address space.
		 */
		if (ctx->c_as == NULL) {
			ttctx = ctx;
			break;
		}
		if (ttctx == NULL || ctx->c_time <= tt) {
			ttctx = ctx;		/* new "best" ctx */
			tt = ctx->c_time;
		}
	}

	TEST_PANIC(ttctx == NULL, "hat_getctx - no ctx's");

	ctxhand = ctx = ttctx;

	/*
	 * Update vmhatstat statistics.
	 */
	if (ctx->c_as) {
		if (ctx->c_clean)
			vmhatstat.vh_ctxstealclean++;
		else
			vmhatstat.vh_ctxstealflush++;
	} else {
		vmhatstat.vh_ctxfree++;
	}

	mmu_setctx(ctx);

	if (ctx->c_as) {
		vac_flushallctx();
		s = splvm();

#ifdef MMU_3LEVEL
		if (mmu_3level) {
			/* invalidate any smgrps already loaded for this ctx */
			for (smg = ctx->c_as->a_hat.hat_smgrps; smg != NULL;
			    smg = smg->smg_next)
				mmu_smginval(smg->smg_base);
		} else {
#endif MMU_3LEVEL
			/* invalidate any pmgrps already loaded for this ctx */
			for (pmg = ctx->c_as->a_hat.hat_pmgrps; pmg != NULL;
			    pmg = pmg->pmg_next) {

				if (pmg->pmg_mapped) {
					ASSERT(pmg->pmg_num != PMGNUM_SW);
					mmu_pmginval(pmg->pmg_base);
					pmg->pmg_mapped = 0;
				}
			}
#ifdef	MMU_3LEVEL
		}
#endif	MMU_3LEVEL
		ctx->c_as->a_hat.hat_ctx = NULL;
		(void)splx(s);
	}

	s = splvm();
	ctx->c_as = as;
	ctx->c_clean = 0;
	ctx->c_time = ctx_time++;
	as->a_hat.hat_ctx = ctx;

#ifdef MMU_3LEVEL
	if (mmu_3level) {
		/* load up any smgrps already allocated to this hat */
		for (smg = as->a_hat.hat_smgrps; smg != NULL;
		    smg = smg->smg_next)
			mmu_setsmg(smg->smg_base, smg);
	} else {
#endif MMU_3LEVEL
		/* load up any pmgrps already allocated to this hat */
		for (pmg = as->a_hat.hat_pmgrps; pmg != NULL;
		    pmg = pmg->pmg_next) {

			if (pmg->pmg_num != PMGNUM_SW) {
				ASSERT(!pmg->pmg_mapped);
				mmu_setpmg(pmg->pmg_base, pmg);
				pmg->pmg_mapped = 1;
				vmhatstat.vh_ctxmappmgs++;
			}
		}
#ifdef	MMU_3LEVEL
	}
#endif	MMU_3LEVEL

	(void)splx(s);
}


/*
 * Used to lock down hat resources for an address range. In this implementation,
 * this means locking down the necessary pmegs. This currently works only
 * for kernel addresses.
 */
void
hat_reserve(seg, addr, len)
	struct seg *seg;
	addr_t addr;
	u_int len;
{
	register	addr_t a;
	addr_t		ea;
	struct pmgrp	*pmg;

	TEST_PANIC(seg->s_as != &kas, "hat_reserve");

	for (a = addr, ea = addr + len; a < ea; a += MMU_PAGESIZE) {
		pmg = hat_pmgalloc(seg, a);
		TEST_PANIC(pmg == NULL, "hat_reserve: pmg == NULL");
		hat_pmgload(pmg);
	}
}

u_int
hat_vtop_prot(vprot)
	u_int vprot;
{

	switch (vprot) {
	case 0:
	case PROT_USER:
		/*
		 * Since 0 might be a valid protection,
		 * the caller should not set valid bit
		 * if vprot == 0 to be sure.
		 */
		return (0);
	case PROT_READ:
	case PROT_EXEC:
	case PROT_READ | PROT_EXEC:
		return (KR);
	case PROT_WRITE:
	case PROT_WRITE | PROT_EXEC:
	case PROT_READ | PROT_WRITE:
	case PROT_READ | PROT_WRITE | PROT_EXEC:
		return (KW);
	case PROT_EXEC | PROT_USER:
	case PROT_READ | PROT_USER:
	case PROT_READ | PROT_EXEC | PROT_USER:
		return (UR);
	case PROT_WRITE | PROT_USER:
	case PROT_WRITE | PROT_EXEC | PROT_USER:
	case PROT_READ | PROT_WRITE | PROT_USER:
	case PROT_READ | PROT_WRITE | PROT_EXEC | PROT_USER:
		return (UW);
	default:
		panic("hat_vtop_prot");
		/* NOTREACHED */
	}
}

#if defined(sun4c) && defined(VAC)
/*
 * Flush all possible cache lines mapping the given physical page. This
 * is used for software cache consistency with I/O, to clean the cache
 * of all data subject to I/O.
 */
void
hat_vacsync(pfnum)
	u_int pfnum;
{
	register struct page	*pp = page_numtopp(pfnum);
	register struct pmgrp	*pmg;
	register struct pment	*pme;
	int			s;
	addr_t			va;
	struct ctx		*ctxsav, *nctx, *curctx;
	struct pte		tpte;

	/*
	 * If the cache is off, the page isn't memory, or the page is
	 * non-cacheable, then none of the page could be in the cache
	 * in the first place, with the exception that a page frame
	 * for kernel .data or .bss objects could be in the cache,
	 * but will have no page structure.
	 */
	if (!vac) {
		return;
	} else if (pp == (struct page *) NULL) {
		extern u_int kpfn_dataseg, kpfn_endbss;
		if (pfnum >= kpfn_dataseg || pfnum <= kpfn_endbss) {
			extern char etext[];

			/*
			 * In sun4c, the page frame number for the start
			 * of the kernel data segment and the page frame
			 * number for end are latched up in kvm_init().
			 * If a page frame number ends up here, then some-
			 * body is doing i/o to an object in kernel .data
			 * or .bss.
			 *
			 * This is a temporary solution, and it does have
			 * some holes in it. It assumes that the page frame
			 * numbers between kernel .data and end are contiguous.
			 *
			 * As a side note, we could go to the effort of
			 * of reading the kernel pte for the calculated
			 * address to check with the passed page frame
			 * number, but it isn't really worth the effort.
			 */

			va = (addr_t) (roundup((u_int) etext, DATA_ALIGN) +
			    ((pfnum - kpfn_dataseg) << MMU_PAGESHIFT));

			vac_pageflush (va);
		}
		return;
	} else if (pp->p_nc) {
		return;
	}


	curctx = ctxsav = mmu_getctx();
	/*
	 * Walk the list of translations for this page, flushing each
	 * one.
	 */
	PP_LIST_OPEN
	{
		/*
		 * If the translation has no context, it can't be
		 * in the cache.
		 */
		if ((nctx = pmg->pmg_as->a_hat.hat_ctx) != NULL &&
		    !nctx->c_clean && pmg->pmg_mapped) {
			/*
			 * Calculate the virtual address, switch to the
			 * correct context, and flush the page.
			 */
			va = pmg->pmg_base + mmu_ptob(pme - pmg->pmg_pme);
			if (nctx != curctx) {
				mmu_setctx(nctx);
				curctx = nctx;
			}
			mmu_getpte(va, &tpte);
			if (tpte.pg_r)
				vac_pageflush(va);
		}
		PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
	}
	PP_LIST_CLOSE
	/*
	 * Restore the original context.
	 */
	if (curctx != ctxsav)
		mmu_setctx(ctxsav);
}
#endif defined(sun4c) && defined(VAC)

#ifdef sun4c
/*
 * Kill any processes that use this page. (Used for parity recovery)
 * If we encounter the kernel's address space, give up (return -1).
 * Otherwise, we return 0.
 */
hat_kill_procs(pp, addr)
	struct	page	*pp;
	addr_t	addr;
{
	register struct pmgrp *pmg;
	register struct pment *pme;
	int	s;
	struct	as	*as;
	struct	proc	*p;
	int	result = 0;

	PP_LIST_OPEN
	{
		/*
		 * Find the address space that contains this pment.
		 */
		as = pmg->pmg_as;

		/*
		 * If the address space is the kernel space, then fail.
		 * The memory is corrupted, and the only thing to do with
		 * corrupted kernel memory is die.
		 */
		if (as == &kas) {
			printf("parity recovery: kernel address space\n");
			result = -1;
		}

		/*
		 * Find the proc that uses this address space and kill
		 * it.	Note that more than one process can share the
		 * same address space, if vfork() was used to create it.
		 * This means that we have to look through the entire
		 * process table and not stop at the first match.
		 */
		for (p = allproc; p; p = p->p_nxt) {
			if (p->p_as == as) {
				printf("pid %d killed: parity error\n",
				    p->p_pid);
				uprintf("pid %d killed: parity error\n",
				    p->p_pid);
				psignal(p, SIGBUS);
				p->p_uarea->u_code = FC_HWERR;
				p->p_uarea->u_addr = addr;

			}
		}
		PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
	}
	PP_LIST_CLOSE

	return (result);
}
#endif /* sun4c */

/*
 * End machine specific interface routines.
 *
 * The remainder of the routines are private to this module and are used
 * by the routines above to implement a service to the outside caller.
 *
 * Start private routines.
 */

/*
 * Unload a pme.  We call hat_ptesync() to unload the translation
 * then remove the pme from the list of pme's mapping the page.
 * Should always be called with the pmgrp for the pme being held.
 */
static void
hat_pteunload(ppmg, ppme, vaddr, flags)
	struct pmgrp		*ppmg;
	register struct pment	*ppme;
	addr_t			vaddr;
	int			flags;
{
	int			s;
	struct page		*pp;
	struct pment		*pmentp;
	u_int			ppmeind;
#ifdef VAC
	struct pment		*pme;	/* temporary for listwalk */
	struct pmgrp		*pmg;	/* temporary for listwalk */
	struct pment		*qpme;	/* second temporary for comparison */
	struct pmgrp		*qpmg;	/* second temporary for comparison */
	addr_t			pa, qa; /* matching address values */
	int			ccf;	/* cache conflict found flag */
	int			s2;
#endif VAC

	s = splvm();
	pp = ppme->pme_page;
	ppmeind	 = PMENXT_INDEX(ppme);
	if (pp != NULL) {
		/*
		 * Remove it from the list of mappings for the page.
		 */
		if (ppme == (struct pment *)(pp->p_mapping)) {
			pp->p_mapping = (caddr_t)PMENXT_PTR(ppme->pme_next);
			ppme->pme_next = PMENXT_NULL;
		} else {
			TEST_PANIC(pp->p_mapping == NULL,
			    "hat_pteunload - no mappings");

			for (pmentp = ((struct pment *) (pp->p_mapping));
			    pmentp->pme_next != ppmeind;
			    pmentp = PMENXT_PTR(pmentp->pme_next))
				TEST_PANIC(pmentp->pme_next == PMENXT_NULL,
				    "hat_pteunload - no mapping");

			pmentp->pme_next = ppme->pme_next;
			ppme->pme_next = PMENXT_NULL;
		}
		(void) splx(s);
		ppme->pme_page = NULL;
		ppmg->pmg_as->a_rss -= 1;
#ifdef VAC
		if (vac && pp->p_nc) {
			ccf = 0;
			PP_LIST_OPEN
			{
				s2 = splvm();
				if ((qpme = PMENXT_PTR(pme->pme_next))
				    != NULL) {
					pa = pmg->pmg_base +
					    mmu_ptob(pme - pmg->pmg_pme);
					qpmg = &pmgrps[(qpme - pments) /
					    NPMENTPERPMGRP];
					qa = qpmg->pmg_base +
					    mmu_ptob(qpme - qpmg->pmg_pme);
					if (!VAC_ALIGNED(pa, qa)) {
						ccf = 1;
						(void) splx(s2);
						PP_LIST_NEXT(NULL)
						continue;
					}
				}
				(void) splx(s2);
				PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
			}
			PP_LIST_CLOSE
			if (!ccf) {
				/*
				 * No more cache conflict.
				 * Use hat_ptesync to resync.
				 */
				pp->p_nc = 0;
				PP_LIST_OPEN
				{
					hat_ptesync(pp, pmg, pme, (addr_t)NULL,
					    HAT_NCSYNC);
					PP_LIST_NEXT(PMENXT_PTR(pme->pme_next))
				}
				PP_LIST_CLOSE
			}
		}
#endif VAC
		s = splvm();
	}
	/*
	 * Invalidate the translation.
	 */
	if (ppme->pme_valid) {
		flags |= HAT_INVSYNC;
		hat_ptesync(pp, ppmg, ppme, vaddr, flags);
		ppme->pme_nosync = ppme->pme_intrep = ppme->pme_valid = 0;
	}
	(void) splx(s);
}

/*
 * Synchronize the hardware and software of a pte.  Used for updating the
 * hardware nocache bit, the software R & M bits, and invalidating ptes.
 */
static void
hat_ptesync(pp, pmg, pme, vaddr, flags)
	struct page		*pp;
	register struct pmgrp	*pmg;
	register struct pment	*pme;
	addr_t			vaddr;
	int			flags;
{
	register struct ctx	*ctxsav, *nctx;
	register addr_t		mapaddr;
	int			s, pmg_off;
	struct pte		pte;
	struct pte		*ppte;	/* pointer to SW pte */
	struct as		*as;
	struct ctx		*ctx;
	int			usetemp;
	int			dommu;
	int			doflush;
	int			didsetpte = 0;
	int			didflush = 0;

	ppte = hat_pmetopte(pmg, pme);
	as = pmg->pmg_as;
	ctx = as->a_hat.hat_ctx;

	TEST_PANIC(pme->pme_valid == 0, "hat_ptesync - invalid pme");

	/*
	 * The HAT_VADDR flag means that the vaddr argument contains a valid
	 * page address.
	 *
	 * It's used as optimization when hat_ptesync is called
	 * from hat_unload(). We know that:
	 *	(1) the pmg context is setup
	 *	(2) we don't need to splvm() here since we will not
	 *	    be using SEGTEMP
	 *
	 */
	if (flags & HAT_VADDR) {

		if (hat_pmgisloaded(pmg)) {
			/*
			 * pmg is loaded in HW - make sure that it is mapped.
			 */
			hat_pmgmap(pmg);

			mapaddr = vaddr;
			dommu = 1;
			usetemp = 0;
			/*
			 * We must flush VAC if the context is not clean.
			 */
			doflush = !ctx->c_clean;

		} else {
			/*
			 * pmg is not loaded in MMU
			 */
			dommu = doflush = usetemp = 0;
			mapaddr = (addr_t)0;
		}
		goto skip;
	}

	pmg_off = mmu_ptob(pme - pmg->pmg_pme);
	vaddr = pmg->pmg_base + pmg_off;
	ctxsav = mmu_getctx();
	/*
	 * We must protect the use of the mapping address,
	 * since it is a shared resource.
	 */
	s = splvm();

	if (!hat_pmgisloaded(pmg)) {
		/*
		 * SW page table is not loaded.
		 */
		dommu = doflush = usetemp = 0;

		/*
		 * XXX - Set mapaddr to 0 so that we get a kernel text fault
		 * and panic if we try to use it in the code below.
		 */
		mapaddr = (addr_t)0;
	} else if (ctx == NULL) {
		/*
		 * pmg is loaded but its as does not have a context.
		 * Set things up so that the pmgrp is mapped into a temporary
		 * segment.  No need to do any VAC flushing since this
		 * was done when we took the ctx away.	Set
		 * up the mapaddr within the temporary segment.
		 */

		/* XXX - this is disgusting! */
#ifdef sun2
		extern struct ctx *kctx;

		nctx = kctx;
		if (nctx != ctxsav)
			mmu_setctx(nctx);
#else sun2
		nctx = ctxsav;		/* no need to switch context */
#endif sun2

		mmu_settpmg(SEGTEMP, pmg);
		mapaddr = SEGTEMP + pmg_off;
		dommu = usetemp = 1;
		doflush = 0;
	} else {
		/*
		 * pmg is loaded and pmg's as has a ctx.
		 * Make sure we are in running in the as context.
		 * Use the virtual address as the mapping address.
		 */
		if ((nctx = ctx) != ctxsav)
			mmu_setctx(nctx);

		/*
		 * Make sure that pmg is mapped.
		 */
		hat_pmgmap(pmg);

		mapaddr = vaddr;
		dommu = 1;
		usetemp = 0;
		/*
		 * We must flush VAC if the context is not clean.
		 */
		doflush = !ctx->c_clean;
	}
skip:
	if (!vac)
		doflush = 0;		/* no VAC on this system */

	if (dommu) {
		/* Assert that the pmg is mapped */
		ASSERTPMGMAPPED(pmg, "ptesync");
	}

	/*
	 * At this point, the flags are set:
	 *
	 * doflush - if the page must be flushed
	 * dommu   - if pte must be get/set from HW MMU
	 * usetemp - if SEGTEDMP is used to access the HW MMU map
	 *
	 * Note that a loaded pmg is also mapped if we have a context.
	 */
	if (pp != NULL) {
		if (flags & HAT_RMSYNC) {

			if (dommu) {
				mmu_getpte(mapaddr, ppte);
			}
			pte = *ppte;

			/*
			 * Call back to inform address space, if turned on.
			 */
			if (as->a_hatcallback) {
			    as_hatsync(as, vaddr, (u_int) pte.pg_r,
				(u_int) pte.pg_m,
				(u_int)(flags & HAT_INVSYNC ? AHAT_UNLOAD : 0));
			}
			pg_setref(pp, pp->p_ref | pte.pg_r);
			pg_setmod(pp, pp->p_mod | pte.pg_m);
#ifdef VAC
			/*
			 * When you zero the modified bit in the MMU
			 * and leave it set in the cache you may not
			 * get it set in the mmu when the line is
			 * re-written.	Writeback caches perform the
			 * setting of the modified bit for a page in
			 * the MMU on the first write miss that happens
			 * to that page. Subsequent writes don't bother
			 * to set the modified bit because the first
			 * write did it.  Therefore if you are zeroing
			 * the modified bit you must flush the cache
			 * so that subsequent writes, see the modified
			 * bit unset in the cache and write it back to
			 * the MMU.
			 */
			if (doflush && pte.pg_r) {
				vac_pageflush(mapaddr);
				didflush = 1;
			}
#endif VAC
			pte.pg_r = pte.pg_m = 0;
		}
#ifdef VAC
		else if (flags & HAT_NCSYNC) {

			if (dommu) {
				mmu_getpte(mapaddr, ppte);
			}
			pte = *ppte;
			/*
			 * N.B.	 The following test assumes that there
			 * are no user addresses at the same virtual
			 * addresses as DVMA and segu in VAC machines.
			 */
			if (mapaddr >= DVMA || (segu != NULL &&
			    mapaddr >= segu->s_base &&
			    mapaddr < segu->s_base + segu->s_size)) {
				/*
				 * To avoid lots of problems, we don't
				 * try to convert anything from cached
				 * to non-cached (or vice-versa) when
				 * it is being loaded for DVMA use.
				 * Also, we refuse to mess with user
				 * areas since it is impossible to
				 * reliably flush when converting
				 * from cached to non-cached and we
				 * don't want to take any performance
				 * hits from using a non-cached stack.
				 */
				didsetpte = 1;
				cache_stats.cs_skip++;

				goto skip_ncsync;
			}

			/*
			 * To avoid lots of problems, we don't try to convert
			 * anything from cached to non-cached (or vice-versa)
			 * when it is being loaded for DVMA use.
			 */
			if (mapaddr < DVMA) {
				if (doflush && !pte.pg_nc && pp->p_nc) {
					int pri, iskas;

					/*
					 * Need to convert from a cached
					 * translation to a non-cached
					 * translation.	 There are lots
					 * of potential races here in the
					 * kernel's address space.  If
					 * some clean line ends up in the
					 * cache after it is flushed here
					 * and is then written to, the
					 * Sirius cache system will end
					 * up giving a memory timeout error.
					 *
					 * For now, we assume that between
					 * time that we flush the virtual
					 * address and reset the MMU that
					 * nothing will be getting into
					 * the cache from things like
					 * ethernet (this is questionable).
					 * We also assume that will never
					 * be converting anything from
					 * cached to non-cached in the
					 * kernel for the current stack,
					 * (i.e., the stack can be accessed
					 * safely w/o it being changed from
					 * cached to non-cached), the interrupt
					 * stack, or anything that might be
					 * touched at interrupts above splhigh
					 * (UARTS, level7 profiling).
					 * The stack being considered safe
					 * will need to be watched if/when
					 * we go away from using a fixed
					 * virtual address for the user area
					 * that is not managed by the hat layer.
					 */
					TEST_PANIC(!pmg->pmg_mapped,
					    "hat_ptesync - pmg not mapped");

					pte.pg_nc = 1;

					iskas = (as == &kas);

					pri = splhigh();

					if (pte.pg_r)
						vac_pageflush(mapaddr);

					/* Change both SW and HW pte */
					*ppte = pte;
					mmu_setpte(mapaddr, pte);
					didsetpte = 1;

					if (iskas) {
						/*
						 * Flush the virtual address
						 * again just in case some IO
						 * got in behind our back
						 * above.  Doing this for
						 * iskas only assumes there
						 * is no UDVMA to worry about.
						 */
						struct pte	tpte;

						mmu_getpte(mapaddr, &tpte);
						if (tpte.pg_r)
							vac_pageflush(mapaddr);

						cache_stats.cs_kchange++;
					} else {
						cache_stats.cs_uchange++;
					}
					(void) splx(pri);
				} else {
					pte.pg_nc = pp->p_nc;
				}
			} else {
				cache_stats.cs_iowantchg++;
			}
		skip_ncsync:
			;
		}
#endif VAC
	}

	if (flags & HAT_INVSYNC) {
#ifdef VAC
		if (vac) {
			if (mapaddr >= DVMA && mapaddr < DVMA + dvmasize) {
				/*
				 * Always flush before invalidating a DVMA
				 * translation because the ref bit may lie.
				 * See bugid 1039410.
				 */
				vac_pageflush(mapaddr);
			} else if (doflush && !didflush) {
				struct pte	tpte;

				mmu_getpte(mapaddr, &tpte);
				if (tpte.pg_r)
					vac_pageflush(mapaddr);
			}
		}
#endif VAC
		pte = mmu_pteinvalid;
	}

	if (!didsetpte) {
		*ppte = pte;
		if (dommu) {
			mmu_setpte(mapaddr, pte);
		}
	}

	/*
	 * Optimized return when hat_ptesync was called from hat_unload.
	 */
	if (flags & HAT_VADDR)
		return;
	if (usetemp)
		mmu_settpmg(SEGTEMP, pmgrp_invalid);
	(void) splx(s);
	if (nctx != ctxsav)
		mmu_setctx(ctxsav);
}

/*
 * Allocate a SW page table for a given address.
 */
static struct pmgrp *
hat_pmgalloc(seg, addr)
	struct seg *seg;
	addr_t addr;
{
	register struct as *as = seg->s_as;
	register struct pmgrp *pmg;
	int s;

	s = splvm();
	if ((pmg = hat_pmgfind(addr, as)) != NULL) {
		vmhatstat.vh_pmgallochas++;
		(void) splx(s);
		return (pmg);
	}

	/*
	 * No pmgrp allocated to this address space contains the address;
	 * allocate a new pmg for this address space.  First, try
	 * the free list.
	 */

	/*
	 * Update vmhatstat statistics.
	 */
	if (pmgrpfree == NULL)
		vmhatstat.vh_pmgallocsteal++;
	else
		vmhatstat.vh_pmgallocfree++;

top:
	if ((pmg = pmgrpfree) == NULL) {
		int try;

		/*
		 * No SW pmg's free, have to take one from someone.
		 * Take from address spaces with no ctx first.
		 */
		pmg = pmgrphand;
		for (try = 1; /* empty */; try++) {
			do {
				pmg++;
				if (pmg == pmgrpsNPMGRPS) {
					/*
					 * Wrap around and skip kernels pmgs.
					 */
					pmg = pmgrpmin;
				}

				if (pmg->pmg_lock == 0 &&
				    pmg->pmg_keepcnt == 0) {

					/*
					 * On the first try, only take a pmg
					 * from an address space with no ctx.
					 */
					if (try == 1 &&
					    pmg->pmg_as->a_hat.hat_ctx != NULL)
						continue;
#ifdef SUNDBE
                                        /*
                                         * Don't steal pmeg with stack
                                         */
                                        if (try == 1 &&
                                            pmg->pmg_base ==
                                            (addr_t)KERNELBASE - PMGRPSIZE)
                                                continue;
#endif /* SUNDBE */
					/*
					 * Found a candidate, free
					 * it up and try again.
					 */

					pmg->pmg_keepcnt++;
					hat_pmgfree(pmg);
					pmgrphand = pmg;
					goto top;
				}
			} while (pmg != pmgrphand);
			/*
			 * Give up after 2 tries.
			 */
			if (try >= 2) {
				panic("hat_pmgalloc out of hat");
			}
		}
	}

	pmgrpfree = pmg->pmg_next;	/* take it off the free list */
	pmg->pmg_lock = 1;
	pmg->pmg_keepcnt = 1;

	hat_pmglink(pmg, as, addr);
	pmg->pmg_lock = 0;

	(void) splx(s);

	return (pmg);
}

/*
 * Load SW pmg in HW
 */
static void
hat_pmgload(swpmg)
	struct pmgrp		*swpmg;
{
	register struct as	*as;
	register struct pmgrp	*pmg = (struct pmgrp *)NULL;
	register struct hwpmg	*hwpmg;
	int			s;
	int			pass;

	if (swpmg->pmg_mapped) {
		ASSERT(swpmg->pmg_num != PMGNUM_SW);
		goto locksmg;
	}

	as = swpmg->pmg_as;

	/*
	 * Loaded, but not mapped.
	 */
	if (swpmg->pmg_num != PMGNUM_SW) {
		ASSERT(as == &kas || mmu_getctx()->c_as == as);
		hat_pmgmap(swpmg);
		goto locksmg;
	}

	hat_setup(as);
	s = splvm();

	if ((hwpmg = hwpmgfree) != NULL) {
		vmhatstat.vh_pmgldfree++;
		goto found_free_hwpmg;
	}

	/*
	 * The strategy for stealing a HW pmeg:
	 * We make 3 passes over the array of HW pmegs (starting at hwpmghand)
	 * and will take the pmeg if:
	 *
	 * Pass 1. pmeg does not require VAC flush
	 *    (we clean all pmegs between Pass 1 and Pass 2).
	 * Pass 2. Take any unlocked pmeg. In pass 2, we will take
	 *    even a dirty pmeg because it may be an unused kernel pmeg.
	 */
	hwpmg = hwpmghand;
	for (pass = 1; pass < 3; pass++) {
		struct ctx	*ctx;

		do {
			hwpmg++;
			if (hwpmg == hwpmgsNHWPMGS) {
				/*
				 * Wrap around and skip kernel pmgs.
				 */
				hwpmg = hwpmgmin;
			}

			pmg = hwpmg->hwp_pmgrp;
			ASSERT(pmg != NULL);
			ASSERT(pmg->pmg_num != PMGNUM_SW);

			/*
			 * Skip locked and kept pmg's
			 */
			if (pmg->pmg_lock || pmg->pmg_keepcnt > 0) {
				continue;
			}

			if (pass == 1 &&
			    (ctx = pmg->pmg_as->a_hat.hat_ctx) != NULL &&
			    !ctx->c_clean && pmg->pmg_mapped)
				continue;

			if (pass == 1) {
				if (ctx == NULL)
					vmhatstat.vh_pmgldnoctx++;
				else if (ctx->c_clean)
					vmhatstat.vh_pmgldcleanctx++;
				else
					vmhatstat.vh_pmgldnomap++;
			}

			/*
			 * Keep the pmg until hat_pmgswapptes().
			 */
			pmg->pmg_keepcnt++;
			hat_pmgunload(pmg, PTESFLAG_SKIP);
			ASSERT(hwpmg = hwpmgfree);
			hwpmghand = hwpmg;
			goto found_free_hwpmg;

		} while (hwpmg != hwpmghand);

		if (pass == 1) {

			/*
			 * We haven't found a pmg that does not require
			 * VAC flush. Clear all pmg's now by flushing
			 * all contexts.
			 */
			vac_flushallctx();
			hat_unmap_aspmgs(as);
			vmhatstat.vh_pmgldflush++;
		}
	}
	panic("hat_pmgload: failed after two  passes");

found_free_hwpmg:
	hat_clrcleanbit();
	swpmg->pmg_num = (hwpmg - hwpmgs);
	hwpmg->hwp_pmgrp = swpmg;
	hwpmgfree = hwpmg->hwp_next;	/* take it off the free list */

	ASSERT(swpmg->pmg_as == &kas || mmu_getctx()->c_as == swpmg->pmg_as);

#ifdef MMU_3LEVEL
	if (mmu_3level)
		hat_smgalloc(as, swpmg->pmg_base, swpmg);
#endif MMU_3LEVEL

	/* We are already at splvm() */
	mmu_setpmg(swpmg->pmg_base, swpmg);
	swpmg->pmg_mapped = 1;

	/*
	 * Load all valid SW PTE's in MMU.
	 *
	 */
	if (pmg == (struct pmgrp *)NULL)
		hat_pmgloadptes(swpmg->pmg_base, swpmg->pmg_pte);
	else {
		hat_pmgswapptes(swpmg->pmg_base, swpmg->pmg_pte, pmg->pmg_pte);
		pmg->pmg_keepcnt--;
	}

	as->a_hat.hat_pmgldcnt++;    /* incr. number of HW pmegs for this as */

	(void) splx(s);
	return;

locksmg:
#ifdef MMU_3LEVEL
	if (mmu_3level) {
		/*
		 * Callers to hat_pmgload assume that hat_pmgload keeps smg.
		 */
		s = splvm();
		hat_pmgtosmg(swpmg)->smg_keepcnt++;
		(void)splx(s);
	}
#endif MMU_3LEVEL
	return;
}

/*
 * Map a pmgrp in segment map.
 */
static void
hat_pmgmap(pmg)
	struct pmgrp	*pmg;
{
	int		s;

	if (pmg->pmg_num != PMGNUM_SW && !pmg->pmg_mapped)  {
		s = splvm();
#ifdef MMU_3LEVEL
		if (mmu_3level) {
			hat_smgalloc(pmg->pmg_as, pmg->pmg_base, pmg);
			hat_pmgtosmg(pmg)->smg_keepcnt--;
		}
#endif MMU_3LEVEL
		mmu_setpmg(pmg->pmg_base, pmg);
		pmg->pmg_mapped = 1;
		(void) splx(s);
	}
}

/*
 * Free the specified SW pmgrp.	 This is done by calling hat_pteunload
 * on all the pme's to process all the referenced and modified bits
 * and to invalidate the pme.  If the hat containing this pmg currently
 * has a ctx, then invalidate that mapping.  Finally we unlink the
 * the pmgrp from the hat pmgrp list and put it on the free list.
 * pmg should be kept (once) when this routine is called.
 */
static void
hat_pmgfree(pmg)
	register struct pmgrp *pmg;
{
	register struct pment	*pme = pmg->pmg_pme;
	register struct as	*as;
	register int		tcnt;
	int			s;

	ASSERT(pmg->pmg_keepcnt == 1);
	ASSERT(pmg->pmg_as != NULL);

	if (hat_pmgisloaded(pmg))
		hat_pmgunload(pmg, PTESFLAG_UNLOAD);

	s = splvm();

	if ((as = pmg->pmg_as) != NULL) {
		for (tcnt = 0; tcnt < NPMENTPERPMGRP; tcnt++, pme++) {
			if (pme->pme_valid)
				hat_pteunload(pmg, pme, (addr_t)NULL,
				    HAT_RMSYNC);
		}
		if (as->a_hat.hat_pmgrps == pmg) {
			as->a_hat.hat_pmgrps = pmg->pmg_next;
			if (pmg->pmg_next)
				pmg->pmg_next->pmg_prev = NULL;
		} else {
			pmg->pmg_prev->pmg_next = pmg->pmg_next;
			if (pmg->pmg_next)
				pmg->pmg_next->pmg_prev = pmg->pmg_prev;
		}
		pmg->pmg_as = NULL;
		pmg->pmg_next = pmg->pmg_prev = NULL;
	}

	pmg->pmg_keepcnt--;
	pmg->pmg_next = pmgrpfree;
	pmgrpfree = pmg;

	(void) splx(s);
}

static void
hat_pmgunload(pmg, ptesflag)
	struct pmgrp	*pmg;
	enum ptesflag	ptesflag;
{
	addr_t		a;
	struct pte	*ppte = pmg->pmg_pte;
	struct hwpmg	*hwpmg;
	struct ctx	*ctx, *ctxsav;
	int		s;

	ASSERT(pmg->pmg_num != PMGNUM_SW);

	s = splvm();			/* using SEGTEMP */

#ifdef notdef				/* 3 level mmu */
	ASSERT(pmg->pmg_as->a_hat.hat_ctx != NULL ||  !pmg->pmg_mapped);
#endif notdef

	/*
	 * Unmap pmg from segment map.
	 */
	if (pmg->pmg_mapped) {

#ifdef MMU_3LEVEL
		if (mmu_3level) {
			ASSERT(pmg->pmg_sme != NULL);
		}
#endif MMU_3LEVEL

		if ((ctx = pmg->pmg_as->a_hat.hat_ctx) != NULL) {
			ctxsav = mmu_getctx();
			if (ctxsav != ctx)
				mmu_setctx(ctx);

			if (!ctx->c_clean)
				vac_segflush(pmg->pmg_base);

			mmu_pmginval(pmg->pmg_base);

			if (ctxsav != ctx)
				mmu_setctx(ctxsav);
		} else {
#ifdef MMU_3LEVEL
			if (mmu_3level) {
				/*
				 * We have to use REGTEMP to map the smg.
				 * Note that REGTEMP may be used only in kctx.
				 */

				struct smgrp	*smg;

				ctxsav = mmu_getctx();
				if (ctxsav != kctx)
					mmu_setctx(kctx);

				smg = &smgrps[(pmg->pmg_sme - sments)
				    >> NSMENTPERSMGRPSHIFT];
				mmu_setsmg(REGTEMP, smg);
				mmu_pmginval(REGTEMP +
				    ((u_int)pmg->pmg_base & SMGRPOFFSET));
				mmu_smginval(REGTEMP);

				if (ctxsav != kctx)
					mmu_setctx(ctxsav);
			} else
				panic("hat_pmgunload");
#else  MMU_3LEVEL
			panic("hat_pmgunload");
#endif MMU_3LEVEL
		}
		pmg->pmg_mapped = 0;

#ifdef MMU_3LEVEL
		if (mmu_3level) {
			pmg->pmg_sme->sme_valid = 0;
			pmg->pmg_sme->sme_pmg = (struct pmgrp *)NULL;
			pmg->pmg_sme = (struct sment *)NULL;
		}
#endif MMU_3LEVEL
	}

	ASSERT(!pmg->pmg_mapped);

	if (ptesflag == PTESFLAG_UNLOAD) {
		/*
		 * Unload all valid SW PTE's in MMU.
		 *
		 */
		map_setsgmap(SEGTEMP, pmg->pmg_num);
		a = SEGTEMP;
		hat_pmgunloadptes(a, ppte);
		map_setsgmap(SEGTEMP, PMGRP_INVALID);
	}

	pmg->pmg_as->a_hat.hat_pmgldcnt--;

	/*
	 * Put hwpmg in HW pmg free list
	 */
	hwpmg = &hwpmgs[pmg->pmg_num];
	hwpmg->hwp_next = hwpmgfree;
	hwpmgfree = hwpmg;
	hwpmg->hwp_pmgrp = NULL;

	pmg->pmg_num = PMGNUM_SW;

	(void) splx(s);
}

/*
 * Add the specified pmgrp to the list of pmgrp's allocated to
 * the specified address space.	 We hang pmgrps off the address
 * space and not the ctx so that we can keep them around even if
 * we don't have a hardware context.
 */
static void
hat_pmglink(pmg, as, addr)
	register struct pmgrp *pmg;
	struct as *as;
	addr_t addr;
{
	int s;

	ASSERT(pmg->pmg_keepcnt > 0);

	s = splvm();
	pmg->pmg_as = as;
	pmg->pmg_next = as->a_hat.hat_pmgrps;
	if (pmg->pmg_next)
		pmg->pmg_next->pmg_prev = pmg;
	pmg->pmg_prev = NULL;
	as->a_hat.hat_pmgrps = pmg;
	pmg->pmg_base = (addr_t)((u_int)addr & ~(PMGRPSIZE - 1));
	(void) splx(s);
}

#ifdef notdef
/*
 * Called when the wrong pmeg is read out from the MMU.
 * Most likely, this is a down rev Carrera CPU board that
 * is missing some pullup registers on the segment RAMs.
 * The ECO for the needed Carrera CPU board fix is 2550.
 */
static void
hat_wrongpmg(pmg, addr, as)
	struct pmgrp *pmg;
	addr_t addr;
	struct as *as;
{
#ifndef sun2
	register struct pmgrp *pmgp;

	addr = (addr_t)((u_int)addr & ~(PMGRPSIZE - 1));
	printf("hardware claims page map entry group number is 0x%x\n",
	    pmg->pmg_num);
	if (addr >= (addr_t)KERNELBASE) {
		/*
		 * Scan the kernel address space for the correct pmg.
		 */
		for (pmgp = kas.a_hat.hat_pmgrps; pmgp != NULL;
		    pmgp = pmgp->pmg_next) {
			if (pmgp->pmg_base == addr)
				break;
		}
	} else {
		for (pmgp = as->a_hat.hat_pmgrps; pmgp != NULL;
		    pmgp = pmgp->pmg_next) {
			if (pmgp->pmg_base == addr)
				break;
		}
	}

	if (pmgp != NULL) {
		printf("  software says page map entry group number is 0x%x\n",
		    pmgp->pmg_num);
	}
#endif
	printf("pmg = %x, pmg base = %x, addr = %x\n", pmg, pmg->pmg_base,
	    addr);
	panic("wrong pmg");
	/* NOTREACHED */
}
#endif notdef

static void
hat_xfree(as)
	register struct as *as;
{
	register int s;
	register struct pmgrp *pmg;

	/*
	 * Free pmgrp's.
	 */
	s = splvm();
	while (pmg = as->a_hat.hat_pmgrps) {
		pmg->pmg_keepcnt++;
		(void) splx(s);
		hat_pmgfree(pmg);
		s = splvm();
	}
	(void) splx(s);

#ifdef MMU_3LEVEL
	/*
	 * If three-level mmu, free smgrp's.
	 */
	if (mmu_3level) {
		register struct smgrp *smg;

		s = splvm();
		while (smg = as->a_hat.hat_smgrps) {
			smg->smg_keepcnt++;
			(void) splx(s);
			hat_smgfree(smg);
			s = splvm();
		}
		(void) splx(s);
	}
#endif MMU_3LEVEL

	ASSERT(as->a_hat.hat_pmgldcnt == 0);
}


/*
 * Find a SW page table.
 *
 * The returned page table is 'kept'.
 *
 * We optimize the search by using a look-aside buffer of mappings
 * from <as, addr> to a pointer to the pmg structure. For each hashed
 * <as, addr> value, we store a pointer to the most recently found pmgrp.
 * 1009 is a prime that was found to be optimal.
 */

#define	hash_asaddr(as, addr) \
	((((u_int)as >> 2) * 1009 + ((u_int)addr >> PMGRPSHIFT)) % (NPMGHASH-1))

static struct pmgrp *
hat_pmgfind(addr, as)
	addr_t		addr;
	struct as	*as;
{
	struct pmgrp	*pmg;
	addr_t		pmgaddr = hat_pmgbase(addr);
	int		s;
	int		hashind;

	hashind = hash_asaddr(as, pmgaddr);

	s = splvm();

	/*
	 * Check if pmg is in the lookaside buffer.
	 */
	if ((pmg = pmghash[hashind]) != NULL &&
	    pmg->pmg_as == as && pmg->pmg_base == pmgaddr) {
		pmg->pmg_keepcnt++;
		pmgfindstat.pf_hit++;
		goto out;
	}

	/*
	 * Exhaustive search of pmg's within the address space.
	 */
	for (pmg = as->a_hat.hat_pmgrps; pmg != NULL &&
	    pmg->pmg_base != pmgaddr; pmg = pmg->pmg_next)
		;

	if (pmg != NULL) {
		pmg->pmg_keepcnt++;
		pmgfindstat.pf_miss++;

		/*
		 * Write the <as, addr> -> pmg entry to look aside buffer
		 */
		pmghash[hashind] = pmg;
	} else {
		pmgfindstat.pf_notfound++;
	}

out:
	(void) splx(s);
	return (pmg);
}


/*
 * Clear running process's context clean bit.
 */
static void
hat_clrcleanbit()
{
	struct	proc	*p;
	struct	as	*a;
	struct	ctx	*c;

	if ((p = u.u_procp) != NULL && (a = p->p_as) != NULL &&
	    (c = a->a_hat.hat_ctx) != NULL) {
		c->c_clean = 0;
	}
}


#ifdef MMU_3LEVEL
/*
 * Code to handle allocation of smegs is cloned from the pmeg versions
 */

int getsmg_check = 1;

/*
 * This routine will return the smgrp structure for the given address
 * in the current ctx.	But unlike mmu_getsmg, this routine will protect
 * against the smgrp being lost by spl'ing and will return a kept smgrp
 * pointer.  The keepcnt should be decremented by the caller when it is
 * done looking at the smgrp contents.
 */
static struct smgrp *
hat_getsmg(addr)
	addr_t addr;
{
	int s;
	struct smgrp *smg;

	s = splvm();
	smg = mmu_getsmg(addr);
	smg->smg_keepcnt++;
	if (getsmg_check && smg != smgrp_invalid && smg->smg_base != 0 &&
	    smg->smg_base != (caddr_t)((u_int)addr & ~(SMGRPSIZE - 1))) {
		printf("hat_getsmg: addr=%x, smg=%x, smg base=%x\n",
		    addr, smg, smg->smg_base);
		call_debug("hat_getsmg");
	}
	(void) splx(s);
	return (smg);
}

/*
 * Free the specified smgrp.  This is done by calling hat_pmgfree
 * on all the sme's to invalidate the smgrp.  If the hat containing
 * this smg currently has a ctx, then invalidate that mapping.
 * Finally we unlink the the smgrp from the hat smgrp list and
 * put it on the free list.
 * smg should be kept (once) when this routine is called.
 */
static void
hat_smgfree(smg)
	register struct smgrp *smg;
{
	register struct sment *sme;
	register struct pmgrp *pmg;
	register struct as *as;
	register int tcnt;
	struct ctx *ctx, *ctxsav, *curctx;
	int s;

	ASSERT(smg->smg_keepcnt == 1);

	if ((as = smg->smg_as) != NULL) {

		/* XXX - we should simplify switching between ctx's */
		ctxsav = curctx = mmu_getctx();
		if ((ctx = smg->smg_as->a_hat.hat_ctx) != NULL &&
		    !ctx->c_clean) {
			mmu_setctx(curctx = ctx);
			vac_rgnflush(smg->smg_base);
		}

		if (curctx != kctx)
			mmu_setctx(curctx = kctx);

#if	defined(SUNDBE) && defined(sun4)
                if (ispseudo_smgrp(smg))
                        goto skip_if_pseudo;
#endif  ISM

		/*
		 * XXX - we may optimize by not using kctx if smg has ctx.
		 */
		s = splvm();
		ASSERT(mmu_getsmg(REGTEMP) == smgrp_invalid);
		mmu_setsmg(REGTEMP, smg);
		sme = smg->smg_sme;
		for (tcnt = 0; tcnt < NSMENTPERSMGRP; tcnt++) {
			if (sme->sme_valid) {
				pmg = sme->sme_pmg;
				ASSERT(((u_int)pmg->pmg_base & SMGRPOFFSET) <
				    SMGRPSIZE);
				mmu_pmginval(REGTEMP +
				    ((u_int)pmg->pmg_base & SMGRPOFFSET));
				pmg->pmg_mapped = 0;
				sme->sme_valid = 0;
				pmg->pmg_sme = (struct sment *)NULL;
				sme->sme_pmg = (struct pmgrp *)NULL;
			}
			sme++;
		}
		mmu_smginval(REGTEMP);
		(void) splx(s);

#if	defined(SUNDBE) && defined(sun4)
        skip_if_pseudo:
#endif  ISM

		if ((ctx = smg->smg_as->a_hat.hat_ctx) != NULL) {
			if (ctx != curctx)
				mmu_setctx(curctx = ctx);
			mmu_smginval(smg->smg_base);
		}
		if (ctxsav != curctx)
			mmu_setctx(ctxsav);

		if (as->a_hat.hat_smgrps == smg) {
			as->a_hat.hat_smgrps = smg->smg_next;
			if (smg->smg_next)
				smg->smg_next->smg_prev = NULL;
		} else {
			smg->smg_prev->smg_next = smg->smg_next;
			if (smg->smg_next)
				smg->smg_next->smg_prev = smg->smg_prev;
		}
		smg->smg_as = NULL;
		smg->smg_next = smg->smg_prev = NULL;
	}

#if	defined(SUNDBE) && defined(sun4)
        if (ispseudo_smgrp(smg)) {
                kmem_free((char *)smg, sizeof(*smg));
                return;
        }
#endif  ISM

	s = splvm();
	smg->smg_keepcnt--;
	smg->smg_next = smgrpfree;
	smgrpfree = smg;
	(void) splx(s);
}

/*
 * Add the specified smgrp to the list of smgrp's allocated to
 * the specified address space.	 We hang smgrps off the address
 * space and not the ctx so that we can keep them around even if
 * we don't have a hardware context.
 */
static void
hat_smglink(smg, as, addr)
	register struct smgrp *smg;
	struct as *as;
	addr_t addr;
{
	int s;

	s = splvm();
	smg->smg_as = as;
	smg->smg_next = as->a_hat.hat_smgrps;
	if (smg->smg_next)
		smg->smg_next->smg_prev = smg;
	smg->smg_prev = NULL;
	as->a_hat.hat_smgrps = smg;
	smg->smg_base = (addr_t)((u_int)addr & ~(SMGRPSIZE - 1));
	(void) splx(s);
}

void
hat_smgreserve(seg, addr)
	struct seg *seg;
	addr_t addr;
{
	register struct as	*as = seg->s_as;
	register struct smgrp	*smg;
	register struct pmgrp	*pmg;
	struct sment		*sme;

	if (!mmu_3level)
		return;

	smg = hat_getsmg(addr);		/* keeps the smg for us */

	/* DEBUGGING */
	if (smg == smgrp_invalid)
		printf("hat_smgreserve: addr 0x%x invalid smg\n", addr);

	if (as->a_hat.hat_ctx == NULL ||
	    (smg->smg_as != NULL && smg->smg_as != as))
		panic("hat_smgreserve");

	if (smg != smgrp_invalid && smg->smg_as == NULL)
		hat_smglink(smg, as, addr);

	smg->smg_lock = 1;	/* if its being reserved, also lock it */

	/*
	 * Set up sme structure.
	 */
	pmg = mmu_getpmg(addr);
	sme = &(smg->smg_sme[(mmu_btop(addr-smg->smg_base)/NPMENTPERPMGRP)]);

	ASSERT(!sme->sme_valid || sme == pmg->pmg_sme);

	if (!sme->sme_valid) {
		pmg = mmu_getpmg(addr);
		ASSERT(pmg->pmg_sme == NULL);
		sme->sme_pmg  = pmg;
		sme->sme_valid = 1;
		pmg->pmg_sme = sme;
	}
}

/*
 * Initialize all the unlocked smgs to have invalid sme's
 * and add them to the free list.
 * This routine is called during startup after all the
 * kernel smgs have been reserved.  This routine will
 * also set the smgrpmin variable for use in hat_smgalloc.
 *
 * REGTEMP is only used here so we temporarily steal
 * the region before KERNELBASE and mark it invalid
 * when we are finished.
 */
void
hat_smginit()
{
	register struct smgrp *smg;
	register addr_t addr;
	int s;

	if (!mmu_3level)
		return;

	s = splvm();

	for (smg = smgrps; smg < smgrpsNSMGRPS; smg++) {
		if (smg->smg_lock || smg->smg_keepcnt != 0)
			continue;

		if (smgrpmin == NULL)
			smgrpmin = smg;

		mmu_settsmg((addr_t)REGTEMP, smg);

		for (addr = (addr_t)REGTEMP;
		    addr < (addr_t)(REGTEMP + SMGRPSIZE);
		    addr += PMGRPSIZE) {
			mmu_pmginval(addr);
		}

		smg->smg_next = smgrpfree;
		smgrpfree = smg;
	}
	smgrphand = smgrpmin;

	mmu_smginval((addr_t)REGTEMP);

	(void) splx(s);
}

/*
 * Allocate a smgrp to map the specified address.
 * Returns w/ the keepcnt incremented for the particular smgrp used.
 * First look for something in the free list and then steal one
 * that is currently being used.
 */
static void
hat_smgalloc(as, addr, pmg)
	struct as		*as;
	addr_t			addr;
	struct pmgrp		*pmg;
{
	register struct smgrp	*smg;
	register struct sment	*sme;
	int			s;
	struct ctx		*ctx;

	if (!mmu_3level)
		return;

	s = splvm();
	if ((smg = mmu_getsmg(addr)) != smgrp_invalid) {
		smg->smg_keepcnt++;
		if (getsmg_check && smg->smg_base !=
		    (caddr_t)((u_int)addr & ~(SMGRPSIZE - 1))) {
			printf("hat_smgalloc: addr=%x, smg=%x, smg base=%x\n",
			    addr, smg, smg->smg_base);
			call_debug("hat_smgalloc");
		}
		sme = &(smg->smg_sme
			[(mmu_btop(addr-smg->smg_base)/NPMENTPERPMGRP)]);
		sme->sme_pmg  = pmg;
		sme->sme_valid = 1;
		pmg->pmg_sme = sme;
		(void) splx(s);
		return;
	}

	/*
	 * No smgrp allocated to this address space contains the pme,
	 * allocate a new smg for this address space.  First, try
	 * the free list.
	 */
	if (smgrpfree != NULL)
		vmhatstat.vh_smgfree++;

top:
	if ((smg = smgrpfree) == NULL) {
		int try;

		/*
		 * No smg's free, have to take one from someone.
		 * Take from address spaces with no ctx first.
		 * XXX - could do it with just one pass.
		 */
		smg = smgrphand;
		try = 1;
		for (;;) {
			do {
				smg++;
				if (smg == smgrpsNSMGRPS) {
					if (smgrpmin) {
						/* skip some kernel smgrps */
						smg = smgrpmin;
					} else {
						smg = smgrps;
					}
				}

				if (smg->smg_lock == 0 &&
				    smg->smg_keepcnt == 0) {
					/*
					 * On the first try, only take a smg
					 * from an address space with no ctx.
					 */
					if (try < 3 &&
					    (ctx = smg->smg_as->a_hat.hat_ctx)
						!= NULL &&
					    !ctx->c_clean)
						continue;

					/*
					 * Found a candidate, free
					 * it up and try again.
					 */
					if (try == 1) {
						if (ctx == NULL)
						    vmhatstat.vh_smgnoctx++;
						else
						    vmhatstat.vh_smgcleanctx++;
					}

					smg->smg_keepcnt++;
					hat_smgfree(smg);
					smgrphand = smg;
					goto top;
				}
			} while (smg != smgrphand);

			if (try == 1) {
				/*
				 * We were not able to find a segment that
				 * would not require flushing.
				 *
				 * Flush all user VAC lines and try again.
				 */
				vac_flushallctx();
				vmhatstat.vh_smgflush++;
			}

			/*
			 * Give up after 2 tries.
			 */
			if (try >= 3) {
				rm_outofhat();
			}
			try++;
		}
	}
	hat_clrcleanbit();
	smgrpfree = smg->smg_next;	/* take it off the free list */
	smg->smg_lock = 1;
	smg->smg_keepcnt = 1;

	hat_smglink(smg, as, addr);

	sme = &(smg->smg_sme[(mmu_btop(addr-smg->smg_base)/NPMENTPERPMGRP)]);
	sme->sme_pmg  = pmg;
	sme->sme_valid = 1;
	pmg->pmg_sme = sme;

	(void) splx(s);

	mmu_setsmg(smg->smg_base, smg);

	smg->smg_lock = 0;

	return;
}


hat_smgcheck_keepcnt(smg)
	struct smgrp	*smg;
{
	int	s;

	/*
	 * smgrp_invalid has smg_keepcnt == 1, but it has no pmgs.
	 */
	if (smg == smgrp_invalid)
		return;

	s = splvm();
	{
		register struct sment *sme;
		register struct pmgrp *pmg;
		int		tcnt;
		int		keepcnt = 0;

		sme = smg->smg_sme;
		for (tcnt = 0; tcnt < NSMENTPERSMGRP; tcnt++) {
			if (sme->sme_valid) {
				pmg = sme->sme_pmg;
				keepcnt += pmg->pmg_keepcnt;
			}
			sme++;
		}

		if (keepcnt != smg->smg_keepcnt) {
			printf(
	"check_smgkeepcnt: keepcnt %d smg_keepcnt %d base 0x%x smg# %d\n",
			    keepcnt, smg->smg_keepcnt,
			    smg->smg_base, smg->smg_num);
			panic("possibly double mapped pmgrp");
		}

	}
	(void) splx(s);
}

hat_smgcheck_keepcntall()
{
	register struct smgrp *smg;

	for (smg = smgrps; smg < smgrpsNSMGRPS; smg++)
		hat_smgcheck_keepcnt(smg);
}
#endif MMU_3LEVEL

/*
 * Unmap all HW pmegs (except for locked pmegs) held by an address space.
 */
static void
hat_unmap_aspmgs(as)
	struct as	*as;
{
	struct pmgrp	*pmg;

	/*
	 * Don't do anything if this is kernel address space.
	 */
	if (as == &kas)
		return;

	/*
	 * We don't unmap pmegs if the address space holds only a
	 * "small number" of HW pmegs. We expect that future pmeg allocations
	 * will steal HW pmegs from other address spaces with a clean
	 * or no context.
	 *
	 * If the address space holds less than 'hatunmaplimit' percent
	 * of the total number of HW pmegs we don't unmap. hatunmaplimit
	 * is set to 30% and may be patched.
	 */

	if (as->a_hat.hat_pmgldcnt * 100 < NPMGRPS * hatunmaplimit)
		return;

	ASSERT(mmu_getctx()->c_as == as);

	for (pmg = as->a_hat.hat_pmgrps; pmg != NULL; pmg = pmg->pmg_next) {
		if (pmg->pmg_mapped && pmg->pmg_keepcnt == 0) {
			ASSERT(pmg->pmg_num != PMGNUM_SW);
			ASSERT(!pmg->pmg_lock);
			mmu_pmginval(pmg->pmg_base);
			pmg->pmg_mapped = 0;
		}
	}
}

#ifdef notdef
/* XXX - temporary (and quite dirty) assertion function */
assertpmgmapped(pmg, msg)
	struct pmgrp	*pmg;
	char		*msg;
{
	addr_t		a = pmg->pmg_base;

	if (pmg == NULL) {
		printf("%s addr 0x%x\n", msg, pmg->pmg_base);
		ASSERT(pmg != NULL);
	}

	if (pmg->pmg_as->a_hat.hat_ctx == NULL) {
#ifdef notdef				/* 3 level mmu */
		if (pmg->pmg_mapped) {
			printf("%s addr 0x%x\n", msg, pmg->pmg_base);
			ASSERT(!pmg->pmg_mapped);
		}
#endif notdef
		a = SEGTEMP;
	} else {
		if (!pmg->pmg_mapped) {
			printf("%s addr 0x%x\n", msg, pmg->pmg_base);
			ASSERT(pmg->pmg_mapped);
		}
	}

	if (pmg->pmg_num == PMGNUM_SW) {
		printf("%s addr 0x%x\n", msg, pmg->pmg_base);
		ASSERT(pmg->pmg_num != PMGNUM_SW);
	}

	if (map_getsgmap(a) == PMGRP_INVALID) {
		printf("%s addr 0x%x\n", msg, pmg->pmg_base);
		/* Force traceback */
		printf("pmg_num %d, pmg_keepcnt %d, ctx 0x%x\n",
		    pmg->pmg_num, pmg->pmg_keepcnt,
		    pmg->pmg_as->a_hat.hat_ctx);
		ASSERT(map_getsgmap(a) != PMGRP_INVALID);
	}
}
#endif notdef

#if	defined(SUNDBE) && defined(sun4)
void
hat_ism_lockpmgs(as)
        struct as       *as;
{
}

void
hat_ism_unlockpmgs(as)
        struct as       *as;
{
}

void
hat_intimate_map(seg, ismseg)
        struct seg      *seg;
        struct seg      *ismseg;
{
        struct as       *as = seg->s_as;
        struct as       *ismas = ismseg->s_as;
        struct smgrp    *smg;
        struct smgrp    *smg2;
        int             offset = seg->s_base - ismseg->s_base;
        struct as       *saveas = mmu_getctx()->c_as;
 
        hat_setup(seg->s_as);
 
        ASSERT(((int)seg->s_base & SMGRPOFFSET) == 0); /* SMEG alignment */
 
        for (smg = ismas->a_hat.hat_smgrps; smg != NULL; smg = smg->smg_next) {
 
                ASSERT(smg->smg_keepcnt > 0);
 
                /* XXX  use kmem_fast_alloc() */
                smg2 = (struct smgrp*) kmem_alloc(sizeof(*smg2));
 
                *smg2 = *smg;        /* this copies smg->smg_num */
 
                smg2->smg_num = smg->smg_num;
                smg2->smg_sme = smg->smg_sme;
                smg2->smg_keepcnt = 1;       /* hat_smglink() asserts this */
 
                /*
                 * Link pseudo smeg into process's hat_smgrps list
                 */
                hat_smglink(smg2, as, smg->smg_base + offset);
                smg2->smg_keepcnt = 0;       /* hat_smgfree() asserts this */
                mmu_setsmg(smg2->smg_base, smg2);
        }
 
        hat_setup(saveas);
}
 
void
hat_intimate_unmap(seg)
        struct seg      *seg;
{
        struct as       *as = seg->s_as;
        addr_t          addr = seg->s_base;
        u_int           len = seg->s_size;
        addr_t          eaddr = addr + len;
        struct smgrp    *smg;
        int             s;
 
        s = splvm();
        smg = as->a_hat.hat_smgrps;
        while (smg != NULL) {
                if (smg->smg_base >= addr && smg->smg_base < eaddr) {
                        (void) splx(s);
                        /*
                         * Free the pseudo pmeg.
                         */
                        ASSERT(ispseudo_smgrp(smg));
                        smg->smg_keepcnt++;  /* hat_smgfree asserts this */
                        hat_smgfree(smg);
                        s = splvm();
                        /*
                         * Go over the entire list again because the list
                         * could have been shuffled.
                         */
                        smg = as->a_hat.hat_smgrps;
                }
                else
                        smg = smg->smg_next;
        }
        (void) splx(s);
}

/*
 * Return 1 if this is a pseudo pmeg
 */
static int
ispseudo_smgrp(smgrp)
        struct smgrp    *smgrp;
{
        return (!(smgrp >= smgrps && smgrp <  smgrpsNSMGRPS));
}
#endif  ISM

