test: mmfault.c Source File

Main Page | Class Hierarchy | Class List | File List | Class Members | File Members

# mmfault.c

Go to the documentation of this file.

    00001 /*++
    00002
    00003 Copyright (c) 1989  Microsoft Corporation
    00004
    00005 Module Name:
    00006
    00007    mmfault.c
    00008
    00009 Abstract:
    00010
    00011     This module contains the handlers for access check, page faults
    00012     and write faults.
    00013
    00014 Author:
    00015
    00016     Lou Perazzoli (loup) 6-Apr-1989
    00017     Landy Wang (landyw) 02-June-1997
    00018
    00019 Revision History:
    00020
    00021 --*/
    00022
    00023 #include "mi.h"
    00024
    00025 #define PROCESS_FOREGROUND_PRIORITY (9)
    00026
    00027 ULONG MiDelayPageFaults;
    00028
    00029 #if DBG
    00030 ULONG MmProtoPteVadLookups = 0;
    00031 ULONG MmProtoPteDirect = 0;
    00032 ULONG MmAutoEvaluate = 0;
    00033
    00034 PMMPTE MmPteHit = NULL;
    00035
    00036 ULONG MmLargePageFaultError;
    00037 #endif
    00038
    00039
    00040 NTSTATUS
    00041 MmAccessFault (
    00042     IN BOOLEAN StoreInstruction,
    00043     IN PVOID VirtualAddress,
    00044     IN KPROCESSOR_MODE PreviousMode,
    00045     IN PVOID TrapInformation
    00046     )
    00047
    00048 /*++
    00049
    00050 Routine Description:
    00051
    00052     This function is called by the kernel on data or instruction
    00053     access faults.  The access fault was detected due to either
    00054     an access violation, a PTE with the present bit clear, or a
    00055     valid PTE with the dirty bit clear and a write operation.
    00056
    00057     Also note that the access violation and the page fault could
    00058     occur because of the Page Directory Entry contents as well.
    00059
    00060     This routine determines what type of fault it is and calls
    00061     the appropriate routine to handle the page fault or the write
    00062     fault.
    00063
    00064 Arguments:
    00065
    00066     StoreInstruction - Supplies TRUE (1) if the operation causes a write into
    00067                        memory.  Note this value must be 1 or 0.
    00068
    00069     VirtualAddress - Supplies the virtual address which caused the fault.
    00070
    00071     PreviousMode - Supplies the mode (kernel or user) in which the fault
    00072                    occurred.
    00073
    00074     TrapInformation - Opaque information about the trap, interpreted by the
    00075                       kernel, not Mm.  Needed to allow fast interlocked access
    00076                       to operate correctly.
    00077
    00078 Return Value:
    00079
    00080     Returns the status of the fault handling operation.  Can be one of:
    00081         - Success.
    00082         - Access Violation.
    00083         - Guard Page Violation.
    00084         - In-page Error.
    00085
    00086 Environment:
    00087
    00088     Kernel mode, APCs disabled.
    00089
    00090 --*/
    00091
    00092 {
    00093     PMMPTE PointerPpe;
    00094     PMMPTE PointerPde;
    00095     PMMPTE PointerPte;
    00096     PMMPTE PointerProtoPte;
    00097     ULONG ProtectionCode;
    00098     MMPTE TempPte;
    00099     PEPROCESS CurrentProcess;
    00100     KIRQL PreviousIrql;
    00101     NTSTATUS status;
    00102     ULONG ProtectCode;
    00103     PFN_NUMBER PageFrameIndex;
    00104     WSLE_NUMBER WorkingSetIndex;
    00105     KIRQL OldIrql;
    00106     PMMPFN Pfn1;
    00107     PPAGE_FAULT_NOTIFY_ROUTINE NotifyRoutine;
    00108     NTSTATUS SessionStatus;
    00109     PEPROCESS FaultProcess;
    00110     PMMSUPPORT Ws;
    00111     BOOLEAN SessionAddress;
    00112     PVOID UsedPageTableHandle;
    00113     ULONG BarrierStamp;
    00114     LOGICAL ApcNeeded;
    00115
    00116 #if defined(_IA64_)
    00117     LOGICAL ExecutionFault = FALSE;
    00118
    00119     //
    00120     // If StoreInstruction indicates it was an execution fault, set
    00121     // ExecutionFault TRUE and StoreInstruction FALSE.
    00122     //
    00123
    00124     if (StoreInstruction == 2) {
    00125         ExecutionFault = TRUE;
    00126         StoreInstruction = FALSE;
    00127     }
    00128 #endif
    00129
    00130     PointerProtoPte = NULL;
    00131
    00132 #if defined (_WIN64)
    00133
    00134     //
    00135     // Perform address sanity checks.
    00136     //
    00137
    00138     if (PreviousMode == UserMode) {
    00139
    00140         if (VirtualAddress >= MM_HIGHEST_USER_ADDRESS) {
    00141             return STATUS_ACCESS_VIOLATION;
    00142         }
    00143
    00144     } else {
    00145
    00146         if (!((VirtualAddress <= (PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS + 1)) ||
    00147
    00148 #if defined (_IA64_)
    00149
    00150             //
    00151             // Page table pages are in the user region space for IA64.
    00152             //
    00153
    00154             (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress)) ||
    00155             (MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress)) ||
    00156             (MI_IS_SESSION_ADDRESS(VirtualAddress)) ||
    00157 #endif
    00158
    00159             ((VirtualAddress >= MM_SYSTEM_RANGE_START) &&
    00160             (VirtualAddress < (PVOID)MM_SYSTEM_SPACE_END)))) {
    00161
    00162             if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00163                 return STATUS_ACCESS_VIOLATION;
    00164             }
    00165
    00166             KeBugCheckEx (MEMORY_MANAGEMENT,
    00167                           (ULONG_PTR) VirtualAddress,
    00168                           StoreInstruction,
    00169                           PreviousMode,
    00170                           0xdead);
    00171         }
    00172     }
    00173
    00174 #endif
    00175
    00176     //
    00177     // Block APCs and acquire the working set mutex.  This prevents any
    00178     // changes to the address space and it prevents valid PTEs from becoming
    00179     // invalid.
    00180     //
    00181
    00182     CurrentProcess = PsGetCurrentProcess ();
    00183
    00184 #if DBG
    00185     if (MmDebug & MM_DBG_SHOW_FAULTS) {
    00186
    00187         PETHREAD CurThread;
    00188
    00189         CurThread = PsGetCurrentThread();
    00190         DbgPrint("MM:**access fault - va %p process %p thread %p\n",
    00191                  VirtualAddress, CurrentProcess, CurThread);
    00192     }
    00193 #endif //DBG
    00194
    00195     PreviousIrql = KeGetCurrentIrql ();
    00196
    00197     //
    00198     // Get the pointer to the PDE and the PTE for this page.
    00199     //
    00200
    00201     PointerPte = MiGetPteAddress (VirtualAddress);
    00202     PointerPde = MiGetPdeAddress (VirtualAddress);
    00203     PointerPpe = MiGetPpeAddress (VirtualAddress);
    00204
    00205 #if PFN_CONSISTENCY
    00206     if (PointerPte >= MiPfnStartPte && PointerPte < MiPfnStartPte + MiPfnPtes) {
    00207         DbgPrint("MM: Unsynchronized access to the PFN database - va %p process %p\n",
    00208                  VirtualAddress, CurrentProcess);
    00209
    00210         KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
    00211         MiMapInPfnDatabase();
    00212         MiPfnProtectionEnabled = FALSE;
    00213         KeLowerIrql (OldIrql);
    00214
    00215         DbgBreakPoint();
    00216     }
    00217 #endif
    00218
    00219 #if DBG
    00220     if (PointerPte == MmPteHit) {
    00221         DbgPrint("MM:pte hit at %p\n", MmPteHit);
    00222         DbgBreakPoint();
    00223     }
    00224 #endif
    00225
    00226     ApcNeeded = FALSE;
    00227
    00228     if (PreviousIrql > APC_LEVEL) {
    00229
    00230         //
    00231         // The PFN database lock is an executive spin-lock.  The pager could
    00232         // get dirty faults or lock faults while servicing and it already owns
    00233         // the PFN database lock.
    00234         //
    00235
    00236 #if !defined (_WIN64)
    00237         MiCheckPdeForPagedPool (VirtualAddress);
    00238 #endif
    00239
    00240 #ifdef _X86_
    00241         if (PointerPde->u.Hard.Valid == 1) {
    00242             if (PointerPde->u.Hard.LargePage == 1) {
    00243 #if DBG
    00244                 if (MmLargePageFaultError < 10) {
    00245                     DbgPrint ("MM - fault on Large page %p\n", VirtualAddress);
    00246                 }
    00247                 MmLargePageFaultError += 1;
    00248 #endif //DBG
    00249                 return STATUS_SUCCESS;
    00250             }
    00251         }
    00252 #endif //X86
    00253
    00254         if (
    00255 #if defined (_WIN64)
    00256             (PointerPpe->u.Hard.Valid == 0) ||
    00257 #endif
    00258             (PointerPde->u.Hard.Valid == 0) ||
    00259             (PointerPte->u.Hard.Valid == 0)) {
    00260
    00261             KdPrint(("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
    00262                      VirtualAddress,
    00263                      PreviousIrql));
    00264
    00265             //
    00266             // use reserved bit to signal fatal error to trap handlers
    00267             //
    00268
    00269             return STATUS_IN_PAGE_ERROR | 0x10000000;
    00270
    00271         }
    00272
    00273         if (StoreInstruction && (PointerPte->u.Hard.CopyOnWrite != 0)) {
    00274             KdPrint(("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
    00275                      VirtualAddress,
    00276                      PreviousIrql));
    00277
    00278             //
    00279             // use reserved bit to signal fatal error to trap handlers
    00280             //
    00281
    00282             return STATUS_IN_PAGE_ERROR | 0x10000000;
    00283         }
    00284
    00285         //
    00286         // The PTE is valid and accessible, another thread must
    00287         // have faulted the PTE in already, or the access bit
    00288         // is clear and this is a access fault; Blindly set the
    00289         // access bit and dismiss the fault.
    00290         //
    00291 #if DBG
    00292         if (MmDebug & MM_DBG_SHOW_FAULTS) {
    00293             DbgPrint("MM:no fault found - pte is %p\n", PointerPte->u.Long);
    00294         }
    00295 #endif //DBG
    00296
    00297         if (StoreInstruction) {
    00298
    00299             Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
    00300
    00301             if (((PointerPte->u.Long & MM_PTE_WRITE_MASK) == 0) &&
    00302                 ((Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) == 0)) {
    00303
    00304                 KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00305                               (ULONG_PTR)VirtualAddress,
    00306                               (ULONG_PTR)PointerPte->u.Long,
    00307                               (ULONG_PTR)TrapInformation,
    00308                               10);
    00309             }
    00310         }
    00311
    00312         MI_NO_FAULT_FOUND (TempPte, PointerPte, VirtualAddress, FALSE);
    00313         return STATUS_SUCCESS;
    00314     }
    00315
    00316     if (VirtualAddress >= MmSystemRangeStart) {
    00317
    00318         //
    00319         // This is a fault in the system address space.  User
    00320         // mode access is not allowed.
    00321         //
    00322
    00323         if (PreviousMode == UserMode) {
    00324             return STATUS_ACCESS_VIOLATION;
    00325         }
    00326
    00327 #if defined (_WIN64)
    00328         if (PointerPpe->u.Hard.Valid == 0) {
    00329
    00330             if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00331                 return STATUS_ACCESS_VIOLATION;
    00332             }
    00333
    00334             KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00335                           (ULONG_PTR)VirtualAddress,
    00336                           StoreInstruction,
    00337                           (ULONG_PTR)TrapInformation,
    00338                           5);
    00339         }
    00340 #endif
    00341
    00342 RecheckPde:
    00343
    00344         if (PointerPde->u.Hard.Valid == 1) {
    00345 #ifdef _X86_
    00346             if (PointerPde->u.Hard.LargePage == 1) {
    00347 #if DBG
    00348                 if (MmLargePageFaultError < 10) {
    00349                     DbgPrint ("MM - fault on Large page %p\n",VirtualAddress);
    00350                 }
    00351                 MmLargePageFaultError += 1;
    00352 #endif //DBG
    00353                 return STATUS_SUCCESS;
    00354             }
    00355 #endif //X86
    00356
    00357             if (PointerPte->u.Hard.Valid == 1) {
    00358
    00359                 //
    00360                 // Session space faults cannot early exit here because
    00361                 // it may be a copy on write which must be checked for
    00362                 // and handled below.
    00363                 //
    00364
    00365                 if (MI_IS_SESSION_ADDRESS (VirtualAddress) == FALSE) {
    00366
    00367                     //
    00368                     // Acquire the PFN lock, check to see if the address is
    00369                     // still valid if writable, update dirty bit.
    00370                     //
    00371
    00372                     LOCK_PFN (OldIrql);
    00373                     TempPte = *(volatile MMPTE *)PointerPte;
    00374                     if (TempPte.u.Hard.Valid == 1) {
    00375
    00376                         Pfn1 = MI_PFN_ELEMENT (TempPte.u.Hard.PageFrameNumber);
    00377
    00378                         if ((StoreInstruction) &&
    00379                             ((TempPte.u.Long & MM_PTE_WRITE_MASK) == 0) &&
    00380                             ((Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) == 0)) {
    00381
    00382                             KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00383                                           (ULONG_PTR)VirtualAddress,
    00384                                           (ULONG_PTR)TempPte.u.Long,
    00385                                           (ULONG_PTR)TrapInformation,
    00386                                           11);
    00387                         }
    00388                         MI_NO_FAULT_FOUND (TempPte, PointerPte, VirtualAddress, TRUE);
    00389                     }
    00390                     UNLOCK_PFN (OldIrql);
    00391                     return STATUS_SUCCESS;
    00392                 }
    00393             }
    00394 #if !defined (_WIN64)
    00395             else {
    00396
    00397                 //
    00398                 // Handle trimmer references to paged pool PTEs where the PDE
    00399                 // might not be present.  Only needed for
    00400                 // MmTrimAllSystemPagable memory.
    00401                 //
    00402
    00403                 MiCheckPdeForPagedPool (VirtualAddress);
    00404                 TempPte = *(volatile MMPTE *)PointerPte;
    00405                 if (TempPte.u.Hard.Valid == 1) {
    00406                     return STATUS_SUCCESS;
    00407                 }
    00408             }
    00409 #endif
    00410         } else {
    00411
    00412             //
    00413             // Due to G-bits in kernel mode code, accesses to paged pool
    00414             // PDEs may not fault even though the PDE is not valid.  Make
    00415             // sure the PDE is valid so PteFrames in the PFN database are
    00416             // tracked properly.
    00417             //
    00418
    00419 #if defined (_WIN64)
    00420             if ((VirtualAddress >= (PVOID)PTE_BASE) && (VirtualAddress < (PVOID)MiGetPteAddress (HYPER_SPACE))) {
    00421                 //
    00422                 // This is a user mode PDE entry being faulted in by the Mm
    00423                 // referencing the page table page.  This needs to be done
    00424                 // with the working set lock so that the PPE validity can be
    00425                 // relied on throughout the fault processing.
    00426                 //
    00427                 // The case when Mm faults in PPE entries by referencing the
    00428                 // page directory page is correctly handled by falling through
    00429                 // the below code.
    00430                 //
    00431
    00432                 goto UserFault;
    00433             }
    00434 #else
    00435             MiCheckPdeForPagedPool (VirtualAddress);
    00436 #endif
    00437
    00438             if (PointerPde->u.Hard.Valid == 0) {
    00439                 if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00440                     return STATUS_ACCESS_VIOLATION;
    00441                 }
    00442                 KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00443                               (ULONG_PTR)VirtualAddress,
    00444                               StoreInstruction,
    00445                               (ULONG_PTR)TrapInformation,
    00446                               2);
    00447                 return STATUS_SUCCESS;
    00448             }
    00449
    00450             //
    00451             // Now that the PDE is valid, go look at the PTE again.
    00452             //
    00453
    00454             goto RecheckPde;
    00455         }
    00456
    00457         if (MiHydra == TRUE) {
    00458
    00459 #if !defined (_WIN64)
    00460
    00461             //
    00462             // First check to see if it's in the session space data
    00463             // structures or page table pages.
    00464             //
    00465
    00466             SessionStatus = MiCheckPdeForSessionSpace (VirtualAddress);
    00467
    00468             if (SessionStatus == STATUS_ACCESS_VIOLATION) {
    00469
    00470                 //
    00471                 // This thread faulted on a session space access, but this
    00472                 // process does not have one.  This could be the system
    00473                 // process attempting to access a working buffer passed
    00474                 // to it from WIN32K or a driver loaded in session space
    00475                 // (video, printer, etc).
    00476                 //
    00477                 // The system process which contains the worker threads
    00478                 // NEVER has a session space - if code accidentally queues a
    00479                 // worker thread that points to a session space buffer, a
    00480                 // fault will occur.  This must be bug checked since drivers
    00481                 // are responsible for making sure this never occurs.
    00482                 //
    00483                 // The only exception to this is when the working set manager
    00484                 // attaches to a session to age or trim it.  However, the
    00485                 // working set manager will never fault and so the bugcheck
    00486                 // below is always valid.  Note that a worker thread can get
    00487                 // away with a bad access if it happens while the working set
    00488                 // manager is attached, but there's really no way to prevent
    00489                 // this case which is a driver bug anyway.
    00490                 //
    00491
    00492                 if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00493                     return STATUS_ACCESS_VIOLATION;
    00494                 }
    00495
    00496                 KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00497                               (ULONG_PTR)VirtualAddress,
    00498                               StoreInstruction,
    00499                               (ULONG_PTR)TrapInformation,
    00500                               6);
    00501             }
    00502
    00503 #endif
    00504
    00505             //
    00506             // Fall though to further fault handling.
    00507             //
    00508
    00509             SessionAddress = MI_IS_SESSION_ADDRESS (VirtualAddress);
    00510         }
    00511         else {
    00512             SessionAddress = FALSE;
    00513         }
    00514
    00515         if (SessionAddress == TRUE ||
    00516             ((!MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress)) &&
    00517              (!MI_IS_HYPER_SPACE_ADDRESS(VirtualAddress)))) {
    00518
    00519             if (SessionAddress == FALSE) {
    00520
    00521                 //
    00522                 // Acquire system working set lock.  While this lock
    00523                 // is held, no pages may go from valid to invalid.
    00524                 //
    00525                 // HOWEVER - transition pages may go to valid, but
    00526                 // may not be added to the working set list.  This
    00527                 // is done in the cache manager support routines to
    00528                 // shortcut faults on transition prototype PTEs.
    00529                 //
    00530
    00531                 if (PsGetCurrentThread() == MmSystemLockOwner) {
    00532
    00533                     //
    00534                     // Recursively trying to acquire the system working set
    00535                     // fast mutex - cause an IRQL > 1 bug check.
    00536                     //
    00537
    00538                     return STATUS_IN_PAGE_ERROR | 0x10000000;
    00539                 }
    00540
    00541                 LOCK_SYSTEM_WS (PreviousIrql);
    00542             }
    00543
    00544             //
    00545             // Note that for session space the below check is done without
    00546             // acquiring the session WSL lock.  This is because this thread
    00547             // may already own it - ie: it may be adding a page to the
    00548             // session space working set and the session's working set list is
    00549             // not mapped in and causes a fault.  The MiCheckPdeForSessionSpace
    00550             // call above will fill in the PDE and then we must check the PTE
    00551             // below - if that's not present then we couldn't possibly be
    00552             // holding the session WSL lock, so we'll acquire it below.
    00553             //
    00554
    00555 #if defined (_X86PAE_)
    00556             //
    00557             // PAE PTEs are subject to write tearing due to the cache manager
    00558             // shortcut routines that insert PTEs without acquiring the working
    00559             // set lock.  Synchronize here via the PFN lock.
    00560             //
    00561             LOCK_PFN (OldIrql);
    00562 #endif
    00563             TempPte = *PointerPte;
    00564 #if defined (_X86PAE_)
    00565             UNLOCK_PFN (OldIrql);
    00566 #endif
    00567
    00568             //
    00569             // If the PTE is valid, make sure we do not have a copy on
    00570             // write.
    00571             //
    00572
    00573             if (TempPte.u.Hard.Valid != 0) {
    00574
    00575                 //
    00576                 // PTE is already valid, return.  Unless it's Hydra where
    00577                 // kernel mode copy-on-write must be handled properly.
    00578                 //
    00579
    00580                 BOOLEAN FaultHandled;
    00581
    00582                 FaultHandled = FALSE;
    00583
    00584                 LOCK_PFN (OldIrql);
    00585                 TempPte = *(volatile MMPTE *)PointerPte;
    00586                 if (TempPte.u.Hard.Valid == 1) {
    00587
    00588                     Pfn1 = MI_PFN_ELEMENT (TempPte.u.Hard.PageFrameNumber);
    00589
    00590                     if ((StoreInstruction) &&
    00591                         (TempPte.u.Hard.CopyOnWrite == 0) &&
    00592                         ((TempPte.u.Long & MM_PTE_WRITE_MASK) == 0) &&
    00593                         ((Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) == 0)) {
    00594
    00595                             KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00596                                           (ULONG_PTR)VirtualAddress,
    00597                                           (ULONG_PTR)TempPte.u.Long,
    00598                                           (ULONG_PTR)TrapInformation,
    00599                                           12);
    00600                     }
    00601
    00602                     //
    00603                     // Set the dirty bit in the PTE and the page frame.
    00604                     //
    00605
    00606 #if defined(_ALPHA_)
    00607                     if (SessionAddress == FALSE || (TempPte.u.Hard.Write == 1 && TempPte.u.Hard.CopyOnWrite == 0))
    00608 #else
    00609                     if (SessionAddress == FALSE || TempPte.u.Hard.Write == 1)
    00610 #endif
    00611                     {
    00612                         FaultHandled = TRUE;
    00613                         MI_NO_FAULT_FOUND (TempPte, PointerPte, VirtualAddress, TRUE);
    00614                     }
    00615                 }
    00616                 UNLOCK_PFN (OldIrql);
    00617                 if (SessionAddress == FALSE) {
    00618                     UNLOCK_SYSTEM_WS (PreviousIrql);
    00619                 }
    00620                 if (SessionAddress == FALSE || FaultHandled == TRUE) {
    00621                     return STATUS_SUCCESS;
    00622                 }
    00623             }
    00624
    00625             if (SessionAddress == TRUE) {
    00626
    00627                 ASSERT (MiHydra == TRUE);
    00628
    00629                 //
    00630                 // Acquire the session space working set lock.  While this lock
    00631                 // is held, no session pages may go from valid to invalid.
    00632                 //
    00633
    00634                 if (PsGetCurrentThread() == MmSessionSpace->WorkingSetLockOwner) {
    00635
    00636                     //
    00637                     // Recursively trying to acquire the session working set
    00638                     // lock - cause an IRQL > 1 bug check.
    00639                     //
    00640
    00641                     return STATUS_IN_PAGE_ERROR | 0x10000000;
    00642                 }
    00643
    00644                 LOCK_SESSION_SPACE_WS (PreviousIrql);
    00645
    00646                 TempPte = *PointerPte;
    00647
    00648                 //
    00649                 // The PTE could have become valid while we waited
    00650                 // for the session space working set lock.
    00651                 //
    00652
    00653                 if (TempPte.u.Hard.Valid == 1) {
    00654
    00655                     LOCK_PFN (OldIrql);
    00656                     TempPte = *(volatile MMPTE *)PointerPte;
    00657
    00658                     //
    00659                     // Check for copy-on-write.
    00660                     //
    00661
    00662                     if (TempPte.u.Hard.Valid == 1) {
    00663
    00664 #if defined(_ALPHA_)
    00665                         if (StoreInstruction && TempPte.u.Hard.CopyOnWrite == 1)
    00666 #else
    00667                         if (StoreInstruction && TempPte.u.Hard.Write == 0)
    00668 #endif
    00669                         {
    00670 #if defined(_ALPHA_)
    00671                             TempPte.u.Hard.Write = 0;
    00672                             MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
    00673 #endif
    00674
    00675                             //
    00676                             // Copy on write only for loaded drivers...
    00677                             //
    00678
    00679                             ASSERT (MI_IS_SESSION_IMAGE_ADDRESS (VirtualAddress));
    00680
    00681                             UNLOCK_PFN (OldIrql);
    00682
    00683                             if (TempPte.u.Hard.CopyOnWrite == 0) {
    00684
    00685                                 KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00686                                               (ULONG_PTR)VirtualAddress,
    00687                                               (ULONG_PTR)TempPte.u.Long,
    00688                                               (ULONG_PTR)TrapInformation,
    00689                                               13);
    00690                             }
    00691
    00692                             MiSessionCopyOnWrite (MmSessionSpace,
    00693                                                   VirtualAddress,
    00694                                                   PointerPte);
    00695
    00696                             UNLOCK_SESSION_SPACE_WS (PreviousIrql);
    00697
    00698                             return STATUS_SUCCESS;
    00699                         }
    00700
    00701 #if DBG
    00702                         //
    00703                         // If we are allowing a store, it better be writable.
    00704                         //
    00705
    00706                         if (StoreInstruction) {
    00707                             ASSERT (TempPte.u.Hard.Write == 1);
    00708                         }
    00709 #endif
    00710                         //
    00711                         // PTE is already valid, return.
    00712                         //
    00713
    00714                         MI_NO_FAULT_FOUND (TempPte, PointerPte, VirtualAddress, TRUE);
    00715                     }
    00716
    00717                     UNLOCK_PFN (OldIrql);
    00718                     UNLOCK_SESSION_SPACE_WS (PreviousIrql);
    00719                     return STATUS_SUCCESS;
    00720                 }
    00721             }
    00722
    00723             if (TempPte.u.Soft.Prototype != 0) {
    00724
    00725                 if (MmProtectFreedNonPagedPool == TRUE) {
    00726
    00727                     PVOID StartVa;
    00728
    00729                     if (MI_IS_PHYSICAL_ADDRESS(MmNonPagedPoolStart)) {
    00730                         StartVa = MmNonPagedPoolExpansionStart;
    00731                     }
    00732                     else {
    00733                         StartVa = MmNonPagedPoolStart;
    00734                     }
    00735
    00736                     if (VirtualAddress >= StartVa && VirtualAddress < MmNonPagedPoolEnd) {
    00737                         //
    00738                         // This is an access to previously freed
    00739                         // non paged pool - bugcheck!
    00740                         //
    00741
    00742                         if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00743                             goto AccessViolation;
    00744                         }
    00745
    00746                         KeBugCheckEx (DRIVER_CAUGHT_MODIFYING_FREED_POOL,
    00747                                       (ULONG_PTR)VirtualAddress,
    00748                                       StoreInstruction,
    00749                                       PreviousMode,
    00750                                       4);
    00751                     }
    00752                 }
    00753
    00754                 //
    00755                 // This is a PTE in prototype format, locate the corresponding
    00756                 // prototype PTE.
    00757                 //
    00758
    00759                 PointerProtoPte = MiPteToProto (&TempPte);
    00760
    00761                 if (SessionAddress == TRUE) {
    00762
    00763                     if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED) {
    00764                         PointerProtoPte = MiCheckVirtualAddress (VirtualAddress,
    00765                                                                  &ProtectionCode);
    00766                         if (PointerProtoPte == NULL) {
    00767                             UNLOCK_SESSION_SPACE_WS (PreviousIrql);
    00768                             return STATUS_IN_PAGE_ERROR | 0x10000000;
    00769                         }
    00770                     }
    00771                     else if (TempPte.u.Proto.ReadOnly == 1) {
    00772
    00773                         //
    00774                         // Writes are not allowed to this page.
    00775                         //
    00776
    00777                     } else if (MI_IS_SESSION_IMAGE_ADDRESS (VirtualAddress)) {
    00778
    00779                         //
    00780                         // Copy on write this page.
    00781                         //
    00782
    00783                         MI_WRITE_INVALID_PTE (PointerPte, PrototypePte);
    00784                         PointerPte->u.Soft.Protection = MM_EXECUTE_WRITECOPY;
    00785                     }
    00786                 }
    00787             } else if ((TempPte.u.Soft.Transition == 0) &&
    00788                         (TempPte.u.Soft.Protection == 0)) {
    00789
    00790                 //
    00791                 // Page file format.  If the protection is ZERO, this
    00792                 // is a page of free system PTEs - bugcheck!
    00793                 //
    00794
    00795                 if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00796                     goto AccessViolation;
    00797                 }
    00798
    00799                 KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00800                               (ULONG_PTR)VirtualAddress,
    00801                               StoreInstruction,
    00802                               (ULONG_PTR)TrapInformation,
    00803                               0);
    00804                 return STATUS_SUCCESS;
    00805             }
    00806             else if (TempPte.u.Soft.Protection == MM_NOACCESS) {
    00807
    00808                 if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00809                     goto AccessViolation;
    00810                 }
    00811
    00812                 KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00813                               (ULONG_PTR)VirtualAddress,
    00814                               StoreInstruction,
    00815                               (ULONG_PTR)TrapInformation,
    00816                               1);
    00817                 return STATUS_SUCCESS;
    00818             }
    00819
    00820 #ifdef PROTECT_KSTACKS
    00821             else {
    00822                  if (TempPte.u.Soft.Protection == MM_KSTACK_OUTSWAPPED) {
    00823
    00824                     if (KeInvalidAccessAllowed(TrapInformation) == TRUE) {
    00825                         goto AccessViolation;
    00826                     }
    00827
    00828                     KeBugCheckEx (PAGE_FAULT_IN_NONPAGED_AREA,
    00829                                   (ULONG_PTR)VirtualAddress,
    00830                                   StoreInstruction,
    00831                                   (ULONG_PTR)TrapInformation,
    00832                                   3);
    00833                  }
    00834             }
    00835 #endif
    00836
    00837             if (SessionAddress == TRUE) {
    00838
    00839                 MM_SESSION_SPACE_WS_LOCK_ASSERT ();
    00840
    00841                 //
    00842                 // If it's a write to a session space page that is ultimately
    00843                 // mapped by a prototype PTE, it's a copy-on-write piece of
    00844                 // a session driver.  Since the page isn't even present yet,
    00845                 // turn the write access into a read access to fault it in.
    00846                 // We'll get a write fault on the present page when we retry
    00847                 // the operation at which point we'll sever the copy on write.
    00848                 //
    00849
    00850                 if (PointerProtoPte &&
    00851                     StoreInstruction &&
    00852                     MI_IS_SESSION_IMAGE_ADDRESS (VirtualAddress)) {
    00853                         StoreInstruction = 0;
    00854                 }
    00855
    00856                 FaultProcess = HYDRA_PROCESS;
    00857             }
    00858             else {
    00859                 FaultProcess = NULL;
    00860
    00861                 if (StoreInstruction) {
    00862
    00863                     if ((TempPte.u.Hard.Valid == 0) && (PointerProtoPte == NULL)) {
    00864                         if (TempPte.u.Soft.Transition == 1) {
    00865
    00866                             if ((TempPte.u.Trans.Protection & MM_READWRITE) == 0) {
    00867                                 KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00868                                               (ULONG_PTR)VirtualAddress,
    00869                                               (ULONG_PTR)TempPte.u.Long,
    00870                                               (ULONG_PTR)TrapInformation,
    00871                                               14);
    00872                             }
    00873                         }
    00874                         else {
    00875                             if ((TempPte.u.Soft.Protection & MM_READWRITE) == 0) {
    00876
    00877                                 KeBugCheckEx (ATTEMPTED_WRITE_TO_READONLY_MEMORY,
    00878                                               (ULONG_PTR)VirtualAddress,
    00879                                               (ULONG_PTR)TempPte.u.Long,
    00880                                               (ULONG_PTR)TrapInformation,
    00881                                               15);
    00882                             }
    00883                         }
    00884                     }
    00885                 }
    00886             }
    00887
    00888             status = MiDispatchFault (StoreInstruction,
    00889                                       VirtualAddress,
    00890                                       PointerPte,
    00891                                       PointerProtoPte,
    00892                                       FaultProcess,
    00893                                       &ApcNeeded);
    00894
    00895             ASSERT (ApcNeeded == FALSE);
    00896             ASSERT (KeGetCurrentIrql() == APC_LEVEL);
    00897
    00898             if (SessionAddress == TRUE) {
    00899                 Ws = &MmSessionSpace->Vm;
    00900                 PageFrameIndex = Ws->PageFaultCount;
    00901                 MM_SESSION_SPACE_WS_LOCK_ASSERT();
    00902             }
    00903             else {
    00904                 Ws = &MmSystemCacheWs;
    00905                 PageFrameIndex = MmSystemCacheWs.PageFaultCount;
    00906             }
    00907
    00908             if (Ws->AllowWorkingSetAdjustment == MM_GROW_WSLE_HASH) {
    00909                 MiGrowWsleHash (Ws);
    00910                 LOCK_EXPANSION_IF_ALPHA (OldIrql);
    00911                 Ws->AllowWorkingSetAdjustment = TRUE;
    00912                 UNLOCK_EXPANSION_IF_ALPHA (OldIrql);
    00913             }
    00914
    00915             if (SessionAddress == TRUE) {
    00916                 UNLOCK_SESSION_SPACE_WS (PreviousIrql);
    00917             }
    00918             else {
    00919                 UNLOCK_SYSTEM_WS (PreviousIrql);
    00920             }
    00921
    00922             if ((PageFrameIndex & 0x3FFFF) == 0x30000) {
    00923
    00924                 //
    00925                 // The system cache or this session is taking too many faults,
    00926                 // delay execution so the modified page writer gets a quick
    00927                 // shot and increase the working set size.
    00928                 //
    00929
    00930                 KeDelayExecutionThread (KernelMode, FALSE, &MmShortTime);
    00931             }
    00932             NotifyRoutine = MmPageFaultNotifyRoutine;
    00933             if (NotifyRoutine) {
    00934                 if (status != STATUS_SUCCESS) {
    00935                     (*NotifyRoutine) (
    00936                         status,
    00937                         VirtualAddress,
    00938                         TrapInformation
    00939                         );
    00940                 }
    00941             }
    00942             return status;
    00943         } else {
    00944 #if !defined (_WIN64)
    00945             if (MiCheckPdeForPagedPool (VirtualAddress) == STATUS_WAIT_1) {
    00946                 return STATUS_SUCCESS;
    00947             }
    00948 #endif
    00949         }
    00950     }
    00951
    00952 #if defined (_WIN64)
    00953 UserFault:
    00954 #endif
    00955
    00956     if (MiDelayPageFaults ||
    00957         ((MmModifiedPageListHead.Total >= (MmModifiedPageMaximum + 100)) &&
    00958         (MmAvailablePages < (1024*1024 / PAGE_SIZE)) &&
    00959             (CurrentProcess->ModifiedPageCount > ((64*1024)/PAGE_SIZE)))) {
    00960
    00961         //
    00962         // This process has placed more than 64k worth of pages on the modified
    00963         // list.  Delay for a short period and set the count to zero.
    00964         //
    00965
    00966         KeDelayExecutionThread (KernelMode,
    00967                                 FALSE,
    00968              (CurrentProcess->Pcb.BasePriority < PROCESS_FOREGROUND_PRIORITY) ?
    00969                                     &MmHalfSecond : &Mm30Milliseconds);
    00970         CurrentProcess->ModifiedPageCount = 0;
    00971     }
    00972
    00973     //
    00974     // FAULT IN USER SPACE OR PAGE DIRECTORY/PAGE TABLE PAGES.
    00975     //
    00976
    00977     //
    00978     // Block APCs and acquire the working set lock.
    00979     //
    00980
    00981     LOCK_WS (CurrentProcess);
    00982
    00983 #if defined (_WIN64)
    00984
    00985     //
    00986     // Locate the Page Directory Parent Entry which maps this virtual
    00987     // address and check for accessibility and validity.  The page directory
    00988     // page must be made valid before any other checks are made.
    00989     //
    00990
    00991     if (PointerPpe->u.Hard.Valid == 0) {
    00992
    00993         //
    00994         // If the PPE is zero, check to see if there is a virtual address
    00995         // mapped at this location, and if so create the necessary
    00996         // structures to map it.
    00997         //
    00998
    00999         if ((PointerPpe->u.Long == MM_ZERO_PTE) ||
    01000             (PointerPpe->u.Long == MM_ZERO_KERNEL_PTE)) {
    01001             PointerProtoPte = MiCheckVirtualAddress (VirtualAddress,
    01002                                                      &ProtectCode);
    01003
    01004 #ifdef LARGE_PAGES
    01005             if (ProtectCode == MM_LARGE_PAGES) {
    01006                 status = STATUS_SUCCESS;
    01007                 goto ReturnStatus2;
    01008             }
    01009 #endif //LARGE_PAGES
    01010
    01011             if (ProtectCode == MM_NOACCESS) {
    01012                 status = STATUS_ACCESS_VIOLATION;
    01013 //                MiCheckPpeForPagedPool (VirtualAddress);
    01014                 if (PointerPpe->u.Hard.Valid == 1) {
    01015                     status = STATUS_SUCCESS;
    01016                 }
    01017
    01018 #if DBG
    01019                 if ((MmDebug & MM_DBG_STOP_ON_ACCVIO) &&
    01020                     (status == STATUS_ACCESS_VIOLATION)) {
    01021                     DbgPrint("MM:access violation - %p\n",VirtualAddress);
    01022                     MiFormatPte(PointerPpe);
    01023                     DbgBreakPoint();
    01024                 }
    01025 #endif //DEBUG
    01026
    01027                 goto ReturnStatus2;
    01028
    01029             } else {
    01030
    01031                 //
    01032                 // Build a demand zero PPE and operate on it.
    01033                 //
    01034
    01035                 *PointerPpe = DemandZeroPde;
    01036             }
    01037         }
    01038
    01039         //
    01040         // The PPE is not valid, call the page fault routine passing
    01041         // in the address of the PPE.  If the PPE is valid, determine
    01042         // the status of the corresponding PDE.
    01043         //
    01044         // Note this call may result in ApcNeeded getting set to TRUE.
    01045         // This is deliberate as there may be another call to MiDispatchFault
    01046         // issued later in this routine and we don't want to lose the APC
    01047         // status.
    01048         //
    01049
    01050         status = MiDispatchFault (TRUE,  //page table page always written
    01051                                   PointerPde,   //Virtual address
    01052                                   PointerPpe,   // PTE (PPE in this case)
    01053                                   NULL,
    01054                                   CurrentProcess,
    01055                                   &ApcNeeded);
    01056
    01057 #if DBG
    01058         if (ApcNeeded == TRUE) {
    01059             ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01060             ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01061         }
    01062 #endif
    01063
    01064         ASSERT (KeGetCurrentIrql() == APC_LEVEL);
    01065         if (PointerPpe->u.Hard.Valid == 0) {
    01066
    01067             //
    01068             // The PPE is not valid, return the status.
    01069             //
    01070             goto ReturnStatus1;
    01071         }
    01072
    01073 #if PFN_CONSISTENCY
    01074         {
    01075             PMMPFN Pfn1;
    01076
    01077             LOCK_PFN (OldIrql);
    01078             Pfn1 = MI_PFN_ELEMENT (PointerPpe->u.Hard.PageFrameNumber);
    01079             Pfn1->u3.e1.PageTablePage = 1;
    01080             UNLOCK_PFN (OldIrql);
    01081         }
    01082 #endif
    01083         //KeFillEntryTb ((PHARDWARE_PTE)PointerPpe, (PVOID)PointerPde, TRUE);
    01084
    01085         MI_SET_PAGE_DIRTY (PointerPpe, PointerPde, FALSE);
    01086
    01087         //
    01088         // Now that the PPE is accessible, get the PDE - let this fall
    01089         // through.
    01090         //
    01091     }
    01092 #endif
    01093
    01094     //
    01095     // Locate the Page Directory Entry which maps this virtual
    01096     // address and check for accessibility and validity.
    01097     //
    01098
    01099     //
    01100     // Check to see if the page table page (PDE entry) is valid.
    01101     // If not, the page table page must be made valid first.
    01102     //
    01103
    01104     if (PointerPde->u.Hard.Valid == 0) {
    01105
    01106         //
    01107         // If the PDE is zero, check to see if there is a virtual address
    01108         // mapped at this location, and if so create the necessary
    01109         // structures to map it.
    01110         //
    01111
    01112         if ((PointerPde->u.Long == MM_ZERO_PTE) ||
    01113             (PointerPde->u.Long == MM_ZERO_KERNEL_PTE)) {
    01114             PointerProtoPte = MiCheckVirtualAddress (VirtualAddress,
    01115                                                      &ProtectCode);
    01116
    01117 #ifdef LARGE_PAGES
    01118             if (ProtectCode == MM_LARGE_PAGES) {
    01119                 status = STATUS_SUCCESS;
    01120                 goto ReturnStatus2;
    01121             }
    01122 #endif //LARGE_PAGES
    01123
    01124             if (ProtectCode == MM_NOACCESS) {
    01125                 status = STATUS_ACCESS_VIOLATION;
    01126 #if !defined (_WIN64)
    01127                 MiCheckPdeForPagedPool (VirtualAddress);
    01128 #endif
    01129
    01130                 if (PointerPde->u.Hard.Valid == 1) {
    01131                     status = STATUS_SUCCESS;
    01132                 }
    01133
    01134 #if DBG
    01135                 if ((MmDebug & MM_DBG_STOP_ON_ACCVIO) &&
    01136                     (status == STATUS_ACCESS_VIOLATION)) {
    01137                     DbgPrint("MM:access violation - %p\n",VirtualAddress);
    01138                     MiFormatPte(PointerPde);
    01139                     DbgBreakPoint();
    01140                 }
    01141 #endif //DEBUG
    01142
    01143                 goto ReturnStatus2;
    01144
    01145             }
    01146
    01147             //
    01148             // Build a demand zero PDE and operate on it.
    01149             //
    01150
    01151             MI_WRITE_INVALID_PTE (PointerPde, DemandZeroPde);
    01152
    01153 #if defined (_WIN64)
    01154
    01155             //
    01156             // Increment the count of non-zero page directory entries for this
    01157             // page directory.
    01158             //
    01159
    01160             if (VirtualAddress <= MM_HIGHEST_USER_ADDRESS) {
    01161                 UsedPageTableHandle = MI_GET_USED_PTES_HANDLE (PointerPte);
    01162                 MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableHandle);
    01163             }
    01164 #endif
    01165
    01166         }
    01167
    01168         //
    01169         // The PDE is not valid, call the page fault routine passing
    01170         // in the address of the PDE.  If the PDE is valid, determine
    01171         // the status of the corresponding PTE.
    01172         //
    01173
    01174         status = MiDispatchFault (TRUE,  //page table page always written
    01175                                   PointerPte,   //Virtual address
    01176                                   PointerPde,   // PTE (PDE in this case)
    01177                                   NULL,
    01178                                   CurrentProcess,
    01179                                   &ApcNeeded);
    01180
    01181 #if DBG
    01182         if (ApcNeeded == TRUE) {
    01183             ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01184             ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01185         }
    01186 #endif
    01187
    01188         ASSERT (KeGetCurrentIrql() == APC_LEVEL);
    01189         if (PointerPde->u.Hard.Valid == 0) {
    01190
    01191             //
    01192             // The PDE is not valid, return the status.
    01193             //
    01194             goto ReturnStatus1;
    01195         }
    01196
    01197 #if PFN_CONSISTENCY
    01198         {
    01199             PMMPFN Pfn1;
    01200
    01201             LOCK_PFN (OldIrql);
    01202             Pfn1 = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
    01203             Pfn1->u3.e1.PageTablePage = 1;
    01204             UNLOCK_PFN (OldIrql);
    01205         }
    01206 #endif
    01207         //KeFillEntryTb ((PHARDWARE_PTE)PointerPde, (PVOID)PointerPte, TRUE);
    01208
    01209         MI_SET_PAGE_DIRTY (PointerPde, PointerPte, FALSE);
    01210
    01211         //
    01212         // Now that the PDE is accessible, get the PTE - let this fall
    01213         // through.
    01214         //
    01215     }
    01216
    01217     //
    01218     // The PDE is valid and accessible, get the PTE contents.
    01219     //
    01220
    01221     TempPte = *PointerPte;
    01222     if (TempPte.u.Hard.Valid != 0) {
    01223
    01224         //
    01225         // The PTE is valid and accessible, is this a write fault
    01226         // copy on write or setting of some dirty bit?
    01227         //
    01228
    01229 #if DBG
    01230         if (MmDebug & MM_DBG_PTE_UPDATE) {
    01231             MiFormatPte(PointerPte);
    01232         }
    01233 #endif //DBG
    01234
    01235         status = STATUS_SUCCESS;
    01236
    01237         if (StoreInstruction) {
    01238
    01239             //
    01240             // This was a write operation.  If the copy on write
    01241             // bit is set in the PTE perform the copy on write,
    01242             // else check to ensure write access to the PTE.
    01243             //
    01244
    01245             if (TempPte.u.Hard.CopyOnWrite != 0) {
    01246                 MiCopyOnWrite (VirtualAddress, PointerPte);
    01247                 status = STATUS_PAGE_FAULT_COPY_ON_WRITE;
    01248                 goto ReturnStatus2;
    01249
    01250             } else {
    01251                 if (TempPte.u.Hard.Write == 0) {
    01252                     status = STATUS_ACCESS_VIOLATION;
    01253                 }
    01254             }
    01255 #if defined(_IA64_)
    01256         } else if (ExecutionFault) {
    01257
    01258             //
    01259             // It also checks to ensure execute access to the PTE.
    01260             //
    01261
    01262             if (TempPte.u.Hard.Execute == 0) {
    01263                 status = STATUS_ACCESS_VIOLATION;
    01264             }
    01265 #endif
    01266 #if DBG
    01267         } else {
    01268
    01269             //
    01270             // The PTE is valid and accessible, another thread must
    01271             // have faulted the PTE in already, or the access bit
    01272             // is clear and this is a access fault; Blindly set the
    01273             // access bit and dismiss the fault.
    01274             //
    01275
    01276             if (MmDebug & MM_DBG_SHOW_FAULTS) {
    01277                 DbgPrint("MM:no fault found - pte is %p\n", PointerPte->u.Long);
    01278             }
    01279 #endif //DBG
    01280         }
    01281
    01282         if (status == STATUS_SUCCESS) {
    01283             LOCK_PFN (OldIrql);
    01284             if (PointerPte->u.Hard.Valid != 0) {
    01285                 MI_NO_FAULT_FOUND (TempPte, PointerPte, VirtualAddress, TRUE);
    01286             }
    01287             UNLOCK_PFN (OldIrql);
    01288         }
    01289
    01290         goto ReturnStatus2;
    01291     }
    01292
    01293     //
    01294     // If the PTE is zero, check to see if there is a virtual address
    01295     // mapped at this location, and if so create the necessary
    01296     // structures to map it.
    01297     //
    01298
    01299     //
    01300     // Check explicitly for demand zero pages.
    01301     //
    01302
    01303     if (TempPte.u.Long == MM_DEMAND_ZERO_WRITE_PTE) {
    01304         MiResolveDemandZeroFault (VirtualAddress,
    01305                                   PointerPte,
    01306                                   CurrentProcess,
    01307                                   0);
    01308
    01309         status = STATUS_PAGE_FAULT_DEMAND_ZERO;
    01310         goto ReturnStatus1;
    01311     }
    01312
    01313     if ((TempPte.u.Long == MM_ZERO_PTE) ||
    01314         (TempPte.u.Long == MM_ZERO_KERNEL_PTE)) {
    01315
    01316         //
    01317         // PTE is needs to be evaluated with respect to its virtual
    01318         // address descriptor (VAD).  At this point there are 3
    01319         // possibilities, bogus address, demand zero, or refers to
    01320         // a prototype PTE.
    01321         //
    01322
    01323         PointerProtoPte = MiCheckVirtualAddress (VirtualAddress,
    01324                                                  &ProtectionCode);
    01325         if (ProtectionCode == MM_NOACCESS) {
    01326             status = STATUS_ACCESS_VIOLATION;
    01327
    01328             //
    01329             // Check to make sure this is not a page table page for
    01330             // paged pool which needs extending.
    01331             //
    01332
    01333 #if !defined (_WIN64)
    01334             MiCheckPdeForPagedPool (VirtualAddress);
    01335 #endif
    01336
    01337             if (PointerPte->u.Hard.Valid == 1) {
    01338                 status = STATUS_SUCCESS;
    01339             }
    01340
    01341 #if DBG
    01342             if ((MmDebug & MM_DBG_STOP_ON_ACCVIO) &&
    01343                 (status == STATUS_ACCESS_VIOLATION)) {
    01344                 DbgPrint("MM:access vio - %p\n",VirtualAddress);
    01345                 MiFormatPte(PointerPte);
    01346                 DbgBreakPoint();
    01347             }
    01348 #endif //DEBUG
    01349             goto ReturnStatus2;
    01350         }
    01351
    01352         //
    01353         // Increment the count of non-zero page table entries for this
    01354         // page table.
    01355         //
    01356
    01357         if (VirtualAddress <= MM_HIGHEST_USER_ADDRESS) {
    01358             UsedPageTableHandle = MI_GET_USED_PTES_HANDLE (VirtualAddress);
    01359             MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableHandle);
    01360         }
    01361
    01362         //
    01363         // Is this page a guard page?
    01364         //
    01365
    01366         if (ProtectionCode & MM_GUARD_PAGE) {
    01367
    01368             //
    01369             // This is a guard page exception.
    01370             //
    01371
    01372             PointerPte->u.Soft.Protection = ProtectionCode & ~MM_GUARD_PAGE;
    01373
    01374             if (PointerProtoPte != NULL) {
    01375
    01376                 //
    01377                 // This is a prototype PTE, build the PTE to not
    01378                 // be a guard page.
    01379                 //
    01380
    01381                 PointerPte->u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
    01382                 PointerPte->u.Soft.Prototype = 1;
    01383             }
    01384
    01385             UNLOCK_WS (CurrentProcess);
    01386             ASSERT (KeGetCurrentIrql() == PreviousIrql);
    01387
    01388             if (ApcNeeded == TRUE) {
    01389                 ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01390                 ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01391                 ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    01392                 KeRaiseIrql (APC_LEVEL, &PreviousIrql);
    01393                 IoRetryIrpCompletions ();
    01394                 KeLowerIrql (PreviousIrql);
    01395             }
    01396
    01397             return MiCheckForUserStackOverflow (VirtualAddress);
    01398         }
    01399
    01400         if (PointerProtoPte == NULL) {
    01401
    01402             //ASSERT (KeReadStateMutant (&CurrentProcess->WorkingSetLock) == 0);
    01403
    01404             //
    01405             // Assert that this is not for a PDE.
    01406             //
    01407
    01408             if (PointerPde == MiGetPdeAddress(PTE_BASE)) {
    01409
    01410                 //
    01411                 // This PTE is really a PDE, set contents as such.
    01412                 //
    01413
    01414                 MI_WRITE_INVALID_PTE (PointerPte, DemandZeroPde);
    01415             } else {
    01416                 PointerPte->u.Soft.Protection = ProtectionCode;
    01417             }
    01418
    01419             LOCK_PFN (OldIrql);
    01420
    01421             //
    01422             // If a fork operation is in progress and the faulting thread
    01423             // is not the thread performing the fork operation, block until
    01424             // the fork is completed.
    01425             //
    01426
    01427             if ((CurrentProcess->ForkInProgress != NULL) &&
    01428                 (CurrentProcess->ForkInProgress != PsGetCurrentThread())) {
    01429                 MiWaitForForkToComplete (CurrentProcess);
    01430                 status = STATUS_SUCCESS;
    01431                 UNLOCK_PFN (OldIrql);
    01432                 goto ReturnStatus1;
    01433             }
    01434
    01435             if (!MiEnsureAvailablePageOrWait (CurrentProcess,
    01436                                               VirtualAddress)) {
    01437
    01438                 ULONG Color;
    01439                 Color = MI_PAGE_COLOR_VA_PROCESS (VirtualAddress,
    01440                                                 &CurrentProcess->NextPageColor);
    01441                 PageFrameIndex = MiRemoveZeroPageIfAny (Color);
    01442                 if (PageFrameIndex == 0) {
    01443                     PageFrameIndex = MiRemoveAnyPage (Color);
    01444                     UNLOCK_PFN (OldIrql);
    01445                     Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    01446                     MiZeroPhysicalPage (PageFrameIndex, Color);
    01447
    01448                     //
    01449                     // Note the stamping must occur after the page is zeroed.
    01450                     //
    01451
    01452                     MI_BARRIER_STAMP_ZEROED_PAGE (&Pfn1->PteFrame);
    01453
    01454                     LOCK_PFN (OldIrql);
    01455                 }
    01456
    01457                 Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    01458
    01459                 CurrentProcess->NumberOfPrivatePages += 1;
    01460                 MmInfoCounters.DemandZeroCount += 1;
    01461
    01462                 //
    01463                 // This barrier check is needed after zeroing the page and
    01464                 // before setting the PTE valid.
    01465                 // Capture it now, check it at the last possible moment.
    01466                 //
    01467
    01468                 BarrierStamp = (ULONG)Pfn1->PteFrame;
    01469
    01470                 MiInitializePfn (PageFrameIndex, PointerPte, 1);
    01471
    01472                 UNLOCK_PFN (OldIrql);
    01473
    01474                 //
    01475                 // As this page is demand zero, set the modified bit in the
    01476                 // PFN database element and set the dirty bit in the PTE.
    01477                 //
    01478
    01479 #if PFN_CONSISTENCY
    01480                 if (PointerPde == MiGetPdeAddress(PTE_BASE)) {
    01481                     LOCK_PFN (OldIrql);
    01482                     Pfn1->u3.e1.PageTablePage = 1;
    01483                     UNLOCK_PFN (OldIrql);
    01484                 }
    01485 #endif
    01486
    01487                 MI_MAKE_VALID_PTE (TempPte,
    01488                                    PageFrameIndex,
    01489                                    PointerPte->u.Soft.Protection,
    01490                                    PointerPte);
    01491
    01492                 if (TempPte.u.Hard.Write != 0) {
    01493                     MI_SET_PTE_DIRTY (TempPte);
    01494                 }
    01495
    01496                 MI_BARRIER_SYNCHRONIZE (BarrierStamp);
    01497
    01498                 MI_WRITE_VALID_PTE (PointerPte, TempPte);
    01499
    01500                 ASSERT (Pfn1->u1.Event == 0);
    01501
    01502                 CONSISTENCY_LOCK_PFN (OldIrql);
    01503
    01504                 Pfn1->u1.Event = (PVOID)PsGetCurrentThread();
    01505
    01506                 CONSISTENCY_UNLOCK_PFN (OldIrql);
    01507
    01508                 WorkingSetIndex = MiLocateAndReserveWsle (&CurrentProcess->Vm);
    01509                 MiUpdateWsle (&WorkingSetIndex,
    01510                               VirtualAddress,
    01511                               MmWorkingSetList,
    01512                               Pfn1);
    01513
    01514                 MI_SET_PTE_IN_WORKING_SET (PointerPte, WorkingSetIndex);
    01515
    01516                 KeFillEntryTb ((PHARDWARE_PTE)PointerPte,
    01517                                 VirtualAddress,
    01518                                 FALSE);
    01519             } else {
    01520                 UNLOCK_PFN (OldIrql);
    01521             }
    01522
    01523             status = STATUS_PAGE_FAULT_DEMAND_ZERO;
    01524             goto ReturnStatus1;
    01525
    01526         } else {
    01527
    01528             //
    01529             // This is a prototype PTE.
    01530             //
    01531
    01532             if (ProtectionCode == MM_UNKNOWN_PROTECTION) {
    01533
    01534                 //
    01535                 // The protection field is stored in the prototype PTE.
    01536                 //
    01537
    01538                 PointerPte->u.Long = MiProtoAddressForPte (PointerProtoPte);
    01539
    01540             } else {
    01541
    01542                 MI_WRITE_INVALID_PTE (PointerPte, PrototypePte);
    01543                 PointerPte->u.Soft.Protection = ProtectionCode;
    01544             }
    01545             TempPte = *PointerPte;
    01546         }
    01547
    01548     } else {
    01549
    01550         //
    01551         // The PTE is non-zero and not valid, see if it is a prototype PTE.
    01552         //
    01553
    01554         ProtectionCode = MI_GET_PROTECTION_FROM_SOFT_PTE(&TempPte);
    01555
    01556         if (TempPte.u.Soft.Prototype != 0) {
    01557             if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED) {
    01558 #if DBG
    01559                 MmProtoPteVadLookups += 1;
    01560 #endif //DBG
    01561                 PointerProtoPte = MiCheckVirtualAddress (VirtualAddress,
    01562                                                          &ProtectCode);
    01563                 if (PointerProtoPte == NULL) {
    01564                     status = STATUS_ACCESS_VIOLATION;
    01565                     goto ReturnStatus1;
    01566                 }
    01567
    01568             } else {
    01569 #if DBG
    01570                 MmProtoPteDirect += 1;
    01571 #endif //DBG
    01572
    01573                 //
    01574                 // Protection is in the prototype PTE, indicate an
    01575                 // access check should not be performed on the current PTE.
    01576                 //
    01577
    01578                 PointerProtoPte = MiPteToProto (&TempPte);
    01579                 ProtectionCode = MM_UNKNOWN_PROTECTION;
    01580
    01581                 //
    01582                 // Check to see if the proto protection has been overridden.
    01583                 //
    01584
    01585                 if (TempPte.u.Proto.ReadOnly != 0) {
    01586                     ProtectionCode = MM_READONLY;
    01587                 }
    01588             }
    01589         }
    01590     }
    01591
    01592     if (ProtectionCode != MM_UNKNOWN_PROTECTION) {
    01593         status = MiAccessCheck (PointerPte,
    01594                                 StoreInstruction,
    01595                                 PreviousMode,
    01596                                 ProtectionCode,
    01597                                 FALSE );
    01598
    01599         if (status != STATUS_SUCCESS) {
    01600 #if DBG
    01601             if ((MmDebug & MM_DBG_STOP_ON_ACCVIO) && (status == STATUS_ACCESS_VIOLATION)) {
    01602                 DbgPrint("MM:access violate - %p\n",VirtualAddress);
    01603                 MiFormatPte(PointerPte);
    01604                 DbgBreakPoint();
    01605             }
    01606 #endif //DEBUG
    01607
    01608             UNLOCK_WS (CurrentProcess);
    01609             ASSERT (KeGetCurrentIrql() == PreviousIrql);
    01610
    01611             if (ApcNeeded == TRUE) {
    01612                 ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01613                 ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01614                 ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    01615                 KeRaiseIrql (APC_LEVEL, &PreviousIrql);
    01616                 IoRetryIrpCompletions ();
    01617                 KeLowerIrql (PreviousIrql);
    01618             }
    01619
    01620             //
    01621             // Check to see if this is a guard page violation
    01622             // and if so, should the user's stack be extended.
    01623             //
    01624
    01625             if (status == STATUS_GUARD_PAGE_VIOLATION) {
    01626                 return MiCheckForUserStackOverflow (VirtualAddress);
    01627             }
    01628
    01629             return status;
    01630         }
    01631     }
    01632
    01633     //
    01634     // This is a page fault, invoke the page fault handler.
    01635     //
    01636
    01637     if (PointerProtoPte != NULL) {
    01638
    01639         //
    01640         // Lock page containing prototype PTEs in memory by
    01641         // incrementing the reference count for the page.
    01642         //
    01643
    01644
    01645         if (!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte)) {
    01646             PointerPde = MiGetPteAddress (PointerProtoPte);
    01647             LOCK_PFN (OldIrql);
    01648             if (PointerPde->u.Hard.Valid == 0) {
    01649                 MiMakeSystemAddressValidPfn (PointerProtoPte);
    01650             }
    01651             Pfn1 = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
    01652             MI_ADD_LOCKED_PAGE_CHARGE(Pfn1, 2);
    01653             Pfn1->u3.e2.ReferenceCount += 1;
    01654             ASSERT (Pfn1->u3.e2.ReferenceCount > 1);
    01655             UNLOCK_PFN (OldIrql);
    01656         }
    01657     }
    01658     status = MiDispatchFault (StoreInstruction,
    01659                               VirtualAddress,
    01660                               PointerPte,
    01661                               PointerProtoPte,
    01662                               CurrentProcess,
    01663                               &ApcNeeded);
    01664
    01665 #if DBG
    01666     if (ApcNeeded == TRUE) {
    01667         ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01668         ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01669     }
    01670 #endif
    01671
    01672     if (PointerProtoPte != NULL) {
    01673
    01674         //
    01675         // Unlock page containing prototype PTEs.
    01676         //
    01677
    01678         if (!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte)) {
    01679             LOCK_PFN (OldIrql);
    01680             ASSERT (Pfn1->u3.e2.ReferenceCount > 1);
    01681             MI_REMOVE_LOCKED_PAGE_CHARGE(Pfn1, 3);
    01682             Pfn1->u3.e2.ReferenceCount -= 1;
    01683             UNLOCK_PFN (OldIrql);
    01684         }
    01685     }
    01686
    01687 ReturnStatus1:
    01688
    01689     ASSERT (KeGetCurrentIrql() <= APC_LEVEL);
    01690     if (CurrentProcess->Vm.AllowWorkingSetAdjustment == MM_GROW_WSLE_HASH) {
    01691         MiGrowWsleHash (&CurrentProcess->Vm);
    01692         LOCK_EXPANSION_IF_ALPHA (OldIrql);
    01693         CurrentProcess->Vm.AllowWorkingSetAdjustment = TRUE;
    01694         UNLOCK_EXPANSION_IF_ALPHA (OldIrql);
    01695     }
    01696
    01697 ReturnStatus2:
    01698
    01699     PageFrameIndex = CurrentProcess->Vm.WorkingSetSize - CurrentProcess->Vm.MinimumWorkingSetSize;
    01700
    01701     UNLOCK_WS (CurrentProcess);
    01702     ASSERT (KeGetCurrentIrql() == PreviousIrql);
    01703
    01704     if (ApcNeeded == TRUE) {
    01705         ASSERT (PsGetCurrentThread()->NestedFaultCount == 0);
    01706         ASSERT (PsGetCurrentThread()->ApcNeeded == 0);
    01707         ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    01708         KeRaiseIrql (APC_LEVEL, &PreviousIrql);
    01709         IoRetryIrpCompletions ();
    01710         KeLowerIrql (PreviousIrql);
    01711     }
    01712
    01713     if (MmAvailablePages < MmMoreThanEnoughFreePages) {
    01714
    01715         if (((SPFN_NUMBER)PageFrameIndex > 100) &&
    01716             (PsGetCurrentThread()->Tcb.Priority >= LOW_REALTIME_PRIORITY)) {
    01717
    01718             //
    01719             // This thread is realtime and is well over the process'
    01720             // working set minimum.  Delay execution so the trimmer & the
    01721             // modified page writer get a quick shot at making pages.
    01722             //
    01723
    01724             KeDelayExecutionThread (KernelMode, FALSE, &MmShortTime);
    01725         }
    01726     }
    01727
    01728     NotifyRoutine = MmPageFaultNotifyRoutine;
    01729     if (NotifyRoutine) {
    01730         if (status != STATUS_SUCCESS) {
    01731             (*NotifyRoutine) (
    01732                 status,
    01733                 VirtualAddress,
    01734                 TrapInformation
    01735                 );
    01736         }
    01737     }
    01738
    01739     return status;
    01740
    01741 AccessViolation:
    01742     if (SessionAddress == TRUE) {
    01743         UNLOCK_SESSION_SPACE_WS (PreviousIrql);
    01744     }
    01745     else {
    01746         UNLOCK_SYSTEM_WS (PreviousIrql);
    01747     }
    01748     return STATUS_ACCESS_VIOLATION;
    01749 }