test: dynmem.c Source File

Main Page | Class Hierarchy | Class List | File List | Class Members | File Members

# dynmem.c

Go to the documentation of this file.

    00001 /*++
    00002
    00003 Copyright (c) 1999  Microsoft Corporation
    00004
    00005 Module Name:
    00006
    00007     dynmem.c
    00008
    00009 Abstract:
    00010
    00011     This module contains the routines which implement dynamically adding
    00012     and removing physical memory from the system.
    00013
    00014 Author:
    00015
    00016     Landy Wang (landyw) 05-Feb-1999
    00017
    00018 Revision History:
    00019
    00020 --*/
    00021
    00022 #include "mi.h"
    00023
    00024 FAST_MUTEX MmDynamicMemoryMutex;
    00025
    00026 LOGICAL MiTrimRemovalPagesOnly = FALSE;
    00027
    00028 #if DBG
    00029 ULONG MiShowStuckPages;
    00030 ULONG MiDynmemData[9];
    00031 #endif
    00032
    00033 #define PFN_REMOVED     ((PMMPTE)(INT_PTR)(int)0x99887766)
    00034
    00035 PFN_COUNT
    00036 MiRemovePhysicalPages (
    00037     IN PFN_NUMBER StartPage,
    00038     IN PFN_NUMBER EndPage
    00039     );
    00040
    00041
    00042 NTSTATUS
    00043 MmAddPhysicalMemory (
    00044     IN PPHYSICAL_ADDRESS StartAddress,
    00045     IN OUT PLARGE_INTEGER NumberOfBytes
    00046     )
    00047
    00048 /*++
    00049
    00050 Routine Description:
    00051
    00052     This routine adds the specified physical address range to the system.
    00053     This includes initializing PFN database entries and adding it to the
    00054     freelists.
    00055
    00056 Arguments:
    00057
    00058     StartAddress  - Supplies the starting physical address.
    00059
    00060     NumberOfBytes  - Supplies a pointer to the number of bytes being added.
    00061                      If any bytes were added (ie: STATUS_SUCCESS is being
    00062                      returned), the actual amount is returned here.
    00063
    00064 Return Value:
    00065
    00066     NTSTATUS.
    00067
    00068 Environment:
    00069
    00070     Kernel mode.  PASSIVE level.  No locks held.
    00071
    00072 --*/
    00073
    00074 {
    00075     ULONG i;
    00076     PMMPFN Pfn1;
    00077     KIRQL OldIrql;
    00078     LOGICAL Inserted;
    00079     LOGICAL Updated;
    00080     MMPTE TempPte;
    00081     PMMPTE PointerPte;
    00082     PMMPTE LastPte;
    00083     PFN_NUMBER NumberOfPages;
    00084     PFN_NUMBER start;
    00085     PFN_NUMBER count;
    00086     PFN_NUMBER StartPage;
    00087     PFN_NUMBER EndPage;
    00088     PFN_NUMBER PageFrameIndex;
    00089     PFN_NUMBER Page;
    00090     PFN_NUMBER LastPage;
    00091     PFN_COUNT PagesNeeded;
    00092     PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    00093     PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    00094     PPHYSICAL_MEMORY_RUN NewRun;
    00095     LOGICAL PfnDatabaseIsPhysical;
    00096
    00097     ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    00098
    00099     ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    00100     ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
    00101
    00102     if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
    00103
    00104         //
    00105         // The system must be configured for dynamic memory addition.  This is
    00106         // critical as only then is the database guaranteed to be non-sparse.
    00107         //
    00108
    00109         if (MmDynamicPfn == FALSE) {
    00110             return STATUS_NOT_SUPPORTED;
    00111         }
    00112
    00113         PfnDatabaseIsPhysical = TRUE;
    00114     }
    00115     else {
    00116         PfnDatabaseIsPhysical = FALSE;
    00117     }
    00118
    00119     StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    00120     NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
    00121
    00122     EndPage = StartPage + NumberOfPages;
    00123
    00124     if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
    00125
    00126         //
    00127         // Truncate the request into something that can be mapped by the PFN
    00128         // database.
    00129         //
    00130
    00131         EndPage = MmHighestPossiblePhysicalPage + 1;
    00132         NumberOfPages = EndPage - StartPage;
    00133     }
    00134
    00135     //
    00136     // The range cannot wrap.
    00137     //
    00138
    00139     if (StartPage >= EndPage) {
    00140         return STATUS_INVALID_PARAMETER_1;
    00141     }
    00142
    00143     ExAcquireFastMutex (&MmDynamicMemoryMutex);
    00144
    00145     i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
    00146          (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
    00147
    00148     NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
    00149                                                     i,
    00150                                                     '  mM');
    00151
    00152     if (NewPhysicalMemoryBlock == NULL) {
    00153         ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00154         return STATUS_INSUFFICIENT_RESOURCES;
    00155     }
    00156
    00157     //
    00158     // The range cannot overlap any ranges that are already present.
    00159     //
    00160
    00161     start = 0;
    00162
    00163     LOCK_PFN (OldIrql);
    00164
    00165     do {
    00166
    00167         count = MmPhysicalMemoryBlock->Run[start].PageCount;
    00168         Page = MmPhysicalMemoryBlock->Run[start].BasePage;
    00169
    00170         if (count != 0) {
    00171
    00172             LastPage = Page + count;
    00173
    00174             if ((StartPage < Page) && (EndPage > Page)) {
    00175                 UNLOCK_PFN (OldIrql);
    00176                 ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00177                 ExFreePool (NewPhysicalMemoryBlock);
    00178                 return STATUS_CONFLICTING_ADDRESSES;
    00179             }
    00180
    00181             if ((StartPage >= Page) && (StartPage < LastPage)) {
    00182                 UNLOCK_PFN (OldIrql);
    00183                 ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00184                 ExFreePool (NewPhysicalMemoryBlock);
    00185                 return STATUS_CONFLICTING_ADDRESSES;
    00186             }
    00187         }
    00188
    00189         start += 1;
    00190
    00191     } while (start != MmPhysicalMemoryBlock->NumberOfRuns);
    00192
    00193     //
    00194     // Fill any gaps in the (sparse) PFN database needed for these pages,
    00195     // unless the PFN database was physically allocated and completely
    00196     // committed up front.
    00197     //
    00198
    00199     PagesNeeded = 0;
    00200
    00201     if (PfnDatabaseIsPhysical == FALSE) {
    00202         PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
    00203         LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
    00204
    00205         while (PointerPte <= LastPte) {
    00206             if (PointerPte->u.Hard.Valid == 0) {
    00207                 PagesNeeded += 1;
    00208             }
    00209             PointerPte += 1;
    00210         }
    00211
    00212         if (MmAvailablePages < PagesNeeded) {
    00213             UNLOCK_PFN (OldIrql);
    00214             ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00215             ExFreePool (NewPhysicalMemoryBlock);
    00216             return STATUS_INSUFFICIENT_RESOURCES;
    00217         }
    00218
    00219         TempPte = ValidKernelPte;
    00220
    00221         PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
    00222
    00223         while (PointerPte <= LastPte) {
    00224             if (PointerPte->u.Hard.Valid == 0) {
    00225
    00226                 PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
    00227
    00228                 MiInitializePfn (PageFrameIndex, PointerPte, 0);
    00229
    00230                 TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
    00231                 *PointerPte = TempPte;
    00232             }
    00233             PointerPte += 1;
    00234         }
    00235         MmResidentAvailablePages -= PagesNeeded;
    00236     }
    00237
    00238     //
    00239     // If the new range is adjacent to an existing range, just merge it into
    00240     // the old block.  Otherwise use the new block as a new entry will have to
    00241     // be used.
    00242     //
    00243
    00244     NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
    00245     NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;
    00246
    00247     NewRun = &NewPhysicalMemoryBlock->Run[0];
    00248     start = 0;
    00249     Inserted = FALSE;
    00250     Updated = FALSE;
    00251
    00252     do {
    00253
    00254         Page = MmPhysicalMemoryBlock->Run[start].BasePage;
    00255         count = MmPhysicalMemoryBlock->Run[start].PageCount;
    00256
    00257         if (Inserted == FALSE) {
    00258
    00259             //
    00260             // Note overlaps into adjacent ranges were already checked above.
    00261             //
    00262
    00263             if (StartPage == Page + count) {
    00264                 MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
    00265                 OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    00266                 MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
    00267
    00268                 //
    00269                 // Coalesce below and above to avoid leaving zero length gaps
    00270                 // as these gaps would prevent callers from removing ranges
    00271                 // the span them.
    00272                 //
    00273
    00274                 if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
    00275
    00276                     start += 1;
    00277                     Page = MmPhysicalMemoryBlock->Run[start].BasePage;
    00278                     count = MmPhysicalMemoryBlock->Run[start].PageCount;
    00279
    00280                     if (StartPage + NumberOfPages == Page) {
    00281                         MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
    00282                             count;
    00283                         MmPhysicalMemoryBlock->NumberOfRuns -= 1;
    00284
    00285                         //
    00286                         // Copy any remaining entries.
    00287                         //
    00288
    00289                         if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
    00290                             RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
    00291                                            &MmPhysicalMemoryBlock->Run[start + 1],
    00292                                            (MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
    00293                         }
    00294                     }
    00295                 }
    00296                 Updated = TRUE;
    00297                 break;
    00298             }
    00299
    00300             if (StartPage + NumberOfPages == Page) {
    00301                 MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
    00302                 MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
    00303                 OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    00304                 MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
    00305                 Updated = TRUE;
    00306                 break;
    00307             }
    00308
    00309             if (StartPage + NumberOfPages <= Page) {
    00310
    00311                 if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
    00312
    00313                     if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
    00314                         //
    00315                         // Don't insert here - the new entry really belongs
    00316                         // (at least) one entry further down.
    00317                         //
    00318
    00319                         continue;
    00320                     }
    00321                 }
    00322
    00323                 NewRun->BasePage = StartPage;
    00324                 NewRun->PageCount = NumberOfPages;
    00325                 NewRun += 1;
    00326                 Inserted = TRUE;
    00327                 Updated = TRUE;
    00328             }
    00329         }
    00330
    00331         *NewRun = MmPhysicalMemoryBlock->Run[start];
    00332         NewRun += 1;
    00333
    00334         start += 1;
    00335
    00336     } while (start != MmPhysicalMemoryBlock->NumberOfRuns);
    00337
    00338     //
    00339     // If the memory block has not been updated, then the new entry must
    00340     // be added at the very end.
    00341     //
    00342
    00343     if (Updated == FALSE) {
    00344         ASSERT (Inserted == FALSE);
    00345         NewRun->BasePage = StartPage;
    00346         NewRun->PageCount = NumberOfPages;
    00347         Inserted = TRUE;
    00348     }
    00349
    00350     //
    00351     // Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
    00352     // releasing the PFN lock.
    00353     //
    00354
    00355     if (Inserted == TRUE) {
    00356         OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
    00357         MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    00358     }
    00359
    00360     //
    00361     // Note that the page directory (page parent entries on Win64) must be
    00362     // filled in at system boot so that already-created processes do not fault
    00363     // when referencing the new PFNs.
    00364     //
    00365
    00366     //
    00367     // Walk through the memory descriptors and add pages to the
    00368     // free list in the PFN database.
    00369     //
    00370
    00371     PageFrameIndex = StartPage;
    00372     Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    00373
    00374     if (EndPage - 1 > MmHighestPhysicalPage) {
    00375         MmHighestPhysicalPage = EndPage - 1;
    00376     }
    00377
    00378     while (PageFrameIndex < EndPage) {
    00379
    00380         ASSERT (Pfn1->u2.ShareCount == 0);
    00381         ASSERT (Pfn1->u3.e2.ShortFlags == 0);
    00382         ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
    00383         ASSERT64 (Pfn1->UsedPageTableEntries == 0);
    00384         ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
    00385         ASSERT (Pfn1->PteFrame == 0);
    00386         ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
    00387                 (Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));
    00388
    00389         //
    00390         // Set the PTE address to the physical page for
    00391         // virtual address alignment checking.
    00392         //
    00393
    00394         Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);
    00395
    00396         MiInsertPageInList (MmPageLocationList[FreePageList],
    00397                             PageFrameIndex);
    00398
    00399         PageFrameIndex += 1;
    00400
    00401         Pfn1 += 1;
    00402     }
    00403
    00404     MmResidentAvailablePages += NumberOfPages;
    00405     MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;
    00406
    00407     UNLOCK_PFN (OldIrql);
    00408
    00409     //
    00410     // Increase all commit limits to reflect the additional memory.
    00411     //
    00412
    00413     ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
    00414
    00415     MmTotalCommitLimit += NumberOfPages;
    00416     MmTotalCommitLimitMaximum += NumberOfPages;
    00417
    00418     MmTotalCommittedPages += PagesNeeded;
    00419
    00420     ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
    00421
    00422     ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00423
    00424     ExFreePool (OldPhysicalMemoryBlock);
    00425
    00426     //
    00427     // Indicate number of bytes actually added to our caller.
    00428     //
    00429
    00430     NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
    00431
    00432     return STATUS_SUCCESS;
    00433 }
    00434
    00435
    00436 NTSTATUS
    00437 MmRemovePhysicalMemory (
    00438     IN PPHYSICAL_ADDRESS StartAddress,
    00439     IN OUT PLARGE_INTEGER NumberOfBytes
    00440     )
    00441
    00442 /*++
    00443
    00444 Routine Description:
    00445
    00446     This routine attempts to remove the specified physical address range
    00447     from the system.
    00448
    00449 Arguments:
    00450
    00451     StartAddress  - Supplies the starting physical address.
    00452
    00453     NumberOfBytes  - Supplies a pointer to the number of bytes being removed.
    00454
    00455 Return Value:
    00456
    00457     NTSTATUS.
    00458
    00459 Environment:
    00460
    00461     Kernel mode.  PASSIVE level.  No locks held.
    00462
    00463 --*/
    00464
    00465 {
    00466     ULONG i;
    00467     ULONG Additional;
    00468     PFN_NUMBER Page;
    00469     PFN_NUMBER LastPage;
    00470     PFN_NUMBER OriginalLastPage;
    00471     PFN_NUMBER start;
    00472     PFN_NUMBER PagesReleased;
    00473     PMMPFN Pfn1;
    00474     PMMPFN StartPfn;
    00475     PMMPFN EndPfn;
    00476     KIRQL OldIrql;
    00477     PFN_NUMBER StartPage;
    00478     PFN_NUMBER EndPage;
    00479     PFN_COUNT NumberOfPages;
    00480     SPFN_NUMBER MaxPages;
    00481     PFN_NUMBER PageFrameIndex;
    00482     PFN_NUMBER RemovedPages;
    00483     LOGICAL Inserted;
    00484     NTSTATUS Status;
    00485     PMMPTE PointerPte;
    00486     PMMPTE EndPte;
    00487     PVOID VirtualAddress;
    00488     PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    00489     PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    00490     PPHYSICAL_MEMORY_RUN NewRun;
    00491     LOGICAL PfnDatabaseIsPhysical;
    00492
    00493     ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    00494
    00495     ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    00496     ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
    00497
    00498     if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
    00499
    00500         //
    00501         // The system must be configured for dynamic memory addition.  This is
    00502         // not strictly required to remove the memory, but it's better to check
    00503         // for it now under the assumption that the administrator is probably
    00504         // going to want to add this range of memory back in - better to give
    00505         // the error now and refuse the removal than to refuse the addition
    00506         // later.
    00507         //
    00508
    00509         if (MmDynamicPfn == FALSE) {
    00510             return STATUS_NOT_SUPPORTED;
    00511         }
    00512
    00513         PfnDatabaseIsPhysical = TRUE;
    00514     }
    00515     else {
    00516         PfnDatabaseIsPhysical = FALSE;
    00517     }
    00518
    00519     StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    00520     NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
    00521
    00522     EndPage = StartPage + NumberOfPages;
    00523
    00524     if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
    00525
    00526         //
    00527         // Truncate the request into something that can be mapped by the PFN
    00528         // database.
    00529         //
    00530
    00531         EndPage = MmHighestPossiblePhysicalPage + 1;
    00532         NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
    00533     }
    00534
    00535     //
    00536     // The range cannot wrap.
    00537     //
    00538
    00539     if (StartPage >= EndPage) {
    00540         return STATUS_INVALID_PARAMETER_1;
    00541     }
    00542
    00543     StartPfn = MI_PFN_ELEMENT (StartPage);
    00544     EndPfn = MI_PFN_ELEMENT (EndPage);
    00545
    00546     ExAcquireFastMutex (&MmDynamicMemoryMutex);
    00547
    00548 #if DBG
    00549     MiDynmemData[0] += 1;
    00550 #endif
    00551
    00552     //
    00553     // Decrease all commit limits to reflect the removed memory.
    00554     //
    00555
    00556     ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
    00557
    00558     ASSERT (MmTotalCommitLimit <= MmTotalCommitLimitMaximum);
    00559
    00560     if ((NumberOfPages + 100 > MmTotalCommitLimit - MmTotalCommittedPages) ||
    00561         (MmTotalCommittedPages > MmTotalCommitLimit)) {
    00562
    00563 #if DBG
    00564         MiDynmemData[1] += 1;
    00565 #endif
    00566         ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
    00567         ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00568         return STATUS_INSUFFICIENT_RESOURCES;
    00569     }
    00570
    00571     MmTotalCommitLimit -= NumberOfPages;
    00572     MmTotalCommitLimitMaximum -= NumberOfPages;
    00573
    00574     ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
    00575
    00576     //
    00577     // Check for outstanding promises that cannot be broken.
    00578     //
    00579
    00580     LOCK_PFN (OldIrql);
    00581
    00582     MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - 100;
    00583
    00584     if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
    00585 #if DBG
    00586         MiDynmemData[2] += 1;
    00587 #endif
    00588         UNLOCK_PFN (OldIrql);
    00589         Status = STATUS_INSUFFICIENT_RESOURCES;
    00590         goto giveup2;
    00591     }
    00592
    00593     MmResidentAvailablePages -= NumberOfPages;
    00594     MmNumberOfPhysicalPages -= NumberOfPages;
    00595
    00596     //
    00597     // The range must be contained in a single entry.  It is permissible for
    00598     // it to be part of a single entry, but it must not cross multiple entries.
    00599     //
    00600
    00601     Additional = (ULONG)-2;
    00602
    00603     start = 0;
    00604     do {
    00605
    00606         Page = MmPhysicalMemoryBlock->Run[start].BasePage;
    00607         LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
    00608
    00609         if ((StartPage >= Page) && (EndPage <= LastPage)) {
    00610             if ((StartPage == Page) && (EndPage == LastPage)) {
    00611                 Additional = (ULONG)-1;
    00612             }
    00613             else if ((StartPage == Page) || (EndPage == LastPage)) {
    00614                 Additional = 0;
    00615             }
    00616             else {
    00617                 Additional = 1;
    00618             }
    00619             break;
    00620         }
    00621
    00622         start += 1;
    00623
    00624     } while (start != MmPhysicalMemoryBlock->NumberOfRuns);
    00625
    00626     if (Additional == (ULONG)-2) {
    00627 #if DBG
    00628         MiDynmemData[3] += 1;
    00629 #endif
    00630         MmResidentAvailablePages += NumberOfPages;
    00631         MmNumberOfPhysicalPages += NumberOfPages;
    00632         UNLOCK_PFN (OldIrql);
    00633         Status = STATUS_CONFLICTING_ADDRESSES;
    00634         goto giveup2;
    00635     }
    00636
    00637     for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
    00638         Pfn1->u3.e1.RemovalRequested = 1;
    00639     }
    00640
    00641     //
    00642     // The free and zero lists must be pruned now before releasing the PFN
    00643     // lock otherwise if another thread allocates the page from these lists,
    00644     // the allocation will clear the RemovalRequested flag forever.
    00645     //
    00646
    00647     RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);
    00648
    00649     if (RemovedPages != NumberOfPages) {
    00650
    00651 #if DBG
    00652 retry:
    00653 #endif
    00654
    00655         Pfn1 = StartPfn;
    00656
    00657         InterlockedIncrement (&MiDelayPageFaults);
    00658
    00659         for (i = 0; i < 5; i += 1) {
    00660
    00661             UNLOCK_PFN (OldIrql);
    00662
    00663             //
    00664             // Attempt to move pages to the standby list.  Note that only the
    00665             // pages with RemovalRequested set are moved.
    00666             //
    00667
    00668             MiTrimRemovalPagesOnly = TRUE;
    00669
    00670             MiEmptyAllWorkingSets ();
    00671
    00672             MiTrimRemovalPagesOnly = FALSE;
    00673
    00674             MiFlushAllPages ();
    00675
    00676             KeDelayExecutionThread (KernelMode, FALSE, &MmHalfSecond);
    00677
    00678             LOCK_PFN (OldIrql);
    00679
    00680             RemovedPages += MiRemovePhysicalPages (StartPage, EndPage);
    00681
    00682             if (RemovedPages == NumberOfPages) {
    00683                 break;
    00684             }
    00685
    00686             //
    00687             // RemovedPages doesn't include pages that were freed directly to
    00688             // the bad page list via MiDecrementReferenceCount.  So use the above
    00689             // check purely as an optimization - and walk here when necessary.
    00690             //
    00691
    00692             for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
    00693                 if (Pfn1->u3.e1.PageLocation != BadPageList) {
    00694                     break;
    00695                 }
    00696             }
    00697
    00698             if (Pfn1 == EndPfn) {
    00699                 RemovedPages = NumberOfPages;
    00700                 break;
    00701             }
    00702         }
    00703
    00704         InterlockedDecrement (&MiDelayPageFaults);
    00705     }
    00706
    00707     if (RemovedPages != NumberOfPages) {
    00708 #if DBG
    00709         MiDynmemData[4] += 1;
    00710         if (MiShowStuckPages != 0) {
    00711
    00712             RemovedPages = 0;
    00713             for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
    00714                 if (Pfn1->u3.e1.PageLocation != BadPageList) {
    00715                     RemovedPages += 1;
    00716                 }
    00717             }
    00718
    00719             ASSERT (RemovedPages != 0);
    00720
    00721             DbgPrint("MmRemovePhysicalMemory : could not get %d of %d pages\n",
    00722                 RemovedPages, NumberOfPages);
    00723
    00724             if (MiShowStuckPages & 0x2) {
    00725
    00726                 ULONG PfnsPrinted;
    00727                 ULONG EnoughShown;
    00728                 PMMPFN FirstPfn;
    00729                 PFN_COUNT PfnCount;
    00730
    00731                 PfnCount = 0;
    00732                 PfnsPrinted = 0;
    00733                 EnoughShown = 100;
    00734
    00735                 if (MiShowStuckPages & 0x4) {
    00736                     EnoughShown = (ULONG)-1;
    00737                 }
    00738
    00739                 DbgPrint("Stuck PFN list: ");
    00740                 for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
    00741                     if (Pfn1->u3.e1.PageLocation != BadPageList) {
    00742                         if (PfnCount == 0) {
    00743                             FirstPfn = Pfn1;
    00744                         }
    00745                         PfnCount += 1;
    00746                     }
    00747                     else {
    00748                         if (PfnCount != 0) {
    00749                             DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
    00750                                                     (FirstPfn - MmPfnDatabase) + PfnCount - 1);
    00751                             PfnsPrinted += 1;
    00752                             if (PfnsPrinted == EnoughShown) {
    00753                                 break;
    00754                             }
    00755                             PfnCount = 0;
    00756                         }
    00757                     }
    00758                 }
    00759                 if (PfnCount != 0) {
    00760                     DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
    00761                                             (FirstPfn - MmPfnDatabase) + PfnCount - 1);
    00762                 }
    00763                 DbgPrint("\n");
    00764             }
    00765             if (MiShowStuckPages & 0x8) {
    00766                 DbgBreakPoint ();
    00767             }
    00768             if (MiShowStuckPages & 0x10) {
    00769                 goto retry;
    00770             }
    00771         }
    00772 #endif
    00773         UNLOCK_PFN (OldIrql);
    00774         Status = STATUS_NO_MEMORY;
    00775         goto giveup;
    00776     }
    00777
    00778 #if DBG
    00779     for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
    00780         ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
    00781     }
    00782 #endif
    00783
    00784     //
    00785     // All the pages in the range have been removed.  Update the physical
    00786     // memory blocks and other associated housekeeping.
    00787     //
    00788
    00789     if (Additional == 0) {
    00790
    00791         //
    00792         // The range can be split off from an end of an existing chunk so no
    00793         // pool growth or shrinkage is required.
    00794         //
    00795
    00796         NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
    00797         OldPhysicalMemoryBlock = NULL;
    00798     }
    00799     else {
    00800
    00801         //
    00802         // The range cannot be split off from an end of an existing chunk so
    00803         // pool growth or shrinkage is required.
    00804         //
    00805
    00806         UNLOCK_PFN (OldIrql);
    00807
    00808         i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
    00809              (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));
    00810
    00811         NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
    00812                                                         i,
    00813                                                         '  mM');
    00814
    00815         if (NewPhysicalMemoryBlock == NULL) {
    00816             Status = STATUS_INSUFFICIENT_RESOURCES;
    00817 #if DBG
    00818             MiDynmemData[5] += 1;
    00819 #endif
    00820             goto giveup;
    00821         }
    00822
    00823         OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
    00824         RtlZeroMemory (NewPhysicalMemoryBlock, i);
    00825
    00826         LOCK_PFN (OldIrql);
    00827     }
    00828
    00829     //
    00830     // Remove or split the requested range from the existing memory block.
    00831     //
    00832
    00833     NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
    00834     NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;
    00835
    00836     NewRun = &NewPhysicalMemoryBlock->Run[0];
    00837     start = 0;
    00838     Inserted = FALSE;
    00839
    00840     do {
    00841
    00842         Page = MmPhysicalMemoryBlock->Run[start].BasePage;
    00843         LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
    00844
    00845         if (Inserted == FALSE) {
    00846
    00847             if ((StartPage >= Page) && (EndPage <= LastPage)) {
    00848
    00849                 if ((StartPage == Page) && (EndPage == LastPage)) {
    00850                     ASSERT (Additional == -1);
    00851                     start += 1;
    00852                     continue;
    00853                 }
    00854                 else if ((StartPage == Page) || (EndPage == LastPage)) {
    00855                     ASSERT (Additional == 0);
    00856                     if (StartPage == Page) {
    00857                         MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
    00858                     }
    00859                     MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
    00860                 }
    00861                 else {
    00862                     ASSERT (Additional == 1);
    00863
    00864                     OriginalLastPage = LastPage;
    00865
    00866                     MmPhysicalMemoryBlock->Run[start].PageCount =
    00867                         StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;
    00868
    00869                     *NewRun = MmPhysicalMemoryBlock->Run[start];
    00870                     NewRun += 1;
    00871
    00872                     NewRun->BasePage = EndPage;
    00873                     NewRun->PageCount = OriginalLastPage - EndPage;
    00874                     NewRun += 1;
    00875
    00876                     start += 1;
    00877                     continue;
    00878                 }
    00879
    00880                 Inserted = TRUE;
    00881             }
    00882         }
    00883
    00884         *NewRun = MmPhysicalMemoryBlock->Run[start];
    00885         NewRun += 1;
    00886         start += 1;
    00887
    00888     } while (start != MmPhysicalMemoryBlock->NumberOfRuns);
    00889
    00890     //
    00891     // Repoint the MmPhysicalMemoryBlock at the new chunk.
    00892     // Free the old block after releasing the PFN lock.
    00893     //
    00894
    00895     MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    00896
    00897     if (EndPage - 1 == MmHighestPhysicalPage) {
    00898         MmHighestPhysicalPage = StartPage - 1;
    00899     }
    00900
    00901     //
    00902     // Throw away all the removed pages that are currently enqueued.
    00903     //
    00904
    00905     for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
    00906
    00907         ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
    00908         ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
    00909
    00910         MiUnlinkPageFromList (Pfn1);
    00911
    00912         ASSERT (Pfn1->u1.Flink == 0);
    00913         ASSERT (Pfn1->u2.Blink == 0);
    00914         ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
    00915         ASSERT64 (Pfn1->UsedPageTableEntries == 0);
    00916
    00917         Pfn1->PteAddress = PFN_REMOVED;
    00918         Pfn1->u3.e2.ShortFlags = 0;
    00919         Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
    00920         Pfn1->PteFrame = 0;
    00921     }
    00922
    00923     //
    00924     // Now that the removed pages have been discarded, eliminate the PFN
    00925     // entries that mapped them.  Straddling entries left over from an
    00926     // adjacent earlier removal are not collapsed at this point.
    00927     //
    00928     //
    00929
    00930     PagesReleased = 0;
    00931
    00932     if (PfnDatabaseIsPhysical == FALSE) {
    00933
    00934         VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
    00935         PointerPte = MiGetPteAddress (VirtualAddress);
    00936         EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));
    00937
    00938         while (PointerPte < EndPte) {
    00939             PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
    00940             Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    00941             ASSERT (Pfn1->u2.ShareCount == 1);
    00942             ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
    00943             Pfn1->u2.ShareCount = 0;
    00944             MI_SET_PFN_DELETED (Pfn1);
    00945 #if DBG
    00946             Pfn1->u3.e1.PageLocation = StandbyPageList;
    00947 #endif //DBG
    00948             MiDecrementReferenceCount (PageFrameIndex);
    00949
    00950             KeFlushSingleTb (VirtualAddress,
    00951                              TRUE,
    00952                              TRUE,
    00953                              (PHARDWARE_PTE)PointerPte,
    00954                              ZeroKernelPte.u.Flush);
    00955
    00956             PagesReleased += 1;
    00957             PointerPte += 1;
    00958             VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
    00959         }
    00960
    00961         MmResidentAvailablePages += PagesReleased;
    00962     }
    00963
    00964 #if DBG
    00965     MiDynmemData[6] += 1;
    00966 #endif
    00967
    00968     UNLOCK_PFN (OldIrql);
    00969
    00970     if (PagesReleased != 0) {
    00971         MiReturnCommitment (PagesReleased);
    00972     }
    00973
    00974     ExReleaseFastMutex (&MmDynamicMemoryMutex);
    00975
    00976     if (OldPhysicalMemoryBlock != NULL) {
    00977         ExFreePool (OldPhysicalMemoryBlock);
    00978     }
    00979
    00980     NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
    00981
    00982     return STATUS_SUCCESS;
    00983
    00984 giveup:
    00985
    00986     //
    00987     // All the pages in the range were not obtained.  Back everything out.
    00988     //
    00989
    00990     PageFrameIndex = StartPage;
    00991     Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    00992
    00993     LOCK_PFN (OldIrql);
    00994
    00995     while (PageFrameIndex < EndPage) {
    00996
    00997         ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
    00998
    00999         Pfn1->u3.e1.RemovalRequested = 0;
    01000
    01001         if ((Pfn1->u3.e1.PageLocation == BadPageList) &&
    01002             (Pfn1->u3.e1.ParityError == 0)) {
    01003
    01004             MiUnlinkPageFromList (Pfn1);
    01005             MiInsertPageInList (MmPageLocationList[FreePageList],
    01006                                 PageFrameIndex);
    01007         }
    01008
    01009         Pfn1 += 1;
    01010         PageFrameIndex += 1;
    01011     }
    01012
    01013     MmResidentAvailablePages += NumberOfPages;
    01014     MmNumberOfPhysicalPages += NumberOfPages;
    01015
    01016     UNLOCK_PFN (OldIrql);
    01017
    01018 giveup2:
    01019
    01020     ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
    01021     MmTotalCommitLimit += NumberOfPages;
    01022     MmTotalCommitLimitMaximum += NumberOfPages;
    01023     ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
    01024
    01025     ExReleaseFastMutex (&MmDynamicMemoryMutex);
    01026
    01027     return Status;
    01028 }
    01029
    01030
    01031 PPHYSICAL_MEMORY_RANGE
    01032 MmGetPhysicalMemoryRanges (
    01033     VOID
    01034     )
    01035
    01036 /*++
    01037
    01038 Routine Description:
    01039
    01040     This routine returns the virtual address of a nonpaged pool block which
    01041     contains the physical memory ranges in the system.
    01042
    01043     The returned block contains physical address and page count pairs.
    01044     The last entry contains zero for both.
    01045
    01046     The caller must understand that this block can change at any point before
    01047     or after this snapshot.
    01048
    01049     It is the caller's responsibility to free this block.
    01050
    01051 Arguments:
    01052
    01053     None.
    01054
    01055 Return Value:
    01056
    01057     NULL on failure.
    01058
    01059 Environment:
    01060
    01061     Kernel mode.  PASSIVE level.  No locks held.
    01062
    01063 --*/
    01064
    01065 {
    01066     ULONG i;
    01067     KIRQL OldIrql;
    01068     PPHYSICAL_MEMORY_RANGE p;
    01069     PPHYSICAL_MEMORY_RANGE PhysicalMemoryBlock;
    01070
    01071     ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
    01072
    01073     ExAcquireFastMutex (&MmDynamicMemoryMutex);
    01074
    01075     i = sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1);
    01076
    01077     PhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
    01078                                                  i,
    01079                                                  'hPmM');
    01080
    01081     if (PhysicalMemoryBlock == NULL) {
    01082         ExReleaseFastMutex (&MmDynamicMemoryMutex);
    01083         return NULL;
    01084     }
    01085
    01086     p = PhysicalMemoryBlock;
    01087
    01088     LOCK_PFN (OldIrql);
    01089
    01090     ASSERT (i == (sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
    01091
    01092     for (i = 0; i < MmPhysicalMemoryBlock->NumberOfRuns; i += 1) {
    01093         p->BaseAddress.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].BasePage * PAGE_SIZE;
    01094         p->NumberOfBytes.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].PageCount * PAGE_SIZE;
    01095         p += 1;
    01096     }
    01097
    01098     p->BaseAddress.QuadPart = 0;
    01099     p->NumberOfBytes.QuadPart = 0;
    01100
    01101     UNLOCK_PFN (OldIrql);
    01102
    01103     ExReleaseFastMutex (&MmDynamicMemoryMutex);
    01104
    01105     return PhysicalMemoryBlock;
    01106 }
    01107
    01108 PFN_COUNT
    01109 MiRemovePhysicalPages (
    01110     IN PFN_NUMBER StartPage,
    01111     IN PFN_NUMBER EndPage
    01112     )
    01113
    01114 /*++
    01115
    01116 Routine Description:
    01117
    01118     This routine searches the PFN database for free, zeroed or standby pages
    01119     that are marked for removal.
    01120
    01121 Arguments:
    01122
    01123     StartPage - Supplies the low physical frame number to remove.
    01124
    01125     EndPage - Supplies the last physical frame number to remove.
    01126
    01127 Return Value:
    01128
    01129     Returns the number of pages removed from the free, zeroed and standby lists.
    01130
    01131 Environment:
    01132
    01133     Kernel mode, PFN lock held.
    01134
    01135 --*/
    01136
    01137 {
    01138     PMMPFN Pfn1;
    01139     PMMPFN Pfn2;
    01140     PMMPFN PfnNextColored;
    01141     PMMPFN PfnNextFlink;
    01142     PMMPFN PfnLastColored;
    01143     PFN_NUMBER Page;
    01144     LOGICAL RemovePage;
    01145     ULONG Color;
    01146     PMMCOLOR_TABLES ColorHead;
    01147     PFN_NUMBER MovedPage;
    01148     MMLISTS MemoryList;
    01149     PFN_NUMBER PageNextColored;
    01150     PFN_NUMBER PageNextFlink;
    01151     PFN_NUMBER PageLastColored;
    01152     PFN_COUNT NumberOfPages;
    01153     PMMPFNLIST ListHead;
    01154     LOGICAL RescanNeeded;
    01155
    01156     MM_PFN_LOCK_ASSERT();
    01157
    01158     NumberOfPages = 0;
    01159
    01160 rescan:
    01161
    01162     //
    01163     // Grab all zeroed (and then free) pages first directly from the
    01164     // colored lists to avoid multiple walks down these singly linked lists.
    01165     // Handle transition pages last.
    01166     //
    01167
    01168     for (MemoryList = ZeroedPageList; MemoryList <= FreePageList; MemoryList += 1) {
    01169
    01170         ListHead = MmPageLocationList[MemoryList];
    01171
    01172         for (Color = 0; Color < MmSecondaryColors; Color += 1) {
    01173             ColorHead = &MmFreePagesByColor[MemoryList][Color];
    01174
    01175             MovedPage = MM_EMPTY_LIST;
    01176
    01177             while (ColorHead->Flink != MM_EMPTY_LIST) {
    01178
    01179                 Page = ColorHead->Flink;
    01180
    01181                 Pfn1 = MI_PFN_ELEMENT(Page);
    01182
    01183                 ASSERT ((MMLISTS)Pfn1->u3.e1.PageLocation == MemoryList);
    01184
    01185                 //
    01186                 // The Flink and Blink must be nonzero here for the page
    01187                 // to be on the listhead.  Only code that scans the
    01188                 // MmPhysicalMemoryBlock has to check for the zero case.
    01189                 //
    01190
    01191                 ASSERT (Pfn1->u1.Flink != 0);
    01192                 ASSERT (Pfn1->u2.Blink != 0);
    01193
    01194                 //
    01195                 // See if the page is desired by the caller.
    01196                 //
    01197
    01198                 if (Pfn1->u3.e1.RemovalRequested == 1) {
    01199
    01200                     ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
    01201
    01202                     MiUnlinkFreeOrZeroedPage (Page);
    01203
    01204                     MiInsertPageInList (MmPageLocationList[BadPageList],
    01205                                         Page);
    01206
    01207                     NumberOfPages += 1;
    01208                 }
    01209                 else {
    01210
    01211                     //
    01212                     // Unwanted so put the page on the end of list.
    01213                     // If first time, save pfn.
    01214                     //
    01215
    01216                     if (MovedPage == MM_EMPTY_LIST) {
    01217                         MovedPage = Page;
    01218                     }
    01219                     else if (Page == MovedPage) {
    01220
    01221                         //
    01222                         // No more pages available in this colored chain.
    01223                         //
    01224
    01225                         break;
    01226                     }
    01227
    01228                     //
    01229                     // If the colored chain has more than one entry then
    01230                     // put this page on the end.
    01231                     //
    01232
    01233                     PageNextColored = (PFN_NUMBER)Pfn1->OriginalPte.u.Long;
    01234
    01235                     if (PageNextColored == MM_EMPTY_LIST) {
    01236
    01237                         //
    01238                         // No more pages available in this colored chain.
    01239                         //
    01240
    01241                         break;
    01242                     }
    01243
    01244                     ASSERT (Pfn1->u1.Flink != 0);
    01245                     ASSERT (Pfn1->u1.Flink != MM_EMPTY_LIST);
    01246                     ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01247
    01248                     PfnNextColored = MI_PFN_ELEMENT(PageNextColored);
    01249                     ASSERT ((MMLISTS)PfnNextColored->u3.e1.PageLocation == MemoryList);
    01250                     ASSERT (PfnNextColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01251
    01252                     //
    01253                     // Adjust the free page list so Page
    01254                     // follows PageNextFlink.
    01255                     //
    01256
    01257                     PageNextFlink = Pfn1->u1.Flink;
    01258                     PfnNextFlink = MI_PFN_ELEMENT(PageNextFlink);
    01259
    01260                     ASSERT ((MMLISTS)PfnNextFlink->u3.e1.PageLocation == MemoryList);
    01261                     ASSERT (PfnNextFlink->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01262
    01263                     PfnLastColored = ColorHead->Blink;
    01264                     ASSERT (PfnLastColored != (PMMPFN)MM_EMPTY_LIST);
    01265                     ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
    01266                     ASSERT (PfnLastColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01267                     ASSERT (PfnLastColored->u2.Blink != MM_EMPTY_LIST);
    01268
    01269                     ASSERT ((MMLISTS)PfnLastColored->u3.e1.PageLocation == MemoryList);
    01270                     PageLastColored = PfnLastColored - MmPfnDatabase;
    01271
    01272                     if (ListHead->Flink == Page) {
    01273
    01274                         ASSERT (Pfn1->u2.Blink == MM_EMPTY_LIST);
    01275                         ASSERT (ListHead->Blink != Page);
    01276
    01277                         ListHead->Flink = PageNextFlink;
    01278
    01279                         PfnNextFlink->u2.Blink = MM_EMPTY_LIST;
    01280                     }
    01281                     else {
    01282
    01283                         ASSERT (Pfn1->u2.Blink != MM_EMPTY_LIST);
    01284                         ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01285                         ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u3.e1.PageLocation == MemoryList);
    01286
    01287                         MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink = PageNextFlink;
    01288                         PfnNextFlink->u2.Blink = Pfn1->u2.Blink;
    01289                     }
    01290
    01291 #if DBG
    01292                     if (PfnLastColored->u1.Flink == MM_EMPTY_LIST) {
    01293                         ASSERT (ListHead->Blink == PageLastColored);
    01294                     }
    01295 #endif
    01296
    01297                     Pfn1->u1.Flink = PfnLastColored->u1.Flink;
    01298                     Pfn1->u2.Blink = PageLastColored;
    01299
    01300                     if (ListHead->Blink == PageLastColored) {
    01301                         ListHead->Blink = Page;
    01302                     }
    01303
    01304                     //
    01305                     // Adjust the colored chains.
    01306                     //
    01307
    01308                     if (PfnLastColored->u1.Flink != MM_EMPTY_LIST) {
    01309                         ASSERT (MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->PteFrame != MI_MAGIC_AWE_PTEFRAME);
    01310                         ASSERT ((MMLISTS)(MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u3.e1.PageLocation) == MemoryList);
    01311                         MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u2.Blink = Page;
    01312                     }
    01313
    01314                     PfnLastColored->u1.Flink = Page;
    01315
    01316                     ColorHead->Flink = PageNextColored;
    01317                     Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
    01318
    01319                     ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
    01320                     PfnLastColored->OriginalPte.u.Long = Page;
    01321                     ColorHead->Blink = Pfn1;
    01322                 }
    01323             }
    01324         }
    01325     }
    01326
    01327     RescanNeeded = FALSE;
    01328     Pfn1 = MI_PFN_ELEMENT (StartPage);
    01329
    01330     do {
    01331
    01332         if ((Pfn1->u3.e1.PageLocation == StandbyPageList) &&
    01333             (Pfn1->u1.Flink != 0) &&
    01334             (Pfn1->u2.Blink != 0) &&
    01335             (Pfn1->u3.e2.ReferenceCount == 0)) {
    01336
    01337             ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
    01338
    01339             RemovePage = TRUE;
    01340
    01341             if (Pfn1->u3.e1.RemovalRequested == 0) {
    01342
    01343                 //
    01344                 // This page is not directly needed for a hot remove - but if
    01345                 // it contains a chunk of prototype PTEs (and this chunk is
    01346                 // in a page that needs to be removed), then any pages
    01347                 // referenced by transition prototype PTEs must also be removed
    01348                 // before the desired page can be removed.
    01349                 //
    01350                 // The same analogy holds for page parent, directory and table
    01351                 // pages.
    01352                 //
    01353
    01354                 Pfn2 = MI_PFN_ELEMENT (Pfn1->PteFrame);
    01355                 if (Pfn2->u3.e1.RemovalRequested == 0) {
    01356 #if defined (_WIN64)
    01357                     Pfn2 = MI_PFN_ELEMENT (Pfn2->PteFrame);
    01358                     if (Pfn2->u3.e1.RemovalRequested == 0) {
    01359                         RemovePage = FALSE;
    01360                     }
    01361                     else if (Pfn2->u2.ShareCount == 1) {
    01362                         RescanNeeded = TRUE;
    01363                     }
    01364 #else
    01365                     RemovePage = FALSE;
    01366 #endif
    01367                 }
    01368                 else if (Pfn2->u2.ShareCount == 1) {
    01369                     RescanNeeded = TRUE;
    01370                 }
    01371             }
    01372
    01373             if (RemovePage == TRUE) {
    01374
    01375                 //
    01376                 // This page is in the desired range - grab it.
    01377                 //
    01378
    01379                 MiUnlinkPageFromList (Pfn1);
    01380                 MiRestoreTransitionPte (StartPage);
    01381                 MiInsertPageInList (MmPageLocationList[BadPageList],
    01382                                     StartPage);
    01383                 NumberOfPages += 1;
    01384             }
    01385         }
    01386
    01387         StartPage += 1;
    01388         Pfn1 += 1;
    01389
    01390     } while (StartPage < EndPage);
    01391
    01392     if (RescanNeeded == TRUE) {
    01393
    01394         //
    01395         // A page table, directory or parent was freed by removing a transition
    01396         // page from the cache.  Rescan from the top to pick it up.
    01397         //
    01398
    01399 #if DBG
    01400         MiDynmemData[7] += 1;
    01401 #endif
    01402
    01403         goto rescan;
    01404     }
    01405 #if DBG
    01406     else {
    01407         MiDynmemData[8] += 1;
    01408     }
    01409 #endif
    01410
    01411     return NumberOfPages;
    01412 }