test: flush.c Source File

Main Page | Class Hierarchy | Class List | File List | Class Members | File Members

# flush.c

Go to the documentation of this file.

    00001 /*++
    00002
    00003 Copyright (c) 1990  Microsoft Corporation
    00004 Copyright (c) 1993  Digital Equipment Corporation
    00005
    00006 Module Name:
    00007
    00008     flush.c
    00009
    00010 Abstract:
    00011
    00012     This module implements Alpha AXP machine dependent kernel functions to flush
    00013     the data and instruction caches and to flush I/O buffers.
    00014
    00015 Author:
    00016
    00017     David N. Cutler (davec) 26-Apr-1990
    00018     Joe Notarangelo  29-Nov-1993
    00019
    00020 Environment:
    00021
    00022     Kernel mode only.
    00023
    00024 Revision History:
    00025
    00026 --*/
    00027
    00028 #include "ki.h"
    00029
    00030
    00031 //
    00032 // Define forward referenced prototypes.
    00033 //
    00034
    00035 VOID
    00036 KiSweepDcacheTarget (
    00037     IN PKIPI_CONTEXT SignalDone,
    00038     IN PVOID Count,
    00039     IN PVOID Parameter2,
    00040     IN PVOID Parameter3
    00041     );
    00042
    00043 VOID
    00044 KiSweepIcacheTarget (
    00045     IN PKIPI_CONTEXT SignalDone,
    00046     IN PVOID Count,
    00047     IN PVOID Parameter2,
    00048     IN PVOID Parameter3
    00049     );
    00050
    00051 VOID
    00052 KiFlushIoBuffersTarget (
    00053     IN PKIPI_CONTEXT SignalDone,
    00054     IN PVOID Mdl,
    00055     IN PVOID ReadOperation,
    00056     IN PVOID DmaOperation
    00057     );
    00058
    00059 VOID
    00060 KiSynchronizeMemoryAccessTarget (
    00061     IN PKIPI_CONTEXT SignalDone,
    00062     IN PVOID Parameter1,
    00063     IN PVOID Parameter2,
    00064     IN PVOID Parameter3
    00065     );
    00066
    00067 ULONG KiSynchronizeMemoryCallCount = 0;
    00068
    00069
    00070 VOID
    00071 KeSweepDcache (
    00072     IN BOOLEAN AllProcessors
    00073     )
    00074
    00075 /*++
    00076
    00077 Routine Description:
    00078
    00079     This function flushes the data cache on all processors that are currently
    00080     running threads which are children of the current process or flushes the
    00081     data cache on all processors in the host configuration.
    00082
    00083 Arguments:
    00084
    00085     AllProcessors - Supplies a boolean value that determines which data
    00086         caches are flushed.
    00087
    00088 Return Value:
    00089
    00090     None.
    00091
    00092 --*/
    00093
    00094 {
    00095
    00096     KIRQL OldIrql;
    00097     KAFFINITY TargetProcessors;
    00098
    00099     ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
    00100
    00101     //
    00102     // Raise IRQL to synchronization level to prevent a context switch.
    00103     //
    00104
    00105 #if !defined(NT_UP)
    00106
    00107     OldIrql = KeRaiseIrqlToSynchLevel();
    00108
    00109     //
    00110     // Compute the set of target processors and send the sweep parameters
    00111     // to the target processors, if any, for execution.
    00112     //
    00113
    00114     TargetProcessors = KeActiveProcessors & PCR->NotMember;
    00115     if (TargetProcessors != 0) {
    00116         KiIpiSendPacket(TargetProcessors,
    00117                         KiSweepDcacheTarget,
    00118                         NULL,
    00119                         NULL,
    00120                         NULL);
    00121     }
    00122
    00123     IPI_INSTRUMENT_COUNT(KeGetCurrentPrcb()->Number, SweepDcache);
    00124
    00125 #endif
    00126
    00127     //
    00128     // Sweep the data cache on the current processor.
    00129     //
    00130
    00131     HalSweepDcache();
    00132
    00133     //
    00134     // Wait until all target processors have finished sweeping the their
    00135     // data cache.
    00136     //
    00137
    00138 #if !defined(NT_UP)
    00139
    00140     if (TargetProcessors != 0) {
    00141         KiIpiStallOnPacketTargets(TargetProcessors);
    00142     }
    00143
    00144     //
    00145     // Lower IRQL to its previous level and return.
    00146     //
    00147
    00148     KeLowerIrql(OldIrql);
    00149
    00150 #endif
    00151
    00152     return;
    00153 }
    00154
    00155 VOID
    00156 KiSweepDcacheTarget (
    00157     IN PKIPI_CONTEXT SignalDone,
    00158     IN PVOID Parameter1,
    00159     IN PVOID Parameter2,
    00160     IN PVOID Parameter3
    00161     )
    00162
    00163 /*++
    00164
    00165 Routine Description:
    00166
    00167     This is the target function for sweeping the data cache on target
    00168     processors.
    00169
    00170 Arguments:
    00171
    00172     SignalDone - Supplies a pointer to a variable that is cleared when the
    00173         requested operation has been performed
    00174
    00175     Parameter1 - Parameter3 - not used
    00176
    00177 Return Value:
    00178
    00179     None.
    00180
    00181 --*/
    00182
    00183 {
    00184
    00185     //
    00186     // Sweep the data cache on the current processor and clear the sweep
    00187     // data cache packet address to signal the source to continue.
    00188     //
    00189
    00190 #if !defined(NT_UP)
    00191
    00192     HalSweepDcache();
    00193     KiIpiSignalPacketDone(SignalDone);
    00194     IPI_INSTRUMENT_COUNT(KeGetCurrentPrcb()->Number, SweepDcache);
    00195
    00196 #endif
    00197
    00198     return;
    00199 }
    00200
    00201 VOID
    00202 KeSweepIcache (
    00203     IN BOOLEAN AllProcessors
    00204     )
    00205
    00206 /*++
    00207
    00208 Routine Description:
    00209
    00210     This function flushes the instruction cache on all processors that are
    00211     currently running threads which are children of the current process or
    00212     flushes the instruction cache on all processors in the host configuration.
    00213
    00214 Arguments:
    00215
    00216     AllProcessors - Supplies a boolean value that determines which instruction
    00217         caches are flushed.
    00218
    00219 Return Value:
    00220
    00221     None.
    00222
    00223 --*/
    00224
    00225 {
    00226
    00227     KIRQL OldIrql;
    00228     KAFFINITY TargetProcessors;
    00229
    00230     ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
    00231
    00232     //
    00233     // Raise IRQL to synchronization level to prevent a context switch.
    00234     //
    00235
    00236 #if !defined(NT_UP)
    00237
    00238     OldIrql = KeRaiseIrqlToSynchLevel();
    00239
    00240     //
    00241     // Compute the set of target processors and send the sweep parameters
    00242     // to the target processors, if any, for execution.
    00243     //
    00244
    00245     TargetProcessors = KeActiveProcessors & PCR->NotMember;
    00246     if (TargetProcessors != 0) {
    00247         KiIpiSendPacket(TargetProcessors,
    00248                         KiSweepIcacheTarget,
    00249                         NULL,
    00250                         NULL,
    00251                         NULL);
    00252     }
    00253
    00254     IPI_INSTRUMENT_COUNT(KeGetCurrentPrcb()->Number, SweepIcache);
    00255
    00256 #endif
    00257
    00258     //
    00259     // Sweep the instruction cache on the current processor.
    00260     //
    00261
    00262     KiImb();
    00263
    00264     //
    00265     // Wait until all target processors have finished sweeping the their
    00266     // instruction cache.
    00267     //
    00268
    00269 #if !defined(NT_UP)
    00270
    00271     if (TargetProcessors != 0) {
    00272         KiIpiStallOnPacketTargets(TargetProcessors);
    00273     }
    00274
    00275     //
    00276     // Lower IRQL to its previous level and return.
    00277     //
    00278
    00279     KeLowerIrql(OldIrql);
    00280
    00281 #endif
    00282
    00283     return;
    00284 }
    00285
    00286 VOID
    00287 KiSweepIcacheTarget (
    00288     IN PKIPI_CONTEXT SignalDone,
    00289     IN PVOID Parameter1,
    00290     IN PVOID Parameter2,
    00291     IN PVOID Parameter3
    00292     )
    00293
    00294 /*++
    00295
    00296 Routine Description:
    00297
    00298     This is the target function for sweeping the instruction cache on
    00299     target processors.
    00300
    00301 Arguments:
    00302
    00303     SignalDone - Supplies a pointer to a variable that is cleared when the
    00304         requested operation has been performed
    00305
    00306     Parameter1 - Parameter3 - not used
    00307
    00308
    00309 Return Value:
    00310
    00311     None.
    00312
    00313 --*/
    00314
    00315 {
    00316
    00317     //
    00318     // Sweep the instruction cache on the current processor and clear
    00319     // the sweep instruction cache packet address to signal the source
    00320     // to continue.
    00321     //
    00322
    00323 #if !defined(NT_UP)
    00324
    00325     KiImb();
    00326     KiIpiSignalPacketDone(SignalDone);
    00327     IPI_INSTRUMENT_COUNT(KeGetCurrentPrcb()->Number, SweepIcache);
    00328
    00329 #endif
    00330
    00331     return;
    00332 }
    00333
    00334 VOID
    00335 KeSweepIcacheRange (
    00336     IN BOOLEAN AllProcessors,
    00337     IN PVOID BaseAddress,
    00338     IN ULONG_PTR Length
    00339     )
    00340
    00341 /*++
    00342
    00343 Routine Description:
    00344
    00345     This function flushes the an range of virtual addresses from the primary
    00346     instruction cache on all processors that are currently running threads
    00347     which are children of the current process or flushes the range of virtual
    00348     addresses from the primary instruction cache on all processors in the host
    00349     configuration.
    00350
    00351 Arguments:
    00352
    00353     AllProcessors - Supplies a boolean value that determines which instruction
    00354         caches are flushed.
    00355
    00356     BaseAddress - Supplies a pointer to the base of the range that is flushed.
    00357
    00358     Length - Supplies the length of the range that is flushed if the base
    00359         address is specified.
    00360
    00361 Return Value:
    00362
    00363     None.
    00364
    00365 --*/
    00366
    00367 {
    00368
    00369     KeSweepIcache(AllProcessors);
    00370     return;
    00371 }
    00372
    00373 VOID
    00374 KeFlushIoBuffers (
    00375     IN PMDL Mdl,
    00376     IN BOOLEAN ReadOperation,
    00377     IN BOOLEAN DmaOperation
    00378     )
    00379
    00380 /*++
    00381
    00382 Routine Description:
    00383
    00384     This function flushes the I/O buffer specified by the memory descriptor
    00385     list from the data cache on all processors.
    00386
    00387     Alpha requires that caches be coherent with respect to I/O. All that
    00388     this routine needs to do is execute a memory barrier on the current
    00389     processor. However, in order to maintain i-stream coherency, all
    00390     processors must execute the IMB PAL call in the case of page reads.
    00391     Thus, all processors are IPI'd to perform the IMB for any flush
    00392     that is a DmaOperation, a ReadOperation, and an MDL_IO_PAGE_READ.
    00393
    00394
    00395 Arguments:
    00396
    00397     Mdl - Supplies a pointer to a memory descriptor list that describes the
    00398         I/O buffer location.
    00399
    00400     ReadOperation - Supplies a boolean value that determines whether the I/O
    00401         operation is a read into memory.
    00402
    00403     DmaOperation - Supplies a boolean value that determines whether the I/O
    00404         operation is a DMA operation.
    00405
    00406 Return Value:
    00407
    00408     None.
    00409
    00410 --*/
    00411
    00412 {
    00413     KIRQL OldIrql;
    00414     KAFFINITY TargetProcessors;
    00415
    00416     ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
    00417
    00418     KiMb();
    00419
    00420     //
    00421     // If the operation is a DMA operation, then check if the flush
    00422     // can be avoided because the host system supports the right set
    00423     // of cache coherency attributes. Otherwise, the flush can also
    00424     // be avoided if the operation is a programmed I/O and not a page
    00425     // read.
    00426     //
    00427
    00428     if (DmaOperation != FALSE) {
    00429         if (ReadOperation != FALSE) {
    00430             if ((KiDmaIoCoherency & DMA_READ_ICACHE_INVALIDATE) != 0) {
    00431
    00432                 ASSERT((KiDmaIoCoherency & DMA_READ_DCACHE_INVALIDATE) != 0);
    00433
    00434                 return;
    00435
    00436             } else if (((Mdl->MdlFlags & MDL_IO_PAGE_READ) == 0) &&
    00437                 ((KiDmaIoCoherency & DMA_READ_DCACHE_INVALIDATE) != 0)) {
    00438                 return;
    00439             }
    00440
    00441         } else if ((KiDmaIoCoherency & DMA_WRITE_DCACHE_SNOOP) != 0) {
    00442             return;
    00443         }
    00444
    00445     } else if ((Mdl->MdlFlags & MDL_IO_PAGE_READ) == 0) {
    00446         return;
    00447     }
    00448
    00449     //
    00450     // Either the operation is a DMA operation and the right coherency
    00451     // attributes are not supported by the host system, or the operation
    00452     // is programmed I/O and a page read.
    00453     //
    00454     // Raise IRQL to synchronization level to prevent a context switch.
    00455     //
    00456
    00457     OldIrql = KeRaiseIrqlToSynchLevel();
    00458
    00459     //
    00460     // Compute the set of target processors, and send the flush I/O
    00461     // parameters to the target processors, if any, for execution.
    00462     //
    00463
    00464 #if !defined(NT_UP)
    00465
    00466     TargetProcessors = KeActiveProcessors & PCR->NotMember;
    00467     if (TargetProcessors != 0) {
    00468         KiIpiSendPacket(TargetProcessors,
    00469                         KiFlushIoBuffersTarget,
    00470                         (PVOID)Mdl,
    00471                         ULongToPtr((ULONG)ReadOperation),
    00472                         ULongToPtr((ULONG)DmaOperation));
    00473     }
    00474
    00475 #endif
    00476
    00477     //
    00478     // Flush I/O buffer on current processor.
    00479     //
    00480
    00481     HalFlushIoBuffers(Mdl, ReadOperation, DmaOperation);
    00482
    00483     //
    00484     // Wait until all target processors have finished flushing the
    00485     // specified I/O buffer.
    00486     //
    00487
    00488 #if !defined(NT_UP)
    00489
    00490     if (TargetProcessors != 0) {
    00491         KiIpiStallOnPacketTargets(TargetProcessors);
    00492     }
    00493
    00494 #endif
    00495
    00496     //
    00497     // Lower IRQL to its previous level and return.
    00498     //
    00499
    00500     KeLowerIrql(OldIrql);
    00501
    00502     return;
    00503 }
    00504
    00505 #if !defined(NT_UP)
    00506
    00507 VOID
    00508 KiFlushIoBuffersTarget (
    00509     IN PKIPI_CONTEXT SignalDone,
    00510     IN PVOID Mdl,
    00511     IN PVOID ReadOperation,
    00512     IN PVOID DmaOperation
    00513     )
    00514
    00515 /*++
    00516
    00517 Routine Description:
    00518
    00519     This is the target function for flushing an I/O buffer on target
    00520     processors.
    00521
    00522 Arguments:
    00523
    00524     SignalDone Supplies a pointer to a variable that is cleared when the
    00525         requested operation has been performed.
    00526
    00527     Mdl - Supplies a pointer to a memory descriptor list that describes the
    00528         I/O buffer location.
    00529
    00530     ReadOperation - Supplies a boolean value that determines whether the I/O
    00531         operation is a read into memory.
    00532
    00533     DmaOperation - Supplies a boolean value that determines whether the I/O
    00534         operation is a DMA operation.
    00535
    00536 Return Value:
    00537
    00538     None.
    00539
    00540 --*/
    00541
    00542 {
    00543
    00544     //
    00545     // Flush the specified I/O buffer on the current processor.
    00546     //
    00547
    00548     HalFlushIoBuffers((PMDL)Mdl,
    00549                       (BOOLEAN)((ULONG_PTR)ReadOperation),
    00550                       (BOOLEAN)((ULONG_PTR)DmaOperation));
    00551
    00552     KiIpiSignalPacketDone(SignalDone);
    00553     IPI_INSTRUMENT_COUNT(KeGetCurrentPrcb()->Number, FlushIoBuffers);
    00554
    00555     return;
    00556 }
    00557
    00558 #endif
    00559
    00560 VOID
    00561 KeSynchronizeMemoryAccess (
    00562     VOID
    00563     )
    00564
    00565 /*++
    00566
    00567 Routine Description:
    00568
    00569     This function synchronizes memory access across all processors in the
    00570     host configurarion.
    00571
    00572 Arguments:
    00573
    00574     None.
    00575
    00576 Return Value:
    00577
    00578     None.
    00579
    00580 --*/
    00581
    00582 {
    00583
    00584     KIRQL OldIrql;
    00585     KAFFINITY TargetProcessors;
    00586
    00587     ASSERT(KeGetCurrentIrql() <= KiSynchIrql);
    00588
    00589     KiSynchronizeMemoryCallCount += 1;
    00590
    00591     //
    00592     // Raise IRQL to synchronization level to prevent a context switch.
    00593     //
    00594
    00595 #if !defined(NT_UP)
    00596
    00597     OldIrql = KeRaiseIrqlToSynchLevel();
    00598
    00599     //
    00600     // Compute the set of target processors and send the synchronize message
    00601     // to the target processors, if any, for execution.
    00602     //
    00603
    00604     TargetProcessors = KeActiveProcessors & PCR->NotMember;
    00605     if (TargetProcessors != 0) {
    00606         KiIpiSendPacket(TargetProcessors,
    00607                         KiSynchronizeMemoryAccessTarget,
    00608                         NULL,
    00609                         NULL,
    00610                         NULL);
    00611     }
    00612
    00613     //
    00614     // On an MP system an implicit memory barrier is executed during the
    00615     // end of the IPI message. On a UP system, a memory barrier must be
    00616     // executed.
    00617     //
    00618
    00619 #else
    00620
    00621     __MB();
    00622
    00623 #endif
    00624
    00625     //
    00626     // Wait until all target processors have finished sweeping the their
    00627     // data cache.
    00628     //
    00629
    00630 #if !defined(NT_UP)
    00631
    00632     if (TargetProcessors != 0) {
    00633         KiIpiStallOnPacketTargets(TargetProcessors);
    00634     }
    00635
    00636     //
    00637     // Lower IRQL to its previous level and return.
    00638     //
    00639
    00640     KeLowerIrql(OldIrql);
    00641
    00642 #endif
    00643
    00644     return;
    00645 }
    00646
    00647 #if !defined(NT_UP)
    00648
    00649
    00650 VOID
    00651 KiSynchronizeMemoryAccessTarget (
    00652     IN PKIPI_CONTEXT SignalDone,
    00653     IN PVOID Parameter1,
    00654     IN PVOID Parameter2,
    00655     IN PVOID Parameter3
    00656     )
    00657
    00658 /*++
    00659
    00660 Routine Description:
    00661
    00662     This function performs no operation, but an implicit memory barrier
    00663     is executed when the IPI message is received.
    00664
    00665 Arguments:
    00666
    00667     SignalDone - Supplies a pointer to a variable that is cleared when the
    00668         requested operation has been performed
    00669
    00670     Parameter1 - Parameter3 - not used
    00671
    00672 Return Value:
    00673
    00674     None.
    00675
    00676 --*/
    00677
    00678 {
    00679
    00680     KiIpiSignalPacketDone(SignalDone);
    00681     return;
    00682 }
    00683
    00684 #endif